├── installation ├── let_jovyan_write.sh ├── docker-stack-scripts │ ├── start-notebook.sh │ ├── start-singleuser.sh │ ├── jupyter_notebook_config.py │ ├── Jupyter-docker-stacks-license.md │ └── start.sh ├── windows_TF_GPU.md ├── miscellaneous_notes.md ├── step_by_step_GCP_install.md ├── step_by_step_MacOSX_install.md └── README.md ├── Dockerfile ├── README.md ├── LICENSE ├── .gitignore ├── notebooks ├── test.ipynb ├── first_tensorflow_graphs.ipynb ├── sigmoid_function.ipynb ├── cross_entropy_cost.ipynb ├── softmax.ipynb ├── stacked_bidirectional_lstm.ipynb ├── vanilla_lstm_in_keras.ipynb ├── bidirectional_lstm.ipynb ├── multi_convnet_architectures.ipynb ├── rnn_in_keras.ipynb ├── intermediate_net_in_keras.ipynb ├── deep_net_in_keras.ipynb ├── intermediate_net_in_tensorflow.ipynb ├── deep_net_in_tensorflow.ipynb ├── first_tensorflow_neurons.ipynb ├── vggnet_in_keras.ipynb ├── alexnet_in_keras.ipynb ├── lenet_in_tensorflow.ipynb ├── generative_adversarial_network.ipynb └── gru_in_keras.ipynb └── Dockerfile-gpu /installation/let_jovyan_write.sh: -------------------------------------------------------------------------------- 1 | # script that provides jovyan, the default Docker container username, permission to write to the directory 2 | sudo chgrp -R 100 nyc-ds-academy/ #to recursively change group to match jovyan 3 | sudo chmod -R g+w nyc-ds-academy/ #to recursively change permissions so jovyan can write to the directory 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/scipy-notebook 2 | 3 | MAINTAINER Jon Krohn 4 | 5 | USER $NB_USER 6 | 7 | # install TensorFlow 8 | RUN pip install --quiet tensorflow==1.3.* 9 | 10 | # install keras: 11 | RUN pip install --quiet keras==2.0.8 12 | 13 | # install NLP packages: 14 | RUN pip install --quiet nltk==3.2.4 15 | RUN pip install --quiet gensim==2.3.0 16 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/start-notebook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | set -e 6 | 7 | if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then 8 | # launched by JupyterHub, use single-user entrypoint 9 | exec /usr/local/bin/start-singleuser.sh $* 10 | else 11 | . /usr/local/bin/start.sh jupyter notebook $* 12 | fi 13 | -------------------------------------------------------------------------------- /installation/windows_TF_GPU.md: -------------------------------------------------------------------------------- 1 | # Install TensorFlow for GPU on Windows 2 | 3 | 1. Install Visual Studio **2015** Community Edition from https://my.visualstudio.com (requires registering a free account). Make sure "Visual C++" is checked. 4 | 2. Install [CUDA 8](https://developer.nvidia.com/cuda-downloads). 5 | 3. Download [cuDNN](https://developer.nvidia.com/rdp/cudnn-download) v5.1, which **isn't** the latest version, and unzip its contents to where you installed CUDA (see details on CUDA and cuDNN installation for Windows 10 [here](https://github.com/philferriere/dlwin)). 6 | 4. Create a conda virtual environment: `C:\> conda create -n tf python=3.5` 7 | 5. Activate the new environment: `C:\> activate tf`. Your prompt should now change to: `(tf) C:\>` 8 | 6. Run `(tf) C:\> pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/windows/gpu/tensorflow_gpu-1.2.1-cp35-cp35m-win_amd64.whl`. 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](http://9ce5546a6ccb7905d04b-7fb50b218d153ea70ad48d487f749494.r12.cf2.rackcdn.com/9c45897cb8616a084ea7d28e4479c64d-22643f7f3bf6f27e0e14a2119c9b0fc3.png) 2 | 3 | # Deep Learning Course 4 | 5 | This repository is for students of the in-classroom [Deep Learning course](nycdatascience.com/courses/deep-learning/) offered by Jon Krohn at the NYC Data Science Academy. 6 | 7 | ## Installation 8 | 9 | Detailed instructions for installing all of the software needed for the course are available [here](https://github.com/the-deep-learners/nyc-ds-academy/tree/master/installation). 10 | 11 | ## Test Notebook 12 | 13 | From the first unit of the course, we will be creating Jupyter notebooks to interactively build Deep Learning models. To test whether your install of TensorFlow was successful, you can try running the test notebook [here](https://github.com/the-deep-learners/nyc-ds-academy/blob/master/notebooks/test.ipynb). 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Deep Learning Study Group 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /installation/miscellaneous_notes.md: -------------------------------------------------------------------------------- 1 | # Miscellaneous Notes 2 | 3 | #### to enter into the Docker container on the command line: 4 | 5 | 1. use `sudo docker ps` to see the `NAME` that was randomly generated 6 | 2. `sudo docker exec -it bash` to enter into the session (N.B.: nothing will be saved) 7 | 8 | 9 | #### to install jupyter vim bindings within Docker container: 10 | 11 | 1. install notebook extensions (as in [here](https://github.com/ipython-contrib/jupyter_contrib_nbextensions#installation)): 12 | 13 | 1. `pip install jupyter_contrib_nbextensions` 14 | 2. `jupyter contrib nbextension install --user` 15 | 16 | 2. following the [instructions](https://github.com/lambdalisue/jupyter-vim-binding/wiki/Installation) doesn't work because you need to activate the notebook after making these changes... I imagine the same is true for themes 17 | 18 | #### to make your own Docker container: 19 | 20 | 1. `vi Dockerfile` 21 | 2. create something with [this](https://raw.githubusercontent.com/jupyter/docker-stacks/master/tensorflow-notebook/Dockerfile) type of structure 22 | 3. `sudo docker build -t .` 23 | 4. `sudo docker run -v ~/TensorFlow-LiveLessons:/home/jovyan/work -it --rm -p 8888:8888 ` 24 | 25 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/start-singleuser.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | set -e 6 | 7 | # set default ip to 0.0.0.0 8 | if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then 9 | NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS" 10 | fi 11 | 12 | # handle some deprecated environment variables 13 | # from DockerSpawner < 0.8. 14 | # These won't be passed from DockerSpawner 0.9, 15 | # so avoid specifying --arg=empty-string 16 | if [ ! -z "$NOTEBOOK_DIR" ]; then 17 | NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS" 18 | fi 19 | if [ ! -z "$JPY_PORT" ]; then 20 | NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS" 21 | fi 22 | if [ ! -z "$JPY_USER" ]; then 23 | NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS" 24 | fi 25 | if [ ! -z "$JPY_COOKIE_NAME" ]; then 26 | NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS" 27 | fi 28 | if [ ! -z "$JPY_BASE_URL" ]; then 29 | NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS" 30 | fi 31 | if [ ! -z "$JPY_HUB_PREFIX" ]; then 32 | NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS" 33 | fi 34 | if [ ! -z "$JPY_HUB_API_URL" ]; then 35 | NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" 36 | fi 37 | 38 | . /usr/local/bin/start.sh jupyterhub-singleuser $NOTEBOOK_ARGS $@ 39 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/jupyter_notebook_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Jupyter Development Team. 2 | # Distributed under the terms of the Modified BSD License. 3 | 4 | from jupyter_core.paths import jupyter_data_dir 5 | import subprocess 6 | import os 7 | import errno 8 | import stat 9 | 10 | c = get_config() 11 | c.NotebookApp.ip = '*' 12 | c.NotebookApp.port = 8888 13 | c.NotebookApp.open_browser = False 14 | 15 | # Generate a self-signed certificate 16 | if 'GEN_CERT' in os.environ: 17 | dir_name = jupyter_data_dir() 18 | pem_file = os.path.join(dir_name, 'notebook.pem') 19 | try: 20 | os.makedirs(dir_name) 21 | except OSError as exc: # Python >2.5 22 | if exc.errno == errno.EEXIST and os.path.isdir(dir_name): 23 | pass 24 | else: 25 | raise 26 | # Generate a certificate if one doesn't exist on disk 27 | subprocess.check_call(['openssl', 'req', '-new', 28 | '-newkey', 'rsa:2048', 29 | '-days', '365', 30 | '-nodes', '-x509', 31 | '-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated', 32 | '-keyout', pem_file, 33 | '-out', pem_file]) 34 | # Restrict access to the file 35 | os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) 36 | c.NotebookApp.certfile = pem_file 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/Jupyter-docker-stacks-license.md: -------------------------------------------------------------------------------- 1 | The files in this directory are licensed under the terms of the Modified BSD License 2 | (also known as New or Revised or 3-Clause BSD), as follows: 3 | 4 | - Copyright (c) 2001-2015, IPython Development Team 5 | - Copyright (c) 2015-, Jupyter Development Team 6 | 7 | All rights reserved. 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are met: 11 | 12 | Redistributions of source code must retain the above copyright notice, this 13 | list of conditions and the following disclaimer. 14 | 15 | Redistributions in binary form must reproduce the above copyright notice, this 16 | list of conditions and the following disclaimer in the documentation and/or 17 | other materials provided with the distribution. 18 | 19 | Neither the name of the Jupyter Development Team nor the names of its 20 | contributors may be used to endorse or promote products derived from this 21 | software without specific prior written permission. 22 | 23 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 24 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 25 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 26 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 27 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 29 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 30 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 31 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | set -e 6 | 7 | # Handle special flags if we're root 8 | if [ $(id -u) == 0 ] ; then 9 | # Handle username change. Since this is cheap, do this unconditionally 10 | usermod -d /home/$NB_USER -l $NB_USER jovyan 11 | 12 | # Change UID of NB_USER to NB_UID if it does not match 13 | if [ "$NB_UID" != $(id -u $NB_USER) ] ; then 14 | echo "Set user UID to: $NB_UID" 15 | usermod -u $NB_UID $NB_USER 16 | # Careful: $HOME might resolve to /root depending on how the 17 | # container is started. Use the $NB_USER home path explicitly. 18 | for d in "$CONDA_DIR" "$JULIA_PKGDIR" "/home/$NB_USER"; do 19 | if [[ ! -z "$d" && -d "$d" ]]; then 20 | echo "Set ownership to uid $NB_UID: $d" 21 | chown -R $NB_UID "$d" 22 | fi 23 | done 24 | fi 25 | 26 | # Change GID of NB_USER to NB_GID if NB_GID is passed as a parameter 27 | if [ "$NB_GID" ] ; then 28 | echo "Change GID to $NB_GID" 29 | groupmod -g $NB_GID -o $(id -g -n $NB_USER) 30 | fi 31 | 32 | # Enable sudo if requested 33 | if [ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]; then 34 | echo "Granting $NB_USER sudo access" 35 | echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook 36 | fi 37 | 38 | # Exec the command as NB_USER 39 | echo "Execute the command as $NB_USER" 40 | exec su $NB_USER -c "env PATH=$PATH $*" 41 | else 42 | if [[ ! -z "$NB_UID" && "$NB_UID" != "$(id -u)" ]]; then 43 | echo 'Container must be run as root to set $NB_UID' 44 | fi 45 | if [[ ! -z "$NB_GID" && "$NB_GID" != "$(id -g)" ]]; then 46 | echo 'Container must be run as root to set $NB_GID' 47 | fi 48 | # if [ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]; then 49 | # echo 'Container must be run as root to grant sudo permissions' 50 | # fi 51 | # Exec the command 52 | echo "Execute the command" 53 | exec $* 54 | fi 55 | -------------------------------------------------------------------------------- /installation/step_by_step_GCP_install.md: -------------------------------------------------------------------------------- 1 | # Step-by-Step Instructions for Installation on a New GCP Ubuntu Instance 2 | 3 | N.B.: These are **rough notes** for creating an Ubuntu machine on Google Cloud Platform that runs Dockerized Jupyter notebooks. 4 | 5 | 1. create an instance in [Google Cloud Platform Compute Engine](https://console.cloud.google.com/compute), Amazon Web Services, or another cloud-computing provider 6 | 1. `us-east1-d` if GPU desired in U.S. on GCP (April 2017) 7 | 2. boot disk: 8 | 1. select `Ubuntu 16.10`, i.e., `amd64 yakkety` for Docker compatibility later 9 | 2. select `SSD` if you're willing to pay for the significant speed benefits 10 | 3. I chose `200 GB`, which should be enough space and costs $34 if you use it for a full month (standard is $0.04/GB/month; SSD is $0.17/GB/month) 11 | 3. this comes out to $0.579/hour, or $422.36 if you run it for a full month (i.e., 730 hours) 12 | 2. ssh into instance, e.g.: 13 | * [like so](https://cloud.google.com/compute/docs/instances/connecting-to-instance) for GCP 14 | 3. from the home directory (which is typically where your ssh session begins): 15 | * `git clone https://github.com/the-deep-learners/nyc-ds-academy.git` 16 | * this command retrieves all of the code for this LiveLessons 17 | * ...and puts it in a new directory called `nyc-ds-academy` 18 | 4. [install Docker](https://docs.docker.com/engine/installation/linux/ubuntu/) (go from `Install using the repository` through to running the `hello-world` image; the free CE version does everything you need; use `amd64` during repository installation) 19 | 5. create a [firewall rule](https://cloud.google.com/compute/docs/networking?hl=en_US&_ga=1.268488185.92442388.1465331838#firewalls) to add a tag to your instance in your [Compute Engine table](https://console.cloud.google.com/compute) 20 | 6. follow steps six through nine from my [instructions for installation on Mac OS X](https://github.com/the-deep-learners/nyc-ds-academy/blob/master/installation/step_by_step_MacOSX_install.md) 21 | 7. **on your local machine**, navigate to `:8888/?token=` in the web browser of your choice 22 | * the `` of the instance you created is shown in your [Compute Engine table](https://console.cloud.google.com/compute) 23 | * `` should have been output by your `docker run` call in the previous step 24 | 25 | 26 | -------------------------------------------------------------------------------- /notebooks/test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Test" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "This notebook tests whether TensorFlow is installed properly." 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import tensorflow as tf" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 2, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "x1 = tf.placeholder(tf.float32)\n", 33 | "x2 = tf.placeholder(tf.float32)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 3, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "sum_op = tf.add(x1, x2)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 4, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "with tf.Session() as session:\n", 52 | " sum_result = session.run(sum_op, feed_dict={x1: 2.0, x2: 1.0})" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 5, 58 | "metadata": {}, 59 | "outputs": [ 60 | { 61 | "data": { 62 | "text/plain": [ 63 | "3.0" 64 | ] 65 | }, 66 | "execution_count": 5, 67 | "metadata": {}, 68 | "output_type": "execute_result" 69 | } 70 | ], 71 | "source": [ 72 | "sum_result" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "metadata": { 79 | "collapsed": true 80 | }, 81 | "outputs": [], 82 | "source": [] 83 | } 84 | ], 85 | "metadata": { 86 | "kernelspec": { 87 | "display_name": "Python 3", 88 | "language": "python", 89 | "name": "python3" 90 | }, 91 | "language_info": { 92 | "codemirror_mode": { 93 | "name": "ipython", 94 | "version": 3 95 | }, 96 | "file_extension": ".py", 97 | "mimetype": "text/x-python", 98 | "name": "python", 99 | "nbconvert_exporter": "python", 100 | "pygments_lexer": "ipython3", 101 | "version": "3.6.2" 102 | } 103 | }, 104 | "nbformat": 4, 105 | "nbformat_minor": 2 106 | } 107 | -------------------------------------------------------------------------------- /installation/step_by_step_MacOSX_install.md: -------------------------------------------------------------------------------- 1 | # Step-by-Step Instructions for Mac OS X 2 | 3 | ## Install 4 | 5 | 1. open the Terminal application ([like this](http://www.wikihow.com/Open-a-Terminal-Window-in-Mac)) 6 | 2. to install in your home directory (this is my recommended default): 7 | * type `cd ~` into the command-line prompt and 8 | * *execute* the line by pressing the **return** key on your keyboard 9 | 3. retrieve all of the code for this LiveLessons by executing `git clone https://github.com/the-deep-learners/nyc-ds-academy.git` (if you haven't used `git` before, you may be prompted to install Xcode -- do it!) 10 | 4. [install the Docker "Stable channel"](https://docs.docker.com/docker-for-mac/install/) 11 | 5. start Docker, e.g., by using Finder to navigate to your Applications folder and double-clicking on the Docker icon 12 | 6. back in Terminal, execute `source nyc-ds-academy/installation/let_jovyan_write.sh` so that you can write to files in the *nyc-ds-academy* directory from inside the Docker container we'll be creating momentarily 13 | 7. move into the *nyc-ds-academy* directory by executing `cd nyc-ds-academy` 14 | 8. build the Docker container by executing `sudo docker build -t nyc-dsa-stack .` (you'll get an error if you miss the final `.`!) 15 | 9. when that build process has finished, run the Docker container by executing `sudo docker run -v ~/nyc-ds-academy:/home/jovyan/work -it --rm -p 8888:8888 nyc-dsa-stack` 16 | 10. in the web browser of your choice (e.g., Chrome), copy and paste the URL created by Docker (this begins with `http://localhost:8888/?token=` and should be visible near the bottom of your Terminal window) 17 | 18 | ## Shutdown 19 | 20 | You can shutdown the Jupyter notebook by returning to the Terminal session that is running it and hitting the **control** and **c** keys simultaneously on your keyboard. 21 | 22 | ## Restart 23 | 24 | You can restart the Jupyter notebook later by following steps nine and ten alone. 25 | 26 | ## Bonus: Training Models with an Nvidia GPU 27 | 28 | You don't need to train your Deep Learning models with a GPU for these LiveLessons, but some of the later notebooks in these LiveLessons will run much more quickly if you do. 29 | 30 | 1. install an [Nvidia GPU](http://www.nvidia.com/content/global/global.php) on your machine or spin up a cloud instance that includes one (typically a Tesla K80) 31 | 1. install CUDA and cuDNN, e.g., per the **Installing CUDA Toolkit** and **Installing cuDNN** sections of [this blog post](https://hackernoon.com/launch-a-gpu-backed-google-compute-engine-instance-and-setup-tensorflow-keras-and-jupyter-902369ed5272) (this step may be tricky if you're relatively new to working with the Unix command line) 32 | 2. in the `nyc-ds-academy/installation/docker-stack-scripts` directory: 33 | * run `chmod 777 jupyter_notebook_config.py start*.sh` 34 | 3. replace step eight of my **Install** section above with `sudo docker build -f Dockerfile-gpu -t nycdsa-gpu-stack .` 35 | 4. replace step nine with `sudo nvidia-docker run -v ~/nyc-ds-academy:/home/jovyan/work -it --rm -p 8888:8888 nycdsa-gpu-stack` 36 | -------------------------------------------------------------------------------- /notebooks/first_tensorflow_graphs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# First Graphs in TF" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "#### Load dependencies" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import numpy as np\n", 24 | "import tensorflow as tf" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "#### Simple arithmetic" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "x1 = tf.placeholder(tf.float32)\n", 41 | "x2 = tf.placeholder(tf.float32)" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 3, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "sum_op = tf.add(x1, x2)\n", 51 | "product_op = tf.multiply(x1, x2)" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 7, 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "with tf.Session() as sess:\n", 61 | " \n", 62 | " sum_result = sess.run(sum_op, feed_dict={x1: 2, x2: [1, 2, 3]})\n", 63 | " \n", 64 | " product_result = sess.run(product_op, feed_dict={x1: 2, x2: [1, 2, 3]})" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 8, 70 | "metadata": {}, 71 | "outputs": [ 72 | { 73 | "data": { 74 | "text/plain": [ 75 | "array([ 3., 4., 5.], dtype=float32)" 76 | ] 77 | }, 78 | "execution_count": 8, 79 | "metadata": {}, 80 | "output_type": "execute_result" 81 | } 82 | ], 83 | "source": [ 84 | "sum_result" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 9, 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "data": { 94 | "text/plain": [ 95 | "array([ 2., 4., 6.], dtype=float32)" 96 | ] 97 | }, 98 | "execution_count": 9, 99 | "metadata": {}, 100 | "output_type": "execute_result" 101 | } 102 | ], 103 | "source": [ 104 | "product_result" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [] 113 | } 114 | ], 115 | "metadata": { 116 | "kernelspec": { 117 | "display_name": "Python 3", 118 | "language": "python", 119 | "name": "python3" 120 | }, 121 | "language_info": { 122 | "codemirror_mode": { 123 | "name": "ipython", 124 | "version": 3 125 | }, 126 | "file_extension": ".py", 127 | "mimetype": "text/x-python", 128 | "name": "python", 129 | "nbconvert_exporter": "python", 130 | "pygments_lexer": "ipython3", 131 | "version": "3.6.3" 132 | } 133 | }, 134 | "nbformat": 4, 135 | "nbformat_minor": 2 136 | } 137 | -------------------------------------------------------------------------------- /notebooks/sigmoid_function.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from math import e" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 3, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "def sigmoid(z):\n", 19 | " return 1/(1+e**-z)" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 4, 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "data": { 29 | "text/plain": [ 30 | "0.5000024999999999" 31 | ] 32 | }, 33 | "execution_count": 4, 34 | "metadata": {}, 35 | "output_type": "execute_result" 36 | } 37 | ], 38 | "source": [ 39 | "sigmoid(.00001)" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 5, 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "text/plain": [ 50 | "1.0" 51 | ] 52 | }, 53 | "execution_count": 5, 54 | "metadata": {}, 55 | "output_type": "execute_result" 56 | } 57 | ], 58 | "source": [ 59 | "sigmoid(1000)" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 7, 65 | "metadata": {}, 66 | "outputs": [ 67 | { 68 | "data": { 69 | "text/plain": [ 70 | "4.539786870243442e-05" 71 | ] 72 | }, 73 | "execution_count": 7, 74 | "metadata": {}, 75 | "output_type": "execute_result" 76 | } 77 | ], 78 | "source": [ 79 | "sigmoid(-10)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 8, 85 | "metadata": {}, 86 | "outputs": [ 87 | { 88 | "data": { 89 | "text/plain": [ 90 | "0.2689414213699951" 91 | ] 92 | }, 93 | "execution_count": 8, 94 | "metadata": {}, 95 | "output_type": "execute_result" 96 | } 97 | ], 98 | "source": [ 99 | "sigmoid(-1)" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": 9, 105 | "metadata": {}, 106 | "outputs": [ 107 | { 108 | "data": { 109 | "text/plain": [ 110 | "0.7310585786300049" 111 | ] 112 | }, 113 | "execution_count": 9, 114 | "metadata": {}, 115 | "output_type": "execute_result" 116 | } 117 | ], 118 | "source": [ 119 | "sigmoid(1)" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [] 128 | } 129 | ], 130 | "metadata": { 131 | "kernelspec": { 132 | "display_name": "Python 3", 133 | "language": "python", 134 | "name": "python3" 135 | }, 136 | "language_info": { 137 | "codemirror_mode": { 138 | "name": "ipython", 139 | "version": 3 140 | }, 141 | "file_extension": ".py", 142 | "mimetype": "text/x-python", 143 | "name": "python", 144 | "nbconvert_exporter": "python", 145 | "pygments_lexer": "ipython3", 146 | "version": "3.6.2" 147 | } 148 | }, 149 | "nbformat": 4, 150 | "nbformat_minor": 2 151 | } 152 | -------------------------------------------------------------------------------- /installation/README.md: -------------------------------------------------------------------------------- 1 | # Installation Instructions 2 | 3 | ## Mac OS X 4 | 5 | Detailed step-by-step instructions for running the code notebooks for these LiveLessons on a Mac [here](https://github.com/the-deep-learners/nyc-ds-academy/blob/master/installation/step_by_step_MacOSX_install.md). 6 | 7 | 8 | ## Unix 9 | 10 | #### Where You Already Have the Dependencies 11 | 12 | The dependencies are provided in this repository's [Dockerfile](https://github.com/the-deep-learners/nyc-ds-academy/blob/master/Dockerfile). They are: 13 | 14 | * the [Jupyter Notebook Scientific Python Stack](https://github.com/jupyter/docker-stacks/tree/master/scipy-notebook) 15 | * TensorFlow 16 | * Keras 17 | 18 | If you have these packages configured as you like them, you can simply: 19 | 20 | `git clone https://github.com/the-deep-learners/nyc-ds-academy.git` 21 | 22 | #### Where You Are Missing Dependencies 23 | 24 | 1. Get Docker CE for, e.g., [Ubuntu](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/) 25 | 2. Follow all the steps in my [Step-by-Step Instructions for Mac](https://github.com/the-deep-learners/nyc-ds-academy/blob/master/installation/step_by_step_MacOSX_install.md) that involve executing code at the command line. That is, execute all steps but one, four and five. 26 | 27 | #### In a Fresh Cloud Instance 28 | 29 | If you'd like to enjoy the power and flexibility of cloud computing, you can spin up a machine with Google Cloud Compute, Amazon Web Services, Microsoft Azure, or other providers. My rough notes for creating an Ubuntu instance with Google Cloud Compute and launching Dockerized Jupyter notebooks are available [here](https://github.com/the-deep-learners/nyc-ds-academy/blob/master/installation/step_by_step_GCP_install.md). 30 | 31 | 32 | ## Windows 33 | 34 | These steps are for users who installed the [Anaconda](https://www.continuum.io/downloads) Python 3 distribution, but other such distributions (e.g., [WinPython](https://winpython.github.io/), [Canopy](https://store.enthought.com/downloads/)) should work too. 35 | 36 | #### Install TensorFlow for CPU 37 | 38 | 1. Create a conda virtual environment: `C:\> conda create -n tf python=3.5` 39 | 2. Activate the new environment: `C:\> activate tf`. Your prompt should now change to: `(tf) C:\>` 40 | 3. Run `(tf) C:\> pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/windows/cpu/tensorflow-1.2.1-cp35-cp35m-win_amd64.whl`. 41 | 42 | #### Running Dockerized Jupyter Notebooks 43 | 44 | *Currently, instructions for running the these LiveLessons' Jupyter notebooks from within a Docker container are unavailable on Windows. If you have experience doing this, please feel free to make a pull request and fill it in!* 45 | 46 | 47 | ## GPU Considerations 48 | 49 | Most of the examples in these LiveLessons involve relatively small (in Deep Learning terms) data sets so you will be in great shape using your CPU alone for training the models. That said, some of the later notebooks in these LiveLessons will train much more quickly if you employ a GPU. Alternatively, you may enjoy leveraging the efficient, highly-parallelised computations that a GPU affords for your own projects. Whatever the reason, here are TensorFlow GPU instructions for [Mac/Unix](https://github.com/the-deep-learners/nyc-ds-academy/blob/master/installation/step_by_step_MacOSX_install.md#bonus-training-models-with-an-nvidia-gpu) or [Windows](https://github.com/the-deep-learners/nyc-ds-academy/blob/master/installation/windows_TF_GPU.md). 50 | 51 | -------------------------------------------------------------------------------- /notebooks/cross_entropy_cost.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from numpy import log" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "def cross_entropy(y, a):\n", 19 | " return -1*(y*log(a) + (1-y)*log(1-a))" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 3, 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "data": { 29 | "text/plain": [ 30 | "0.10536051565782628" 31 | ] 32 | }, 33 | "execution_count": 3, 34 | "metadata": {}, 35 | "output_type": "execute_result" 36 | } 37 | ], 38 | "source": [ 39 | "cross_entropy(0, 0.1)" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 4, 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "data": { 49 | "text/plain": [ 50 | "0.010050335853501451" 51 | ] 52 | }, 53 | "execution_count": 4, 54 | "metadata": {}, 55 | "output_type": "execute_result" 56 | } 57 | ], 58 | "source": [ 59 | "cross_entropy(0, 0.01)" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 5, 65 | "metadata": {}, 66 | "outputs": [ 67 | { 68 | "data": { 69 | "text/plain": [ 70 | "0.010050335853501451" 71 | ] 72 | }, 73 | "execution_count": 5, 74 | "metadata": {}, 75 | "output_type": "execute_result" 76 | } 77 | ], 78 | "source": [ 79 | "cross_entropy(1, 0.99)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 6, 85 | "metadata": {}, 86 | "outputs": [ 87 | { 88 | "data": { 89 | "text/plain": [ 90 | "0.35667494393873245" 91 | ] 92 | }, 93 | "execution_count": 6, 94 | "metadata": {}, 95 | "output_type": "execute_result" 96 | } 97 | ], 98 | "source": [ 99 | "cross_entropy(0, 0.3)" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": 7, 105 | "metadata": {}, 106 | "outputs": [ 107 | { 108 | "data": { 109 | "text/plain": [ 110 | "0.916290731874155" 111 | ] 112 | }, 113 | "execution_count": 7, 114 | "metadata": {}, 115 | "output_type": "execute_result" 116 | } 117 | ], 118 | "source": [ 119 | "cross_entropy(0, 0.6)" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 8, 125 | "metadata": {}, 126 | "outputs": [ 127 | { 128 | "data": { 129 | "text/plain": [ 130 | "2.3025850929940459" 131 | ] 132 | }, 133 | "execution_count": 8, 134 | "metadata": {}, 135 | "output_type": "execute_result" 136 | } 137 | ], 138 | "source": [ 139 | "cross_entropy(0, 0.9)" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "metadata": {}, 146 | "outputs": [], 147 | "source": [] 148 | } 149 | ], 150 | "metadata": { 151 | "kernelspec": { 152 | "display_name": "Python 3", 153 | "language": "python", 154 | "name": "python3" 155 | }, 156 | "language_info": { 157 | "codemirror_mode": { 158 | "name": "ipython", 159 | "version": 3 160 | }, 161 | "file_extension": ".py", 162 | "mimetype": "text/x-python", 163 | "name": "python", 164 | "nbconvert_exporter": "python", 165 | "pygments_lexer": "ipython3", 166 | "version": "3.6.2" 167 | } 168 | }, 169 | "nbformat": 4, 170 | "nbformat_minor": 2 171 | } 172 | -------------------------------------------------------------------------------- /notebooks/softmax.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from math import exp" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [ 17 | { 18 | "data": { 19 | "text/plain": [ 20 | "0.36787944117144233" 21 | ] 22 | }, 23 | "execution_count": 2, 24 | "metadata": {}, 25 | "output_type": "execute_result" 26 | } 27 | ], 28 | "source": [ 29 | "exp(-1)" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 3, 35 | "metadata": {}, 36 | "outputs": [ 37 | { 38 | "data": { 39 | "text/plain": [ 40 | "1.0" 41 | ] 42 | }, 43 | "execution_count": 3, 44 | "metadata": {}, 45 | "output_type": "execute_result" 46 | } 47 | ], 48 | "source": [ 49 | "exp(0)" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 4, 55 | "metadata": {}, 56 | "outputs": [ 57 | { 58 | "data": { 59 | "text/plain": [ 60 | "7.38905609893065" 61 | ] 62 | }, 63 | "execution_count": 4, 64 | "metadata": {}, 65 | "output_type": "execute_result" 66 | } 67 | ], 68 | "source": [ 69 | "exp(2)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 5, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "sigma = exp(-1) + exp(0) + exp(2)" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 6, 84 | "metadata": {}, 85 | "outputs": [ 86 | { 87 | "data": { 88 | "text/plain": [ 89 | "8.756935540102093" 90 | ] 91 | }, 92 | "execution_count": 6, 93 | "metadata": {}, 94 | "output_type": "execute_result" 95 | } 96 | ], 97 | "source": [ 98 | "sigma" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 7, 104 | "metadata": {}, 105 | "outputs": [ 106 | { 107 | "data": { 108 | "text/plain": [ 109 | "0.8437947344813395" 110 | ] 111 | }, 112 | "execution_count": 7, 113 | "metadata": {}, 114 | "output_type": "execute_result" 115 | } 116 | ], 117 | "source": [ 118 | "exp(2)/sigma" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 8, 124 | "metadata": {}, 125 | "outputs": [ 126 | { 127 | "data": { 128 | "text/plain": [ 129 | "0.11419519938459446" 130 | ] 131 | }, 132 | "execution_count": 8, 133 | "metadata": {}, 134 | "output_type": "execute_result" 135 | } 136 | ], 137 | "source": [ 138 | "exp(0)/sigma" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 9, 144 | "metadata": {}, 145 | "outputs": [ 146 | { 147 | "data": { 148 | "text/plain": [ 149 | "0.04201006613406605" 150 | ] 151 | }, 152 | "execution_count": 9, 153 | "metadata": {}, 154 | "output_type": "execute_result" 155 | } 156 | ], 157 | "source": [ 158 | "exp(-1)/sigma" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [] 167 | } 168 | ], 169 | "metadata": { 170 | "kernelspec": { 171 | "display_name": "Python 3", 172 | "language": "python", 173 | "name": "python3" 174 | }, 175 | "language_info": { 176 | "codemirror_mode": { 177 | "name": "ipython", 178 | "version": 3 179 | }, 180 | "file_extension": ".py", 181 | "mimetype": "text/x-python", 182 | "name": "python", 183 | "nbconvert_exporter": "python", 184 | "pygments_lexer": "ipython3", 185 | "version": "3.6.2" 186 | } 187 | }, 188 | "nbformat": 4, 189 | "nbformat_minor": 2 190 | } 191 | -------------------------------------------------------------------------------- /Dockerfile-gpu: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:8.0-cudnn5-devel 2 | 3 | MAINTAINER Jon Krohn 4 | 5 | ENV PATH="/usr/local/cuda-8.0/bin:${PATH}" 6 | ENV LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64:${LD_LIBRARY_PATH}" 7 | 8 | 9 | ## from core/Dockerfile-gpu 10 | # Setup locales 11 | RUN apt-get clean && apt-get update && apt-get install -y locales 12 | RUN locale-gen en_US.UTF-8 13 | ENV LANG en_US.UTF-8 14 | ENV LANGUAGE en_US:en 15 | ENV LC_ALL en_US.UTF-8 16 | 17 | # Purge existing python installs 18 | RUN apt-get purge -y python.* 19 | 20 | # Install prereqs 21 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential checkinstall \ 22 | wget \ 23 | bzip2 \ 24 | ca-certificates \ 25 | sudo \ 26 | locales \ 27 | fonts-liberation \ 28 | && apt-get clean \ 29 | && rm -rf /var/lib/apt/lists/* 30 | 31 | 32 | ## below mostly from jupyter/minimal-notebook (there's a bit of it in the line above too) 33 | ## -- see https://github.com/jupyter/docker-stacks/blob/master/LICENSE.md 34 | # Install Tini 35 | RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.10.0/tini && \ 36 | echo "1361527f39190a7338a0b434bd8c88ff7233ce7b9a4876f3315c22fce7eca1b0 *tini" | sha256sum -c - && \ 37 | mv tini /usr/local/bin/tini && \ 38 | chmod +x /usr/local/bin/tini 39 | 40 | # Configure environment 41 | ENV CONDA_DIR /opt/conda 42 | ENV PATH $CONDA_DIR/bin:$PATH 43 | ENV SHELL /bin/bash 44 | ENV NB_USER jovyan 45 | ENV NB_UID 1000 46 | ENV HOME /home/$NB_USER 47 | 48 | # Create jovyan user with UID=1000 and in the 'users' group 49 | RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ 50 | mkdir -p $CONDA_DIR && \ 51 | chown $NB_USER $CONDA_DIR 52 | 53 | USER $NB_USER 54 | 55 | # Setup work directory for backward-compatibility 56 | RUN mkdir /home/$NB_USER/work 57 | 58 | # Install conda as jovyan and check the md5 sum provided on the download site 59 | ENV MINICONDA_VERSION 4.3.21 60 | RUN cd /tmp && \ 61 | mkdir -p $CONDA_DIR && \ 62 | wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ 63 | echo "c1c15d3baba15bf50293ae963abef853 *Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \ 64 | /bin/bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ 65 | rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ 66 | $CONDA_DIR/bin/conda config --system --prepend channels conda-forge && \ 67 | $CONDA_DIR/bin/conda config --system --set auto_update_conda false && \ 68 | $CONDA_DIR/bin/conda config --system --set show_channel_urls true && \ 69 | $CONDA_DIR/bin/conda update --all && \ 70 | conda clean -tipsy 71 | 72 | # Install Jupyter Notebook and Hub 73 | RUN conda install --quiet --yes \ 74 | 'notebook=5.0.*' \ 75 | 'jupyterhub=0.7.*' \ 76 | 'jupyterlab=0.24.*' \ 77 | && conda clean -tipsy 78 | 79 | USER root 80 | 81 | EXPOSE 8888 82 | WORKDIR $HOME 83 | 84 | # Configure container startup 85 | ENTRYPOINT ["tini", "--"] 86 | CMD ["start-notebook.sh"] 87 | 88 | # Add local files as late as possible to avoid cache busting 89 | COPY installation/docker-stack-scripts/start.sh /usr/local/bin/ 90 | COPY installation/docker-stack-scripts/start-notebook.sh /usr/local/bin/ 91 | COPY installation/docker-stack-scripts/start-singleuser.sh /usr/local/bin/ 92 | COPY installation/docker-stack-scripts/jupyter_notebook_config.py /etc/jupyter/ 93 | RUN chown -R $NB_USER:users /etc/jupyter/ 94 | 95 | 96 | ## the next chunk of code is from jupyter/minimal-notebook 97 | 98 | # Install all OS dependencies for fully functional notebook server 99 | RUN apt-get update && apt-get install -yq --no-install-recommends \ 100 | build-essential \ 101 | emacs \ 102 | git \ 103 | inkscape \ 104 | jed \ 105 | libsm6 \ 106 | libxext-dev \ 107 | libxrender1 \ 108 | lmodern \ 109 | pandoc \ 110 | python-dev \ 111 | texlive-fonts-extra \ 112 | texlive-fonts-recommended \ 113 | texlive-generic-recommended \ 114 | texlive-latex-base \ 115 | texlive-latex-extra \ 116 | texlive-xetex \ 117 | vim \ 118 | unzip \ 119 | && apt-get clean && \ 120 | rm -rf /var/lib/apt/lists/* 121 | 122 | 123 | ## the next chunk of code is from jupyter/scipy-notebook 124 | 125 | # libav-tools for matplotlib anim 126 | RUN apt-get update && \ 127 | apt-get install -y --no-install-recommends libav-tools && \ 128 | apt-get clean && \ 129 | rm -rf /var/lib/apt/lists/* 130 | 131 | USER $NB_USER 132 | 133 | # Install Python 3 packages 134 | # Remove pyqt and qt pulled in for matplotlib since we're only ever going to 135 | # use notebook-friendly backends in these images 136 | RUN conda install --quiet --yes \ 137 | 'nomkl' \ 138 | 'ipywidgets=6.0*' \ 139 | 'pandas=0.19*' \ 140 | 'numexpr=2.6*' \ 141 | 'matplotlib=2.0*' \ 142 | 'scipy=0.19*' \ 143 | 'seaborn=0.7*' \ 144 | 'scikit-learn=0.18*' \ 145 | 'scikit-image=0.12*' \ 146 | 'sympy=1.0*' \ 147 | 'cython=0.25*' \ 148 | 'patsy=0.4*' \ 149 | 'statsmodels=0.8*' \ 150 | 'cloudpickle=0.2*' \ 151 | 'dill=0.2*' \ 152 | 'numba=0.31*' \ 153 | 'bokeh=0.12*' \ 154 | 'sqlalchemy=1.1*' \ 155 | 'hdf5=1.8.17' \ 156 | 'h5py=2.6*' \ 157 | 'vincent=0.4.*' \ 158 | 'beautifulsoup4=4.5.*' \ 159 | 'xlrd' && \ 160 | conda remove --quiet --yes --force qt pyqt && \ 161 | conda clean -tipsy 162 | 163 | # Activate ipywidgets extension in the environment that runs the notebook server 164 | RUN jupyter nbextension enable --py widgetsnbextension --sys-prefix 165 | 166 | # Import matplotlib the first time to build the font cache. 167 | ENV XDG_CACHE_HOME /home/$NB_USER/.cache/ 168 | RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" 169 | 170 | USER $NB_USER 171 | 172 | 173 | 174 | ## Install TensorFlow for GPU 175 | RUN conda install --quiet --yes 'tensorflow-gpu=1.3*' 176 | 177 | ## Install high-level TensorFlow APIs 178 | RUN pip install keras==2.0.8 179 | 180 | ## Install NLP packages 181 | RUN pip install nltk==3.2.4 182 | RUN pip install gensim==2.3.0 183 | -------------------------------------------------------------------------------- /notebooks/stacked_bidirectional_lstm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Stacked Bidirectional LSTM in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we *stack* LSTM layers to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import keras\n", 31 | "from keras.datasets import imdb\n", 32 | "from keras.preprocessing.sequence import pad_sequences\n", 33 | "from keras.models import Sequential\n", 34 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, LSTM\n", 35 | "from keras.layers.wrappers import Bidirectional \n", 36 | "from keras.callbacks import ModelCheckpoint\n", 37 | "import os\n", 38 | "from sklearn.metrics import roc_auc_score \n", 39 | "import matplotlib.pyplot as plt \n", 40 | "%matplotlib inline" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "#### Set hyperparameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": { 54 | "collapsed": true 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "# output directory name:\n", 59 | "output_dir = 'model_output/stackedLSTM'\n", 60 | "\n", 61 | "# training:\n", 62 | "epochs = 4\n", 63 | "batch_size = 128\n", 64 | "\n", 65 | "# vector-space embedding: \n", 66 | "n_dim = 64 \n", 67 | "n_unique_words = 10000 \n", 68 | "max_review_length = 200 \n", 69 | "pad_type = trunc_type = 'pre'\n", 70 | "drop_embed = 0.2 \n", 71 | "\n", 72 | "# LSTM layer architecture:\n", 73 | "n_lstm_1 = 64 # lower\n", 74 | "n_lstm_2 = 64 # new!\n", 75 | "drop_lstm = 0.2" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": {}, 81 | "source": [ 82 | "#### Load data" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": { 89 | "collapsed": true 90 | }, 91 | "outputs": [], 92 | "source": [ 93 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "#### Preprocess data" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": { 107 | "collapsed": true 108 | }, 109 | "outputs": [], 110 | "source": [ 111 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 112 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": { 118 | "collapsed": true 119 | }, 120 | "source": [ 121 | "#### Design neural network architecture" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": { 128 | "collapsed": true 129 | }, 130 | "outputs": [], 131 | "source": [ 132 | "model = Sequential()\n", 133 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n", 134 | "\n", 135 | "# CODE HERE\n", 136 | "\n", 137 | "model.add(Dense(1, activation='sigmoid'))" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": null, 143 | "metadata": {}, 144 | "outputs": [], 145 | "source": [ 146 | "# LSTM layer parameters double due to both reading directions\n", 147 | "model.summary() " 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "#### Configure model" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": { 161 | "collapsed": true 162 | }, 163 | "outputs": [], 164 | "source": [ 165 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": { 172 | "collapsed": true 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 177 | "if not os.path.exists(output_dir):\n", 178 | " os.makedirs(output_dir)" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "#### Train!" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "# 87.6% validation accuracy in epoch 2\n", 195 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 196 | ] 197 | }, 198 | { 199 | "cell_type": "markdown", 200 | "metadata": { 201 | "collapsed": true 202 | }, 203 | "source": [ 204 | "#### Evaluate" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": { 211 | "collapsed": true 212 | }, 213 | "outputs": [], 214 | "source": [ 215 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": null, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [ 224 | "y_hat = model.predict_proba(x_valid)" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": null, 230 | "metadata": {}, 231 | "outputs": [], 232 | "source": [ 233 | "plt.hist(y_hat)\n", 234 | "_ = plt.axvline(x=0.5, color='orange')" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": null, 240 | "metadata": {}, 241 | "outputs": [], 242 | "source": [ 243 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": null, 249 | "metadata": { 250 | "collapsed": true 251 | }, 252 | "outputs": [], 253 | "source": [] 254 | } 255 | ], 256 | "metadata": { 257 | "kernelspec": { 258 | "display_name": "Python 3", 259 | "language": "python", 260 | "name": "python3" 261 | }, 262 | "language_info": { 263 | "codemirror_mode": { 264 | "name": "ipython", 265 | "version": 3 266 | }, 267 | "file_extension": ".py", 268 | "mimetype": "text/x-python", 269 | "name": "python", 270 | "nbconvert_exporter": "python", 271 | "pygments_lexer": "ipython3", 272 | "version": "3.6.2" 273 | } 274 | }, 275 | "nbformat": 4, 276 | "nbformat_minor": 2 277 | } 278 | -------------------------------------------------------------------------------- /notebooks/vanilla_lstm_in_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Vanilla LSTM in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we use an LSTM to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import keras\n", 31 | "from keras.datasets import imdb\n", 32 | "from keras.preprocessing.sequence import pad_sequences\n", 33 | "from keras.models import Sequential\n", 34 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D\n", 35 | "from keras.layers import LSTM # new! \n", 36 | "from keras.callbacks import ModelCheckpoint\n", 37 | "import os\n", 38 | "from sklearn.metrics import roc_auc_score \n", 39 | "import matplotlib.pyplot as plt \n", 40 | "%matplotlib inline" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "#### Set hyperparameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": { 54 | "collapsed": true 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "# output directory name:\n", 59 | "output_dir = 'model_output/vanillaLSTM'\n", 60 | "\n", 61 | "# training:\n", 62 | "epochs = 4\n", 63 | "batch_size = 128\n", 64 | "\n", 65 | "# vector-space embedding: \n", 66 | "n_dim = 64 \n", 67 | "n_unique_words = 10000 \n", 68 | "max_review_length = 100 # lowered due to vanishing gradient over time\n", 69 | "pad_type = trunc_type = 'pre'\n", 70 | "drop_embed = 0.2 \n", 71 | "\n", 72 | "# LSTM layer architecture:\n", 73 | "n_lstm = 256 \n", 74 | "drop_lstm = 0.2\n", 75 | "\n", 76 | "# dense layer architecture: \n", 77 | "# n_dense = 256\n", 78 | "# dropout = 0.2" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "#### Load data" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": { 92 | "collapsed": true 93 | }, 94 | "outputs": [], 95 | "source": [ 96 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "#### Preprocess data" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": { 110 | "collapsed": true 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 115 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": { 121 | "collapsed": true 122 | }, 123 | "source": [ 124 | "#### Design neural network architecture" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": { 131 | "collapsed": true 132 | }, 133 | "outputs": [], 134 | "source": [ 135 | "model = Sequential()\n", 136 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n", 137 | "model.add(SpatialDropout1D(drop_embed))\n", 138 | "\n", 139 | "model.add(LSTM(n_lstm, dropout=drop_lstm))\n", 140 | "\n", 141 | "model.add(Dense(1, activation='sigmoid'))" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "model.summary() " 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "#### Configure model" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": null, 163 | "metadata": { 164 | "collapsed": true 165 | }, 166 | "outputs": [], 167 | "source": [ 168 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "metadata": { 175 | "collapsed": true 176 | }, 177 | "outputs": [], 178 | "source": [ 179 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 180 | "if not os.path.exists(output_dir):\n", 181 | " os.makedirs(output_dir)" 182 | ] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": {}, 187 | "source": [ 188 | "#### Train!" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": null, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "# go have a gander at nvidia-smi\n", 198 | "# 85.2% validation accuracy in epoch 2\n", 199 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": { 205 | "collapsed": true 206 | }, 207 | "source": [ 208 | "#### Evaluate" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": null, 214 | "metadata": { 215 | "collapsed": true 216 | }, 217 | "outputs": [], 218 | "source": [ 219 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "metadata": {}, 226 | "outputs": [], 227 | "source": [ 228 | "y_hat = model.predict_proba(x_valid)" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": {}, 235 | "outputs": [], 236 | "source": [ 237 | "plt.hist(y_hat)\n", 238 | "_ = plt.axvline(x=0.5, color='orange')" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [ 247 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": null, 253 | "metadata": { 254 | "collapsed": true 255 | }, 256 | "outputs": [], 257 | "source": [] 258 | } 259 | ], 260 | "metadata": { 261 | "kernelspec": { 262 | "display_name": "Python 3", 263 | "language": "python", 264 | "name": "python3" 265 | }, 266 | "language_info": { 267 | "codemirror_mode": { 268 | "name": "ipython", 269 | "version": 3 270 | }, 271 | "file_extension": ".py", 272 | "mimetype": "text/x-python", 273 | "name": "python", 274 | "nbconvert_exporter": "python", 275 | "pygments_lexer": "ipython3", 276 | "version": "3.6.2" 277 | } 278 | }, 279 | "nbformat": 4, 280 | "nbformat_minor": 2 281 | } 282 | -------------------------------------------------------------------------------- /notebooks/bidirectional_lstm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Bidirectional LSTM in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we use a *bidirectional* LSTM to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import keras\n", 31 | "from keras.datasets import imdb\n", 32 | "from keras.preprocessing.sequence import pad_sequences\n", 33 | "from keras.models import Sequential\n", 34 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, LSTM\n", 35 | "from keras.layers.wrappers import Bidirectional # new! \n", 36 | "from keras.callbacks import ModelCheckpoint\n", 37 | "import os\n", 38 | "from sklearn.metrics import roc_auc_score \n", 39 | "import matplotlib.pyplot as plt \n", 40 | "%matplotlib inline" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "#### Set hyperparameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": { 54 | "collapsed": true 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "# output directory name:\n", 59 | "output_dir = 'model_output/biLSTM'\n", 60 | "\n", 61 | "# training:\n", 62 | "epochs = 6\n", 63 | "batch_size = 128\n", 64 | "\n", 65 | "# vector-space embedding: \n", 66 | "n_dim = 64 \n", 67 | "n_unique_words = 10000 \n", 68 | "max_review_length = 200 # doubled!\n", 69 | "pad_type = trunc_type = 'pre'\n", 70 | "drop_embed = 0.2 \n", 71 | "\n", 72 | "# LSTM layer architecture:\n", 73 | "n_lstm = 256 \n", 74 | "drop_lstm = 0.2" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "#### Load data" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": { 88 | "collapsed": true 89 | }, 90 | "outputs": [], 91 | "source": [ 92 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "#### Preprocess data" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "metadata": { 106 | "collapsed": true 107 | }, 108 | "outputs": [], 109 | "source": [ 110 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 111 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": { 117 | "collapsed": true 118 | }, 119 | "source": [ 120 | "#### Design neural network architecture" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": { 127 | "collapsed": true 128 | }, 129 | "outputs": [], 130 | "source": [ 131 | "model = Sequential()\n", 132 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n", 133 | "model.add(SpatialDropout1D(drop_embed))\n", 134 | "\n", 135 | "model.add(Bidirectional(LSTM(n_lstm, dropout=drop_lstm)))\n", 136 | "\n", 137 | "model.add(Dense(1, activation='sigmoid'))" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": null, 143 | "metadata": {}, 144 | "outputs": [], 145 | "source": [ 146 | "# LSTM layer parameters double due to both reading directions\n", 147 | "model.summary() " 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "#### Configure model" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": { 161 | "collapsed": true 162 | }, 163 | "outputs": [], 164 | "source": [ 165 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": { 172 | "collapsed": true 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 177 | "if not os.path.exists(output_dir):\n", 178 | " os.makedirs(output_dir)" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "#### Train!" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "# - we see 87.0% validation accuracy in epoch 2\n", 195 | "# - with this toy dataset, the complex interplay of words over long sentence segments, won't be learned much\n", 196 | "# - so our CNN picking up location-invariant segments of two to four words that predict review sentiment\n", 197 | "# - these are simpler and so easier to learn from the data\n", 198 | "# - CNN therefore outperforms on the IMDB data set\n", 199 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": { 205 | "collapsed": true 206 | }, 207 | "source": [ 208 | "#### Evaluate" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": null, 214 | "metadata": { 215 | "collapsed": true 216 | }, 217 | "outputs": [], 218 | "source": [ 219 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "metadata": {}, 226 | "outputs": [], 227 | "source": [ 228 | "y_hat = model.predict_proba(x_valid)" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": {}, 235 | "outputs": [], 236 | "source": [ 237 | "plt.hist(y_hat)\n", 238 | "_ = plt.axvline(x=0.5, color='orange')" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [ 247 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": null, 253 | "metadata": { 254 | "collapsed": true 255 | }, 256 | "outputs": [], 257 | "source": [] 258 | } 259 | ], 260 | "metadata": { 261 | "kernelspec": { 262 | "display_name": "Python 3", 263 | "language": "python", 264 | "name": "python3" 265 | }, 266 | "language_info": { 267 | "codemirror_mode": { 268 | "name": "ipython", 269 | "version": 3 270 | }, 271 | "file_extension": ".py", 272 | "mimetype": "text/x-python", 273 | "name": "python", 274 | "nbconvert_exporter": "python", 275 | "pygments_lexer": "ipython3", 276 | "version": "3.6.2" 277 | } 278 | }, 279 | "nbformat": 4, 280 | "nbformat_minor": 2 281 | } 282 | -------------------------------------------------------------------------------- /notebooks/multi_convnet_architectures.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Multi-ConvNet Architectures" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we concatenate *multiple parallel convolutional nets together* to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import keras\n", 31 | "from keras.datasets import imdb\n", 32 | "from keras.preprocessing.sequence import pad_sequences\n", 33 | "from keras.models import Model # new!\n", 34 | "from keras.layers import Input, concatenate # new! \n", 35 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, Conv1D, GlobalMaxPooling1D\n", 36 | "from keras.callbacks import ModelCheckpoint\n", 37 | "import os\n", 38 | "from sklearn.metrics import roc_auc_score \n", 39 | "import matplotlib.pyplot as plt \n", 40 | "%matplotlib inline" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "#### Set hyperparameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": { 54 | "collapsed": true 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "# output directory name:\n", 59 | "output_dir = 'model_output/multiconv'\n", 60 | "\n", 61 | "# training:\n", 62 | "epochs = 4\n", 63 | "batch_size = 128\n", 64 | "\n", 65 | "# vector-space embedding: \n", 66 | "n_dim = 64\n", 67 | "n_unique_words = 5000 \n", 68 | "max_review_length = 400\n", 69 | "pad_type = trunc_type = 'pre'\n", 70 | "drop_embed = 0.2 \n", 71 | "\n", 72 | "# convolutional layer architecture:\n", 73 | "n_conv_1 = n_conv_2 = n_conv_3 = 256 \n", 74 | "k_conv_1 = 3\n", 75 | "k_conv_2 = 2\n", 76 | "k_conv_3 = 4\n", 77 | "\n", 78 | "# dense layer architecture: \n", 79 | "n_dense = 256\n", 80 | "dropout = 0.2" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "#### Load data" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": { 94 | "collapsed": true 95 | }, 96 | "outputs": [], 97 | "source": [ 98 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) " 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "#### Preprocess data" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": { 112 | "collapsed": true 113 | }, 114 | "outputs": [], 115 | "source": [ 116 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 117 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": { 123 | "collapsed": true 124 | }, 125 | "source": [ 126 | "#### Design neural network architecture" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": { 133 | "collapsed": true 134 | }, 135 | "outputs": [], 136 | "source": [ 137 | "input_layer = Input(shape=(max_review_length,), dtype='int16', name='input') # supports integers +/- 32.7k \n", 138 | "embedding_layer = Embedding(n_unique_words, n_dim, input_length=max_review_length, name='embedding')(input_layer)\n", 139 | "drop_embed_layer = SpatialDropout1D(drop_embed, name='drop_embed')(embedding_layer)\n", 140 | "\n", 141 | "# CODE HERE\n", 142 | "\n", 143 | "# start with conv_1 only and no concat\n", 144 | "# add conv_2\n", 145 | "# add conv_3\n", 146 | "\n", 147 | "dense_layer = Dense(n_dense, activation='relu', name='dense')(concat)\n", 148 | "drop_dense_layer = Dropout(dropout, name='drop_dense')(dense_layer)\n", 149 | "dense_2 = Dense(64, activation='relu', name='dense_2')(drop_dense_layer)\n", 150 | "dropout_2 = Dropout(dropout, name='drop_dense_2')(dense_2)\n", 151 | "\n", 152 | "predictions = Dense(1, activation='sigmoid', name='output')(dropout_2)\n", 153 | "\n", 154 | "model = Model(input_layer, predictions)" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "model.summary() " 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "#### Configure model" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": null, 176 | "metadata": { 177 | "collapsed": true 178 | }, 179 | "outputs": [], 180 | "source": [ 181 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": { 188 | "collapsed": true 189 | }, 190 | "outputs": [], 191 | "source": [ 192 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 193 | "if not os.path.exists(output_dir):\n", 194 | " os.makedirs(output_dir)" 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": {}, 200 | "source": [ 201 | "#### Train!" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "# start with conv_1 only and no concat: 89.1% validation accuracy in epoch 2, as earlier notebook\n", 211 | "# add conv_2: 89.5% in epoch 3\n", 212 | "# add conv_3: ditto\n", 213 | "# add dense_2: ditto in epoch 2\n", 214 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 215 | ] 216 | }, 217 | { 218 | "cell_type": "markdown", 219 | "metadata": { 220 | "collapsed": true 221 | }, 222 | "source": [ 223 | "#### Evaluate" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": null, 229 | "metadata": { 230 | "collapsed": true 231 | }, 232 | "outputs": [], 233 | "source": [ 234 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": null, 240 | "metadata": { 241 | "collapsed": true 242 | }, 243 | "outputs": [], 244 | "source": [ 245 | "y_hat = model.predict(x_valid)" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": null, 251 | "metadata": {}, 252 | "outputs": [], 253 | "source": [ 254 | "plt.hist(y_hat)\n", 255 | "_ = plt.axvline(x=0.5, color='orange')" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": null, 270 | "metadata": { 271 | "collapsed": true 272 | }, 273 | "outputs": [], 274 | "source": [] 275 | } 276 | ], 277 | "metadata": { 278 | "kernelspec": { 279 | "display_name": "Python 3", 280 | "language": "python", 281 | "name": "python3" 282 | }, 283 | "language_info": { 284 | "codemirror_mode": { 285 | "name": "ipython", 286 | "version": 3 287 | }, 288 | "file_extension": ".py", 289 | "mimetype": "text/x-python", 290 | "name": "python", 291 | "nbconvert_exporter": "python", 292 | "pygments_lexer": "ipython3", 293 | "version": "3.6.2" 294 | } 295 | }, 296 | "nbformat": 4, 297 | "nbformat_minor": 2 298 | } 299 | -------------------------------------------------------------------------------- /notebooks/rnn_in_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Recurrent Neural Network in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we use an RNN to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stderr", 31 | "output_type": "stream", 32 | "text": [ 33 | "Using TensorFlow backend.\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "import keras\n", 39 | "from keras.datasets import imdb\n", 40 | "from keras.preprocessing.sequence import pad_sequences\n", 41 | "from keras.models import Sequential\n", 42 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D\n", 43 | "from keras.layers import SimpleRNN # new! \n", 44 | "from keras.callbacks import ModelCheckpoint\n", 45 | "import os\n", 46 | "from sklearn.metrics import roc_auc_score \n", 47 | "import matplotlib.pyplot as plt \n", 48 | "%matplotlib inline" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "#### Set hyperparameters" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 2, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "# output directory name:\n", 65 | "output_dir = 'model_output/rnn'\n", 66 | "\n", 67 | "# training:\n", 68 | "epochs = 16 # way more!\n", 69 | "batch_size = 128\n", 70 | "\n", 71 | "# vector-space embedding: \n", 72 | "n_dim = 64 \n", 73 | "n_unique_words = 10000 \n", 74 | "max_review_length = 100 # lowered due to vanishing gradient over time\n", 75 | "pad_type = trunc_type = 'pre'\n", 76 | "drop_embed = 0.2 \n", 77 | "\n", 78 | "# RNN layer architecture:\n", 79 | "n_rnn = 256 \n", 80 | "drop_rnn = 0.2\n", 81 | "\n", 82 | "# dense layer architecture: \n", 83 | "# n_dense = 256\n", 84 | "# dropout = 0.2" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "#### Load data" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 3, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "#### Preprocess data" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 4, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 117 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": { 123 | "collapsed": true 124 | }, 125 | "source": [ 126 | "#### Design neural network architecture" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 6, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "# CODE HERE\n", 136 | "model = Sequential()\n", 137 | "\n", 138 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length))\n", 139 | "model.add(SpatialDropout1D(drop_embed))\n", 140 | "\n", 141 | "model.add(SimpleRNN(n_rnn, dropout=drop_rnn))\n", 142 | "\n", 143 | "model.add(Dense(1, activation='sigmoid'))" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 7, 149 | "metadata": {}, 150 | "outputs": [ 151 | { 152 | "name": "stdout", 153 | "output_type": "stream", 154 | "text": [ 155 | "_________________________________________________________________\n", 156 | "Layer (type) Output Shape Param # \n", 157 | "=================================================================\n", 158 | "embedding_2 (Embedding) (None, 100, 64) 640000 \n", 159 | "_________________________________________________________________\n", 160 | "spatial_dropout1d_1 (Spatial (None, 100, 64) 0 \n", 161 | "_________________________________________________________________\n", 162 | "simple_rnn_1 (SimpleRNN) (None, 256) 82176 \n", 163 | "_________________________________________________________________\n", 164 | "dense_1 (Dense) (None, 1) 257 \n", 165 | "=================================================================\n", 166 | "Total params: 722,433\n", 167 | "Trainable params: 722,433\n", 168 | "Non-trainable params: 0\n", 169 | "_________________________________________________________________\n" 170 | ] 171 | } 172 | ], 173 | "source": [ 174 | "model.summary() " 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "#### Configure model" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": { 188 | "collapsed": true 189 | }, 190 | "outputs": [], 191 | "source": [ 192 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "metadata": { 199 | "collapsed": true 200 | }, 201 | "outputs": [], 202 | "source": [ 203 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 204 | "if not os.path.exists(output_dir):\n", 205 | " os.makedirs(output_dir)" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "#### Train!" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": null, 218 | "metadata": {}, 219 | "outputs": [], 220 | "source": [ 221 | "# 80.6% validation accuracy in epoch 4\n", 222 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 223 | ] 224 | }, 225 | { 226 | "cell_type": "markdown", 227 | "metadata": { 228 | "collapsed": true 229 | }, 230 | "source": [ 231 | "#### Evaluate" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": null, 237 | "metadata": { 238 | "collapsed": true 239 | }, 240 | "outputs": [], 241 | "source": [ 242 | "model.load_weights(output_dir+\"/weights.03.hdf5\") # zero-indexed" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [ 251 | "y_hat = model.predict_proba(x_valid)" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": null, 257 | "metadata": {}, 258 | "outputs": [], 259 | "source": [ 260 | "plt.hist(y_hat)\n", 261 | "_ = plt.axvline(x=0.5, color='orange')" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": null, 267 | "metadata": {}, 268 | "outputs": [], 269 | "source": [ 270 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": null, 276 | "metadata": { 277 | "collapsed": true 278 | }, 279 | "outputs": [], 280 | "source": [] 281 | } 282 | ], 283 | "metadata": { 284 | "kernelspec": { 285 | "display_name": "Python 3", 286 | "language": "python", 287 | "name": "python3" 288 | }, 289 | "language_info": { 290 | "codemirror_mode": { 291 | "name": "ipython", 292 | "version": 3 293 | }, 294 | "file_extension": ".py", 295 | "mimetype": "text/x-python", 296 | "name": "python", 297 | "nbconvert_exporter": "python", 298 | "pygments_lexer": "ipython3", 299 | "version": "3.6.2" 300 | } 301 | }, 302 | "nbformat": 4, 303 | "nbformat_minor": 2 304 | } 305 | -------------------------------------------------------------------------------- /notebooks/intermediate_net_in_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Intermediate Neural Network in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "#### Load dependencies" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "name": "stderr", 24 | "output_type": "stream", 25 | "text": [ 26 | "Using TensorFlow backend.\n" 27 | ] 28 | } 29 | ], 30 | "source": [ 31 | "import keras\n", 32 | "from keras.datasets import mnist\n", 33 | "from keras.models import Sequential\n", 34 | "from keras.layers import Dense\n", 35 | "from keras.optimizers import SGD" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": {}, 41 | "source": [ 42 | "#### Load data" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 2, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "(X_train, y_train), (X_test, y_test) = mnist.load_data()" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "#### Preprocess Data" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 3, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "X_train = X_train.reshape(60000, 784).astype('float32')\n", 68 | "X_test = X_test.reshape(10000, 784).astype('float32')" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 4, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "X_train /= 255\n", 78 | "X_test /= 255" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 5, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "n_classes = 10\n", 88 | "y_train = keras.utils.to_categorical(y_train, n_classes)\n", 89 | "y_test = keras.utils.to_categorical(y_test, n_classes)" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": {}, 95 | "source": [ 96 | "#### Design neural network architecture" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 6, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "model = Sequential()\n", 106 | "model.add(Dense(64, activation='relu', input_shape=(784,)))\n", 107 | "model.add(Dense(64, activation='relu'))\n", 108 | "model.add(Dense(10, activation='softmax'))" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 7, 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "name": "stdout", 118 | "output_type": "stream", 119 | "text": [ 120 | "_________________________________________________________________\n", 121 | "Layer (type) Output Shape Param # \n", 122 | "=================================================================\n", 123 | "dense_1 (Dense) (None, 64) 50240 \n", 124 | "_________________________________________________________________\n", 125 | "dense_2 (Dense) (None, 64) 4160 \n", 126 | "_________________________________________________________________\n", 127 | "dense_3 (Dense) (None, 10) 650 \n", 128 | "=================================================================\n", 129 | "Total params: 55,050\n", 130 | "Trainable params: 55,050\n", 131 | "Non-trainable params: 0\n", 132 | "_________________________________________________________________\n" 133 | ] 134 | } 135 | ], 136 | "source": [ 137 | "model.summary()" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 9, 143 | "metadata": {}, 144 | "outputs": [ 145 | { 146 | "data": { 147 | "text/plain": [ 148 | "50240" 149 | ] 150 | }, 151 | "execution_count": 9, 152 | "metadata": {}, 153 | "output_type": "execute_result" 154 | } 155 | ], 156 | "source": [ 157 | "(64*784)+64" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 8, 163 | "metadata": {}, 164 | "outputs": [ 165 | { 166 | "data": { 167 | "text/plain": [ 168 | "4160" 169 | ] 170 | }, 171 | "execution_count": 8, 172 | "metadata": {}, 173 | "output_type": "execute_result" 174 | } 175 | ], 176 | "source": [ 177 | "(64*64)+64" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 10, 183 | "metadata": {}, 184 | "outputs": [ 185 | { 186 | "data": { 187 | "text/plain": [ 188 | "650" 189 | ] 190 | }, 191 | "execution_count": 10, 192 | "metadata": {}, 193 | "output_type": "execute_result" 194 | } 195 | ], 196 | "source": [ 197 | "(10*64)+10" 198 | ] 199 | }, 200 | { 201 | "cell_type": "markdown", 202 | "metadata": {}, 203 | "source": [ 204 | "#### Configure model" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": 11, 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [ 213 | "model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.01), metrics=['accuracy'])" 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": {}, 219 | "source": [ 220 | "#### Train" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": 12, 226 | "metadata": {}, 227 | "outputs": [ 228 | { 229 | "name": "stdout", 230 | "output_type": "stream", 231 | "text": [ 232 | "Train on 60000 samples, validate on 10000 samples\n", 233 | "Epoch 1/10\n", 234 | "60000/60000 [==============================] - 1s - loss: 1.3285 - acc: 0.6546 - val_loss: 0.6498 - val_acc: 0.8395\n", 235 | "Epoch 2/10\n", 236 | "60000/60000 [==============================] - 1s - loss: 0.5260 - acc: 0.8626 - val_loss: 0.4210 - val_acc: 0.8868\n", 237 | "Epoch 3/10\n", 238 | "60000/60000 [==============================] - 1s - loss: 0.4026 - acc: 0.8880 - val_loss: 0.3545 - val_acc: 0.9000\n", 239 | "Epoch 4/10\n", 240 | "60000/60000 [==============================] - 1s - loss: 0.3545 - acc: 0.8998 - val_loss: 0.3221 - val_acc: 0.9092\n", 241 | "Epoch 5/10\n", 242 | "60000/60000 [==============================] - 1s - loss: 0.3262 - acc: 0.9074 - val_loss: 0.2993 - val_acc: 0.9140\n", 243 | "Epoch 6/10\n", 244 | "60000/60000 [==============================] - 1s - loss: 0.3059 - acc: 0.9134 - val_loss: 0.2845 - val_acc: 0.9186\n", 245 | "Epoch 7/10\n", 246 | "60000/60000 [==============================] - 1s - loss: 0.2902 - acc: 0.9177 - val_loss: 0.2723 - val_acc: 0.9233\n", 247 | "Epoch 8/10\n", 248 | "60000/60000 [==============================] - 1s - loss: 0.2768 - acc: 0.9213 - val_loss: 0.2619 - val_acc: 0.9239\n", 249 | "Epoch 9/10\n", 250 | "60000/60000 [==============================] - 1s - loss: 0.2658 - acc: 0.9247 - val_loss: 0.2521 - val_acc: 0.9265\n", 251 | "Epoch 10/10\n", 252 | "60000/60000 [==============================] - 1s - loss: 0.2558 - acc: 0.9276 - val_loss: 0.2432 - val_acc: 0.9297\n" 253 | ] 254 | }, 255 | { 256 | "data": { 257 | "text/plain": [ 258 | "" 259 | ] 260 | }, 261 | "execution_count": 12, 262 | "metadata": {}, 263 | "output_type": "execute_result" 264 | } 265 | ], 266 | "source": [ 267 | "model.fit(X_train, y_train, validation_data=(X_test, y_test), batch_size=128, epochs=10, verbose=1)" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": null, 273 | "metadata": {}, 274 | "outputs": [], 275 | "source": [] 276 | } 277 | ], 278 | "metadata": { 279 | "kernelspec": { 280 | "display_name": "Python 3", 281 | "language": "python", 282 | "name": "python3" 283 | }, 284 | "language_info": { 285 | "codemirror_mode": { 286 | "name": "ipython", 287 | "version": 3 288 | }, 289 | "file_extension": ".py", 290 | "mimetype": "text/x-python", 291 | "name": "python", 292 | "nbconvert_exporter": "python", 293 | "pygments_lexer": "ipython3", 294 | "version": "3.6.2" 295 | } 296 | }, 297 | "nbformat": 4, 298 | "nbformat_minor": 2 299 | } 300 | -------------------------------------------------------------------------------- /notebooks/deep_net_in_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Deep Neural Network in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "#### Load dependencies" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import keras\n", 24 | "from keras.datasets import mnist\n", 25 | "from keras.models import Sequential\n", 26 | "from keras.layers import Dense\n", 27 | "from keras.layers import Dropout # new\n", 28 | "from keras.layers.normalization import BatchNormalization # new \n", 29 | "from keras import regularizers # new\n", 30 | "from keras.optimizers import SGD" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "metadata": {}, 36 | "source": [ 37 | "#### Load data" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 2, 43 | "metadata": {}, 44 | "outputs": [ 45 | { 46 | "name": "stdout", 47 | "output_type": "stream", 48 | "text": [ 49 | "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n", 50 | "11059200/11490434 [===========================>..] - ETA: 0s" 51 | ] 52 | } 53 | ], 54 | "source": [ 55 | "(X_train, y_train), (X_test, y_test) = mnist.load_data()" 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "#### Preprocess Data" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 3, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "X_train = X_train.reshape(60000, 784).astype('float32')\n", 72 | "X_test = X_test.reshape(10000, 784).astype('float32')" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 4, 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [ 81 | "X_train /= 255\n", 82 | "X_test /= 255" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 5, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "n_classes = 10\n", 92 | "y_train = keras.utils.to_categorical(y_train, n_classes)\n", 93 | "y_test = keras.utils.to_categorical(y_test, n_classes)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "#### Design neural network architecture" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 6, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "model = Sequential()\n", 110 | "\n", 111 | "model.add(Dense(64, activation='relu', input_shape=(784,)))\n", 112 | "model.add(BatchNormalization())\n", 113 | "model.add(Dropout(0.5))\n", 114 | "\n", 115 | "model.add(Dense(64, activation='relu'))\n", 116 | "model.add(BatchNormalization())\n", 117 | "model.add(Dropout(0.5))\n", 118 | "\n", 119 | "model.add(Dense(64, activation='relu'))\n", 120 | "model.add(BatchNormalization())\n", 121 | "model.add(Dropout(0.5))\n", 122 | "\n", 123 | "model.add(Dense(10, activation='softmax'))" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 7, 129 | "metadata": {}, 130 | "outputs": [ 131 | { 132 | "name": "stdout", 133 | "output_type": "stream", 134 | "text": [ 135 | "_________________________________________________________________\n", 136 | "Layer (type) Output Shape Param # \n", 137 | "=================================================================\n", 138 | "dense_1 (Dense) (None, 64) 50240 \n", 139 | "_________________________________________________________________\n", 140 | "batch_normalization_1 (Batch (None, 64) 256 \n", 141 | "_________________________________________________________________\n", 142 | "dropout_1 (Dropout) (None, 64) 0 \n", 143 | "_________________________________________________________________\n", 144 | "dense_2 (Dense) (None, 64) 4160 \n", 145 | "_________________________________________________________________\n", 146 | "batch_normalization_2 (Batch (None, 64) 256 \n", 147 | "_________________________________________________________________\n", 148 | "dropout_2 (Dropout) (None, 64) 0 \n", 149 | "_________________________________________________________________\n", 150 | "dense_3 (Dense) (None, 64) 4160 \n", 151 | "_________________________________________________________________\n", 152 | "batch_normalization_3 (Batch (None, 64) 256 \n", 153 | "_________________________________________________________________\n", 154 | "dropout_3 (Dropout) (None, 64) 0 \n", 155 | "_________________________________________________________________\n", 156 | "dense_4 (Dense) (None, 10) 650 \n", 157 | "=================================================================\n", 158 | "Total params: 59,978\n", 159 | "Trainable params: 59,594\n", 160 | "Non-trainable params: 384\n", 161 | "_________________________________________________________________\n" 162 | ] 163 | } 164 | ], 165 | "source": [ 166 | "model.summary()" 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "metadata": {}, 172 | "source": [ 173 | "#### Configure model" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 8, 179 | "metadata": {}, 180 | "outputs": [], 181 | "source": [ 182 | "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": {}, 188 | "source": [ 189 | "#### Train" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 9, 195 | "metadata": {}, 196 | "outputs": [ 197 | { 198 | "name": "stdout", 199 | "output_type": "stream", 200 | "text": [ 201 | "Train on 60000 samples, validate on 10000 samples\n", 202 | "Epoch 1/10\n", 203 | "60000/60000 [==============================] - 3s - loss: 1.1909 - acc: 0.6221 - val_loss: 0.3125 - val_acc: 0.9085\n", 204 | "Epoch 2/10\n", 205 | "60000/60000 [==============================] - 3s - loss: 0.5716 - acc: 0.8319 - val_loss: 0.2476 - val_acc: 0.9263\n", 206 | "Epoch 3/10\n", 207 | "60000/60000 [==============================] - 3s - loss: 0.4562 - acc: 0.8704 - val_loss: 0.2076 - val_acc: 0.9380\n", 208 | "Epoch 4/10\n", 209 | "60000/60000 [==============================] - 3s - loss: 0.4044 - acc: 0.8878 - val_loss: 0.1841 - val_acc: 0.9461\n", 210 | "Epoch 5/10\n", 211 | "60000/60000 [==============================] - 3s - loss: 0.3685 - acc: 0.8981 - val_loss: 0.1808 - val_acc: 0.9471\n", 212 | "Epoch 6/10\n", 213 | "60000/60000 [==============================] - 3s - loss: 0.3438 - acc: 0.9044 - val_loss: 0.1701 - val_acc: 0.9516\n", 214 | "Epoch 7/10\n", 215 | "60000/60000 [==============================] - 3s - loss: 0.3288 - acc: 0.9089 - val_loss: 0.1648 - val_acc: 0.9531\n", 216 | "Epoch 8/10\n", 217 | "60000/60000 [==============================] - 3s - loss: 0.3096 - acc: 0.9146 - val_loss: 0.1459 - val_acc: 0.9584\n", 218 | "Epoch 9/10\n", 219 | "60000/60000 [==============================] - 3s - loss: 0.2981 - acc: 0.9168 - val_loss: 0.1480 - val_acc: 0.9576\n", 220 | "Epoch 10/10\n", 221 | "60000/60000 [==============================] - 3s - loss: 0.2913 - acc: 0.9194 - val_loss: 0.1475 - val_acc: 0.9575\n" 222 | ] 223 | }, 224 | { 225 | "data": { 226 | "text/plain": [ 227 | "" 228 | ] 229 | }, 230 | "execution_count": 9, 231 | "metadata": {}, 232 | "output_type": "execute_result" 233 | } 234 | ], 235 | "source": [ 236 | "model.fit(X_train, y_train, validation_data=(X_test, y_test), batch_size=128, epochs=10, verbose=1)" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": null, 242 | "metadata": {}, 243 | "outputs": [], 244 | "source": [] 245 | } 246 | ], 247 | "metadata": { 248 | "kernelspec": { 249 | "display_name": "Python 3", 250 | "language": "python", 251 | "name": "python3" 252 | }, 253 | "language_info": { 254 | "codemirror_mode": { 255 | "name": "ipython", 256 | "version": 3 257 | }, 258 | "file_extension": ".py", 259 | "mimetype": "text/x-python", 260 | "name": "python", 261 | "nbconvert_exporter": "python", 262 | "pygments_lexer": "ipython3", 263 | "version": "3.6.2" 264 | } 265 | }, 266 | "nbformat": 4, 267 | "nbformat_minor": 2 268 | } 269 | -------------------------------------------------------------------------------- /notebooks/intermediate_net_in_tensorflow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Intermediate Neural Network in TensorFlow" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we convert our [intermediate-depth MNIST-classifying neural network](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/intermediate_net_in_keras.ipynb) from Keras to TensorFlow (compare them side by side) following Aymeric Damien's [Multi-Layer Perceptron Notebook](https://github.com/aymericdamien/TensorFlow-Examples/blob/master/notebooks/3_NeuralNetworks/multilayer_perceptron.ipynb) style." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import numpy as np\n", 31 | "np.random.seed(42)\n", 32 | "import tensorflow as tf\n", 33 | "tf.set_random_seed(42)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Load data" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "name": "stdout", 50 | "output_type": "stream", 51 | "text": [ 52 | "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", 53 | "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", 54 | "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", 55 | "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n" 56 | ] 57 | } 58 | ], 59 | "source": [ 60 | "from tensorflow.examples.tutorials.mnist import input_data\n", 61 | "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "#### Set neural network hyperparameters (tidier at top of file!)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "lr = 0.1\n", 78 | "epochs = 10\n", 79 | "batch_size = 128\n", 80 | "weight_initializer = tf.contrib.layers.xavier_initializer()" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "#### Set number of neurons for each layer" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 4, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "n_input = 784\n", 97 | "n_dense_1 = 64\n", 98 | "n_dense_2 = 64\n", 99 | "n_classes = 10" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "#### Define placeholders Tensors for inputs and labels" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 5, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "x = tf.placeholder(tf.float32, [None, n_input])\n", 116 | "y = tf.placeholder(tf.float32, [None, n_classes])" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": {}, 122 | "source": [ 123 | "#### Define types of layers" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 6, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "# dense layer with ReLU activation:\n", 133 | "def dense(x, W, b):\n", 134 | " z = tf.add(tf.matmul(x, W), b)\n", 135 | " a = tf.nn.relu(z)\n", 136 | " return a" 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "metadata": {}, 142 | "source": [ 143 | "#### Define dictionaries for storing weights and biases for each layer -- and initialize" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 7, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "bias_dict = {\n", 153 | " 'b1': tf.Variable(tf.zeros([n_dense_1])),\n", 154 | " 'b2': tf.Variable(tf.zeros([n_dense_2])),\n", 155 | " 'b_out': tf.Variable(tf.zeros([n_classes]))\n", 156 | "}\n", 157 | "\n", 158 | "weight_dict = {\n", 159 | " 'W1': tf.get_variable('W1', [n_input, n_dense_1], initializer=weight_initializer),\n", 160 | " 'W2': tf.get_variable('W2', [n_dense_1, n_dense_2], initializer=weight_initializer),\n", 161 | " 'W_out': tf.get_variable('W_out', [n_dense_2, n_classes], initializer=weight_initializer),\n", 162 | "}" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "#### Design neural network architecture" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 8, 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "def network(x, weights, biases):\n", 179 | " \n", 180 | " # two dense hidden layers:\n", 181 | " dense_1 = dense(x, weights['W1'], biases['b1'])\n", 182 | " dense_2 = dense(dense_1, weights['W2'], biases['b2'])\n", 183 | " \n", 184 | " # linear output layer (softmax):\n", 185 | " out_layer_z = tf.add(tf.matmul(dense_2, weights['W_out']), biases['b_out'])\n", 186 | " \n", 187 | " return out_layer_z" 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "metadata": {}, 193 | "source": [ 194 | "#### Build model" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 9, 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "predictions = network(x, weights=weight_dict, biases=bias_dict)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "metadata": {}, 209 | "source": [ 210 | "#### Define model's loss and its optimizer" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 10, 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predictions, labels=y))\n", 220 | "optimizer = tf.train.GradientDescentOptimizer(learning_rate=lr).minimize(cost)" 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "metadata": {}, 226 | "source": [ 227 | "#### Define evaluation metrics" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": 11, 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [ 236 | "# calculate accuracy by identifying test cases where the model's highest-probability class matches the true y label: \n", 237 | "correct_prediction = tf.equal(tf.argmax(predictions, 1), tf.argmax(y, 1))\n", 238 | "accuracy_pct = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) * 100" 239 | ] 240 | }, 241 | { 242 | "cell_type": "markdown", 243 | "metadata": {}, 244 | "source": [ 245 | "#### Create op for variable initialization" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": 12, 251 | "metadata": {}, 252 | "outputs": [], 253 | "source": [ 254 | "initializer_op = tf.global_variables_initializer()" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": {}, 260 | "source": [ 261 | "#### Train the network in a session" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 13, 267 | "metadata": {}, 268 | "outputs": [ 269 | { 270 | "name": "stdout", 271 | "output_type": "stream", 272 | "text": [ 273 | "Training for 10 epochs.\n", 274 | "Epoch 001: cost = 0.501, accuracy = 85.40%\n", 275 | "Epoch 002: cost = 0.251, accuracy = 92.71%\n", 276 | "Epoch 003: cost = 0.194, accuracy = 94.35%\n", 277 | "Epoch 004: cost = 0.159, accuracy = 95.38%\n", 278 | "Epoch 005: cost = 0.134, accuracy = 96.07%\n", 279 | "Epoch 006: cost = 0.117, accuracy = 96.58%\n", 280 | "Epoch 007: cost = 0.104, accuracy = 96.96%\n", 281 | "Epoch 008: cost = 0.093, accuracy = 97.21%\n", 282 | "Epoch 009: cost = 0.084, accuracy = 97.51%\n", 283 | "Epoch 010: cost = 0.077, accuracy = 97.74%\n", 284 | "Training Complete. Testing Model.\n", 285 | "\n", 286 | "Test Cost: 0.088\n", 287 | "Test Accuracy: 97.13%\n" 288 | ] 289 | } 290 | ], 291 | "source": [ 292 | "with tf.Session() as session:\n", 293 | " session.run(initializer_op)\n", 294 | " \n", 295 | " print(\"Training for\", epochs, \"epochs.\")\n", 296 | " \n", 297 | " # loop over epochs: \n", 298 | " for epoch in range(epochs):\n", 299 | " \n", 300 | " avg_cost = 0.0 # track cost to monitor performance during training\n", 301 | " avg_accuracy_pct = 0.0\n", 302 | " \n", 303 | " # loop over all batches of the epoch:\n", 304 | " n_batches = int(mnist.train.num_examples / batch_size)\n", 305 | " for i in range(n_batches):\n", 306 | " \n", 307 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 308 | " \n", 309 | " # feed batch data to run optimization and fetching cost and accuracy: \n", 310 | " _, batch_cost, batch_acc = session.run([optimizer, cost, accuracy_pct], feed_dict={x: batch_x, y: batch_y})\n", 311 | " \n", 312 | " # accumulate mean loss and accuracy over epoch: \n", 313 | " avg_cost += batch_cost / n_batches\n", 314 | " avg_accuracy_pct += batch_acc / n_batches\n", 315 | " \n", 316 | " # output logs at end of each epoch of training:\n", 317 | " print(\"Epoch \", '%03d' % (epoch+1), \n", 318 | " \": cost = \", '{:.3f}'.format(avg_cost), \n", 319 | " \", accuracy = \", '{:.2f}'.format(avg_accuracy_pct), \"%\", \n", 320 | " sep='')\n", 321 | " \n", 322 | " print(\"Training Complete. Testing Model.\\n\")\n", 323 | " \n", 324 | " test_cost = cost.eval({x: mnist.test.images, y: mnist.test.labels})\n", 325 | " test_accuracy_pct = accuracy_pct.eval({x: mnist.test.images, y: mnist.test.labels})\n", 326 | " \n", 327 | " print(\"Test Cost:\", '{:.3f}'.format(test_cost))\n", 328 | " print(\"Test Accuracy: \", '{:.2f}'.format(test_accuracy_pct), \"%\", sep='')" 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "execution_count": null, 334 | "metadata": {}, 335 | "outputs": [], 336 | "source": [] 337 | } 338 | ], 339 | "metadata": { 340 | "kernelspec": { 341 | "display_name": "Python 3", 342 | "language": "python", 343 | "name": "python3" 344 | }, 345 | "language_info": { 346 | "codemirror_mode": { 347 | "name": "ipython", 348 | "version": 3 349 | }, 350 | "file_extension": ".py", 351 | "mimetype": "text/x-python", 352 | "name": "python", 353 | "nbconvert_exporter": "python", 354 | "pygments_lexer": "ipython3", 355 | "version": "3.6.3" 356 | } 357 | }, 358 | "nbformat": 4, 359 | "nbformat_minor": 2 360 | } 361 | -------------------------------------------------------------------------------- /notebooks/deep_net_in_tensorflow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Deep Neural Network in TensorFlow" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we convert our [intermediate-depth MNIST-classifying neural network](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/intermediate_net_in_keras.ipynb) from Keras to TensorFlow (compare them side by side) following Aymeric Damien's [Multi-Layer Perceptron Notebook](https://github.com/aymericdamien/TensorFlow-Examples/blob/master/notebooks/3_NeuralNetworks/multilayer_perceptron.ipynb) style." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import numpy as np\n", 31 | "np.random.seed(42)\n", 32 | "import tensorflow as tf\n", 33 | "tf.set_random_seed(42)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Load data" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "name": "stdout", 50 | "output_type": "stream", 51 | "text": [ 52 | "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", 53 | "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", 54 | "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", 55 | "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n" 56 | ] 57 | } 58 | ], 59 | "source": [ 60 | "from tensorflow.examples.tutorials.mnist import input_data\n", 61 | "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "#### Set neural network hyperparameters (tidier at top of file!)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "lr = 0.1\n", 78 | "epochs = 10\n", 79 | "batch_size = 128\n", 80 | "weight_initializer = tf.contrib.layers.xavier_initializer()" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "#### Set number of neurons for each layer" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 4, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "n_input = 784\n", 97 | "n_dense_1 = 64\n", 98 | "n_dense_2 = 64\n", 99 | "n_dense_3 = 64\n", 100 | "n_classes = 10" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "#### Define placeholders Tensors for inputs and labels" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 5, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "x = tf.placeholder(tf.float32, [None, n_input])\n", 117 | "y = tf.placeholder(tf.float32, [None, n_classes])" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "#### Define types of layers" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 6, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "# dense layer with ReLU activation:\n", 134 | "def dense(x, W, b):\n", 135 | " z = tf.add(tf.matmul(x, W), b)\n", 136 | " a = tf.nn.relu(z)\n", 137 | " return a" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": {}, 143 | "source": [ 144 | "#### Define dictionaries for storing weights and biases for each layer -- and initialize" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": 7, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "bias_dict = {\n", 154 | " 'b1': tf.Variable(tf.zeros([n_dense_1])),\n", 155 | " 'b2': tf.Variable(tf.zeros([n_dense_2])),\n", 156 | " 'b3': tf.Variable(tf.zeros([n_dense_3])),\n", 157 | " 'b_out': tf.Variable(tf.zeros([n_classes]))\n", 158 | "}\n", 159 | "\n", 160 | "weight_dict = {\n", 161 | " 'W1': tf.get_variable('W1', [n_input, n_dense_1], initializer=weight_initializer),\n", 162 | " 'W2': tf.get_variable('W2', [n_dense_1, n_dense_2], initializer=weight_initializer),\n", 163 | " 'W3': tf.get_variable('W3', [n_dense_2, n_dense_3], initializer=weight_initializer),\n", 164 | " 'W_out': tf.get_variable('W_out', [n_dense_3, n_classes], initializer=weight_initializer),\n", 165 | "}" 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "metadata": {}, 171 | "source": [ 172 | "#### Design neural network architecture" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 8, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "def network(x, weights, biases):\n", 182 | " \n", 183 | " # two dense hidden layers:\n", 184 | " dense_1 = dense(x, weights['W1'], biases['b1'])\n", 185 | " dense_2 = dense(dense_1, weights['W2'], biases['b2'])\n", 186 | " dense_3 = dense(dense_2, weights['W3'], biases['b3'])\n", 187 | " \n", 188 | " # linear output layer (softmax):\n", 189 | " out_layer_z = tf.add(tf.matmul(dense_3, weights['W_out']), biases['b_out'])\n", 190 | " \n", 191 | " return out_layer_z" 192 | ] 193 | }, 194 | { 195 | "cell_type": "markdown", 196 | "metadata": {}, 197 | "source": [ 198 | "#### Build model" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": 9, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [ 207 | "predictions = network(x, weights=weight_dict, biases=bias_dict)" 208 | ] 209 | }, 210 | { 211 | "cell_type": "markdown", 212 | "metadata": {}, 213 | "source": [ 214 | "#### Define model's loss and its optimizer" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": 10, 220 | "metadata": {}, 221 | "outputs": [], 222 | "source": [ 223 | "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predictions, labels=y))\n", 224 | "optimizer = tf.train.GradientDescentOptimizer(learning_rate=lr).minimize(cost)" 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": {}, 230 | "source": [ 231 | "#### Define evaluation metrics" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 11, 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "# calculate accuracy by identifying test cases where the model's highest-probability class matches the true y label: \n", 241 | "correct_prediction = tf.equal(tf.argmax(predictions, 1), tf.argmax(y, 1))\n", 242 | "accuracy_pct = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) * 100" 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "#### Create op for variable initialization" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 12, 255 | "metadata": {}, 256 | "outputs": [], 257 | "source": [ 258 | "initializer_op = tf.global_variables_initializer()" 259 | ] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": {}, 264 | "source": [ 265 | "#### Train the network in a session" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": 13, 271 | "metadata": {}, 272 | "outputs": [ 273 | { 274 | "name": "stdout", 275 | "output_type": "stream", 276 | "text": [ 277 | "Training for 10 epochs.\n", 278 | "Epoch 001: cost = 0.542, accuracy = 83.35%\n", 279 | "Epoch 002: cost = 0.211, accuracy = 93.73%\n", 280 | "Epoch 003: cost = 0.158, accuracy = 95.32%\n", 281 | "Epoch 004: cost = 0.130, accuracy = 96.14%\n", 282 | "Epoch 005: cost = 0.110, accuracy = 96.68%\n", 283 | "Epoch 006: cost = 0.096, accuracy = 97.16%\n", 284 | "Epoch 007: cost = 0.085, accuracy = 97.50%\n", 285 | "Epoch 008: cost = 0.076, accuracy = 97.77%\n", 286 | "Epoch 009: cost = 0.067, accuracy = 97.91%\n", 287 | "Epoch 010: cost = 0.061, accuracy = 98.19%\n", 288 | "Training Complete. Testing Model.\n", 289 | "\n", 290 | "Test Cost: 0.095\n", 291 | "Test Accuracy: 97.01%\n" 292 | ] 293 | } 294 | ], 295 | "source": [ 296 | "with tf.Session() as session:\n", 297 | " session.run(initializer_op)\n", 298 | " \n", 299 | " print(\"Training for\", epochs, \"epochs.\")\n", 300 | " \n", 301 | " # loop over epochs: \n", 302 | " for epoch in range(epochs):\n", 303 | " \n", 304 | " avg_cost = 0.0 # track cost to monitor performance during training\n", 305 | " avg_accuracy_pct = 0.0\n", 306 | " \n", 307 | " # loop over all batches of the epoch:\n", 308 | " n_batches = int(mnist.train.num_examples / batch_size)\n", 309 | " for i in range(n_batches):\n", 310 | " \n", 311 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 312 | " \n", 313 | " # feed batch data to run optimization and fetching cost and accuracy: \n", 314 | " _, batch_cost, batch_acc = session.run([optimizer, cost, accuracy_pct], feed_dict={x: batch_x, y: batch_y})\n", 315 | " \n", 316 | " # accumulate mean loss and accuracy over epoch: \n", 317 | " avg_cost += batch_cost / n_batches\n", 318 | " avg_accuracy_pct += batch_acc / n_batches\n", 319 | " \n", 320 | " # output logs at end of each epoch of training:\n", 321 | " print(\"Epoch \", '%03d' % (epoch+1), \n", 322 | " \": cost = \", '{:.3f}'.format(avg_cost), \n", 323 | " \", accuracy = \", '{:.2f}'.format(avg_accuracy_pct), \"%\", \n", 324 | " sep='')\n", 325 | " \n", 326 | " print(\"Training Complete. Testing Model.\\n\")\n", 327 | " \n", 328 | " test_cost = cost.eval({x: mnist.test.images, y: mnist.test.labels})\n", 329 | " test_accuracy_pct = accuracy_pct.eval({x: mnist.test.images, y: mnist.test.labels})\n", 330 | " \n", 331 | " print(\"Test Cost:\", '{:.3f}'.format(test_cost))\n", 332 | " print(\"Test Accuracy: \", '{:.2f}'.format(test_accuracy_pct), \"%\", sep='')" 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": null, 338 | "metadata": {}, 339 | "outputs": [], 340 | "source": [] 341 | } 342 | ], 343 | "metadata": { 344 | "kernelspec": { 345 | "display_name": "Python 3", 346 | "language": "python", 347 | "name": "python3" 348 | }, 349 | "language_info": { 350 | "codemirror_mode": { 351 | "name": "ipython", 352 | "version": 3 353 | }, 354 | "file_extension": ".py", 355 | "mimetype": "text/x-python", 356 | "name": "python", 357 | "nbconvert_exporter": "python", 358 | "pygments_lexer": "ipython3", 359 | "version": "3.6.3" 360 | } 361 | }, 362 | "nbformat": 4, 363 | "nbformat_minor": 2 364 | } 365 | -------------------------------------------------------------------------------- /notebooks/first_tensorflow_neurons.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# First TensorFlow Neurons" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "#### Load dependencies" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import numpy as np\n", 24 | "np.random.seed(42)\n", 25 | "import tensorflow as tf\n", 26 | "tf.set_random_seed(42)\n", 27 | "import matplotlib.pyplot as plt\n", 28 | "%matplotlib inline" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "#### Set number of neurons" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 2, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "n_input = 784\n", 45 | "n_dense = 128" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "#### Define placeholder Tensor for simulated MNIST digits" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 3, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "x = tf.placeholder(tf.float32, [None, n_input])" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "#### Create Variable Tensors for neuron biases `b` and weight matrix `W`" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 4, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "b = tf.Variable(tf.zeros([n_dense]))\n", 78 | "# W = tf.Variable(tf.random_uniform([n_input, n_dense])) # 1.\n", 79 | "# W = tf.Variable(tf.random_normal([n_input, n_dense])) # 2.\n", 80 | "W = tf.get_variable('W', [n_input, n_dense], \n", 81 | " initializer=tf.contrib.layers.xavier_initializer())" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "#### Design the computational graph" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 5, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "z = tf.add(tf.matmul(x, W), b)\n", 98 | "a = tf.nn.relu(z) # first with tf.sigmoid(), then stick with tf.tanh() or tf.nn.relu()" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "#### Create op for variable initialization" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 6, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "initializer_op = tf.global_variables_initializer()" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "#### Execute the graph in a session" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 7, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "with tf.Session() as session:\n", 131 | " session.run(initializer_op)\n", 132 | " \n", 133 | " layer_output = session.run(a, {x: np.random.random([1, n_input])})" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 8, 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "data": { 143 | "text/plain": [ 144 | "array([[ 0.28484702, 0.66296649, 0.48465875, 0.58361048, 0. ,\n", 145 | " 0.02700119, 0.50025719, 0.35395339, 0.33801708, 0.65332055,\n", 146 | " 0. , 0.10179006, 0. , 0.61159277, 0. ,\n", 147 | " 1.29969335, 0. , 0.0700454 , 0.52458507, 0.60666955,\n", 148 | " 0.45962104, 0.84671533, 0.10901574, 0.43929845, 0. ,\n", 149 | " 0.33268207, 0.10732757, 0. , 0. , 0.38768527,\n", 150 | " 0.17347404, 0. , 0. , 0. , 0.04758957,\n", 151 | " 0.45743263, 0.04072431, 0. , 0.27846065, 0. ,\n", 152 | " 0.64920127, 0. , 0. , 0. , 0. ,\n", 153 | " 0.20801233, 0. , 0. , 0. , 0.54332167,\n", 154 | " 1.74375212, 0.19492842, 0.25929692, 0. , 0. ,\n", 155 | " 0.58499128, 0.36324847, 0. , 0. , 0. ,\n", 156 | " 0. , 0.8660351 , 0. , 0. , 0.83429128,\n", 157 | " 1.06981802, 0.64549267, 0. , 0. , 0. ,\n", 158 | " 0. , 0.56380248, 0. , 0. , 0. ,\n", 159 | " 0.24550769, 0.72879362, 0.28983656, 0. , 0.22116758,\n", 160 | " 0. , 0. , 0.16525199, 0.92048723, 0. ,\n", 161 | " 0. , 0. , 0.71630067, 0.50095582, 0.62799764,\n", 162 | " 0.18321247, 0. , 0. , 0.91466761, 0. ,\n", 163 | " 0.07857732, 1.25968564, 0. , 0.49644947, 0.00613178,\n", 164 | " 0. , 0.70885855, 0.22226629, 0.35115728, 0. ,\n", 165 | " 0.22060682, 0. , 0.83496559, 0.82810038, 0.05427774,\n", 166 | " 0. , 0.13187447, 0. , 0.02490139, 0. ,\n", 167 | " 0.08315995, 1.03106618, 0. , 0.30102578, 0. ,\n", 168 | " 0. , 0. , 0.05216391, 0. , 0. ,\n", 169 | " 0. , 0. , 0. ]], dtype=float32)" 170 | ] 171 | }, 172 | "execution_count": 8, 173 | "metadata": {}, 174 | "output_type": "execute_result" 175 | } 176 | ], 177 | "source": [ 178 | "layer_output" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 9, 184 | "metadata": { 185 | "scrolled": true 186 | }, 187 | "outputs": [ 188 | { 189 | "data": { 190 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEHVJREFUeJzt3X+MZWV9x/H3p6xopbbsykA3IC4kG/yRFNAJQWmsgrQo\nrbtNtcHYZrXbbGzVaGxa15I0bdOk6z/VNjZttmi7JpYfopStv+p2gZjWgg7Kb8SFFXEDZUcElZpg\nod/+MWf1suxwz8zcO3fm4f1KJvec5zxnzneeOfnMmXPuOTdVhSRp9fupSRcgSRoNA12SGmGgS1Ij\nDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUiDXLubHjjjuuNmzYsJyblKRV78Ybb/xOVU0N67es\ngb5hwwZmZmaWc5OStOol+Vaffp5ykaRGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXC\nQJekRizrnaJLsWH7Zyay3Xt3XDiR7UrSQnmELkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtS\nIwx0SWqEgS5JjTDQJakRBrokNWJooCc5LclNA1/fT/KeJOuS7Emyr3tduxwFS5KObGigV9VdVXVG\nVZ0BvBz4IXAVsB3YW1Ubgb3dvCRpQhZ6yuU84J6q+hawCdjVte8CNo+yMEnSwiw00C8CLu2mT6iq\nBwC61+NHWZgkaWF6B3qSo4E3AJ9YyAaSbEsyk2RmdnZ2ofVJknpayBH664CvVtWD3fyDSdYDdK8H\nj7RSVe2squmqmp6amlpatZKkeS0k0N/MT063AOwGtnTTW4CrR1WUJGnhegV6kucC5wOfGmjeAZyf\nZF+3bMfoy5Mk9dXrM0Wr6ofA8w9re4i5d71IklYA7xSVpEYY6JLUCANdkhphoEtSIwx0SWqEgS5J\njTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQI\nA12SGtH3Q6KPTXJlkq8nuTPJK5KsS7Inyb7ude24i5Ukza/vEfpfA5+vqhcBpwN3AtuBvVW1Edjb\nzUuSJmRooCf5WeBVwEcAqupHVfUIsAnY1XXbBWweV5GSpOH6HKGfCswC/5jka0kuSXIMcEJVPQDQ\nvR5/pJWTbEsyk2RmdnZ2ZIVLkp6sT6CvAV4G/F1VnQn8Dws4vVJVO6tquqqmp6amFlmmJGmYPoF+\nADhQVTd081cyF/APJlkP0L0eHE+JkqQ+hgZ6Vf038O0kp3VN5wF3ALuBLV3bFuDqsVQoSeplTc9+\n7wI+nuRoYD/wNub+GFyRZCtwH/Cm8ZQoSeqjV6BX1U3A9BEWnTfaciRJi+WdopLUCANdkhphoEtS\nIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXC\nQJekRhjoktQIA12SGtHrI+iS3Av8AHgCeLyqppOsAy4HNgD3Ar9ZVQ+Pp0xJ0jALOUJ/TVWdUVWH\nPlt0O7C3qjYCe7t5SdKELOWUyyZgVze9C9i89HIkSYvVN9AL+EKSG5Ns69pOqKoHALrX48dRoCSp\nn17n0IFzqur+JMcDe5J8ve8Guj8A2wBOPvnkRZQoSeqj1xF6Vd3fvR4ErgLOAh5Msh6gez04z7o7\nq2q6qqanpqZGU7Uk6SmGBnqSY5I879A08MvAbcBuYEvXbQtw9biKlCQN1+eUywnAVUkO9f/nqvp8\nkq8AVyTZCtwHvGl8ZUqShhka6FW1Hzj9CO0PAeeNoyhJ0sJ5p6gkNcJAl6RGGOiS1AgDXZIaYaBL\nUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1\nwkCXpEb0DvQkRyX5WpJPd/OnJLkhyb4klyc5enxlSpKGWcgR+ruBOwfmPwB8sKo2Ag8DW0dZmCRp\nYXoFepKTgAuBS7r5AOcCV3ZddgGbx1GgJKmfvkfoHwL+CPi/bv75wCNV9Xg3fwA4ccS1SZIWYGig\nJ/lV4GBV3TjYfISuNc/625LMJJmZnZ1dZJmSpGH6HKGfA7whyb3AZcydavkQcGySNV2fk4D7j7Ry\nVe2squmqmp6amhpByZKkIxka6FX1/qo6qao2ABcB11TVW4BrgTd23bYAV4+tSknSUEt5H/r7gPcm\nuZu5c+ofGU1JkqTFWDO8y09U1XXAdd30fuCs0ZckSVoM7xSVpEYY6JLUCANdkhphoEtSIwx0SWqE\ngS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjo\nktSIoYGe5DlJvpzk5iS3J/mzrv2UJDck2Zfk8iRHj79cSdJ8+hyhPwacW1WnA2cAFyQ5G/gA8MGq\n2gg8DGwdX5mSpGGGBnrNebSbfVb3VcC5wJVd+y5g81gqlCT10uscepKjktwEHAT2APcAj1TV412X\nA8CJ4ylRktRHr0Cvqieq6gzgJOAs4MVH6nakdZNsSzKTZGZ2dnbxlUqSntaC3uVSVY8A1wFnA8cm\nWdMtOgm4f551dlbVdFVNT01NLaVWSdLT6PMul6kkx3bTPw28FrgTuBZ4Y9dtC3D1uIqUJA23ZngX\n1gO7khzF3B+AK6rq00nuAC5L8hfA14CPjLFOSdIQQwO9qm4BzjxC+37mzqdLklYA7xSVpEYY6JLU\nCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w\n0CWpEQa6JDXCQJekRhjoktQIA12SGjE00JO8IMm1Se5McnuSd3ft65LsSbKve107/nIlSfPpc4T+\nOPAHVfVi4GzgHUleAmwH9lbVRmBvNy9JmpChgV5VD1TVV7vpHwB3AicCm4BdXbddwOZxFSlJGm5B\n59CTbADOBG4ATqiqB2Au9IHj51lnW5KZJDOzs7NLq1aSNK/egZ7kZ4BPAu+pqu/3Xa+qdlbVdFVN\nT01NLaZGSVIPvQI9ybOYC/OPV9WnuuYHk6zvlq8HDo6nRElSH33e5RLgI8CdVfVXA4t2A1u66S3A\n1aMvT5LU15oefc4Bfhu4NclNXdsfAzuAK5JsBe4D3jSeEiVJfQwN9Kr6DyDzLD5vtOVIkhbLO0Ul\nqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RG9Hk41zPahu2f\nmdi2791x4cS2LWn18QhdkhphoEtSIwx0SWqEgS5JjfCiqJ7CC8HS6uQRuiQ1os+HRH80ycEktw20\nrUuyJ8m+7nXteMuUJA3T55TLPwEfBj420LYd2FtVO5Js7+bfN/ryntkmeepD0uoz9Ai9qr4IfPew\n5k3Arm56F7B5xHVJkhZosefQT6iqBwC61+NHV5IkaTHGflE0ybYkM0lmZmdnx705SXrGWmygP5hk\nPUD3enC+jlW1s6qmq2p6ampqkZuTJA2z2EDfDWzpprcAV4+mHEnSYvV52+KlwH8BpyU5kGQrsAM4\nP8k+4PxuXpI0QUPftlhVb55n0XkjrkWStATe+q8VZVLvvfeRA2qBt/5LUiMMdElqhIEuSY0w0CWp\nEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wqctSkzuKY/gkx41Oh6h\nS1IjPEKXnqH8r6Q9HqFLUiOWFOhJLkhyV5K7k2wfVVGSpIVb9CmXJEcBf8vch0QfAL6SZHdV3TGq\n4qRngkme+nimaf0jDpdyhH4WcHdV7a+qHwGXAZtGU5YkaaGWEugnAt8emD/QtUmSJmAp73LJEdrq\nKZ2SbcC2bvbRJHctcnvHAd9Z5LqTsJrqXU21gvWO07LUmg+M7FutirHtft6l1PrCPp2WEugHgBcM\nzJ8E3H94p6raCexcwnYASDJTVdNL/T7LZTXVu5pqBesdp9VUK6yuepej1qWccvkKsDHJKUmOBi4C\ndo+mLEnSQi36CL2qHk/yTuDfgKOAj1bV7SOrTJK0IEu6U7SqPgt8dkS1DLPk0zbLbDXVu5pqBesd\np9VUK6yuesdea6qech1TkrQKeeu/JDViRQT6sEcIJHl2ksu75Tck2TCw7P1d+11JfmUF1PreJHck\nuSXJ3iQvHFj2RJKbuq9luYDco963JpkdqOt3B5ZtSbKv+9qyAmr94ECd30jyyMCySYztR5McTHLb\nPMuT5G+6n+eWJC8bWLbcYzus1rd0Nd6S5EtJTh9Ydm+SW7uxnRl3rT3rfXWS7w38zv9kYNmyPpKk\nR61/OFDnbd2+uq5bNtqxraqJfjF3QfUe4FTgaOBm4CWH9fl94O+76YuAy7vpl3T9nw2c0n2foyZc\n62uA53bTv3eo1m7+0RU4tm8FPnyEddcB+7vXtd302knWelj/dzF3IX4iY9tt81XAy4Db5ln+euBz\nzN2zcTZwwyTGtmetrzxUA/C6Q7V28/cCx62wsX018Oml7kfLUethfX8NuGZcY7sSjtD7PEJgE7Cr\nm74SOC9JuvbLquqxqvomcHf3/SZWa1VdW1U/7GavZ+79+ZOylMcz/Aqwp6q+W1UPA3uAC8ZUJyy8\n1jcDl46xnqGq6ovAd5+myybgYzXneuDYJOtZ/rEdWmtVfamrBSa/3/YZ2/ks+yNJFljrWPfblRDo\nfR4h8OM+VfU48D3g+T3XHaWFbm8rc0dohzwnyUyS65NsHkeBh+lb7290/2pfmeTQzWIrdmy701in\nANcMNC/32PYx38+00h+bcfh+W8AXktyYuTu/V4pXJLk5yeeSvLRrW7Fjm+S5zP3h/uRA80jHdiV8\nwEWfRwjM16fX4wdGqPf2kvwWMA380kDzyVV1f5JTgWuS3FpV94yhzh+XcYS2w+v9V+DSqnosyduZ\n+0/o3J7rjtJCtncRcGVVPTHQttxj28dK2W97S/Ia5gL9Fweaz+nG9nhgT5Kvd0elk/RV4IVV9WiS\n1wP/AmxkBY8tc6db/rOqBo/mRzq2K+EIvc8jBH7cJ8ka4OeY+xen1+MHRqjX9pK8FrgYeENVPXao\nvaru7173A9cBZ46xVuhRb1U9NFDjPwAv77vuiC1kexdx2L+tExjbPub7mZZ7bHtJ8gvAJcCmqnro\nUPvA2B4ErmK8pzV7qarvV9Wj3fRngWclOY4VOradp9tvRzO247xY0POCwhrmLgqdwk8uYrz0sD7v\n4MkXRa/opl/Kky+K7me8F0X71HomcxdlNh7WvhZ4djd9HLCP8V+s6VPv+oHpXweu76bXAd/s6l7b\nTa+bZK1dv9OYu5CUSY7twLY3MP+Fuwt58kXRL09ibHvWejJz16BeeVj7McDzBqa/BFywAsb25w/t\nA8yF4H3dOPfaj5az1m75oYPQY8Y5tmP/pfQcjNcD3+iC8OKu7c+ZO8IFeA7wiW6H+zJw6sC6F3fr\n3QW8bgXU+u/Ag8BN3dfurv2VwK3dDnYrsHWFjO1fArd3dV0LvGhg3d/pxvxu4G2TrrWb/1Ngx2Hr\nTWpsLwUeAP6XuSPDrcDbgbd3y8Pch8Dc09U1PcGxHVbrJcDDA/vtTNd+ajeuN3f7ycUrZGzfObDf\nXs/AH6Ij7UeTrLXr81bm3sAxuN7Ix9Y7RSWpESvhHLokaQQMdElqhIEuSY0w0CWpEQa6JDXCQJek\nRhjoktQIA12SGvH/eacggB4hyJoAAAAASUVORK5CYII=\n", 191 | "text/plain": [ 192 | "" 193 | ] 194 | }, 195 | "metadata": {}, 196 | "output_type": "display_data" 197 | } 198 | ], 199 | "source": [ 200 | "_ = plt.hist(np.transpose(layer_output))" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [] 209 | } 210 | ], 211 | "metadata": { 212 | "kernelspec": { 213 | "display_name": "Python 3", 214 | "language": "python", 215 | "name": "python3" 216 | }, 217 | "language_info": { 218 | "codemirror_mode": { 219 | "name": "ipython", 220 | "version": 3 221 | }, 222 | "file_extension": ".py", 223 | "mimetype": "text/x-python", 224 | "name": "python", 225 | "nbconvert_exporter": "python", 226 | "pygments_lexer": "ipython3", 227 | "version": "3.6.3" 228 | } 229 | }, 230 | "nbformat": 4, 231 | "nbformat_minor": 2 232 | } 233 | -------------------------------------------------------------------------------- /notebooks/vggnet_in_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# VGGNet in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we fit a model inspired by the \"very deep\" convolutional network [VGGNet](https://arxiv.org/pdf/1409.1556.pdf) to classify flowers into the 17 categories of the Oxford Flowers data set. Derived from [these](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/old/L3-3c__TFLearn_VGGNet.ipynb) [two](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/alexnet_in_keras.ipynb) earlier notebooks. " 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Set seed for reproducibility" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "np.random.seed(42)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Load dependencies" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "name": "stderr", 50 | "output_type": "stream", 51 | "text": [ 52 | "Using TensorFlow backend.\n" 53 | ] 54 | } 55 | ], 56 | "source": [ 57 | "import keras\n", 58 | "from keras.models import Sequential\n", 59 | "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n", 60 | "from keras.layers.normalization import BatchNormalization\n", 61 | "from keras.callbacks import TensorBoard # for part 3.5 on TensorBoard" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "#### Load *and preprocess* data" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": { 75 | "collapsed": true 76 | }, 77 | "outputs": [], 78 | "source": [ 79 | "import tflearn.datasets.oxflower17 as oxflower17\n", 80 | "X, Y = oxflower17.load_data(one_hot=True)" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "#### Design neural network architecture" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 4, 93 | "metadata": { 94 | "collapsed": true 95 | }, 96 | "outputs": [], 97 | "source": [ 98 | "model = Sequential()\n", 99 | "\n", 100 | "model.add(Conv2D(64, 3, activation='relu', input_shape=(224, 224, 3)))\n", 101 | "model.add(Conv2D(64, 3, activation='relu'))\n", 102 | "model.add(MaxPooling2D(2, 2))\n", 103 | "model.add(BatchNormalization())\n", 104 | "\n", 105 | "model.add(Conv2D(128, 3, activation='relu'))\n", 106 | "model.add(Conv2D(128, 3, activation='relu'))\n", 107 | "model.add(MaxPooling2D(2, 2))\n", 108 | "model.add(BatchNormalization())\n", 109 | "\n", 110 | "model.add(Conv2D(256, 3, activation='relu'))\n", 111 | "model.add(Conv2D(256, 3, activation='relu'))\n", 112 | "model.add(Conv2D(256, 3, activation='relu'))\n", 113 | "model.add(MaxPooling2D(2, 2))\n", 114 | "model.add(BatchNormalization())\n", 115 | "\n", 116 | "model.add(Conv2D(512, 3, activation='relu'))\n", 117 | "model.add(Conv2D(512, 3, activation='relu'))\n", 118 | "model.add(Conv2D(512, 3, activation='relu'))\n", 119 | "model.add(MaxPooling2D(2, 2))\n", 120 | "model.add(BatchNormalization())\n", 121 | "\n", 122 | "model.add(Conv2D(512, 3, activation='relu'))\n", 123 | "model.add(Conv2D(512, 3, activation='relu'))\n", 124 | "model.add(Conv2D(512, 3, activation='relu'))\n", 125 | "model.add(MaxPooling2D(2, 2))\n", 126 | "model.add(BatchNormalization())\n", 127 | "\n", 128 | "model.add(Flatten())\n", 129 | "model.add(Dense(4096, activation='relu'))\n", 130 | "model.add(Dropout(0.5))\n", 131 | "model.add(Dense(4096, activation='relu'))\n", 132 | "model.add(Dropout(0.5))\n", 133 | "\n", 134 | "model.add(Dense(17, activation='softmax'))" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": 5, 140 | "metadata": {}, 141 | "outputs": [ 142 | { 143 | "name": "stdout", 144 | "output_type": "stream", 145 | "text": [ 146 | "_________________________________________________________________\n", 147 | "Layer (type) Output Shape Param # \n", 148 | "=================================================================\n", 149 | "conv2d_1 (Conv2D) (None, 222, 222, 64) 1792 \n", 150 | "_________________________________________________________________\n", 151 | "conv2d_2 (Conv2D) (None, 220, 220, 64) 36928 \n", 152 | "_________________________________________________________________\n", 153 | "max_pooling2d_1 (MaxPooling2 (None, 110, 110, 64) 0 \n", 154 | "_________________________________________________________________\n", 155 | "batch_normalization_1 (Batch (None, 110, 110, 64) 256 \n", 156 | "_________________________________________________________________\n", 157 | "conv2d_3 (Conv2D) (None, 108, 108, 128) 73856 \n", 158 | "_________________________________________________________________\n", 159 | "conv2d_4 (Conv2D) (None, 106, 106, 128) 147584 \n", 160 | "_________________________________________________________________\n", 161 | "max_pooling2d_2 (MaxPooling2 (None, 53, 53, 128) 0 \n", 162 | "_________________________________________________________________\n", 163 | "batch_normalization_2 (Batch (None, 53, 53, 128) 512 \n", 164 | "_________________________________________________________________\n", 165 | "conv2d_5 (Conv2D) (None, 51, 51, 256) 295168 \n", 166 | "_________________________________________________________________\n", 167 | "conv2d_6 (Conv2D) (None, 49, 49, 256) 590080 \n", 168 | "_________________________________________________________________\n", 169 | "conv2d_7 (Conv2D) (None, 47, 47, 256) 590080 \n", 170 | "_________________________________________________________________\n", 171 | "max_pooling2d_3 (MaxPooling2 (None, 23, 23, 256) 0 \n", 172 | "_________________________________________________________________\n", 173 | "batch_normalization_3 (Batch (None, 23, 23, 256) 1024 \n", 174 | "_________________________________________________________________\n", 175 | "conv2d_8 (Conv2D) (None, 21, 21, 512) 1180160 \n", 176 | "_________________________________________________________________\n", 177 | "conv2d_9 (Conv2D) (None, 19, 19, 512) 2359808 \n", 178 | "_________________________________________________________________\n", 179 | "conv2d_10 (Conv2D) (None, 17, 17, 512) 2359808 \n", 180 | "_________________________________________________________________\n", 181 | "max_pooling2d_4 (MaxPooling2 (None, 8, 8, 512) 0 \n", 182 | "_________________________________________________________________\n", 183 | "batch_normalization_4 (Batch (None, 8, 8, 512) 2048 \n", 184 | "_________________________________________________________________\n", 185 | "conv2d_11 (Conv2D) (None, 6, 6, 512) 2359808 \n", 186 | "_________________________________________________________________\n", 187 | "conv2d_12 (Conv2D) (None, 4, 4, 512) 2359808 \n", 188 | "_________________________________________________________________\n", 189 | "conv2d_13 (Conv2D) (None, 2, 2, 512) 2359808 \n", 190 | "_________________________________________________________________\n", 191 | "max_pooling2d_5 (MaxPooling2 (None, 1, 1, 512) 0 \n", 192 | "_________________________________________________________________\n", 193 | "batch_normalization_5 (Batch (None, 1, 1, 512) 2048 \n", 194 | "_________________________________________________________________\n", 195 | "flatten_1 (Flatten) (None, 512) 0 \n", 196 | "_________________________________________________________________\n", 197 | "dense_1 (Dense) (None, 4096) 2101248 \n", 198 | "_________________________________________________________________\n", 199 | "dropout_1 (Dropout) (None, 4096) 0 \n", 200 | "_________________________________________________________________\n", 201 | "dense_2 (Dense) (None, 4096) 16781312 \n", 202 | "_________________________________________________________________\n", 203 | "dropout_2 (Dropout) (None, 4096) 0 \n", 204 | "_________________________________________________________________\n", 205 | "dense_3 (Dense) (None, 17) 69649 \n", 206 | "=================================================================\n", 207 | "Total params: 33,672,785\n", 208 | "Trainable params: 33,669,841\n", 209 | "Non-trainable params: 2,944\n", 210 | "_________________________________________________________________\n" 211 | ] 212 | } 213 | ], 214 | "source": [ 215 | "model.summary()" 216 | ] 217 | }, 218 | { 219 | "cell_type": "markdown", 220 | "metadata": {}, 221 | "source": [ 222 | "#### Configure model" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 6, 228 | "metadata": { 229 | "collapsed": true 230 | }, 231 | "outputs": [], 232 | "source": [ 233 | "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])" 234 | ] 235 | }, 236 | { 237 | "cell_type": "markdown", 238 | "metadata": {}, 239 | "source": [ 240 | "#### Configure TensorBoard (for part 5 of lesson 3)" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "metadata": { 247 | "collapsed": true 248 | }, 249 | "outputs": [], 250 | "source": [ 251 | "tensorbrd = TensorBoard('logs/vggnet')" 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "metadata": {}, 257 | "source": [ 258 | "#### Train!" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": null, 264 | "metadata": {}, 265 | "outputs": [ 266 | { 267 | "name": "stdout", 268 | "output_type": "stream", 269 | "text": [ 270 | "Train on 1224 samples, validate on 136 samples\n", 271 | "Epoch 1/16\n", 272 | "1224/1224 [==============================] - 784s - loss: 4.1356 - acc: 0.0735 - val_loss: 12.5035 - val_acc: 0.0368\n", 273 | "Epoch 2/16\n", 274 | "1224/1224 [==============================] - 774s - loss: 3.4962 - acc: 0.1062 - val_loss: 13.7122 - val_acc: 0.0735\n", 275 | "Epoch 3/16\n", 276 | "1224/1224 [==============================] - 773s - loss: 2.9282 - acc: 0.1373 - val_loss: 13.0526 - val_acc: 0.0441\n", 277 | "Epoch 4/16\n", 278 | "1224/1224 [==============================] - 774s - loss: 3.1878 - acc: 0.1381 - val_loss: 3.1900 - val_acc: 0.0956\n", 279 | "Epoch 5/16\n", 280 | "1224/1224 [==============================] - 773s - loss: 3.4662 - acc: 0.0752 - val_loss: 2.8582 - val_acc: 0.0588\n", 281 | "Epoch 6/16\n", 282 | "1224/1224 [==============================] - 770s - loss: 3.2509 - acc: 0.1119 - val_loss: 2.8061 - val_acc: 0.0441\n", 283 | "Epoch 7/16\n", 284 | "1224/1224 [==============================] - 770s - loss: 2.8882 - acc: 0.1193 - val_loss: 2.6081 - val_acc: 0.1029\n", 285 | "Epoch 8/16\n", 286 | "1224/1224 [==============================] - 772s - loss: 3.1455 - acc: 0.1144 - val_loss: 3.9969 - val_acc: 0.0588\n", 287 | "Epoch 9/16\n", 288 | "1224/1224 [==============================] - 773s - loss: 3.2672 - acc: 0.1185 - val_loss: 5.0395 - val_acc: 0.0515\n", 289 | "Epoch 10/16\n", 290 | "1224/1224 [==============================] - 771s - loss: 2.8959 - acc: 0.1324 - val_loss: 2.8850 - val_acc: 0.0662\n", 291 | "Epoch 11/16\n", 292 | "1224/1224 [==============================] - 771s - loss: 2.6121 - acc: 0.1765 - val_loss: 2.7125 - val_acc: 0.0588\n", 293 | "Epoch 12/16\n", 294 | "1224/1224 [==============================] - 771s - loss: 2.5183 - acc: 0.1675 - val_loss: 2.5720 - val_acc: 0.1176\n", 295 | "Epoch 13/16\n", 296 | "1224/1224 [==============================] - 771s - loss: 2.6153 - acc: 0.1822 - val_loss: 6.8114 - val_acc: 0.1103\n", 297 | "Epoch 14/16\n", 298 | "1224/1224 [==============================] - 770s - loss: 2.6700 - acc: 0.1748 - val_loss: 4.6305 - val_acc: 0.1324\n", 299 | "Epoch 15/16\n", 300 | "1088/1224 [=========================>....] - ETA: 82s - loss: 2.7326 - acc: 0.1967 " 301 | ] 302 | } 303 | ], 304 | "source": [ 305 | "model.fit(X, Y, batch_size=64, epochs=16, verbose=1, validation_split=0.1, shuffle=True,\n", 306 | " callbacks=[tensorbrd])" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": null, 312 | "metadata": { 313 | "collapsed": true 314 | }, 315 | "outputs": [], 316 | "source": [] 317 | } 318 | ], 319 | "metadata": { 320 | "kernelspec": { 321 | "display_name": "Python 3", 322 | "language": "python", 323 | "name": "python3" 324 | }, 325 | "language_info": { 326 | "codemirror_mode": { 327 | "name": "ipython", 328 | "version": 3 329 | }, 330 | "file_extension": ".py", 331 | "mimetype": "text/x-python", 332 | "name": "python", 333 | "nbconvert_exporter": "python", 334 | "pygments_lexer": "ipython3", 335 | "version": "3.6.1" 336 | } 337 | }, 338 | "nbformat": 4, 339 | "nbformat_minor": 2 340 | } 341 | -------------------------------------------------------------------------------- /notebooks/alexnet_in_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# AlexNet in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we leverage an [AlexNet](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks)-like deep, convolutional neural network to classify flowers into the 17 categories of the [Oxford Flowers](http://www.robots.ox.ac.uk/~vgg/data/flowers/17/) data set. Derived from [this earlier notebook](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/old/L3-3b__TFLearn_AlexNet.ipynb)." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Set seed for reproducibility" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "np.random.seed(42)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Load dependencies" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 4, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "import keras\n", 50 | "from keras.models import Sequential\n", 51 | "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n", 52 | "from keras.layers.normalization import BatchNormalization\n", 53 | "from keras.callbacks import TensorBoard # for part 3.5 on TensorBoard" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "#### Load *and preprocess* data" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 5, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "import tflearn.datasets.oxflower17 as oxflower17\n", 70 | "X, Y = oxflower17.load_data(one_hot=True)" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "#### Design neural network architecture" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 6, 83 | "metadata": { 84 | "collapsed": true 85 | }, 86 | "outputs": [], 87 | "source": [ 88 | "model = Sequential()\n", 89 | "\n", 90 | "model.add(Conv2D(96, kernel_size=(11, 11), strides=(4, 4), activation='relu', input_shape=(224, 224, 3)))\n", 91 | "model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))\n", 92 | "model.add(BatchNormalization())\n", 93 | "\n", 94 | "model.add(Conv2D(256, kernel_size=(5, 5), activation='relu'))\n", 95 | "model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))\n", 96 | "model.add(BatchNormalization())\n", 97 | "\n", 98 | "model.add(Conv2D(256, kernel_size=(3, 3), activation='relu'))\n", 99 | "model.add(Conv2D(384, kernel_size=(3, 3), activation='relu'))\n", 100 | "model.add(Conv2D(384, kernel_size=(3, 3), activation='relu'))\n", 101 | "model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))\n", 102 | "model.add(BatchNormalization())\n", 103 | "\n", 104 | "model.add(Flatten())\n", 105 | "model.add(Dense(4096, activation='tanh'))\n", 106 | "model.add(Dropout(0.5))\n", 107 | "model.add(Dense(4096, activation='tanh'))\n", 108 | "model.add(Dropout(0.5))\n", 109 | "\n", 110 | "model.add(Dense(17, activation='softmax'))" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 7, 116 | "metadata": {}, 117 | "outputs": [ 118 | { 119 | "name": "stdout", 120 | "output_type": "stream", 121 | "text": [ 122 | "_________________________________________________________________\n", 123 | "Layer (type) Output Shape Param # \n", 124 | "=================================================================\n", 125 | "conv2d_1 (Conv2D) (None, 54, 54, 96) 34944 \n", 126 | "_________________________________________________________________\n", 127 | "max_pooling2d_1 (MaxPooling2 (None, 26, 26, 96) 0 \n", 128 | "_________________________________________________________________\n", 129 | "batch_normalization_1 (Batch (None, 26, 26, 96) 384 \n", 130 | "_________________________________________________________________\n", 131 | "conv2d_2 (Conv2D) (None, 22, 22, 256) 614656 \n", 132 | "_________________________________________________________________\n", 133 | "max_pooling2d_2 (MaxPooling2 (None, 10, 10, 256) 0 \n", 134 | "_________________________________________________________________\n", 135 | "batch_normalization_2 (Batch (None, 10, 10, 256) 1024 \n", 136 | "_________________________________________________________________\n", 137 | "conv2d_3 (Conv2D) (None, 8, 8, 256) 590080 \n", 138 | "_________________________________________________________________\n", 139 | "conv2d_4 (Conv2D) (None, 6, 6, 384) 885120 \n", 140 | "_________________________________________________________________\n", 141 | "conv2d_5 (Conv2D) (None, 4, 4, 384) 1327488 \n", 142 | "_________________________________________________________________\n", 143 | "max_pooling2d_3 (MaxPooling2 (None, 1, 1, 384) 0 \n", 144 | "_________________________________________________________________\n", 145 | "batch_normalization_3 (Batch (None, 1, 1, 384) 1536 \n", 146 | "_________________________________________________________________\n", 147 | "flatten_1 (Flatten) (None, 384) 0 \n", 148 | "_________________________________________________________________\n", 149 | "dense_1 (Dense) (None, 4096) 1576960 \n", 150 | "_________________________________________________________________\n", 151 | "dropout_1 (Dropout) (None, 4096) 0 \n", 152 | "_________________________________________________________________\n", 153 | "dense_2 (Dense) (None, 4096) 16781312 \n", 154 | "_________________________________________________________________\n", 155 | "dropout_2 (Dropout) (None, 4096) 0 \n", 156 | "_________________________________________________________________\n", 157 | "dense_3 (Dense) (None, 17) 69649 \n", 158 | "=================================================================\n", 159 | "Total params: 21,883,153\n", 160 | "Trainable params: 21,881,681\n", 161 | "Non-trainable params: 1,472\n", 162 | "_________________________________________________________________\n" 163 | ] 164 | } 165 | ], 166 | "source": [ 167 | "model.summary()" 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": {}, 173 | "source": [ 174 | "#### Configure model" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 8, 180 | "metadata": { 181 | "collapsed": true 182 | }, 183 | "outputs": [], 184 | "source": [ 185 | "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])" 186 | ] 187 | }, 188 | { 189 | "cell_type": "markdown", 190 | "metadata": {}, 191 | "source": [ 192 | "#### Configure TensorBoard (for part 5 of lesson 3)" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 9, 198 | "metadata": { 199 | "collapsed": true 200 | }, 201 | "outputs": [], 202 | "source": [ 203 | "tensorbrd = TensorBoard('logs/alexnet') " 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "metadata": {}, 209 | "source": [ 210 | "#### Train!" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 10, 216 | "metadata": {}, 217 | "outputs": [ 218 | { 219 | "name": "stdout", 220 | "output_type": "stream", 221 | "text": [ 222 | "Train on 1224 samples, validate on 136 samples\n", 223 | "Epoch 1/32\n", 224 | "1224/1224 [==============================] - 43s - loss: 5.4860 - acc: 0.2092 - val_loss: 10.5151 - val_acc: 0.1029\n", 225 | "Epoch 2/32\n", 226 | "1224/1224 [==============================] - 41s - loss: 4.0177 - acc: 0.2525 - val_loss: 4.1294 - val_acc: 0.1838\n", 227 | "Epoch 3/32\n", 228 | "1224/1224 [==============================] - 41s - loss: 2.8861 - acc: 0.3113 - val_loss: 4.2866 - val_acc: 0.1838\n", 229 | "Epoch 4/32\n", 230 | "1224/1224 [==============================] - 41s - loss: 2.7104 - acc: 0.3194 - val_loss: 5.1511 - val_acc: 0.1471\n", 231 | "Epoch 5/32\n", 232 | "1224/1224 [==============================] - 42s - loss: 2.6687 - acc: 0.3154 - val_loss: 4.1556 - val_acc: 0.1838\n", 233 | "Epoch 6/32\n", 234 | "1224/1224 [==============================] - 41s - loss: 2.4847 - acc: 0.3358 - val_loss: 2.7126 - val_acc: 0.2647\n", 235 | "Epoch 7/32\n", 236 | "1224/1224 [==============================] - 41s - loss: 2.4848 - acc: 0.3358 - val_loss: 3.5959 - val_acc: 0.2574\n", 237 | "Epoch 8/32\n", 238 | "1224/1224 [==============================] - 41s - loss: 2.3969 - acc: 0.3922 - val_loss: 3.5869 - val_acc: 0.2868\n", 239 | "Epoch 9/32\n", 240 | "1224/1224 [==============================] - 41s - loss: 2.7176 - acc: 0.3472 - val_loss: 2.8241 - val_acc: 0.3309\n", 241 | "Epoch 10/32\n", 242 | "1224/1224 [==============================] - 41s - loss: 2.3563 - acc: 0.3693 - val_loss: 2.6689 - val_acc: 0.3676\n", 243 | "Epoch 11/32\n", 244 | "1224/1224 [==============================] - 41s - loss: 1.9205 - acc: 0.4281 - val_loss: 2.8030 - val_acc: 0.3824\n", 245 | "Epoch 12/32\n", 246 | "1224/1224 [==============================] - 39s - loss: 2.1723 - acc: 0.4101 - val_loss: 2.4830 - val_acc: 0.3603\n", 247 | "Epoch 13/32\n", 248 | "1224/1224 [==============================] - 39s - loss: 1.9932 - acc: 0.4542 - val_loss: 2.6662 - val_acc: 0.3824\n", 249 | "Epoch 14/32\n", 250 | "1224/1224 [==============================] - 38s - loss: 2.0622 - acc: 0.4404 - val_loss: 2.0595 - val_acc: 0.4485\n", 251 | "Epoch 15/32\n", 252 | "1224/1224 [==============================] - 38s - loss: 2.0491 - acc: 0.4706 - val_loss: 2.1866 - val_acc: 0.4632\n", 253 | "Epoch 16/32\n", 254 | "1224/1224 [==============================] - 39s - loss: 2.0746 - acc: 0.4534 - val_loss: 2.5153 - val_acc: 0.4559\n", 255 | "Epoch 17/32\n", 256 | "1224/1224 [==============================] - 40s - loss: 2.2386 - acc: 0.4297 - val_loss: 2.6147 - val_acc: 0.4265\n", 257 | "Epoch 18/32\n", 258 | "1224/1224 [==============================] - 42s - loss: 2.0306 - acc: 0.4812 - val_loss: 2.8573 - val_acc: 0.3824\n", 259 | "Epoch 19/32\n", 260 | "1224/1224 [==============================] - 41s - loss: 1.9511 - acc: 0.4771 - val_loss: 2.1484 - val_acc: 0.4632\n", 261 | "Epoch 20/32\n", 262 | "1224/1224 [==============================] - 41s - loss: 1.8286 - acc: 0.5123 - val_loss: 2.3557 - val_acc: 0.4853\n", 263 | "Epoch 21/32\n", 264 | "1224/1224 [==============================] - 41s - loss: 2.2471 - acc: 0.4493 - val_loss: 2.3572 - val_acc: 0.4118\n", 265 | "Epoch 22/32\n", 266 | "1224/1224 [==============================] - 41s - loss: 1.6955 - acc: 0.5278 - val_loss: 2.2848 - val_acc: 0.4779\n", 267 | "Epoch 23/32\n", 268 | "1224/1224 [==============================] - 41s - loss: 1.5541 - acc: 0.5498 - val_loss: 1.6983 - val_acc: 0.5956\n", 269 | "Epoch 24/32\n", 270 | "1224/1224 [==============================] - 40s - loss: 1.5198 - acc: 0.5523 - val_loss: 1.9519 - val_acc: 0.5588\n", 271 | "Epoch 25/32\n", 272 | "1224/1224 [==============================] - 41s - loss: 1.6150 - acc: 0.5588 - val_loss: 2.6820 - val_acc: 0.4853\n", 273 | "Epoch 26/32\n", 274 | "1224/1224 [==============================] - 39s - loss: 1.4255 - acc: 0.5940 - val_loss: 2.8716 - val_acc: 0.4118\n", 275 | "Epoch 27/32\n", 276 | "1224/1224 [==============================] - 39s - loss: 1.6967 - acc: 0.5408 - val_loss: 2.7123 - val_acc: 0.4485\n", 277 | "Epoch 28/32\n", 278 | "1224/1224 [==============================] - 44s - loss: 1.6957 - acc: 0.5474 - val_loss: 2.1038 - val_acc: 0.4779\n", 279 | "Epoch 29/32\n", 280 | "1224/1224 [==============================] - 37s - loss: 1.5395 - acc: 0.5703 - val_loss: 2.5981 - val_acc: 0.5074\n", 281 | "Epoch 30/32\n", 282 | "1224/1224 [==============================] - 37s - loss: 1.5090 - acc: 0.5940 - val_loss: 2.2708 - val_acc: 0.5368\n", 283 | "Epoch 31/32\n", 284 | "1224/1224 [==============================] - 37s - loss: 1.5410 - acc: 0.5678 - val_loss: 2.3326 - val_acc: 0.4853\n", 285 | "Epoch 32/32\n", 286 | "1224/1224 [==============================] - 37s - loss: 1.6714 - acc: 0.5531 - val_loss: 1.8566 - val_acc: 0.5000\n" 287 | ] 288 | }, 289 | { 290 | "data": { 291 | "text/plain": [ 292 | "" 293 | ] 294 | }, 295 | "execution_count": 10, 296 | "metadata": {}, 297 | "output_type": "execute_result" 298 | } 299 | ], 300 | "source": [ 301 | "model.fit(X, Y, batch_size=64, epochs=32, verbose=1, validation_split=0.1, shuffle=True, \n", 302 | " callbacks=[tensorbrd])" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "metadata": { 309 | "collapsed": true 310 | }, 311 | "outputs": [], 312 | "source": [] 313 | } 314 | ], 315 | "metadata": { 316 | "kernelspec": { 317 | "display_name": "Python 3", 318 | "language": "python", 319 | "name": "python3" 320 | }, 321 | "language_info": { 322 | "codemirror_mode": { 323 | "name": "ipython", 324 | "version": 3 325 | }, 326 | "file_extension": ".py", 327 | "mimetype": "text/x-python", 328 | "name": "python", 329 | "nbconvert_exporter": "python", 330 | "pygments_lexer": "ipython3", 331 | "version": "3.6.1" 332 | } 333 | }, 334 | "nbformat": 4, 335 | "nbformat_minor": 2 336 | } 337 | -------------------------------------------------------------------------------- /notebooks/lenet_in_tensorflow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Deep Convolutional Neural Network in TensorFlow" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we convert our [LeNet-5-inspired, MNIST-classifying, deep convolutional network](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/lenet_in_keras.ipynb) from Keras to TensorFlow (compare them side by side) following Aymeric Damien's [style](https://github.com/aymericdamien/TensorFlow-Examples/blob/master/notebooks/3_NeuralNetworks/multilayer_perceptron.ipynb)." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import numpy as np\n", 31 | "np.random.seed(42)\n", 32 | "import tensorflow as tf\n", 33 | "tf.set_random_seed(42)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Load data" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "name": "stdout", 50 | "output_type": "stream", 51 | "text": [ 52 | "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", 53 | "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", 54 | "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", 55 | "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n" 56 | ] 57 | } 58 | ], 59 | "source": [ 60 | "from tensorflow.examples.tutorials.mnist import input_data\n", 61 | "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "#### Set neural network hyperparameters" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "epochs = 1\n", 78 | "batch_size = 128\n", 79 | "display_progress = 10 # after this many batches, output progress to screen\n", 80 | "wt_init = tf.contrib.layers.xavier_initializer() # weight initializer" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "#### Set parameters for each layer" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 4, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "# input layer: \n", 97 | "n_input = 784\n", 98 | "\n", 99 | "# first convolutional layer: \n", 100 | "n_conv_1 = 32\n", 101 | "k_conv_1 = 3\n", 102 | "\n", 103 | "# second convolutional layer: \n", 104 | "n_conv_2 = 64\n", 105 | "k_conv_2 = 3\n", 106 | "\n", 107 | "# max pooling layer:\n", 108 | "pool_size = 2\n", 109 | "mp_layer_dropout = 0.25\n", 110 | "\n", 111 | "# dense layer: \n", 112 | "n_dense = 128\n", 113 | "dense_layer_dropout = 0.5\n", 114 | "\n", 115 | "# output layer: \n", 116 | "n_classes = 10" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": {}, 122 | "source": [ 123 | "#### Define placeholder Tensors for inputs and labels" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 5, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "x = tf.placeholder(tf.float32, [None, n_input])\n", 133 | "y = tf.placeholder(tf.float32, [None, n_classes])" 134 | ] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "metadata": {}, 139 | "source": [ 140 | "#### Define types of layers" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 7, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "# dense layer with ReLU activation:\n", 150 | "def dense(x, W, b):\n", 151 | " z = tf.add(tf.matmul(x, W), b)\n", 152 | " a = tf.nn.relu(z)\n", 153 | " return a\n", 154 | "\n", 155 | "# convolutional layer with ReLU activation:\n", 156 | "def conv2d(x, W, b, stride_length=1):\n", 157 | " xW = tf.nn.conv2d(x, W, strides=[1, stride_length, stride_length, 1], padding='SAME')\n", 158 | " z = tf.nn.bias_add(xW, b)\n", 159 | " a = tf.nn.relu(z)\n", 160 | " return a\n", 161 | "\n", 162 | "# max-pooling layer: \n", 163 | "def maxpooling2d(x, p_size):\n", 164 | " return tf.nn.max_pool(x, \n", 165 | " ksize=[1, p_size, p_size, 1],\n", 166 | " strides=[1, p_size, p_size, 1],\n", 167 | " padding='SAME'\n", 168 | " )" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "#### Define dictionaries for storing weights and biases for each layer -- and initialize" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 8, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "bias_dict = {\n", 185 | " 'b_c1': tf.Variable(tf.zeros([n_conv_1])),\n", 186 | " 'b_c2': tf.Variable(tf.zeros([n_conv_2])),\n", 187 | " 'b_d1': tf.Variable(tf.zeros([n_dense])),\n", 188 | " 'b_out': tf.Variable(tf.zeros([n_classes]))\n", 189 | "}\n", 190 | "\n", 191 | "# calculate number of inputs to dense layer: \n", 192 | "full_square_length = np.sqrt(n_input)\n", 193 | "pooled_square_length = int(full_square_length / pool_size)\n", 194 | "dense_inputs = pooled_square_length**2 * n_conv_2\n", 195 | "\n", 196 | "weight_dict = {\n", 197 | " 'W_c1': tf.get_variable('W_c1', \n", 198 | " [k_conv_1, k_conv_1, 1, n_conv_1], initializer=wt_init),\n", 199 | " 'W_c2': tf.get_variable('W_c2', \n", 200 | " [k_conv_2, k_conv_2, n_conv_1, n_conv_2], initializer=wt_init),\n", 201 | " 'W_d1': tf.get_variable('W_d1', \n", 202 | " [dense_inputs, n_dense], initializer=wt_init),\n", 203 | " 'W_out': tf.get_variable('W_out', \n", 204 | " [n_dense, n_classes], initializer=wt_init)\n", 205 | "}" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "#### Design neural network architecture" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": 9, 218 | "metadata": {}, 219 | "outputs": [], 220 | "source": [ 221 | "def network(x, weights, biases, n_in, mp_psize, mp_dropout, dense_dropout):\n", 222 | "\n", 223 | " # reshape linear MNIST pixel input into square image: \n", 224 | " square_dimensions = int(np.sqrt(n_in))\n", 225 | " square_x = tf.reshape(x, shape=[-1, square_dimensions, square_dimensions, 1])\n", 226 | " \n", 227 | " # convolutional and max-pooling layers:\n", 228 | " conv_1 = conv2d(square_x, weights['W_c1'], biases['b_c1'])\n", 229 | " conv_2 = conv2d(conv_1, weights['W_c2'], biases['b_c2'])\n", 230 | " pool_1 = maxpooling2d(conv_2, mp_psize)\n", 231 | " pool_1 = tf.nn.dropout(pool_1, 1-mp_dropout)\n", 232 | " \n", 233 | " # dense layer: \n", 234 | " flat = tf.reshape(pool_1, [-1, weight_dict['W_d1'].get_shape().as_list()[0]])\n", 235 | " dense_1 = dense(flat, weights['W_d1'], biases['b_d1'])\n", 236 | " dense_1 = tf.nn.dropout(dense_1, 1-dense_dropout)\n", 237 | " \n", 238 | " # output layer: \n", 239 | " out_layer_z = tf.add(tf.matmul(dense_1, weights['W_out']), biases['b_out'])\n", 240 | " \n", 241 | " return out_layer_z" 242 | ] 243 | }, 244 | { 245 | "cell_type": "markdown", 246 | "metadata": {}, 247 | "source": [ 248 | "#### Build model" 249 | ] 250 | }, 251 | { 252 | "cell_type": "code", 253 | "execution_count": 10, 254 | "metadata": {}, 255 | "outputs": [], 256 | "source": [ 257 | "predictions = network(x, weight_dict, bias_dict, n_input, \n", 258 | " pool_size, mp_layer_dropout, dense_layer_dropout)" 259 | ] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": {}, 264 | "source": [ 265 | "#### Define model's loss and its optimizer" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": 11, 271 | "metadata": {}, 272 | "outputs": [], 273 | "source": [ 274 | "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predictions, labels=y))\n", 275 | "optimizer = tf.train.AdamOptimizer().minimize(cost)" 276 | ] 277 | }, 278 | { 279 | "cell_type": "markdown", 280 | "metadata": {}, 281 | "source": [ 282 | "#### Define evaluation metrics" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": 12, 288 | "metadata": {}, 289 | "outputs": [], 290 | "source": [ 291 | "correct_prediction = tf.equal(tf.argmax(predictions, 1), tf.argmax(y, 1))\n", 292 | "accuracy_pct = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) * 100" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "#### Create op for variable initialization" 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": 13, 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [ 308 | "initializer_op = tf.global_variables_initializer()" 309 | ] 310 | }, 311 | { 312 | "cell_type": "markdown", 313 | "metadata": {}, 314 | "source": [ 315 | "#### Train the network in a session (identical to `intermediate_net_in_tensorflow.ipynb` except addition of `display_progress`)" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": null, 321 | "metadata": {}, 322 | "outputs": [ 323 | { 324 | "name": "stdout", 325 | "output_type": "stream", 326 | "text": [ 327 | "Training for 1 epochs.\n", 328 | "Step 1 of 429 in epoch 1.\n", 329 | "Step 11 of 429 in epoch 1.\n", 330 | "Step 21 of 429 in epoch 1.\n", 331 | "Step 31 of 429 in epoch 1.\n", 332 | "Step 41 of 429 in epoch 1.\n", 333 | "Step 51 of 429 in epoch 1.\n", 334 | "Step 61 of 429 in epoch 1.\n", 335 | "Step 71 of 429 in epoch 1.\n", 336 | "Step 81 of 429 in epoch 1.\n", 337 | "Step 91 of 429 in epoch 1.\n", 338 | "Step 101 of 429 in epoch 1.\n", 339 | "Step 111 of 429 in epoch 1.\n", 340 | "Step 121 of 429 in epoch 1.\n", 341 | "Step 131 of 429 in epoch 1.\n", 342 | "Step 141 of 429 in epoch 1.\n", 343 | "Step 151 of 429 in epoch 1.\n", 344 | "Step 161 of 429 in epoch 1.\n", 345 | "Step 171 of 429 in epoch 1.\n", 346 | "Step 181 of 429 in epoch 1.\n", 347 | "Step 191 of 429 in epoch 1.\n", 348 | "Step 201 of 429 in epoch 1.\n", 349 | "Step 211 of 429 in epoch 1.\n", 350 | "Step 221 of 429 in epoch 1.\n", 351 | "Step 231 of 429 in epoch 1.\n", 352 | "Step 241 of 429 in epoch 1.\n", 353 | "Step 251 of 429 in epoch 1.\n", 354 | "Step 261 of 429 in epoch 1.\n", 355 | "Step 271 of 429 in epoch 1.\n", 356 | "Step 281 of 429 in epoch 1.\n", 357 | "Step 291 of 429 in epoch 1.\n", 358 | "Step 301 of 429 in epoch 1.\n", 359 | "Step 311 of 429 in epoch 1.\n", 360 | "Step 321 of 429 in epoch 1.\n", 361 | "Step 331 of 429 in epoch 1.\n", 362 | "Step 341 of 429 in epoch 1.\n", 363 | "Step 351 of 429 in epoch 1.\n", 364 | "Step 361 of 429 in epoch 1.\n", 365 | "Step 371 of 429 in epoch 1.\n", 366 | "Step 381 of 429 in epoch 1.\n", 367 | "Step 391 of 429 in epoch 1.\n", 368 | "Step 401 of 429 in epoch 1.\n", 369 | "Step 411 of 429 in epoch 1.\n", 370 | "Step 421 of 429 in epoch 1.\n", 371 | "Epoch 001: cost = 0.247, accuracy = 92.72%\n", 372 | "Training Complete. Testing Model.\n", 373 | "\n" 374 | ] 375 | } 376 | ], 377 | "source": [ 378 | "with tf.Session() as session:\n", 379 | " session.run(initializer_op)\n", 380 | " \n", 381 | " print(\"Training for\", epochs, \"epochs.\")\n", 382 | " \n", 383 | " # loop over epochs: \n", 384 | " for epoch in range(epochs):\n", 385 | " \n", 386 | " avg_cost = 0.0 # track cost to monitor performance during training\n", 387 | " avg_accuracy_pct = 0.0\n", 388 | " \n", 389 | " # loop over all batches of the epoch:\n", 390 | " n_batches = int(mnist.train.num_examples / batch_size)\n", 391 | " for i in range(n_batches):\n", 392 | "\n", 393 | " # to reassure you something's happening! \n", 394 | " if i % display_progress == 0:\n", 395 | " print(\"Step \", i+1, \" of \", n_batches, \" in epoch \", epoch+1, \".\", sep='')\n", 396 | " \n", 397 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 398 | " \n", 399 | " # feed batch data to run optimization and fetching cost and accuracy: \n", 400 | " _, batch_cost, batch_acc = session.run([optimizer, cost, accuracy_pct], \n", 401 | " feed_dict={x: batch_x, y: batch_y})\n", 402 | " \n", 403 | " # accumulate mean loss and accuracy over epoch: \n", 404 | " avg_cost += batch_cost / n_batches\n", 405 | " avg_accuracy_pct += batch_acc / n_batches\n", 406 | " \n", 407 | " # output logs at end of each epoch of training:\n", 408 | " print(\"Epoch \", '%03d' % (epoch+1), \n", 409 | " \": cost = \", '{:.3f}'.format(avg_cost), \n", 410 | " \", accuracy = \", '{:.2f}'.format(avg_accuracy_pct), \"%\", \n", 411 | " sep='')\n", 412 | " \n", 413 | " print(\"Training Complete. Testing Model.\\n\")\n", 414 | " \n", 415 | " test_cost = cost.eval({x: mnist.test.images, y: mnist.test.labels})\n", 416 | " test_accuracy_pct = accuracy_pct.eval({x: mnist.test.images, y: mnist.test.labels})\n", 417 | " \n", 418 | " print(\"Test Cost:\", '{:.3f}'.format(test_cost))\n", 419 | " print(\"Test Accuracy: \", '{:.2f}'.format(test_accuracy_pct), \"%\", sep='')" 420 | ] 421 | }, 422 | { 423 | "cell_type": "markdown", 424 | "metadata": {}, 425 | "source": [ 426 | "#### Compare with LeNet Keras results" 427 | ] 428 | }, 429 | { 430 | "cell_type": "markdown", 431 | "metadata": { 432 | "collapsed": true 433 | }, 434 | "source": [ 435 | "#### Increase dropout probability(/ies) or add dropout to other conv layer? Stop earlier? Coming up in Lecture 5 :)" 436 | ] 437 | }, 438 | { 439 | "cell_type": "code", 440 | "execution_count": null, 441 | "metadata": {}, 442 | "outputs": [], 443 | "source": [] 444 | }, 445 | { 446 | "cell_type": "code", 447 | "execution_count": null, 448 | "metadata": {}, 449 | "outputs": [], 450 | "source": [] 451 | }, 452 | { 453 | "cell_type": "markdown", 454 | "metadata": {}, 455 | "source": [ 456 | "#### As an exercise, try converting our [AlexNet](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/alexnet_in_keras.ipynb) from Keras to TensorFlow following the same style as this LeNet-5 notebook. " 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": null, 462 | "metadata": {}, 463 | "outputs": [], 464 | "source": [] 465 | } 466 | ], 467 | "metadata": { 468 | "kernelspec": { 469 | "display_name": "Python 3", 470 | "language": "python", 471 | "name": "python3" 472 | }, 473 | "language_info": { 474 | "codemirror_mode": { 475 | "name": "ipython", 476 | "version": 3 477 | }, 478 | "file_extension": ".py", 479 | "mimetype": "text/x-python", 480 | "name": "python", 481 | "nbconvert_exporter": "python", 482 | "pygments_lexer": "ipython3", 483 | "version": "3.6.3" 484 | } 485 | }, 486 | "nbformat": 4, 487 | "nbformat_minor": 2 488 | } 489 | -------------------------------------------------------------------------------- /notebooks/generative_adversarial_network.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# \"Quick, Draw!\" GAN" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "* code based directly on [Grant Beyleveld's](https://github.com/grantbey/quickdraw-GAN/blob/master/octopus-v1.0.ipynb), which is derived from [Rowel Atienza's](https://towardsdatascience.com/gan-by-example-using-keras-on-tensorflow-backend-1a6d515a60d0) under [MIT License](https://github.com/roatienza/Deep-Learning-Experiments/blob/master/LICENSE)\n", 15 | "* data provided by [Google](https://github.com/googlecreativelab/quickdraw-dataset) under [Creative Commons Attribution 4.0 license](https://creativecommons.org/licenses/by/4.0/)" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 1, 21 | "metadata": {}, 22 | "outputs": [ 23 | { 24 | "name": "stderr", 25 | "output_type": "stream", 26 | "text": [ 27 | "Using TensorFlow backend.\n" 28 | ] 29 | } 30 | ], 31 | "source": [ 32 | "import numpy as np\n", 33 | "import h5py\n", 34 | "\n", 35 | "import keras\n", 36 | "from keras.models import Model, Sequential\n", 37 | "from keras.layers import Input, Activation, Conv2D, Reshape, Dense, BatchNormalization, Dropout, Flatten\n", 38 | "from keras.layers import UpSampling2D, Conv2DTranspose, AveragePooling2D # new! \n", 39 | "from keras.optimizers import RMSprop\n", 40 | "\n", 41 | "from matplotlib import pyplot as plt\n", 42 | "%matplotlib inline" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 2, 48 | "metadata": {}, 49 | "outputs": [ 50 | { 51 | "data": { 52 | "text/plain": [ 53 | "(126845, 28, 28, 1)" 54 | ] 55 | }, 56 | "execution_count": 2, 57 | "metadata": {}, 58 | "output_type": "execute_result" 59 | } 60 | ], 61 | "source": [ 62 | "data = np.load('../quickdraw/baseball.npy')\n", 63 | "data = data/255\n", 64 | "data = np.reshape(data,(data.shape[0],28,28,1))\n", 65 | "img_w,img_h = data.shape[1:3]\n", 66 | "data.shape" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 12, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "def discriminator_builder(dim=64,p=0.4):\n", 76 | " \n", 77 | " # Define inputs\n", 78 | " inputs = Input((img_w, img_h, 1))\n", 79 | " \n", 80 | " # Convolutional layers\n", 81 | " conv1 = Conv2D(dim*1, 5, strides=2, padding='same', activation='relu')(inputs)\n", 82 | " conv1 = Dropout(p)(conv1)\n", 83 | " \n", 84 | " conv2 = Conv2D(dim*2, 5, strides=2, padding='same', activation='relu')(conv1)\n", 85 | " conv2 = Dropout(p)(conv2)\n", 86 | " \n", 87 | " conv3 = Conv2D(dim*4, 5, strides=2, padding='same', activation='relu')(conv2)\n", 88 | " conv3 = Dropout(p)(conv3)\n", 89 | " \n", 90 | " conv4 = Conv2D(dim*8, 5, strides=1, padding='same', activation='relu')(conv3)\n", 91 | " conv4 = Dropout(p)(conv4)\n", 92 | " conv4 = Flatten()(conv4)\n", 93 | " \n", 94 | " outputs = Dense(1, activation='sigmoid')(conv4)\n", 95 | " \n", 96 | " model = Model(inputs=inputs, outputs=outputs)\n", 97 | " model.summary()\n", 98 | " \n", 99 | " return model" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": 13, 105 | "metadata": {}, 106 | "outputs": [ 107 | { 108 | "name": "stdout", 109 | "output_type": "stream", 110 | "text": [ 111 | "_________________________________________________________________\n", 112 | "Layer (type) Output Shape Param # \n", 113 | "=================================================================\n", 114 | "input_4 (InputLayer) (None, 28, 28, 1) 0 \n", 115 | "_________________________________________________________________\n", 116 | "conv2d_10 (Conv2D) (None, 14, 14, 64) 1664 \n", 117 | "_________________________________________________________________\n", 118 | "dropout_10 (Dropout) (None, 14, 14, 64) 0 \n", 119 | "_________________________________________________________________\n", 120 | "conv2d_11 (Conv2D) (None, 7, 7, 128) 204928 \n", 121 | "_________________________________________________________________\n", 122 | "dropout_11 (Dropout) (None, 7, 7, 128) 0 \n", 123 | "_________________________________________________________________\n", 124 | "conv2d_12 (Conv2D) (None, 4, 4, 256) 819456 \n", 125 | "_________________________________________________________________\n", 126 | "dropout_12 (Dropout) (None, 4, 4, 256) 0 \n", 127 | "_________________________________________________________________\n", 128 | "conv2d_13 (Conv2D) (None, 4, 4, 512) 3277312 \n", 129 | "_________________________________________________________________\n", 130 | "dropout_13 (Dropout) (None, 4, 4, 512) 0 \n", 131 | "_________________________________________________________________\n", 132 | "flatten_3 (Flatten) (None, 8192) 0 \n", 133 | "_________________________________________________________________\n", 134 | "dense_4 (Dense) (None, 1) 8193 \n", 135 | "=================================================================\n", 136 | "Total params: 4,311,553\n", 137 | "Trainable params: 4,311,553\n", 138 | "Non-trainable params: 0\n", 139 | "_________________________________________________________________\n" 140 | ] 141 | } 142 | ], 143 | "source": [ 144 | "# Compile discriminator: \n", 145 | "discriminator_model = discriminator_builder()" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 5, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "discriminator_model.compile(loss='binary_crossentropy', optimizer=RMSprop(lr=0.0002, decay=6e-18))" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": 6, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "def generator_builder(z_dim=100,dim=64,p=0.4):\n", 164 | " \n", 165 | " # Define inputs\n", 166 | " inputs = Input((z_dim,))\n", 167 | " \n", 168 | " # First dense layer\n", 169 | " dense1 = Dense(7*7*64)(inputs)\n", 170 | " dense1 = BatchNormalization(axis=-1,momentum=0.9)(dense1)\n", 171 | " dense1 = Activation(activation='relu')(dense1)\n", 172 | " dense1 = Reshape((7,7,64))(dense1)\n", 173 | " dense1 = Dropout(p)(dense1)\n", 174 | " \n", 175 | " # Deconvolutional layers\n", 176 | " conv1 = UpSampling2D()(dense1)\n", 177 | " conv1 = Conv2DTranspose(int(dim/2), kernel_size=5, padding='same', activation=None)(conv1)\n", 178 | " conv1 = BatchNormalization(axis=-1, momentum=0.9)(conv1)\n", 179 | " conv1 = Activation(activation='relu')(conv1)\n", 180 | " \n", 181 | " conv2 = UpSampling2D()(conv1)\n", 182 | " conv2 = Conv2DTranspose(int(dim/4), kernel_size=5, padding='same', activation=None)(conv2)\n", 183 | " conv2 = BatchNormalization(axis=-1, momentum=0.9)(conv2)\n", 184 | " conv2 = Activation(activation='relu')(conv2)\n", 185 | " \n", 186 | "# conv3 = UpSampling2D()(conv2)\n", 187 | " conv3 = Conv2DTranspose(int(dim/8), kernel_size=5, padding='same', activation=None)(conv2)\n", 188 | " conv3 = BatchNormalization(axis=-1, momentum=0.9)(conv3)\n", 189 | " conv3 = Activation(activation='relu')(conv3)\n", 190 | " \n", 191 | " # Define output layers\n", 192 | " outputs = Conv2D(1, kernel_size=5, padding='same', activation='sigmoid')(conv3)\n", 193 | " \n", 194 | " # Model definition \n", 195 | " model = Model(inputs=inputs, outputs=outputs)\n", 196 | " \n", 197 | " model.summary()\n", 198 | " \n", 199 | " return model" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": 7, 205 | "metadata": {}, 206 | "outputs": [ 207 | { 208 | "name": "stdout", 209 | "output_type": "stream", 210 | "text": [ 211 | "_________________________________________________________________\n", 212 | "Layer (type) Output Shape Param # \n", 213 | "=================================================================\n", 214 | "input_2 (InputLayer) (None, 100) 0 \n", 215 | "_________________________________________________________________\n", 216 | "dense_2 (Dense) (None, 3136) 316736 \n", 217 | "_________________________________________________________________\n", 218 | "batch_normalization_1 (Batch (None, 3136) 12544 \n", 219 | "_________________________________________________________________\n", 220 | "activation_1 (Activation) (None, 3136) 0 \n", 221 | "_________________________________________________________________\n", 222 | "reshape_1 (Reshape) (None, 7, 7, 64) 0 \n", 223 | "_________________________________________________________________\n", 224 | "dropout_5 (Dropout) (None, 7, 7, 64) 0 \n", 225 | "_________________________________________________________________\n", 226 | "up_sampling2d_1 (UpSampling2 (None, 14, 14, 64) 0 \n", 227 | "_________________________________________________________________\n", 228 | "conv2d_transpose_1 (Conv2DTr (None, 14, 14, 32) 51232 \n", 229 | "_________________________________________________________________\n", 230 | "batch_normalization_2 (Batch (None, 14, 14, 32) 128 \n", 231 | "_________________________________________________________________\n", 232 | "activation_2 (Activation) (None, 14, 14, 32) 0 \n", 233 | "_________________________________________________________________\n", 234 | "up_sampling2d_2 (UpSampling2 (None, 28, 28, 32) 0 \n", 235 | "_________________________________________________________________\n", 236 | "conv2d_transpose_2 (Conv2DTr (None, 28, 28, 16) 12816 \n", 237 | "_________________________________________________________________\n", 238 | "batch_normalization_3 (Batch (None, 28, 28, 16) 64 \n", 239 | "_________________________________________________________________\n", 240 | "activation_3 (Activation) (None, 28, 28, 16) 0 \n", 241 | "_________________________________________________________________\n", 242 | "conv2d_transpose_3 (Conv2DTr (None, 28, 28, 8) 3208 \n", 243 | "_________________________________________________________________\n", 244 | "batch_normalization_4 (Batch (None, 28, 28, 8) 32 \n", 245 | "_________________________________________________________________\n", 246 | "activation_4 (Activation) (None, 28, 28, 8) 0 \n", 247 | "_________________________________________________________________\n", 248 | "conv2d_5 (Conv2D) (None, 28, 28, 1) 201 \n", 249 | "=================================================================\n", 250 | "Total params: 396,961\n", 251 | "Trainable params: 390,577\n", 252 | "Non-trainable params: 6,384\n", 253 | "_________________________________________________________________\n" 254 | ] 255 | } 256 | ], 257 | "source": [ 258 | "generator = generator_builder()" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": 14, 264 | "metadata": {}, 265 | "outputs": [], 266 | "source": [ 267 | "def adversarial_builder(z_dim=100):\n", 268 | " \n", 269 | " model = Sequential()\n", 270 | "\n", 271 | " model.add(generator)\n", 272 | " model.add(discriminator_model)\n", 273 | " \n", 274 | " model.compile(loss='binary_crossentropy', optimizer=RMSprop(lr=0.0001, decay=3e-8), metrics=['accuracy'])\n", 275 | " \n", 276 | " model.summary()\n", 277 | "\n", 278 | " return model" 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": 15, 284 | "metadata": {}, 285 | "outputs": [ 286 | { 287 | "name": "stdout", 288 | "output_type": "stream", 289 | "text": [ 290 | "_________________________________________________________________\n", 291 | "Layer (type) Output Shape Param # \n", 292 | "=================================================================\n", 293 | "model_2 (Model) (None, 28, 28, 1) 396961 \n", 294 | "_________________________________________________________________\n", 295 | "model_4 (Model) (None, 1) 4311553 \n", 296 | "=================================================================\n", 297 | "Total params: 4,708,514\n", 298 | "Trainable params: 4,702,130\n", 299 | "Non-trainable params: 6,384\n", 300 | "_________________________________________________________________\n" 301 | ] 302 | } 303 | ], 304 | "source": [ 305 | "AM = adversarial_builder()" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 16, 311 | "metadata": { 312 | "scrolled": true 313 | }, 314 | "outputs": [], 315 | "source": [ 316 | "import os\n", 317 | "output_dir = './images'\n", 318 | "if not os.path.exists(output_dir):\n", 319 | " os.makedirs(output_dir)" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": null, 325 | "metadata": { 326 | "scrolled": false 327 | }, 328 | "outputs": [], 329 | "source": [ 330 | "def train(epochs=2000,batch=128):\n", 331 | " for i in range(epochs):\n", 332 | "\n", 333 | " real_imgs = np.reshape(data[np.random.choice(data.shape[0],batch,replace=False)],(batch,28,28,1))\n", 334 | " fake_imgs = generator.predict(np.random.uniform(-1.0, 1.0, size=[batch, 100]))\n", 335 | " \n", 336 | " x = np.concatenate((real_imgs,fake_imgs))\n", 337 | " y = np.ones([2*batch,1])\n", 338 | " y[batch:,:] = 0\n", 339 | " \n", 340 | " d_loss = discriminator_model.train_on_batch(x,y)\n", 341 | " \n", 342 | " noise = np.random.uniform(-1.0, 1.0, size=[batch, 100])\n", 343 | " y = np.ones([batch,1])\n", 344 | " a_loss = AM.train_on_batch(noise,y)\n", 345 | " \n", 346 | " if (i+1)%1000 == 0:\n", 347 | " print('Epoch #{}'.format(i+1))\n", 348 | " log_mesg = \"%d: [D loss: %f, acc: %f]\" % (i, d_loss[0], d_loss[1])\n", 349 | " log_mesg = \"%s [A loss: %f, acc: %f]\" % (log_mesg, a_loss[0], a_loss[1])\n", 350 | " print(log_mesg)\n", 351 | " noise = np.random.uniform(-1.0, 1.0, size=[16, 100])\n", 352 | " gen_imgs = generator.predict(noise)\n", 353 | " plt.figure(figsize=(5,5))\n", 354 | " \n", 355 | " for k in range(gen_imgs.shape[0]):\n", 356 | " plt.subplot(4, 4, k+1)\n", 357 | " plt.imshow(gen_imgs[k, :, :, 0], cmap='gray')\n", 358 | " plt.axis('off')\n", 359 | " \n", 360 | " plt.tight_layout()\n", 361 | " plt.show()\n", 362 | " plt.savefig('./images/baseball_{}.png'.format(i+1))" 363 | ] 364 | }, 365 | { 366 | "cell_type": "code", 367 | "execution_count": null, 368 | "metadata": {}, 369 | "outputs": [], 370 | "source": [ 371 | "train(epochs=20000)" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": null, 377 | "metadata": {}, 378 | "outputs": [], 379 | "source": [] 380 | } 381 | ], 382 | "metadata": { 383 | "anaconda-cloud": {}, 384 | "kernelspec": { 385 | "display_name": "Python 3", 386 | "language": "python", 387 | "name": "python3" 388 | }, 389 | "language_info": { 390 | "codemirror_mode": { 391 | "name": "ipython", 392 | "version": 3 393 | }, 394 | "file_extension": ".py", 395 | "mimetype": "text/x-python", 396 | "name": "python", 397 | "nbconvert_exporter": "python", 398 | "pygments_lexer": "ipython3", 399 | "version": "3.6.3" 400 | } 401 | }, 402 | "nbformat": 4, 403 | "nbformat_minor": 2 404 | } 405 | -------------------------------------------------------------------------------- /notebooks/gru_in_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# GRU in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we use a GRU to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stderr", 31 | "output_type": "stream", 32 | "text": [ 33 | "Using TensorFlow backend.\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "import keras\n", 39 | "from keras.datasets import imdb\n", 40 | "from keras.preprocessing.sequence import pad_sequences\n", 41 | "from keras.models import Sequential\n", 42 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D\n", 43 | "from keras.layers import GRU # new! \n", 44 | "from keras.callbacks import ModelCheckpoint\n", 45 | "import os\n", 46 | "from sklearn.metrics import roc_auc_score \n", 47 | "import matplotlib.pyplot as plt \n", 48 | "%matplotlib inline" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "#### Set hyperparameters" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 2, 61 | "metadata": { 62 | "collapsed": true 63 | }, 64 | "outputs": [], 65 | "source": [ 66 | "# output directory name:\n", 67 | "output_dir = 'model_output/gru'\n", 68 | "\n", 69 | "# training:\n", 70 | "epochs = 4\n", 71 | "batch_size = 128\n", 72 | "\n", 73 | "# vector-space embedding: \n", 74 | "n_dim = 64 \n", 75 | "n_unique_words = 10000 \n", 76 | "max_review_length = 100 # lowered due to vanishing gradient over time\n", 77 | "pad_type = trunc_type = 'pre'\n", 78 | "drop_embed = 0.2 \n", 79 | "\n", 80 | "# GRU layer architecture:\n", 81 | "n_gru = 256 \n", 82 | "drop_gru = 0.2\n", 83 | "\n", 84 | "# dense layer architecture: \n", 85 | "# n_dense = 256\n", 86 | "# dropout = 0.2" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "#### Load data" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 3, 99 | "metadata": { 100 | "collapsed": true 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "#### Preprocess data" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": 4, 117 | "metadata": { 118 | "collapsed": true 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 123 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": { 129 | "collapsed": true 130 | }, 131 | "source": [ 132 | "#### Design neural network architecture" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 5, 138 | "metadata": { 139 | "collapsed": true 140 | }, 141 | "outputs": [], 142 | "source": [ 143 | "model = Sequential()\n", 144 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n", 145 | "model.add(SpatialDropout1D(drop_embed))\n", 146 | "model.add(GRU(n_gru, dropout=drop_gru))\n", 147 | "# model.add(Dense(n_dense, activation='relu')) # typically don't see top dense layer in NLP like in \n", 148 | "# model.add(Dropout(dropout))\n", 149 | "model.add(Dense(1, activation='sigmoid'))" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 6, 155 | "metadata": {}, 156 | "outputs": [ 157 | { 158 | "name": "stdout", 159 | "output_type": "stream", 160 | "text": [ 161 | "_________________________________________________________________\n", 162 | "Layer (type) Output Shape Param # \n", 163 | "=================================================================\n", 164 | "embedding_1 (Embedding) (None, 100, 64) 640000 \n", 165 | "_________________________________________________________________\n", 166 | "spatial_dropout1d_1 (Spatial (None, 100, 64) 0 \n", 167 | "_________________________________________________________________\n", 168 | "gru_1 (GRU) (None, 256) 246528 \n", 169 | "_________________________________________________________________\n", 170 | "dense_1 (Dense) (None, 1) 257 \n", 171 | "=================================================================\n", 172 | "Total params: 886,785\n", 173 | "Trainable params: 886,785\n", 174 | "Non-trainable params: 0\n", 175 | "_________________________________________________________________\n" 176 | ] 177 | } 178 | ], 179 | "source": [ 180 | "model.summary() " 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "#### Configure model" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 7, 193 | "metadata": { 194 | "collapsed": true 195 | }, 196 | "outputs": [], 197 | "source": [ 198 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": 8, 204 | "metadata": { 205 | "collapsed": true 206 | }, 207 | "outputs": [], 208 | "source": [ 209 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 210 | "if not os.path.exists(output_dir):\n", 211 | " os.makedirs(output_dir)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": {}, 217 | "source": [ 218 | "#### Train!" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 9, 224 | "metadata": {}, 225 | "outputs": [ 226 | { 227 | "name": "stdout", 228 | "output_type": "stream", 229 | "text": [ 230 | "Train on 25000 samples, validate on 25000 samples\n", 231 | "Epoch 1/4\n", 232 | "25000/25000 [==============================] - 31s - loss: 0.5238 - acc: 0.7241 - val_loss: 0.3771 - val_acc: 0.8350\n", 233 | "Epoch 2/4\n", 234 | "25000/25000 [==============================] - 29s - loss: 0.3062 - acc: 0.8731 - val_loss: 0.3390 - val_acc: 0.8504\n", 235 | "Epoch 3/4\n", 236 | "25000/25000 [==============================] - 29s - loss: 0.2484 - acc: 0.9004 - val_loss: 0.3680 - val_acc: 0.8452\n", 237 | "Epoch 4/4\n", 238 | "25000/25000 [==============================] - 28s - loss: 0.2123 - acc: 0.9176 - val_loss: 0.3837 - val_acc: 0.8396\n" 239 | ] 240 | }, 241 | { 242 | "data": { 243 | "text/plain": [ 244 | "" 245 | ] 246 | }, 247 | "execution_count": 9, 248 | "metadata": {}, 249 | "output_type": "execute_result" 250 | } 251 | ], 252 | "source": [ 253 | "# XXX% validation accuracy in epoch X\n", 254 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": { 260 | "collapsed": true 261 | }, 262 | "source": [ 263 | "#### Evaluate" 264 | ] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": 10, 269 | "metadata": { 270 | "collapsed": true 271 | }, 272 | "outputs": [], 273 | "source": [ 274 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": 11, 280 | "metadata": {}, 281 | "outputs": [ 282 | { 283 | "name": "stdout", 284 | "output_type": "stream", 285 | "text": [ 286 | "25000/25000 [==============================] - 24s \n" 287 | ] 288 | } 289 | ], 290 | "source": [ 291 | "y_hat = model.predict_proba(x_valid)" 292 | ] 293 | }, 294 | { 295 | "cell_type": "code", 296 | "execution_count": 12, 297 | "metadata": {}, 298 | "outputs": [ 299 | { 300 | "data": { 301 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE5NJREFUeJzt3X+s3fV93/HnKzgka5vEJlwQss1MVDcLrRTCLCCK1NE4\nM8apMH+EydE6XGTNU0erdqu2kO0Pb9BMZNPGhtTSucWridoQypZhJazMc0DZpplgCqUBinxDKFyZ\nYReDsw4lHel7f5yPkwO51/dc+95zcvk8H9LR9/t9fz/f8/18fO37Ot9fx6kqJEn9edukOyBJmgwD\nQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktSpFZPuwKmce+65tW7dukl3Q/pB33pm\nMH33+yfbD2kWjz766J9V1dR87X6oA2DdunUcOnRo0t2QftB/u3Iw/dhDk+yFNKskfzpKO08BSVKn\nDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUqXkDIMn7kzw+9PpWkl9Jck6S/UkOt+mq1j5Jbk8y\nneSJJJcOvdf21v5wku1LOTBJ0qnNGwBV9UxVXVJVlwB/HXgN+CJwE3CgqtYDB9oywNXA+vbaCdwB\nkOQcYBdwOXAZsOtkaEiSxm+hTwJvBL5RVX+aZCtwZavvBR4CPgVsBe6qwf82fzDJyiQXtLb7q+o4\nQJL9wGbg82c6CElaCutu+vLE9v3crR9f8n0s9BrANr7/C/v8qnoRoE3Pa/XVwAtD28y02lz1N0iy\nM8mhJIeOHTu2wO5JkkY1cgAkORu4Bvj9+ZrOUqtT1N9YqNpdVRuqasPU1LzfZSRJOk0LOQK4GvjD\nqnqpLb/UTu3QpkdbfQZYO7TdGuDIKeqSpAlYSAB8kjeer98HnLyTZztw31D9+nY30BXAiXaK6AFg\nU5JV7eLvplaTJE3ASBeBk/wI8DeBvzdUvhW4J8kO4Hngula/H9gCTDO4Y+gGgKo6nuQW4JHW7uaT\nF4QlSeM3UgBU1WvAe99Ue5nBXUFvblvAjXO8zx5gz8K7eXomdQV/HFfvJelM+SSwJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJ\nnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6NVIAJFmZ5N4kf5Lk6SQfTnJOkv1JDrfpqtY2SW5PMp3k\niSSXDr3P9tb+cJLtSzUoSdL8Rj0C+HfAH1TVXwM+CDwN3AQcqKr1wIG2DHA1sL69dgJ3ACQ5B9gF\nXA5cBuw6GRqSpPGbNwCSvBv4aeBOgKr6i6p6FdgK7G3N9gLXtvmtwF01cBBYmeQC4Cpgf1Udr6pX\ngP3A5kUdjSRpZKMcAbwPOAb8hySPJfntJD8KnF9VLwK06Xmt/WrghaHtZ1ptrrokaQJGCYAVwKXA\nHVX1IeD/8v3TPbPJLLU6Rf2NGyc7kxxKcujYsWMjdE+SdDpGCYAZYKaqHm7L9zIIhJfaqR3a9OhQ\n+7VD268Bjpyi/gZVtbuqNlTVhqmpqYWMRZK0APMGQFX9b+CFJO9vpY3AU8A+4OSdPNuB+9r8PuD6\ndjfQFcCJdoroAWBTklXt4u+mVpMkTcCKEdv9EvC7Sc4GngVuYBAe9yTZATwPXNfa3g9sAaaB11pb\nqup4kluAR1q7m6vq+KKMQpK0YCMFQFU9DmyYZdXGWdoWcOMc77MH2LOQDkqSloZPAktSpwwASeqU\nASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMjBUCS55L8cZLHkxxqtXOS7E9yuE1XtXqS3J5kOskT\nSS4dep/trf3hJNuXZkiSpFEs5AjgZ6rqkqra0JZvAg5U1XrgQFsGuBpY3147gTtgEBjALuBy4DJg\n18nQkCSN35mcAtoK7G3ze4Frh+p31cBBYGWSC4CrgP1VdbyqXgH2A5vPYP+SpDMwagAU8F+TPJpk\nZ6udX1UvArTpea2+GnhhaNuZVpurLkmagBUjtvtIVR1Jch6wP8mfnKJtZqnVKepv3HgQMDsBLrzw\nwhG7J0laqJGOAKrqSJseBb7I4Bz+S+3UDm16tDWfAdYObb4GOHKK+pv3tbuqNlTVhqmpqYWNRpI0\nsnkDIMmPJnnXyXlgE/B1YB9w8k6e7cB9bX4fcH27G+gK4EQ7RfQAsCnJqnbxd1OrSZImYJRTQOcD\nX0xysv3vVdUfJHkEuCfJDuB54LrW/n5gCzANvAbcAFBVx5PcAjzS2t1cVccXbSSSpAWZNwCq6lng\ng7PUXwY2zlIv4MY53msPsGfh3ZQkLTafBJakThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMG\ngCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBI\nUqcMAEnq1MgBkOSsJI8l+VJbvijJw0kOJ/lCkrNb/R1tebqtXzf0Hp9u9WeSXLXYg5EkjW4hRwC/\nDDw9tPxZ4LaqWg+8Auxo9R3AK1X148BtrR1JLga2AT8JbAZ+I8lZZ9Z9SdLpGikAkqwBPg78dlsO\n8FHg3tZkL3Btm9/almnrN7b2W4G7q+o7VfVNYBq4bDEGIUlauFGPAP4t8I+Bv2zL7wVerarX2/IM\nsLrNrwZeAGjrT7T236vPss33JNmZ5FCSQ8eOHVvAUCRJCzFvACT5WeBoVT06XJ6lac2z7lTbfL9Q\ntbuqNlTVhqmpqfm6J0k6TStGaPMR4JokW4B3Au9mcESwMsmK9il/DXCktZ8B1gIzSVYA7wGOD9VP\nGt5GkjRm8x4BVNWnq2pNVa1jcBH3K1X1t4EHgU+0ZtuB+9r8vrZMW/+VqqpW39buEroIWA98bdFG\nIklakFGOAObyKeDuJL8GPAbc2ep3Ap9LMs3gk/82gKp6Msk9wFPA68CNVfXdM9i/JOkMLCgAquoh\n4KE2/yyz3MVTVd8Grptj+88An1loJyVJi88ngSWpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ\n6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKRO\nGQCS1Kl5AyDJO5N8LckfJXkyyT9v9YuSPJzkcJIvJDm71d/Rlqfb+nVD7/XpVn8myVVLNShJ0vxG\nOQL4DvDRqvogcAmwOckVwGeB26pqPfAKsKO13wG8UlU/DtzW2pHkYmAb8JPAZuA3kpy1mIORJI1u\n3gCogT9vi29vrwI+Ctzb6nuBa9v81rZMW78xSVr97qr6TlV9E5gGLluUUUiSFmykawBJzkryOHAU\n2A98A3i1ql5vTWaA1W1+NfACQFt/AnjvcH2WbSRJYzZSAFTVd6vqEmANg0/tH5itWZtmjnVz1d8g\nyc4kh5IcOnbs2CjdkySdhgXdBVRVrwIPAVcAK5OsaKvWAEfa/AywFqCtfw9wfLg+yzbD+9hdVRuq\nasPU1NRCuidJWoBR7gKaSrKyzf8V4GPA08CDwCdas+3AfW1+X1umrf9KVVWrb2t3CV0ErAe+tlgD\nkSQtzIr5m3ABsLfdsfM24J6q+lKSp4C7k/wa8BhwZ2t/J/C5JNMMPvlvA6iqJ5PcAzwFvA7cWFXf\nXdzhSJJGNW8AVNUTwIdmqT/LLHfxVNW3gevmeK/PAJ9ZeDclSYvNJ4ElqVMGgCR1ygCQpE4ZAJLU\nKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0y\nACSpUwaAJHVqlP8TWAu07qYvT2S/z9368YnsV1pqk/o39VbnEYAkdWreAEiyNsmDSZ5O8mSSX271\nc5LsT3K4TVe1epLcnmQ6yRNJLh16r+2t/eEk25duWJKk+YxyBPA68KtV9QHgCuDGJBcDNwEHqmo9\ncKAtA1wNrG+vncAdMAgMYBdwOXAZsOtkaEiSxm/eAKiqF6vqD9v8/wGeBlYDW4G9rdle4No2vxW4\nqwYOAiuTXABcBeyvquNV9QqwH9i8qKORJI1sQdcAkqwDPgQ8DJxfVS/CICSA81qz1cALQ5vNtNpc\ndUnSBIwcAEl+DPiPwK9U1bdO1XSWWp2i/ub97ExyKMmhY8eOjdo9SdICjRQASd7O4Jf/71bVf2rl\nl9qpHdr0aKvPAGuHNl8DHDlF/Q2qandVbaiqDVNTUwsZiyRpAUa5CyjAncDTVfVvhlbtA07eybMd\nuG+ofn27G+gK4EQ7RfQAsCnJqnbxd1OrSZImYJQHwT4C/B3gj5M83mr/BLgVuCfJDuB54Lq27n5g\nCzANvAbcAFBVx5PcAjzS2t1cVccXZRSSpAWbNwCq6n8w+/l7gI2ztC/gxjneaw+wZyEdlCQtDZ8E\nlqROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ\n6tQoXwctSQCsu+nLk+6CFpEB8BYyyX+cz9368YntW9Lp8RSQJHXKAJCkThkAktQpA0CSOmUASFKn\nDABJ6tS8AZBkT5KjSb4+VDsnyf4kh9t0Vasnye1JppM8keTSoW22t/aHk2xfmuFIkkY1yhHA7wCb\n31S7CThQVeuBA20Z4GpgfXvtBO6AQWAAu4DLgcuAXSdDQ5I0GfM+CFZVX02y7k3lrcCVbX4v8BDw\nqVa/q6oKOJhkZZILWtv9VXUcIMl+BqHy+TMegdQZn8bVYjndJ4HPr6oXAarqxSTntfpq4IWhdjOt\nNlf9ByTZyeDogQsvvPA0u6dxm9QvJZ9Alk7fYl8Eziy1OkX9B4tVu6tqQ1VtmJqaWtTOSZK+73SP\nAF5KckH79H8BcLTVZ4C1Q+3WAEda/co31R86zX1LPxQOPvsy2zwdo2XsdANgH7AduLVN7xuq/2KS\nuxlc8D3RQuIB4F8MXfjdBHz69LstDUzq1NPd73t5IvuVFtO8AZDk8ww+vZ+bZIbB3Ty3Avck2QE8\nD1zXmt8PbAGmgdeAGwCq6niSW4BHWrubT14QliRNxih3AX1yjlUbZ2lbwI1zvM8eYM+CeidJWjI+\nCSxJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE6NPQCSbE7yTJLpJDeNe/+SpIGxBkCS\ns4BfB64GLgY+meTicfZBkjQw7iOAy4Dpqnq2qv4CuBvYOuY+SJIYfwCsBl4YWp5pNUnSmK0Y8/4y\nS63e0CDZCexsi3+e5JnT3Ne5wJ+d5rbLlWMekw9/b+5nx71r8OfchXz2jMb8V0dpNO4AmAHWDi2v\nAY4MN6iq3cDuM91RkkNVteFM32c5ccx9cMx9GMeYx30K6BFgfZKLkpwNbAP2jbkPkiTGfARQVa8n\n+UXgAeAsYE9VPTnOPkiSBsZ9Coiquh+4fwy7OuPTSMuQY+6DY+7Dko85VTV/K0nSW45fBSFJnVr2\nATDfV0skeUeSL7T1DydZN/5eLq4RxvwPkzyV5IkkB5KMdEvYD7NRv0IkySeSVJJlf8fIKGNO8rfa\nz/rJJL837j4uthH+bl+Y5MEkj7W/31sm0c/FkmRPkqNJvj7H+iS5vf15PJHk0kXtQFUt2xeDC8nf\nAN4HnA38EXDxm9r8feA32/w24AuT7vcYxvwzwI+0+V/oYcyt3buArwIHgQ2T7vcYfs7rgceAVW35\nvEn3ewxj3g38Qpu/GHhu0v0+wzH/NHAp8PU51m8B/guDZ6iuAB5ezP0v9yOAUb5aYiuwt83fC2xM\nMtsDacvFvGOuqger6rW2eJDB8xbL2ahfIXIL8C+Bb4+zc0tklDH/XeDXq+oVgKo6OuY+LrZRxlzA\nu9v8e3jTc0TLTVV9FTh+iiZbgbtq4CCwMskFi7X/5R4Ao3y1xPfaVNXrwAngvWPp3dJY6Ndp7GDw\nCWI5m3fMST4ErK2qL42zY0tolJ/zTwA/keR/JjmYZPPYerc0RhnzPwN+LskMg7sJf2k8XZuYJf36\nnLHfBrrI5v1qiRHbLCcjjyfJzwEbgL+xpD1aeqccc5K3AbcBPz+uDo3BKD/nFQxOA13J4Cjvvyf5\nqap6dYn7tlRGGfMngd+pqn+d5MPA59qY/3LpuzcRS/r7a7kfAcz71RLDbZKsYHDYeKpDrh92o4yZ\nJB8D/ilwTVV9Z0x9WyrzjfldwE8BDyV5jsG50n3L/ELwqH+376uq/1dV3wSeYRAIy9UoY94B3ANQ\nVf8LeCeD7wl6qxrp3/vpWu4BMMpXS+wDtrf5TwBfqXZ1ZZmad8ztdMi/Z/DLf7mfF4Z5xlxVJ6rq\n3KpaV1XrGFz3uKaqDk2mu4tilL/b/5nBBX+SnMvglNCzY+3l4hplzM8DGwGSfIBBABwbay/Hax9w\nfbsb6ArgRFW9uFhvvqxPAdUcXy2R5GbgUFXtA+5kcJg4zeCT/7bJ9fjMjTjmfwX8GPD77Xr381V1\nzcQ6fYZGHPNbyohjfgDYlOQp4LvAP6qqlyfX6zMz4ph/FfitJP+AwamQn1/OH+iSfJ7BKbxz23WN\nXcDbAarqNxlc59gCTAOvATcs6v6X8Z+dJOkMLPdTQJKk02QASFKnDABJ6pQBIEmdMgAkqVMGgCR1\nygCQpE4ZAJLUqf8PwRa4eRxq3rIAAAAASUVORK5CYII=\n", 302 | "text/plain": [ 303 | "" 304 | ] 305 | }, 306 | "metadata": {}, 307 | "output_type": "display_data" 308 | } 309 | ], 310 | "source": [ 311 | "plt.hist(y_hat)\n", 312 | "_ = plt.axvline(x=0.5, color='orange')" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": 13, 318 | "metadata": {}, 319 | "outputs": [ 320 | { 321 | "data": { 322 | "text/plain": [ 323 | "'92.98'" 324 | ] 325 | }, 326 | "execution_count": 13, 327 | "metadata": {}, 328 | "output_type": "execute_result" 329 | } 330 | ], 331 | "source": [ 332 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": null, 338 | "metadata": { 339 | "collapsed": true 340 | }, 341 | "outputs": [], 342 | "source": [] 343 | } 344 | ], 345 | "metadata": { 346 | "kernelspec": { 347 | "display_name": "Python 3", 348 | "language": "python", 349 | "name": "python3" 350 | }, 351 | "language_info": { 352 | "codemirror_mode": { 353 | "name": "ipython", 354 | "version": 3 355 | }, 356 | "file_extension": ".py", 357 | "mimetype": "text/x-python", 358 | "name": "python", 359 | "nbconvert_exporter": "python", 360 | "pygments_lexer": "ipython3", 361 | "version": "3.6.2" 362 | } 363 | }, 364 | "nbformat": 4, 365 | "nbformat_minor": 2 366 | } 367 | --------------------------------------------------------------------------------