├── .gitignore ├── Dockerfile ├── Dockerfile-gpu ├── LICENSE ├── README.md ├── book.jpg ├── installation ├── README.md ├── conda_TensorFlow_install.md ├── docker-stack-scripts │ ├── Jupyter-docker-stacks-license.md │ ├── jupyter_notebook_config.py │ ├── start-notebook.sh │ ├── start-singleuser.sh │ └── start.sh ├── let_jovyan_write.sh ├── miscellaneous_notes.md ├── simple_Windows_Anaconda_install.md ├── step_by_step_GCP_install.md ├── step_by_step_MacOSX_install.md ├── step_by_step_Windows_Docker_install.md └── windows_TF_GPU.md └── notebooks ├── alexnet_in_keras.ipynb ├── bidirectional_lstm.ipynb ├── cartpole_dqn.ipynb ├── convolutional_sentiment_classifier.ipynb ├── creating_word_vectors_with_word2vec.ipynb ├── cross_entropy_cost.ipynb ├── deep_net_in_keras.ipynb ├── dense_sentiment_classifier.ipynb ├── first_tensorflow_graphs.ipynb ├── first_tensorflow_neurons.ipynb ├── generative_adversarial_network.ipynb ├── gru_in_keras.ipynb ├── intermediate_net_in_keras.ipynb ├── intermediate_net_in_tensorflow.ipynb ├── intro_to_tensorflow_times_a_million.ipynb ├── lenet_in_keras.ipynb ├── lenet_in_tensorflow.ipynb ├── live_training ├── convolutional_sentiment_classifier_LT.ipynb ├── deep_net_in_keras_LT.ipynb ├── deep_net_in_tensorflow_LT.ipynb ├── dense_sentiment_classifier_LT.ipynb ├── first_tensorflow_graphs_LT.ipynb ├── first_tensorflow_neurons_LT.ipynb ├── multi_convnet_architectures_LT.ipynb ├── natural_language_preprocessing_best_practices_LT.ipynb ├── point_by_point_intro_to_tensorflow_LT.ipynb ├── rnn_in_keras_LT.ipynb ├── shallow_net_in_keras_LT.ipynb ├── tensor-fied_intro_to_tensorflow_LT.ipynb └── vanilla_lstm_in_keras_LT.ipynb ├── multi_convnet_architectures.ipynb ├── natural_language_preprocessing_best_practices.ipynb ├── point_by_point_intro_to_tensorflow.ipynb ├── rnn_in_keras.ipynb ├── shallow_net_in_keras.ipynb ├── sigmoid_function.ipynb ├── softmax_demo.ipynb ├── stacked_bidirectional_lstm.ipynb ├── tensor-fied_intro_to_tensorflow.ipynb ├── vanilla_lstm_in_keras.ipynb ├── vggnet_in_keras.ipynb └── ye_olde_conv_lstm_stackeroo.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyter/scipy-notebook 2 | 3 | MAINTAINER Jon Krohn 4 | 5 | USER $NB_USER 6 | 7 | # install TensorFlow 8 | RUN conda install --quiet --yes 'tensorflow=1.0*' 9 | 10 | # install tflearn and keras: 11 | RUN pip install tflearn==0.3.2 12 | RUN pip install keras==2.0.8 13 | 14 | # install NLP packages: 15 | RUN pip install nltk==3.2.4 16 | RUN pip install gensim==2.3.0 17 | 18 | # install Reinforcement Learning packages: 19 | RUN pip install gym==0.9.4 20 | -------------------------------------------------------------------------------- /Dockerfile-gpu: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:8.0-cudnn5-devel 2 | 3 | MAINTAINER Jon Krohn 4 | 5 | ENV PATH="/usr/local/cuda-8.0/bin:${PATH}" 6 | ENV LD_LIBRARY_PATH="/usr/local/cuda-8.0/lib64:${LD_LIBRARY_PATH}" 7 | 8 | 9 | ## from core/Dockerfile-gpu 10 | # Setup locales 11 | RUN apt-get clean && apt-get update && apt-get install -y locales 12 | RUN locale-gen en_US.UTF-8 13 | ENV LANG en_US.UTF-8 14 | ENV LANGUAGE en_US:en 15 | ENV LC_ALL en_US.UTF-8 16 | 17 | # Purge existing python installs 18 | RUN apt-get purge -y python.* 19 | 20 | # Install prereqs 21 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential checkinstall \ 22 | wget \ 23 | bzip2 \ 24 | ca-certificates \ 25 | sudo \ 26 | locales \ 27 | fonts-liberation \ 28 | && apt-get clean \ 29 | && rm -rf /var/lib/apt/lists/* 30 | 31 | 32 | ## below mostly from jupyter/minimal-notebook (there's a bit of it in the line above too) 33 | ## -- see https://github.com/jupyter/docker-stacks/blob/master/LICENSE.md 34 | # Install Tini 35 | RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.10.0/tini && \ 36 | echo "1361527f39190a7338a0b434bd8c88ff7233ce7b9a4876f3315c22fce7eca1b0 *tini" | sha256sum -c - && \ 37 | mv tini /usr/local/bin/tini && \ 38 | chmod +x /usr/local/bin/tini 39 | 40 | # Configure environment 41 | ENV CONDA_DIR /opt/conda 42 | ENV PATH $CONDA_DIR/bin:$PATH 43 | ENV SHELL /bin/bash 44 | ENV NB_USER jovyan 45 | ENV NB_UID 1000 46 | ENV HOME /home/$NB_USER 47 | 48 | # Create jovyan user with UID=1000 and in the 'users' group 49 | RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ 50 | mkdir -p $CONDA_DIR && \ 51 | chown $NB_USER $CONDA_DIR 52 | 53 | USER $NB_USER 54 | 55 | # Setup work directory for backward-compatibility 56 | RUN mkdir /home/$NB_USER/work 57 | 58 | # Install conda as jovyan and check the md5 sum provided on the download site 59 | ENV MINICONDA_VERSION 4.3.21 60 | RUN cd /tmp && \ 61 | mkdir -p $CONDA_DIR && \ 62 | wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ 63 | echo "c1c15d3baba15bf50293ae963abef853 *Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \ 64 | /bin/bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ 65 | rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ 66 | $CONDA_DIR/bin/conda config --system --prepend channels conda-forge && \ 67 | $CONDA_DIR/bin/conda config --system --set auto_update_conda false && \ 68 | $CONDA_DIR/bin/conda config --system --set show_channel_urls true && \ 69 | $CONDA_DIR/bin/conda update --all && \ 70 | conda clean -tipsy 71 | 72 | # Install Jupyter Notebook and Hub 73 | RUN conda install --quiet --yes \ 74 | 'notebook=5.0.*' \ 75 | 'jupyterhub=0.7.*' \ 76 | 'jupyterlab=0.24.*' \ 77 | && conda clean -tipsy 78 | 79 | USER root 80 | 81 | EXPOSE 8888 82 | WORKDIR $HOME 83 | 84 | # Configure container startup 85 | ENTRYPOINT ["tini", "--"] 86 | CMD ["start-notebook.sh"] 87 | 88 | # Add local files as late as possible to avoid cache busting 89 | COPY installation/docker-stack-scripts/start.sh /usr/local/bin/ 90 | COPY installation/docker-stack-scripts/start-notebook.sh /usr/local/bin/ 91 | COPY installation/docker-stack-scripts/start-singleuser.sh /usr/local/bin/ 92 | COPY installation/docker-stack-scripts/jupyter_notebook_config.py /etc/jupyter/ 93 | RUN chown -R $NB_USER:users /etc/jupyter/ 94 | 95 | 96 | ## the next chunk of code is from jupyter/minimal-notebook 97 | 98 | # Install all OS dependencies for fully functional notebook server 99 | RUN apt-get update && apt-get install -yq --no-install-recommends \ 100 | build-essential \ 101 | emacs \ 102 | git \ 103 | inkscape \ 104 | jed \ 105 | libsm6 \ 106 | libxext-dev \ 107 | libxrender1 \ 108 | lmodern \ 109 | pandoc \ 110 | python-dev \ 111 | texlive-fonts-extra \ 112 | texlive-fonts-recommended \ 113 | texlive-generic-recommended \ 114 | texlive-latex-base \ 115 | texlive-latex-extra \ 116 | texlive-xetex \ 117 | vim \ 118 | unzip \ 119 | && apt-get clean && \ 120 | rm -rf /var/lib/apt/lists/* 121 | 122 | 123 | ## the next chunk of code is from jupyter/scipy-notebook 124 | 125 | # libav-tools for matplotlib anim 126 | RUN apt-get update && \ 127 | apt-get install -y --no-install-recommends libav-tools && \ 128 | apt-get clean && \ 129 | rm -rf /var/lib/apt/lists/* 130 | 131 | USER $NB_USER 132 | 133 | # Install Python 3 packages 134 | # Remove pyqt and qt pulled in for matplotlib since we're only ever going to 135 | # use notebook-friendly backends in these images 136 | RUN conda install --quiet --yes \ 137 | 'nomkl' \ 138 | 'ipywidgets=6.0*' \ 139 | 'pandas=0.19*' \ 140 | 'numexpr=2.6*' \ 141 | 'matplotlib=2.0*' \ 142 | 'scipy=0.19*' \ 143 | 'seaborn=0.7*' \ 144 | 'scikit-learn=0.18*' \ 145 | 'scikit-image=0.12*' \ 146 | 'sympy=1.0*' \ 147 | 'cython=0.25*' \ 148 | 'patsy=0.4*' \ 149 | 'statsmodels=0.8*' \ 150 | 'cloudpickle=0.2*' \ 151 | 'dill=0.2*' \ 152 | 'numba=0.31*' \ 153 | 'bokeh=0.12*' \ 154 | 'sqlalchemy=1.1*' \ 155 | 'hdf5=1.8.17' \ 156 | 'h5py=2.6*' \ 157 | 'vincent=0.4.*' \ 158 | 'beautifulsoup4=4.5.*' \ 159 | 'xlrd' && \ 160 | conda remove --quiet --yes --force qt pyqt && \ 161 | conda clean -tipsy 162 | 163 | # Activate ipywidgets extension in the environment that runs the notebook server 164 | RUN jupyter nbextension enable --py widgetsnbextension --sys-prefix 165 | 166 | # Import matplotlib the first time to build the font cache. 167 | ENV XDG_CACHE_HOME /home/$NB_USER/.cache/ 168 | RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" 169 | 170 | USER $NB_USER 171 | 172 | 173 | 174 | ## Install TensorFlow for GPU 175 | RUN conda install --quiet --yes 'tensorflow-gpu=1.0*' 176 | 177 | ## Install high-level TensorFlow APIs 178 | RUN pip install tflearn==0.3.2 179 | RUN pip install keras==2.0.8 180 | 181 | ## Install NLP packages 182 | RUN pip install nltk==3.2.4 183 | RUN pip install gensim==2.3.0 184 | 185 | # install Reinforcement Learning packages: 186 | RUN pip install gym==0.9.4 187 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017-2018 Jon Krohn 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /book.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jonkrohn/TensorFlow-LiveLessons/0bddd44c368e0160e4b70faeb2750b3344639ad3/book.jpg -------------------------------------------------------------------------------- /installation/README.md: -------------------------------------------------------------------------------- 1 | # Installation Instructions 2 | 3 | ## Mac OS X 4 | 5 | Detailed step-by-step instructions for running the code notebooks for these LiveLessons on a Mac [here](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/installation/step_by_step_MacOSX_install.md). 6 | 7 | 8 | ## Unix 9 | 10 | #### Where You Already Have the Dependencies 11 | 12 | The dependencies are provided in this repository's [Dockerfile](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/Dockerfile). They are: 13 | 14 | * the [Jupyter Notebook Scientific Python Stack](https://github.com/jupyter/docker-stacks/tree/master/scipy-notebook) 15 | * TensorFlow 16 | * Keras 17 | * tflearn 18 | 19 | If you have these packages configured as you like them, you can simply: 20 | 21 | `git clone https://github.com/the-deep-learners/TensorFlow-LiveLessons.git` 22 | 23 | #### Where You Are Missing Dependencies 24 | 25 | 1. Get Docker CE for, e.g., [Ubuntu](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/) 26 | 2. Follow all the steps in my [Step-by-Step Instructions for Mac](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/installation/step_by_step_MacOSX_install.md) that involve executing code at the command line. That is, execute all steps but one, four and five. 27 | 28 | #### In a Fresh Cloud Instance 29 | 30 | If you'd like to enjoy the power and flexibility of cloud computing, you can spin up a machine with Google Cloud Compute, Amazon Web Services (who conveniently offer a [Deep Learning-specific machine image](https://aws.amazon.com/amazon-ai/amis/)), Microsoft Azure, or other providers. Rough notes for creating an Ubuntu instance with Google Cloud Compute and launching Dockerized Jupyter notebooks are available [here](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/installation/step_by_step_GCP_install.md). 31 | 32 | 33 | ## Windows 34 | 35 | Community members have kindly contributed several different sets of Windows installation instructions, each suited to a different use-case: 36 | 37 | 1. If you have a 64-bit installation of Windows 10 Professional or Enterprise, you can follow the [full Docker container installation](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/installation/step_by_step_Windows_Docker_install.md), which will ensure that you have all the dependencies. 38 | 2. If you've never heard of *Anaconda* as being anything other than a snake, you can follow the simple step-by-step instructions [here](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/installation/simple_Windows_Anaconda_install.md). 39 | 3. If you already have Anaconda or a similar Python 3 distribution set up on your machine (e.g., WinPython, Canopy), then you can install TensorFlow in a virtual environment as outlined [here](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/installation/conda_TensorFlow_install.md). 40 | 41 | ## GPU Considerations 42 | 43 | Most of the examples in these LiveLessons involve relatively small (in Deep Learning terms) data sets so you will be in great shape using your CPU alone for training the models. That said, some of the later notebooks in these LiveLessons will train much more quickly if you employ a GPU. Alternatively, you may enjoy leveraging the efficient, highly-parallelised computations that a GPU affords for your own projects. Whatever the reason, here are TensorFlow GPU instructions for [Mac/Unix](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/installation/step_by_step_MacOSX_install.md#bonus-training-models-with-an-nvidia-gpu) or [Windows](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/installation/windows_TF_GPU.md). 44 | 45 | -------------------------------------------------------------------------------- /installation/conda_TensorFlow_install.md: -------------------------------------------------------------------------------- 1 | # Install TensorFlow in a Conda Virtual Environment 2 | 3 | These steps are for users who have already installed the [Anaconda](https://www.continuum.io/downloads) Python 3 distribution, but other such distributions (e.g., [WinPython](https://winpython.github.io/), [Canopy](https://store.enthought.com/downloads/)) should work as well. 4 | 5 | 1. Create a conda virtual environment: `C:\> conda create -n tf python=3.5` 6 | 2. Activate the new environment: `C:\> activate tf`. Your prompt should now change to: `(tf) C:\>` 7 | 3. Run `(tf) C:\> pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/windows/cpu/tensorflow-1.2.1-cp35-cp35m-win_amd64.whl`. 8 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/Jupyter-docker-stacks-license.md: -------------------------------------------------------------------------------- 1 | The files in this directory are licensed under the terms of the Modified BSD License 2 | (also known as New or Revised or 3-Clause BSD), as follows: 3 | 4 | - Copyright (c) 2001-2015, IPython Development Team 5 | - Copyright (c) 2015-, Jupyter Development Team 6 | 7 | All rights reserved. 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are met: 11 | 12 | Redistributions of source code must retain the above copyright notice, this 13 | list of conditions and the following disclaimer. 14 | 15 | Redistributions in binary form must reproduce the above copyright notice, this 16 | list of conditions and the following disclaimer in the documentation and/or 17 | other materials provided with the distribution. 18 | 19 | Neither the name of the Jupyter Development Team nor the names of its 20 | contributors may be used to endorse or promote products derived from this 21 | software without specific prior written permission. 22 | 23 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 24 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 25 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 26 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 27 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 29 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 30 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 31 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/jupyter_notebook_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Jupyter Development Team. 2 | # Distributed under the terms of the Modified BSD License. 3 | 4 | from jupyter_core.paths import jupyter_data_dir 5 | import subprocess 6 | import os 7 | import errno 8 | import stat 9 | 10 | c = get_config() 11 | c.NotebookApp.ip = '*' 12 | c.NotebookApp.port = 8888 13 | c.NotebookApp.open_browser = False 14 | 15 | # Generate a self-signed certificate 16 | if 'GEN_CERT' in os.environ: 17 | dir_name = jupyter_data_dir() 18 | pem_file = os.path.join(dir_name, 'notebook.pem') 19 | try: 20 | os.makedirs(dir_name) 21 | except OSError as exc: # Python >2.5 22 | if exc.errno == errno.EEXIST and os.path.isdir(dir_name): 23 | pass 24 | else: 25 | raise 26 | # Generate a certificate if one doesn't exist on disk 27 | subprocess.check_call(['openssl', 'req', '-new', 28 | '-newkey', 'rsa:2048', 29 | '-days', '365', 30 | '-nodes', '-x509', 31 | '-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated', 32 | '-keyout', pem_file, 33 | '-out', pem_file]) 34 | # Restrict access to the file 35 | os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) 36 | c.NotebookApp.certfile = pem_file 37 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/start-notebook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | set -e 6 | 7 | if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then 8 | # launched by JupyterHub, use single-user entrypoint 9 | exec /usr/local/bin/start-singleuser.sh $* 10 | else 11 | . /usr/local/bin/start.sh jupyter notebook $* 12 | fi 13 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/start-singleuser.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | set -e 6 | 7 | # set default ip to 0.0.0.0 8 | if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then 9 | NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS" 10 | fi 11 | 12 | # handle some deprecated environment variables 13 | # from DockerSpawner < 0.8. 14 | # These won't be passed from DockerSpawner 0.9, 15 | # so avoid specifying --arg=empty-string 16 | if [ ! -z "$NOTEBOOK_DIR" ]; then 17 | NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS" 18 | fi 19 | if [ ! -z "$JPY_PORT" ]; then 20 | NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS" 21 | fi 22 | if [ ! -z "$JPY_USER" ]; then 23 | NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS" 24 | fi 25 | if [ ! -z "$JPY_COOKIE_NAME" ]; then 26 | NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS" 27 | fi 28 | if [ ! -z "$JPY_BASE_URL" ]; then 29 | NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS" 30 | fi 31 | if [ ! -z "$JPY_HUB_PREFIX" ]; then 32 | NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS" 33 | fi 34 | if [ ! -z "$JPY_HUB_API_URL" ]; then 35 | NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" 36 | fi 37 | 38 | . /usr/local/bin/start.sh jupyterhub-singleuser $NOTEBOOK_ARGS $@ 39 | -------------------------------------------------------------------------------- /installation/docker-stack-scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | set -e 6 | 7 | # Handle special flags if we're root 8 | if [ $(id -u) == 0 ] ; then 9 | # Handle username change. Since this is cheap, do this unconditionally 10 | usermod -d /home/$NB_USER -l $NB_USER jovyan 11 | 12 | # Change UID of NB_USER to NB_UID if it does not match 13 | if [ "$NB_UID" != $(id -u $NB_USER) ] ; then 14 | echo "Set user UID to: $NB_UID" 15 | usermod -u $NB_UID $NB_USER 16 | # Careful: $HOME might resolve to /root depending on how the 17 | # container is started. Use the $NB_USER home path explicitly. 18 | for d in "$CONDA_DIR" "$JULIA_PKGDIR" "/home/$NB_USER"; do 19 | if [[ ! -z "$d" && -d "$d" ]]; then 20 | echo "Set ownership to uid $NB_UID: $d" 21 | chown -R $NB_UID "$d" 22 | fi 23 | done 24 | fi 25 | 26 | # Change GID of NB_USER to NB_GID if NB_GID is passed as a parameter 27 | if [ "$NB_GID" ] ; then 28 | echo "Change GID to $NB_GID" 29 | groupmod -g $NB_GID -o $(id -g -n $NB_USER) 30 | fi 31 | 32 | # Enable sudo if requested 33 | if [ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]; then 34 | echo "Granting $NB_USER sudo access" 35 | echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook 36 | fi 37 | 38 | # Exec the command as NB_USER 39 | echo "Execute the command as $NB_USER" 40 | exec su $NB_USER -c "env PATH=$PATH $*" 41 | else 42 | if [[ ! -z "$NB_UID" && "$NB_UID" != "$(id -u)" ]]; then 43 | echo 'Container must be run as root to set $NB_UID' 44 | fi 45 | if [[ ! -z "$NB_GID" && "$NB_GID" != "$(id -g)" ]]; then 46 | echo 'Container must be run as root to set $NB_GID' 47 | fi 48 | # if [ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]; then 49 | # echo 'Container must be run as root to grant sudo permissions' 50 | # fi 51 | # Exec the command 52 | echo "Execute the command" 53 | exec $* 54 | fi 55 | -------------------------------------------------------------------------------- /installation/let_jovyan_write.sh: -------------------------------------------------------------------------------- 1 | # script that provides jovyan, the default Docker container username, permission to write to the directory 2 | sudo chgrp -R 100 TensorFlow-LiveLessons/ #to recursively change group to match jovyan 3 | sudo chmod -R g+w TensorFlow-LiveLessons/ #to recursively change permissions so jovyan can write to the directory 4 | -------------------------------------------------------------------------------- /installation/miscellaneous_notes.md: -------------------------------------------------------------------------------- 1 | # Miscellaneous Notes 2 | 3 | #### to enter into the Docker container on the command line: 4 | 5 | 1. use `sudo docker ps` to see the `NAME` that was randomly generated 6 | 2. `sudo docker exec -it bash` to enter into the session (N.B.: nothing will be saved) 7 | 8 | 9 | #### to install jupyter vim bindings within Docker container: 10 | 11 | 1. install notebook extensions (as in [here](https://github.com/ipython-contrib/jupyter_contrib_nbextensions#installation)): 12 | 13 | 1. `pip install jupyter_contrib_nbextensions` 14 | 2. `jupyter contrib nbextension install --user` 15 | 16 | 2. following the [instructions](https://github.com/lambdalisue/jupyter-vim-binding/wiki/Installation) doesn't work because you need to activate the notebook after making these changes... I imagine the same is true for themes 17 | 18 | #### to make your own Docker container: 19 | 20 | 1. `vi Dockerfile` 21 | 2. create something with [this](https://raw.githubusercontent.com/jupyter/docker-stacks/master/tensorflow-notebook/Dockerfile) type of structure 22 | 3. `sudo docker build -t .` 23 | 4. `sudo docker run -v ~/TensorFlow-LiveLessons:/home/jovyan/work -it --rm -p 8888:8888 ` 24 | 25 | -------------------------------------------------------------------------------- /installation/simple_Windows_Anaconda_install.md: -------------------------------------------------------------------------------- 1 | # Simple Step-by-Step Anaconda Install 2 | 3 | [Gavriel Markado](https://www.linkedin.com/in/gavriel-merkado-17759917/) has kindly provided step-by-step instructions for a simple installation of Anaconda for using Python, TensorFlow, and Jupyter Notebooks on a Windows machine: 4 | 5 | 1. Install [Anaconda for Windows](https://conda.io/docs/user-guide/install/windows.html) 6 | 2. Open *Anaconda Prompt* (a black command prompt screen should appear) and execute the following commands: 7 | * `pip install tensorflow` 8 | * `pip install keras` 9 | 3. Open *Anaconda Navigator* 10 | 4. Select *Jupyter Notebooks* 11 | -------------------------------------------------------------------------------- /installation/step_by_step_GCP_install.md: -------------------------------------------------------------------------------- 1 | # Step-by-Step Instructions for Installation on a New GCP Ubuntu Instance 2 | 3 | N.B.: These are **rough notes** for creating an Ubuntu machine on Google Cloud Platform that runs Dockerized Jupyter notebooks. 4 | 5 | 1. create an instance in [Google Cloud Platform Compute Engine](https://console.cloud.google.com/compute), Amazon Web Services, or another cloud-computing provider 6 | 1. `us-east1-d` if GPU desired in U.S. on GCP (April 2017) 7 | 2. boot disk: 8 | 1. select `Ubuntu 16.10`, i.e., `amd64 yakkety` for Docker compatibility later 9 | 2. select `SSD` if you're willing to pay for the significant speed benefits 10 | 3. I chose `200 GB`, which should be enough space and costs $34 if you use it for a full month (standard is $0.04/GB/month; SSD is $0.17/GB/month) 11 | 3. this comes out to $0.579/hour, or $422.36 if you run it for a full month (i.e., 730 hours) 12 | 2. ssh into instance, e.g.: 13 | * [like so](https://cloud.google.com/compute/docs/instances/connecting-to-instance) for GCP 14 | 3. from the home directory (which is typically where your ssh session begins): 15 | * `git clone https://github.com/the-deep-learners/TensorFlow-LiveLessons.git` 16 | * this command retrieves all of the code for this LiveLessons 17 | * ...and puts it in a new directory called `TensorFlow-Livelessons` 18 | 4. [install Docker](https://docs.docker.com/engine/installation/linux/ubuntu/) (go from `Install using the repository` through to running the `hello-world` image; the free CE version does everything you need; use `amd64` during repository installation) 19 | 5. create a [firewall rule](https://cloud.google.com/compute/docs/networking?hl=en_US&_ga=1.268488185.92442388.1465331838#firewalls) to add a tag to your instance in your [Compute Engine table](https://console.cloud.google.com/compute) 20 | 6. follow steps six through nine from my [instructions for installation on Mac OS X](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/installation/step_by_step_MacOSX_install.md) 21 | 7. **on your local machine**, navigate to `:8888/?token=` in the web browser of your choice 22 | * the `` of the instance you created is shown in your [Compute Engine table](https://console.cloud.google.com/compute) 23 | * `` should have been output by your `docker run` call in the previous step 24 | 25 | 26 | -------------------------------------------------------------------------------- /installation/step_by_step_MacOSX_install.md: -------------------------------------------------------------------------------- 1 | # Step-by-Step Instructions for Mac OS X 2 | 3 | These instructions enable you to run TensorFlow code from the comfort of interactive Jupyter notebooks. Jupyter is itself run from within a Docker container because this ensures that you'll have all of the software dependencies you need while simultaneously preventing these dependencies from clashing with the software you already have installed on your system. 4 | 5 | ## Install 6 | 7 | 1. Open the Terminal application ([like this](http://www.wikihow.com/Open-a-Terminal-Window-in-Mac)) 8 | 2. To install in your home directory (this is my recommended default): 9 | * Type `cd ~` into the command-line prompt and 10 | * *Execute* the line by pressing the **return** key on your keyboard 11 | 3. Retrieve all of the code for this LiveLessons by executing `git clone https://github.com/the-deep-learners/TensorFlow-LiveLessons.git` (if you haven't used `git` before, you may be prompted to install Xcode -- do it!) 12 | 4. [Install the Docker "Stable channel"](https://docs.docker.com/docker-for-mac/install/) (if you are already using an older version of Docker and run into installation issues downstream, try updating to the latest version of Docker) 13 | 5. Start Docker, e.g., by using Finder to navigate to your Applications folder and double-clicking on the Docker icon 14 | 6. Back in Terminal, execute `source TensorFlow-LiveLessons/installation/let_jovyan_write.sh` so that you can write to files in the *TensorFlow-LiveLessons* directory from inside the Docker container we'll be creating momentarily 15 | 7. Move into the *TensorFlow-LiveLessons* directory by executing `cd TensorFlow-LiveLessons` 16 | 8. Build the Docker container by executing `sudo docker build -t tensorflow-ll-stack .` (you'll get an error if you miss the final `.`!) 17 | 9. When that build process has finished, run the Docker container by executing `sudo docker run -v ~/TensorFlow-LiveLessons:/home/jovyan/work -it --rm -p 8888:8888 tensorflow-ll-stack` (if you modified steps two or three above, e.g., by running `git clone` somewhere other than your home `~` directory, then you will need to similarly modify the `~/TensorFlow-LiveLessons` directory location portion of this command) 18 | 10. In the web browser of your choice (e.g., Chrome), copy and paste the URL created by Docker (this begins with `http://localhost:8888/?token=` and should be visible near the bottom of your Terminal window) 19 | 20 | ## Shutdown 21 | 22 | You can shutdown the Jupyter notebook by returning to the Terminal session that is running it and hitting the **control** and **c** keys simultaneously on your keyboard. 23 | 24 | ## Restart 25 | 26 | You can restart the Jupyter notebook later by following steps nine and ten alone. 27 | 28 | ## Bonus: Training Models with an Nvidia GPU 29 | 30 | You don't need to train your Deep Learning models with a GPU for these LiveLessons, but some of the later notebooks in these LiveLessons will run much more quickly if you do. 31 | 32 | 1. Install an [Nvidia GPU](http://www.nvidia.com/content/global/global.php) on your machine or spin up a cloud instance that includes one (typically a Tesla K80) 33 | 1. Install CUDA and cuDNN, e.g., per the **Installing CUDA Toolkit** and **Installing cuDNN** sections of [this blog post](https://hackernoon.com/launch-a-gpu-backed-google-compute-engine-instance-and-setup-tensorflow-keras-and-jupyter-902369ed5272) (this step may be tricky if you're relatively new to working with the Unix command line) 34 | 2. In the `TensorFlow-LiveLessons/installation/docker-stack-scripts` directory: 35 | * run `chmod 777 jupyter_notebook_config.py start*.sh` 36 | 3. Replace step eight of my **Install** section above with `sudo docker build -f Dockerfile-gpu -t tfll-gpu-stack .` 37 | 4. Replace step nine with `sudo nvidia-docker run -v ~/TensorFlow-LiveLessons:/home/jovyan/work -it --rm -p 8888:8888 tfll-gpu-stack` 38 | -------------------------------------------------------------------------------- /installation/step_by_step_Windows_Docker_install.md: -------------------------------------------------------------------------------- 1 | # Step-by-Step Instructions for Windows 2 | 3 | These instructions enable you to run TensorFlow code from the comfort of interactive Jupyter notebooks. Jupyter is itself run from within a Docker container because this ensures that you'll have all of the software dependencies you need while simultaneously preventing these dependencies from clashing with the software you already have installed on your system. 4 | 5 | Please note that **to use Docker on Windows**, you will need a 64-bit installation of Windows 10 Professional or Enterprise. 6 | 7 | 1. Install [Docker Community Edition for Windows](https://store.docker.com/editions/community/docker-ce-desktop-windows). You may need to log out and/or reboot to complete the installation. 8 | 2. Install [SourceTree](https://www.sourcetreeapp.com/). 9 | 3. Using SourceTree, clone the repository at `https://github.com/the-deep-learners/TensorFlow-LiveLessons.git`. Note the directory which you used for this clone. 10 | 4. Right-click on the Docker "whale" icon in the system tray and select "Settings..." followed by "Shared Drives". Ensure that the drive which you used for the checkout is marked as shared; you will need to enter your Windows password and restart Docker at this point. 11 | 5. Start a PowerShell prompt and change into the directory where you cloned the repository. 12 | 6. Build the Docker container by executing `docker build -t tensorflow-ll-stack .` (you'll get an error if you miss the final `.`!) 13 | 7. When that build process has finished, run the Docker container by executing `docker run -v c:/full/path/to/the/clone:/home/jovyan/work -it --rm -p 8888:8888 tensorflow-ll-stack`. 14 | 8. In the web browser of your choice (e.g., Chrome), copy and paste the URL created by Docker (this begins with `http://localhost:8888/?token=` and should be visible near the bottom of your Terminal window) 15 | -------------------------------------------------------------------------------- /installation/windows_TF_GPU.md: -------------------------------------------------------------------------------- 1 | # Install TensorFlow for GPU on Windows 2 | 3 | 1. Install Visual Studio **2015** Community Edition from https://my.visualstudio.com (requires registering a free account). Make sure "Visual C++" is checked. 4 | 2. Install [CUDA 8](https://developer.nvidia.com/cuda-downloads). 5 | 3. Download [cuDNN](https://developer.nvidia.com/rdp/cudnn-download) v5.1, which **isn't** the latest version, and unzip its contents to where you installed CUDA. 6 | 4. Create a conda virtual environment: `C:\> conda create -n tf python=3.5` 7 | 5. Activate the new environment: `C:\> activate tf`. Your prompt should now change to: `(tf) C:\>` 8 | 6. Run `(tf) C:\> pip install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/windows/gpu/tensorflow_gpu-1.2.1-cp35-cp35m-win_amd64.whl`. 9 | -------------------------------------------------------------------------------- /notebooks/bidirectional_lstm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Bidirectional LSTM in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we use a *bidirectional* LSTM to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stderr", 31 | "output_type": "stream", 32 | "text": [ 33 | "Using TensorFlow backend.\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "import keras\n", 39 | "from keras.datasets import imdb\n", 40 | "from keras.preprocessing.sequence import pad_sequences\n", 41 | "from keras.models import Sequential\n", 42 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, LSTM\n", 43 | "from keras.layers.wrappers import Bidirectional # new! \n", 44 | "from keras.callbacks import ModelCheckpoint\n", 45 | "import os\n", 46 | "from sklearn.metrics import roc_auc_score \n", 47 | "import matplotlib.pyplot as plt \n", 48 | "%matplotlib inline" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "#### Set hyperparameters" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 2, 61 | "metadata": { 62 | "collapsed": true 63 | }, 64 | "outputs": [], 65 | "source": [ 66 | "# output directory name:\n", 67 | "output_dir = 'model_output/biLSTM'\n", 68 | "\n", 69 | "# training:\n", 70 | "epochs = 6\n", 71 | "batch_size = 128\n", 72 | "\n", 73 | "# vector-space embedding: \n", 74 | "n_dim = 64 \n", 75 | "n_unique_words = 10000 \n", 76 | "max_review_length = 200 # doubled!\n", 77 | "pad_type = trunc_type = 'pre'\n", 78 | "drop_embed = 0.2 \n", 79 | "\n", 80 | "# LSTM layer architecture:\n", 81 | "n_lstm = 256 \n", 82 | "drop_lstm = 0.2" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "#### Load data" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 3, 95 | "metadata": { 96 | "collapsed": true 97 | }, 98 | "outputs": [], 99 | "source": [ 100 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "#### Preprocess data" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 4, 113 | "metadata": { 114 | "collapsed": true 115 | }, 116 | "outputs": [], 117 | "source": [ 118 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 119 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": { 125 | "collapsed": true 126 | }, 127 | "source": [ 128 | "#### Design neural network architecture" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 5, 134 | "metadata": { 135 | "collapsed": true 136 | }, 137 | "outputs": [], 138 | "source": [ 139 | "model = Sequential()\n", 140 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n", 141 | "model.add(SpatialDropout1D(drop_embed))\n", 142 | "model.add(Bidirectional(LSTM(n_lstm, dropout=drop_lstm)))\n", 143 | "model.add(Dense(1, activation='sigmoid'))" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 6, 149 | "metadata": {}, 150 | "outputs": [ 151 | { 152 | "name": "stdout", 153 | "output_type": "stream", 154 | "text": [ 155 | "_________________________________________________________________\n", 156 | "Layer (type) Output Shape Param # \n", 157 | "=================================================================\n", 158 | "embedding_1 (Embedding) (None, 200, 64) 640000 \n", 159 | "_________________________________________________________________\n", 160 | "spatial_dropout1d_1 (Spatial (None, 200, 64) 0 \n", 161 | "_________________________________________________________________\n", 162 | "bidirectional_1 (Bidirection (None, 512) 657408 \n", 163 | "_________________________________________________________________\n", 164 | "dense_1 (Dense) (None, 1) 513 \n", 165 | "=================================================================\n", 166 | "Total params: 1,297,921\n", 167 | "Trainable params: 1,297,921\n", 168 | "Non-trainable params: 0\n", 169 | "_________________________________________________________________\n" 170 | ] 171 | } 172 | ], 173 | "source": [ 174 | "# LSTM layer parameters double due to both reading directions\n", 175 | "model.summary() " 176 | ] 177 | }, 178 | { 179 | "cell_type": "markdown", 180 | "metadata": {}, 181 | "source": [ 182 | "#### Configure model" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": 7, 188 | "metadata": { 189 | "collapsed": true 190 | }, 191 | "outputs": [], 192 | "source": [ 193 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": 8, 199 | "metadata": { 200 | "collapsed": true 201 | }, 202 | "outputs": [], 203 | "source": [ 204 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 205 | "if not os.path.exists(output_dir):\n", 206 | " os.makedirs(output_dir)" 207 | ] 208 | }, 209 | { 210 | "cell_type": "markdown", 211 | "metadata": {}, 212 | "source": [ 213 | "#### Train!" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": 9, 219 | "metadata": {}, 220 | "outputs": [ 221 | { 222 | "name": "stdout", 223 | "output_type": "stream", 224 | "text": [ 225 | "Train on 25000 samples, validate on 25000 samples\n", 226 | "Epoch 1/6\n", 227 | "25000/25000 [==============================] - 118s - loss: 0.5368 - acc: 0.7193 - val_loss: 0.3578 - val_acc: 0.8480\n", 228 | "Epoch 2/6\n", 229 | "25000/25000 [==============================] - 112s - loss: 0.2881 - acc: 0.8835 - val_loss: 0.3141 - val_acc: 0.8719\n", 230 | "Epoch 3/6\n", 231 | "25000/25000 [==============================] - 114s - loss: 0.2198 - acc: 0.9164 - val_loss: 0.3167 - val_acc: 0.8651\n", 232 | "Epoch 4/6\n", 233 | "25000/25000 [==============================] - 114s - loss: 0.1772 - acc: 0.9344 - val_loss: 0.3469 - val_acc: 0.8598\n", 234 | "Epoch 5/6\n", 235 | "25000/25000 [==============================] - 114s - loss: 0.1492 - acc: 0.9445 - val_loss: 0.3802 - val_acc: 0.8676\n", 236 | "Epoch 6/6\n", 237 | "25000/25000 [==============================] - 115s - loss: 0.1260 - acc: 0.9536 - val_loss: 0.4241 - val_acc: 0.8620\n" 238 | ] 239 | }, 240 | { 241 | "data": { 242 | "text/plain": [ 243 | "" 244 | ] 245 | }, 246 | "execution_count": 9, 247 | "metadata": {}, 248 | "output_type": "execute_result" 249 | } 250 | ], 251 | "source": [ 252 | "# - we see 87.0% validation accuracy in epoch 2\n", 253 | "# - with this toy dataset, the complex interplay of words over long sentence segments, won't be learned much\n", 254 | "# - so our CNN picking up location-invariant segments of two to four words that predict review sentiment\n", 255 | "# - these are simpler and so easier to learn from the data\n", 256 | "# - CNN therefore outperforms on the IMDB data set\n", 257 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 258 | ] 259 | }, 260 | { 261 | "cell_type": "markdown", 262 | "metadata": { 263 | "collapsed": true 264 | }, 265 | "source": [ 266 | "#### Evaluate" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": 10, 272 | "metadata": { 273 | "collapsed": true 274 | }, 275 | "outputs": [], 276 | "source": [ 277 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 278 | ] 279 | }, 280 | { 281 | "cell_type": "code", 282 | "execution_count": 11, 283 | "metadata": {}, 284 | "outputs": [ 285 | { 286 | "name": "stdout", 287 | "output_type": "stream", 288 | "text": [ 289 | "25000/25000 [==============================] - 92s \n" 290 | ] 291 | } 292 | ], 293 | "source": [ 294 | "y_hat = model.predict_proba(x_valid)" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 12, 300 | "metadata": {}, 301 | "outputs": [ 302 | { 303 | "data": { 304 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAERpJREFUeJzt3X+sX3V9x/HnSyr+VooU51q2i7E6kWQRbxBn4pw1/NJY\n/hBTM0clzZo45pwzm7gtYUFJ0P1ASfyxTjrBOIExMxrFkQ4hbotFL+JUYIQOGHQwudpadcQf1ff+\n+H5wVz63vd/e773329s+H0nzPedzPuec94dbeN3zOed7SFUhSdJMTxh3AZKkQ4/hIEnqGA6SpI7h\nIEnqGA6SpI7hIEnqGA6SpI7hIEnqzBkOSbYmeSTJN2a0HZtke5J72ufK1p4klyfZmeRrSU6Zsc/G\n1v+eJBtntL80ydfbPpcnyUIPUpJ0cDLXN6STvBL4PnBVVZ3c2t4P7K6qS5NcCKysqnclORt4G3A2\n8DLgg1X1siTHAlPAJFDAbcBLq2pPki8Bbwd2ADcAl1fV5+Yq/LjjjquJiYl5DVpaNN+9e/D5zBeO\ntw5pFrfddtu3qmrVMH1XzNWhqr6QZOJxzeuBV7XlK4FbgHe19qtqkDg7khyT5Lmt7/aq2g2QZDtw\nZpJbgGdW1Rdb+1XAOcCc4TAxMcHU1NRc3aSl9c+vGny+5pZxViHNKsl/Ddt3vvccnlNVDwO0z+Nb\n+2rgwRn9drW2A7XvmqVdkjRGC31Derb7BTWP9tkPnmxOMpVkanp6ep4lSpLmMt9w+GabLqJ9PtLa\ndwEnzOi3BnhojvY1s7TPqqq2VNVkVU2uWjXUtJkkaR7mGw7bgMeeONoIXD+j/bz21NJpwN427XQj\ncHqSle3JptOBG9u27yU5rT2ldN6MY0mSxmTOG9JJPsXghvJxSXYBFwGXAtcm2QQ8AJzbut/A4Eml\nncCjwPkAVbU7yXuAL7d+Fz92cxp4K/Bx4CkMbkTPeTNakrS4hnla6U372bRulr4FXLCf42wFts7S\nPgWcPFcdkqSl4zekJUkdw0GS1DEcJEmdOe85HI4mLvzsWM57/6WvHct5JelgeeUgSeoYDpKkjuEg\nSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoY\nDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKk\njuEgSeoYDpKkzkjhkOQdSe5I8o0kn0ry5CQnJrk1yT1JrklydOv7pLa+s22fmHGcd7f2u5OcMdqQ\nJEmjmnc4JFkN/B4wWVUnA0cBG4D3AZdV1VpgD7Cp7bIJ2FNVzwcua/1IclLb78XAmcCHkxw137ok\nSaMbdVppBfCUJCuApwIPA68GrmvbrwTOacvr2zpt+7okae1XV9UPq+o+YCdw6oh1SZJGMO9wqKr/\nBv4CeIBBKOwFbgO+U1X7WrddwOq2vBp4sO27r/V/9sz2WfaRJI3BKNNKKxn81n8i8IvA04CzZula\nj+2yn237a5/tnJuTTCWZmp6ePviiJUlDGWVa6TXAfVU1XVU/Bj4N/BpwTJtmAlgDPNSWdwEnALTt\nzwJ2z2yfZZ+fU1VbqmqyqiZXrVo1QumSpAMZJRweAE5L8tR272AdcCdwM/CG1mcjcH1b3tbWads/\nX1XV2je0p5lOBNYCXxqhLknSiFbM3WV2VXVrkuuArwD7gNuBLcBngauTvLe1XdF2uQL4RJKdDK4Y\nNrTj3JHkWgbBsg+4oKp+Mt+6JEmjm3c4AFTVRcBFj2u+l1meNqqqHwDn7uc4lwCXjFKLJGnh+A1p\nSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLH\ncJAkdUZ6ZbckHakmLvzsWM57/6WvXZLzeOUgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoY\nDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeqMFA5J\njklyXZL/SHJXkpcnOTbJ9iT3tM+VrW+SXJ5kZ5KvJTllxnE2tv73JNk46qAkSaMZ9crhg8A/VdWv\nAL8K3AVcCNxUVWuBm9o6wFnA2vZnM/ARgCTHAhcBLwNOBS56LFAkSeMx73BI8kzglcAVAFX1o6r6\nDrAeuLJ1uxI4py2vB66qgR3AMUmeC5wBbK+q3VW1B9gOnDnfuiRJoxvlyuF5wDTwt0luT/KxJE8D\nnlNVDwO0z+Nb/9XAgzP239Xa9tcuSRqTUcJhBXAK8JGqegnwv/z/FNJsMktbHaC9P0CyOclUkqnp\n6emDrVeSNKRRwmEXsKuqbm3r1zEIi2+26SLa5yMz+p8wY/81wEMHaO9U1ZaqmqyqyVWrVo1QuiTp\nQOYdDlX1P8CDSV7YmtYBdwLbgMeeONoIXN+WtwHntaeWTgP2tmmnG4HTk6xsN6JPb22SpDFZMeL+\nbwM+meRo4F7gfAaBc22STcADwLmt7w3A2cBO4NHWl6raneQ9wJdbv4uraveIdUmSRjBSOFTVV4HJ\nWTatm6VvARfs5zhbga2j1CJJWjh+Q1qS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkd\nw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS\n1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEc\nJEmdkcMhyVFJbk/ymbZ+YpJbk9yT5JokR7f2J7X1nW37xIxjvLu1353kjFFrkiSNZiGuHN4O3DVj\n/X3AZVW1FtgDbGrtm4A9VfV84LLWjyQnARuAFwNnAh9OctQC1CVJmqeRwiHJGuC1wMfaeoBXA9e1\nLlcC57Tl9W2dtn1d678euLqqflhV9wE7gVNHqUuSNJpRrxw+APwR8NO2/mzgO1W1r63vAla35dXA\ngwBt+97W/2fts+wjSRqDeYdDktcBj1TVbTObZ+lac2w70D6PP+fmJFNJpqanpw+qXknS8Ea5cngF\n8Pok9wNXM5hO+gBwTJIVrc8a4KG2vAs4AaBtfxawe2b7LPv8nKraUlWTVTW5atWqEUqXJB3IvMOh\nqt5dVWuqaoLBDeXPV9VvAjcDb2jdNgLXt+VtbZ22/fNVVa19Q3ua6URgLfCl+dYlSRrdirm7HLR3\nAVcneS9wO3BFa78C+ESSnQyuGDYAVNUdSa4F7gT2ARdU1U8WoS5J0pAWJByq6hbglrZ8L7M8bVRV\nPwDO3c/+lwCXLEQtkqTR+Q1pSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLH\ncJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAk\ndQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdVbM\nd8ckJwBXAb8A/BTYUlUfTHIscA0wAdwPvLGq9iQJ8EHgbOBR4C1V9ZV2rI3An7ZDv7eqrpxvXZKO\nHBMXfnbcJRy2Rrly2Ae8s6peBJwGXJDkJOBC4KaqWgvc1NYBzgLWtj+bgY8AtDC5CHgZcCpwUZKV\nI9QlSRrRvMOhqh5+7Df/qvoecBewGlgPPPab/5XAOW15PXBVDewAjknyXOAMYHtV7a6qPcB24Mz5\n1iVJGt2C3HNIMgG8BLgVeE5VPQyDAAGOb91WAw/O2G1Xa9tfuyRpTEYOhyRPB/4B+P2q+u6Bus7S\nVgdon+1cm5NMJZmanp4++GIlSUMZKRySPJFBMHyyqj7dmr/Zpoton4+09l3ACTN2XwM8dID2TlVt\nqarJqppctWrVKKVLkg5g3uHQnj66Arirqv5qxqZtwMa2vBG4fkb7eRk4Ddjbpp1uBE5PsrLdiD69\ntUmSxmTej7ICrwB+C/h6kq+2tj8GLgWuTbIJeAA4t227gcFjrDsZPMp6PkBV7U7yHuDLrd/FVbV7\nhLokSSOadzhU1b8y+/0CgHWz9C/ggv0cayuwdb61SJIWlt+QliR1DAdJUsdwkCR1DAdJUsdwkCR1\nDAdJUmeU7znoII3z9cL3X/rasZ1b0vLjlYMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqeP3\nHCSNbJzf4dHi8MpBktQxHCRJHcNBktQxHCRJHW9IHyHGdcPQF/5Jy5NXDpKkjlcO0mHCx0m1kLxy\nkCR1DAdJUsdpJS2qI/H/frfj3m+zwSkeLXOGgw5b4wimq5/37SU/p7QYnFaSJHUMB0lSx3CQJHUM\nB0lSx3CQJHUMB0lSx3CQJHUMB0lS55AJhyRnJrk7yc4kF467Hkk6kh0S4ZDkKOBDwFnAScCbkpw0\n3qok6ch1SIQDcCqws6ruraofAVcD68dckyQdsQ6VcFgNPDhjfVdrkySNwaHy4r3M0lZdp2QzsLmt\nfj/J3fM833HAt+a573LlmJfAy3+29LqlPO1j/BkfAfK+kcb8y8N2PFTCYRdwwoz1NcBDj+9UVVuA\nLaOeLMlUVU2OepzlxDEf/o608YJjXkyHyrTSl4G1SU5McjSwAdg25pok6Yh1SFw5VNW+JL8L3Agc\nBWytqjvGXJYkHbEOiXAAqKobgBuW6HQjT00tQ4758HekjRcc86JJVXffV5J0hDtU7jlIkg4hh204\nzPU6jiRPSnJN235rkomlr3JhDTHmP0hyZ5KvJbkpydCPtR2qhn3tSpI3JKkky/7JlmHGnOSN7Wd9\nR5K/W+oaF9oQf7d/KcnNSW5vf7/PHkedCynJ1iSPJPnGfrYnyeXtn8nXkpyyoAVU1WH3h8FN7f8E\nngccDfw7cNLj+vwO8NG2vAG4Ztx1L8GYfwN4alt+65Ew5tbvGcAXgB3A5LjrXoKf81rgdmBlWz9+\n3HUvwZi3AG9tyycB94+77gUY9yuBU4Bv7Gf72cDnGHxP7DTg1oU8/+F65TDM6zjWA1e25euAdUlm\n+zLecjHnmKvq5qp6tK3uYPB9kuVs2NeuvAd4P/CDpSxukQwz5t8GPlRVewCq6pElrnGhDTPmAp7Z\nlp/FLN+TWm6q6gvA7gN0WQ9cVQM7gGOSPHehzn+4hsMwr+P4WZ+q2gfsBZ69JNUtjoN9BckmBr91\nLGdzjjnJS4ATquozS1nYIhrm5/wC4AVJ/i3JjiRnLll1i2OYMf8Z8OYkuxg89fi2pSltrBb1tUOH\nzKOsC2yY13EM9cqOZWTo8SR5MzAJ/PqiVrT4DjjmJE8ALgPeslQFLYFhfs4rGEwtvYrB1eG/JDm5\nqr6zyLUtlmHG/Cbg41X1l0leDnyijfmni1/e2Czqf8MO1yuHYV7H8bM+SVYwuBQ90CXcoW6oV5Ak\neQ3wJ8Drq+qHS1TbYplrzM8ATgZuSXI/g3nZbcv8pvSwf7evr6ofV9V9wN0MwmK5GmbMm4BrAarq\ni8CTGbx36XA21L/z83W4hsMwr+PYBmxsy28APl/tLs8yNeeY2xTLXzMIhuU+Dw1zjLmq9lbVcVU1\nUVUTDO6zvL6qpsZT7oIY5u/2PzJ4+IAkxzGYZrp3SatcWMOM+QFgHUCSFzEIh+klrXLpbQPOa08t\nnQbsraqHF+rgh+W0Uu3ndRxJLgamqmobcAWDS8+dDK4YNoyv4tENOeY/B54O/H279/5AVb1+bEWP\naMgxH1aGHPONwOlJ7gR+AvxhVX17fFWPZsgxvxP4myTvYDC18pZl/sseST7FYGrwuHYv5SLgiQBV\n9VEG91bOBnYCjwLnL+j5l/k/P0nSIjhcp5UkSSMwHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQx\nHCRJnf8DaFnnSTEPqCgAAAAASUVORK5CYII=\n", 305 | "text/plain": [ 306 | "" 307 | ] 308 | }, 309 | "metadata": {}, 310 | "output_type": "display_data" 311 | } 312 | ], 313 | "source": [ 314 | "plt.hist(y_hat)\n", 315 | "_ = plt.axvline(x=0.5, color='orange')" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": 13, 321 | "metadata": {}, 322 | "outputs": [ 323 | { 324 | "data": { 325 | "text/plain": [ 326 | "'94.39'" 327 | ] 328 | }, 329 | "execution_count": 13, 330 | "metadata": {}, 331 | "output_type": "execute_result" 332 | } 333 | ], 334 | "source": [ 335 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": null, 341 | "metadata": { 342 | "collapsed": true 343 | }, 344 | "outputs": [], 345 | "source": [] 346 | } 347 | ], 348 | "metadata": { 349 | "kernelspec": { 350 | "display_name": "Python 3", 351 | "language": "python", 352 | "name": "python3" 353 | }, 354 | "language_info": { 355 | "codemirror_mode": { 356 | "name": "ipython", 357 | "version": 3 358 | }, 359 | "file_extension": ".py", 360 | "mimetype": "text/x-python", 361 | "name": "python", 362 | "nbconvert_exporter": "python", 363 | "pygments_lexer": "ipython3", 364 | "version": "3.6.2" 365 | } 366 | }, 367 | "nbformat": 4, 368 | "nbformat_minor": 2 369 | } 370 | -------------------------------------------------------------------------------- /notebooks/convolutional_sentiment_classifier.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Convolutional Sentiment Classifier" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we build a *convolutional* neural net to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stderr", 31 | "output_type": "stream", 32 | "text": [ 33 | "Using TensorFlow backend.\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "import keras\n", 39 | "from keras.datasets import imdb\n", 40 | "from keras.preprocessing.sequence import pad_sequences\n", 41 | "from keras.models import Sequential\n", 42 | "from keras.layers import Dense, Dropout, Embedding\n", 43 | "from keras.layers import SpatialDropout1D, Conv1D, GlobalMaxPooling1D # new! \n", 44 | "from keras.callbacks import ModelCheckpoint\n", 45 | "import os\n", 46 | "from sklearn.metrics import roc_auc_score \n", 47 | "import matplotlib.pyplot as plt \n", 48 | "%matplotlib inline" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "#### Set hyperparameters" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 2, 61 | "metadata": { 62 | "collapsed": true 63 | }, 64 | "outputs": [], 65 | "source": [ 66 | "# output directory name:\n", 67 | "output_dir = 'model_output/conv'\n", 68 | "\n", 69 | "# training:\n", 70 | "epochs = 4\n", 71 | "batch_size = 128\n", 72 | "\n", 73 | "# vector-space embedding: \n", 74 | "n_dim = 64\n", 75 | "n_unique_words = 5000 \n", 76 | "max_review_length = 400\n", 77 | "pad_type = trunc_type = 'pre'\n", 78 | "drop_embed = 0.2 # new!\n", 79 | "\n", 80 | "# convolutional layer architecture:\n", 81 | "n_conv = 256 # filters, a.k.a. kernels\n", 82 | "k_conv = 3 # kernel length\n", 83 | "\n", 84 | "# dense layer architecture: \n", 85 | "n_dense = 256\n", 86 | "dropout = 0.2" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "#### Load data" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 3, 99 | "metadata": { 100 | "collapsed": true 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "#### Preprocess data" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": 4, 117 | "metadata": { 118 | "collapsed": true 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 123 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": { 129 | "collapsed": true 130 | }, 131 | "source": [ 132 | "#### Design neural network architecture" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 5, 138 | "metadata": { 139 | "collapsed": true 140 | }, 141 | "outputs": [], 142 | "source": [ 143 | "model = Sequential()\n", 144 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n", 145 | "model.add(SpatialDropout1D(drop_embed))\n", 146 | "model.add(Conv1D(n_conv, k_conv, activation='relu'))\n", 147 | "# model.add(Conv1D(n_conv, k_conv, activation='relu'))\n", 148 | "model.add(GlobalMaxPooling1D())\n", 149 | "model.add(Dense(n_dense, activation='relu'))\n", 150 | "model.add(Dropout(dropout))\n", 151 | "model.add(Dense(1, activation='sigmoid'))" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 6, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "name": "stdout", 161 | "output_type": "stream", 162 | "text": [ 163 | "_________________________________________________________________\n", 164 | "Layer (type) Output Shape Param # \n", 165 | "=================================================================\n", 166 | "embedding_1 (Embedding) (None, 400, 64) 320000 \n", 167 | "_________________________________________________________________\n", 168 | "spatial_dropout1d_1 (Spatial (None, 400, 64) 0 \n", 169 | "_________________________________________________________________\n", 170 | "conv1d_1 (Conv1D) (None, 398, 256) 49408 \n", 171 | "_________________________________________________________________\n", 172 | "global_max_pooling1d_1 (Glob (None, 256) 0 \n", 173 | "_________________________________________________________________\n", 174 | "dense_1 (Dense) (None, 256) 65792 \n", 175 | "_________________________________________________________________\n", 176 | "dropout_1 (Dropout) (None, 256) 0 \n", 177 | "_________________________________________________________________\n", 178 | "dense_2 (Dense) (None, 1) 257 \n", 179 | "=================================================================\n", 180 | "Total params: 435,457\n", 181 | "Trainable params: 435,457\n", 182 | "Non-trainable params: 0\n", 183 | "_________________________________________________________________\n" 184 | ] 185 | } 186 | ], 187 | "source": [ 188 | "model.summary() " 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": {}, 194 | "source": [ 195 | "#### Configure model" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 7, 201 | "metadata": { 202 | "collapsed": true 203 | }, 204 | "outputs": [], 205 | "source": [ 206 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 8, 212 | "metadata": { 213 | "collapsed": true 214 | }, 215 | "outputs": [], 216 | "source": [ 217 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 218 | "if not os.path.exists(output_dir):\n", 219 | " os.makedirs(output_dir)" 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "metadata": {}, 225 | "source": [ 226 | "#### Train!" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": 9, 232 | "metadata": {}, 233 | "outputs": [ 234 | { 235 | "name": "stdout", 236 | "output_type": "stream", 237 | "text": [ 238 | "Train on 25000 samples, validate on 25000 samples\n", 239 | "Epoch 1/4\n", 240 | "25000/25000 [==============================] - 45s - loss: 0.4800 - acc: 0.7472 - val_loss: 0.2931 - val_acc: 0.8754\n", 241 | "Epoch 2/4\n", 242 | "25000/25000 [==============================] - 3s - loss: 0.2502 - acc: 0.8974 - val_loss: 0.2602 - val_acc: 0.8918\n", 243 | "Epoch 3/4\n", 244 | "25000/25000 [==============================] - 3s - loss: 0.1719 - acc: 0.9357 - val_loss: 0.2811 - val_acc: 0.8848\n", 245 | "Epoch 4/4\n", 246 | "25000/25000 [==============================] - 3s - loss: 0.1147 - acc: 0.9609 - val_loss: 0.3094 - val_acc: 0.8834\n" 247 | ] 248 | }, 249 | { 250 | "data": { 251 | "text/plain": [ 252 | "" 253 | ] 254 | }, 255 | "execution_count": 9, 256 | "metadata": {}, 257 | "output_type": "execute_result" 258 | } 259 | ], 260 | "source": [ 261 | "# 89.1% validation accuracy in epoch 2\n", 262 | "# ...with second convolutional layer is essentially the same at 89.0%\n", 263 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 264 | ] 265 | }, 266 | { 267 | "cell_type": "markdown", 268 | "metadata": { 269 | "collapsed": true 270 | }, 271 | "source": [ 272 | "#### Evaluate" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": 10, 278 | "metadata": { 279 | "collapsed": true 280 | }, 281 | "outputs": [], 282 | "source": [ 283 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": 11, 289 | "metadata": {}, 290 | "outputs": [ 291 | { 292 | "name": "stdout", 293 | "output_type": "stream", 294 | "text": [ 295 | "24544/25000 [============================>.] - ETA: 0s" 296 | ] 297 | } 298 | ], 299 | "source": [ 300 | "y_hat = model.predict_proba(x_valid)" 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": 12, 306 | "metadata": { 307 | "scrolled": true 308 | }, 309 | "outputs": [ 310 | { 311 | "data": { 312 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD4FJREFUeJzt3X+sX3ddx/Hni5WB/NyghWBbvSMUpJAYlpsxJEGkZD/J\nuj82UyJSSGMTnIhI1KF/1ABLhr+GJPyw0mkhSDcncQ2bLnM/gho36BhOtrmsbnOrm+xCu6IugIW3\nf3w/m3fjtvfc9t7vd3ef5yNp7jmf8znf83n33t7X93zO+Z6mqpAk9ecZkx6AJGkyDABJ6pQBIEmd\nMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSp1ZMegBHsnLlypqampr0MKQf9Z27Rl9f8KrJjkOa\nwy233PKtqlo1X7+ndABMTU2xZ8+eSQ9D+lF/9+bR17feOMlRSHNK8u9D+jkFJEmdMgAkqVMGgCR1\nygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnXpKfxJYkiZp6sKrJnbs+y4+e8mP4RmAJHXKAJCk\nThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqU\nASBJnTIAJKlTBoAkdcoAkKRODQqAJO9PcnuSbyT5QpJnJzkpyc1J7k5yWZLjW99ntfW9bfvUrNf5\nYGu/K8npS1OSJGmIeQMgyWrgV4HpqnotcBywCfgocElVrQMOAFvaLluAA1X1CuCS1o8k69t+rwHO\nAD6Z5LjFLUeSNNTQKaAVwI8lWQE8B3gIeAtwRdu+Ezi3LW9s67TtG5Kkte+qqu9V1b3AXuCUYy9B\nknQ05g2AqvoP4A+A+xn94j8I3AI8UlWHWrd9wOq2vBp4oO17qPV/8ez2OfaRJI3ZkCmgExm9ez8J\n+HHgucCZc3Stx3Y5zLbDtT/5eFuT7EmyZ2ZmZr7hSZKO0ooBfd4K3FtVMwBJvgj8DHBCkhXtXf4a\n4MHWfx+wFtjXpoxeCOyf1f6Y2fs8rqq2A9sBpqenfyQgFmLqwquOZfejdt/FZ0/kuJK0EEOuAdwP\nnJrkOW0ufwNwB3ADcF7rsxm4si3vbuu07ddXVbX2Te0uoZOAdcBXFqcMSdJCzXsGUFU3J7kC+Bpw\nCLiV0Tv0q4BdST7S2na0XXYAn0uyl9E7/03tdW5Pcjmj8DgEXFBVP1jkeiRJAw2ZAqKqtgHbntR8\nD3PcxVNV3wXOP8zrXARctMAxSpKWgJ8ElqROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaA\nJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhS\npwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXK\nAJCkThkAktQpA0CSOmUASFKnDABJ6tSgAEhyQpIrkvxrkjuTvCHJi5Jcm+Tu9vXE1jdJPp5kb5Lb\nkpw863U2t/53J9m8VEVJkuY39Azgj4G/raqfAn4auBO4ELiuqtYB17V1gDOBde3PVuBTAEleBGwD\nXg+cAmx7LDQkSeM3bwAkeQHwJmAHQFV9v6oeATYCO1u3ncC5bXkj8NkauQk4IcnLgNOBa6tqf1Ud\nAK4FzljUaiRJgw05A3g5MAP8WZJbk3wmyXOBl1bVQwDt60ta/9XAA7P239faDtcuSZqAIQGwAjgZ\n+FRVvQ74H/5/umcumaOtjtD+xJ2TrUn2JNkzMzMzYHiSpKMxJAD2Afuq6ua2fgWjQPhmm9qhfX14\nVv+1s/ZfAzx4hPYnqKrtVTVdVdOrVq1aSC2SpAWYNwCq6j+BB5K8qjVtAO4AdgOP3cmzGbiyLe8G\n3tnuBjoVONimiK4BTktyYrv4e1prkyRNwIqB/d4LfD7J8cA9wLsZhcflSbYA9wPnt75XA2cBe4FH\nW1+qan+SDwNfbf0+VFX7F6UKSdKCDQqAqvo6MD3Hpg1z9C3ggsO8zqXApQsZoCRpafhJYEnqlAEg\nSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLU\nKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0y\nACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4NDoAkxyW5\nNcmX2vpJSW5OcneSy5Ic39qf1db3tu1Ts17jg639riSnL3YxkqThFnIG8D7gzlnrHwUuqap1wAFg\nS2vfAhyoqlcAl7R+JFkPbAJeA5wBfDLJccc2fEnS0RoUAEnWAGcDn2nrAd4CXNG67ATObcsb2zpt\n+4bWfyOwq6q+V1X3AnuBUxajCEnSwg09A/gY8JvAD9v6i4FHqupQW98HrG7Lq4EHANr2g63/4+1z\n7CNJGrN5AyDJ24CHq+qW2c1zdK15th1pn9nH25pkT5I9MzMz8w1PknSUhpwBvBE4J8l9wC5GUz8f\nA05IsqL1WQM82Jb3AWsB2vYXAvtnt8+xz+OqantVTVfV9KpVqxZckCRpmHkDoKo+WFVrqmqK0UXc\n66vqF4AbgPNat83AlW15d1unbb++qqq1b2p3CZ0ErAO+smiVSJIWZMX8XQ7rt4BdST4C3ArsaO07\ngM8l2cvonf8mgKq6PcnlwB3AIeCCqvrBMRxfknQMFhQAVXUjcGNbvoc57uKpqu8C5x9m/4uAixY6\nSEnS4vOTwJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcM\nAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQ\npE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnq\nlAEgSZ2aNwCSrE1yQ5I7k9ye5H2t/UVJrk1yd/t6YmtPko8n2ZvktiQnz3qtza3/3Uk2L11ZkqT5\nDDkDOAR8oKpeDZwKXJBkPXAhcF1VrQOua+sAZwLr2p+twKdgFBjANuD1wCnAtsdCQ5I0fvMGQFU9\nVFVfa8v/BdwJrAY2Ajtbt53AuW15I/DZGrkJOCHJy4DTgWuran9VHQCuBc5Y1GokSYMt6BpAking\ndcDNwEur6iEYhQTwktZtNfDArN32tbbDtUuSJmBwACR5HvBXwK9V1XeO1HWOtjpC+5OPszXJniR7\nZmZmhg5PkrRAgwIgyTMZ/fL/fFV9sTV/s03t0L4+3Nr3AWtn7b4GePAI7U9QVdurarqqpletWrWQ\nWiRJCzDkLqAAO4A7q+qPZm3aDTx2J89m4MpZ7e9sdwOdChxsU0TXAKclObFd/D2ttUmSJmDFgD5v\nBH4R+JckX29tvw1cDFyeZAtwP3B+23Y1cBawF3gUeDdAVe1P8mHgq63fh6pq/6JUIUlasHkDoKr+\ngbnn7wE2zNG/gAsO81qXApcuZICSpKXhJ4ElqVMGgCR1ygCQpE4NuQgsSRM1deFVkx7C05JnAJLU\nKc8AlsCk3q3cd/HZEzmupOXJMwBJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwA\nSeqUASBJnTIAJKlTPgtI0mA+lfPpxTMASeqUASBJnXIK6GlkkqfnPopaWn48A5CkThkAktQpp4C0\nKPxf0MbHO3G0WDwDkKROeQagZW1S74Z3vfzbAGzy3biWMc8AJKlTBoAkdcoAkKROGQCS1CkDQJI6\nZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkTo09AJKckeSuJHuTXDju40uSRsYaAEmOAz4BnAms\nB96eZP04xyBJGhn3GcApwN6quqeqvg/sAjaOeQySJMYfAKuBB2at72ttkqQxG/f/B5A52uoJHZKt\nwNa2+t9J7jqG460EvnUM+y83vdULE6r5DY8vvW3chwa/z13IR4+p5p8c0mncAbAPWDtrfQ3w4OwO\nVbUd2L4YB0uyp6qmF+O1loPe6gVr7oU1L41xTwF9FViX5KQkxwObgN1jHoMkiTGfAVTVoSS/AlwD\nHAdcWlW3j3MMkqSRsf+fwFV1NXD1mA63KFNJy0hv9YI198Kal0Cqav5ekqSnHR8FIUmdWvYBMN+j\nJZI8K8llbfvNSabGP8rFNaDmX09yR5LbklyXZNAtYU9lQx8hkuS8JJVk2d8xMqTmJD/fvte3J/mL\ncY9xsQ342f6JJDckubX9fJ81iXEuliSXJnk4yTcOsz1JPt7+Pm5LcvKiDqCqlu0fRheS/w14OXA8\n8M/A+if1+WXg0215E3DZpMc9hpp/DnhOW35PDzW3fs8HvgzcBExPetxj+D6vA24FTmzrL5n0uMdQ\n83bgPW15PXDfpMd9jDW/CTgZ+MZhtp8F/A2jz1CdCty8mMdf7mcAQx4tsRHY2ZavADYkmesDacvF\nvDVX1Q1V9WhbvYnR5y2Ws6GPEPkw8HvAd8c5uCUypOZfAj5RVQcAqurhMY9xsQ2puYAXtOUX8qTP\nES03VfVlYP8RumwEPlsjNwEnJHnZYh1/uQfAkEdLPN6nqg4BB4EXj2V0S2Ohj9PYwugdxHI2b81J\nXgesraovjXNgS2jI9/mVwCuT/GOSm5KcMbbRLY0hNf8u8I4k+xjdTfje8QxtYpb08Tljvw10kc37\naImBfZaTwfUkeQcwDfzsko5o6R2x5iTPAC4B3jWuAY3BkO/zCkbTQG9mdJb390leW1WPLPHYlsqQ\nmt8O/HlV/WGSNwCfazX/cOmHNxFL+vtruZ8BzPtoidl9kqxgdNp4pFOup7ohNZPkrcDvAOdU1ffG\nNLalMl/NzwdeC9yY5D5Gc6W7l/mF4KE/21dW1f9W1b3AXYwCYbkaUvMW4HKAqvon4NmMnhP0dDXo\n3/vRWu4BMOTREruBzW35POD6aldXlql5a27TIX/C6Jf/cp8XhnlqrqqDVbWyqqaqaorRdY9zqmrP\nZIa7KIb8bP81owv+JFnJaEronrGOcnENqfl+YANAklczCoCZsY5yvHYD72x3A50KHKyqhxbrxZf1\nFFAd5tESST4E7Kmq3cAORqeJexm98980uREfu4E1/z7wPOAv2/Xu+6vqnIkN+hgNrPlpZWDN1wCn\nJbkD+AHwG1X17cmN+tgMrPkDwJ8meT+jqZB3Lec3dEm+wGgKb2W7rrENeCZAVX2a0XWOs4C9wKPA\nuxf1+Mv4706SdAyW+xSQJOkoGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXq/wDB1xm9\njnP+YAAAAABJRU5ErkJggg==\n", 313 | "text/plain": [ 314 | "" 315 | ] 316 | }, 317 | "metadata": {}, 318 | "output_type": "display_data" 319 | } 320 | ], 321 | "source": [ 322 | "plt.hist(y_hat)\n", 323 | "_ = plt.axvline(x=0.5, color='orange')" 324 | ] 325 | }, 326 | { 327 | "cell_type": "code", 328 | "execution_count": 13, 329 | "metadata": {}, 330 | "outputs": [ 331 | { 332 | "data": { 333 | "text/plain": [ 334 | "'95.97'" 335 | ] 336 | }, 337 | "execution_count": 13, 338 | "metadata": {}, 339 | "output_type": "execute_result" 340 | } 341 | ], 342 | "source": [ 343 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 344 | ] 345 | }, 346 | { 347 | "cell_type": "code", 348 | "execution_count": null, 349 | "metadata": { 350 | "collapsed": true 351 | }, 352 | "outputs": [], 353 | "source": [] 354 | } 355 | ], 356 | "metadata": { 357 | "kernelspec": { 358 | "display_name": "Python 3", 359 | "language": "python", 360 | "name": "python3" 361 | }, 362 | "language_info": { 363 | "codemirror_mode": { 364 | "name": "ipython", 365 | "version": 3 366 | }, 367 | "file_extension": ".py", 368 | "mimetype": "text/x-python", 369 | "name": "python", 370 | "nbconvert_exporter": "python", 371 | "pygments_lexer": "ipython3", 372 | "version": "3.6.2" 373 | } 374 | }, 375 | "nbformat": 4, 376 | "nbformat_minor": 2 377 | } 378 | -------------------------------------------------------------------------------- /notebooks/cross_entropy_cost.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from numpy import log" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "def cross_entropy(y, a):\n", 21 | " return -1*(y*log(a) + (1-y)*log(1-a))" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 3, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "data": { 31 | "text/plain": [ 32 | "0.010050335853501451" 33 | ] 34 | }, 35 | "execution_count": 3, 36 | "metadata": {}, 37 | "output_type": "execute_result" 38 | } 39 | ], 40 | "source": [ 41 | "cross_entropy(0, 0.01)" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 4, 47 | "metadata": {}, 48 | "outputs": [ 49 | { 50 | "data": { 51 | "text/plain": [ 52 | "0.010050335853501451" 53 | ] 54 | }, 55 | "execution_count": 4, 56 | "metadata": {}, 57 | "output_type": "execute_result" 58 | } 59 | ], 60 | "source": [ 61 | "cross_entropy(1, 0.99)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 5, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "data": { 71 | "text/plain": [ 72 | "0.35667494393873245" 73 | ] 74 | }, 75 | "execution_count": 5, 76 | "metadata": {}, 77 | "output_type": "execute_result" 78 | } 79 | ], 80 | "source": [ 81 | "cross_entropy(0, 0.3)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 6, 87 | "metadata": {}, 88 | "outputs": [ 89 | { 90 | "data": { 91 | "text/plain": [ 92 | "0.916290731874155" 93 | ] 94 | }, 95 | "execution_count": 6, 96 | "metadata": {}, 97 | "output_type": "execute_result" 98 | } 99 | ], 100 | "source": [ 101 | "cross_entropy(0, 0.6)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 7, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "data": { 111 | "text/plain": [ 112 | "2.3025850929940459" 113 | ] 114 | }, 115 | "execution_count": 7, 116 | "metadata": {}, 117 | "output_type": "execute_result" 118 | } 119 | ], 120 | "source": [ 121 | "cross_entropy(0, 0.9)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": { 128 | "collapsed": true 129 | }, 130 | "outputs": [], 131 | "source": [] 132 | } 133 | ], 134 | "metadata": { 135 | "kernelspec": { 136 | "display_name": "Python 3", 137 | "language": "python", 138 | "name": "python3" 139 | }, 140 | "language_info": { 141 | "codemirror_mode": { 142 | "name": "ipython", 143 | "version": 3 144 | }, 145 | "file_extension": ".py", 146 | "mimetype": "text/x-python", 147 | "name": "python", 148 | "nbconvert_exporter": "python", 149 | "pygments_lexer": "ipython3", 150 | "version": "3.5.2" 151 | } 152 | }, 153 | "nbformat": 4, 154 | "nbformat_minor": 2 155 | } 156 | -------------------------------------------------------------------------------- /notebooks/first_tensorflow_graphs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# First TensorFlow Graphs" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we execute elementary TensorFlow computational graphs." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "import tensorflow as tf" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Simple arithmetic" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": { 47 | "collapsed": true 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "x1 = tf.placeholder(tf.float32)\n", 52 | "x2 = tf.placeholder(tf.float32)" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 3, 58 | "metadata": { 59 | "collapsed": true 60 | }, 61 | "outputs": [], 62 | "source": [ 63 | "sum_op = tf.add(x1, x2)\n", 64 | "product_op = tf.multiply(x1, x2)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 4, 70 | "metadata": { 71 | "collapsed": true 72 | }, 73 | "outputs": [], 74 | "source": [ 75 | "with tf.Session() as session:\n", 76 | " sum_result = session.run(sum_op, feed_dict={x1: 2.0, x2: 0.5}) # run again with {x1: [2.0, 2.0, 2.0], x2: [0.5, 1.0, 2.0]}\n", 77 | " product_result = session.run(product_op, feed_dict={x1: 2.0, x2: 0.5}) # ...and with {x1: [2.0, 4.0], x2: 0.5}" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 5, 83 | "metadata": {}, 84 | "outputs": [ 85 | { 86 | "data": { 87 | "text/plain": [ 88 | "2.5" 89 | ] 90 | }, 91 | "execution_count": 5, 92 | "metadata": {}, 93 | "output_type": "execute_result" 94 | } 95 | ], 96 | "source": [ 97 | "sum_result" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 6, 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "data": { 107 | "text/plain": [ 108 | "1.0" 109 | ] 110 | }, 111 | "execution_count": 6, 112 | "metadata": {}, 113 | "output_type": "execute_result" 114 | } 115 | ], 116 | "source": [ 117 | "product_result" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "#### Simple array arithmetic" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 7, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "with tf.Session() as session:\n", 134 | " sum_result = session.run(sum_op, feed_dict={x1: [2.0, 2.0, 2.0], x2: [0.5, 1.0, 2.0]})\n", 135 | " product_result = session.run(product_op, feed_dict={x1: [2.0, 4.0], x2: 0.5})" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 8, 141 | "metadata": {}, 142 | "outputs": [ 143 | { 144 | "data": { 145 | "text/plain": [ 146 | "array([ 2.5, 3. , 4. ], dtype=float32)" 147 | ] 148 | }, 149 | "execution_count": 8, 150 | "metadata": {}, 151 | "output_type": "execute_result" 152 | } 153 | ], 154 | "source": [ 155 | "sum_result" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": 9, 161 | "metadata": {}, 162 | "outputs": [ 163 | { 164 | "data": { 165 | "text/plain": [ 166 | "array([ 1., 2.], dtype=float32)" 167 | ] 168 | }, 169 | "execution_count": 9, 170 | "metadata": {}, 171 | "output_type": "execute_result" 172 | } 173 | ], 174 | "source": [ 175 | "product_result" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "metadata": { 182 | "collapsed": true 183 | }, 184 | "outputs": [], 185 | "source": [] 186 | } 187 | ], 188 | "metadata": { 189 | "kernelspec": { 190 | "display_name": "Python 3", 191 | "language": "python", 192 | "name": "python3" 193 | }, 194 | "language_info": { 195 | "codemirror_mode": { 196 | "name": "ipython", 197 | "version": 3 198 | }, 199 | "file_extension": ".py", 200 | "mimetype": "text/x-python", 201 | "name": "python", 202 | "nbconvert_exporter": "python", 203 | "pygments_lexer": "ipython3", 204 | "version": "3.5.2" 205 | } 206 | }, 207 | "nbformat": 4, 208 | "nbformat_minor": 2 209 | } 210 | -------------------------------------------------------------------------------- /notebooks/first_tensorflow_neurons.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# First TensorFlow Neurons" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "#### Load dependencies" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": { 21 | "collapsed": true 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "import numpy as np\n", 26 | "np.random.seed(42)\n", 27 | "import tensorflow as tf\n", 28 | "tf.set_random_seed(42)\n", 29 | "import matplotlib.pyplot as plt\n", 30 | "%matplotlib inline" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "metadata": {}, 36 | "source": [ 37 | "#### Set number of neurons" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 2, 43 | "metadata": { 44 | "collapsed": true 45 | }, 46 | "outputs": [], 47 | "source": [ 48 | "n_input = 784\n", 49 | "n_dense = 128" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "#### Define placeholder Tensor for simulated MNIST digits" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 3, 62 | "metadata": { 63 | "collapsed": true 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "x = tf.placeholder(tf.float32, [None, n_input])" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "#### Create Variable Tensors for neuron biases `b` and weight matrix `W`" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 4, 80 | "metadata": { 81 | "collapsed": true 82 | }, 83 | "outputs": [], 84 | "source": [ 85 | "b = tf.Variable(tf.zeros([n_dense]))\n", 86 | "W = tf.Variable(tf.random_uniform([n_input, n_dense])) # 1.\n", 87 | "# W = tf.Variable(tf.random_normal([n_input, n_dense])) # 2.\n", 88 | "# W = tf.get_variable('W', [n_input, n_dense], \n", 89 | "# initializer=tf.contrib.layers.xavier_initializer())" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": {}, 95 | "source": [ 96 | "#### Design the computational graph" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 5, 102 | "metadata": { 103 | "collapsed": true 104 | }, 105 | "outputs": [], 106 | "source": [ 107 | "z = tf.add(tf.matmul(x, W), b)\n", 108 | "a = tf.sigmoid(z) # first with tf.sigmoid(), then stick with tf.tanh() or tf.nn.relu()" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "#### Create op for variable initialization" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 6, 121 | "metadata": { 122 | "collapsed": true 123 | }, 124 | "outputs": [], 125 | "source": [ 126 | "initializer_op = tf.global_variables_initializer()" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": {}, 132 | "source": [ 133 | "#### Execute the graph in a session" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 7, 139 | "metadata": { 140 | "collapsed": true 141 | }, 142 | "outputs": [], 143 | "source": [ 144 | "with tf.Session() as session:\n", 145 | " session.run(initializer_op)\n", 146 | " \n", 147 | " layer_output = session.run(a, {x: np.random.random([1, n_input])})" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 8, 153 | "metadata": {}, 154 | "outputs": [ 155 | { 156 | "data": { 157 | "text/plain": [ 158 | "array([[ 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", 159 | " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", 160 | " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", 161 | " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", 162 | " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", 163 | " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", 164 | " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", 165 | " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", 166 | " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n", 167 | " 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.]], dtype=float32)" 168 | ] 169 | }, 170 | "execution_count": 8, 171 | "metadata": {}, 172 | "output_type": "execute_result" 173 | } 174 | ], 175 | "source": [ 176 | "layer_output" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 9, 182 | "metadata": {}, 183 | "outputs": [ 184 | { 185 | "data": { 186 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADhJJREFUeJzt3H+M5PVdx/HnS64UW1MPuIXgHfWuyVV7Nm1KVoJt0tBi\nFGjTwwQMBNsTL15UxNo2CmgixqQJRGOxSX/kBORqEEoolotSlVAaovXQpVTKj1JOQFg5ua388AeJ\n7eHbP+aL2V72bvbmO7PDfng+ks3OfOczM+/vsTzve9+dmVQVkqR2fd+0B5AkTZahl6TGGXpJapyh\nl6TGGXpJapyhl6TGGXpJapyhl6TGGXpJatyaaQ8AsG7dutq4ceO0x5CkVeXee+/9dlXNDFv3igj9\nxo0bmZubm/YYkrSqJPmX5azz1I0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0k\nNe4V8c5Y6ZVs42V/OZXnfeLK903ledUej+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGG\nXpIaNzT0Sa5Lsj/JA4u2/X6Sbya5P8mfJ1m76LbLk+xN8kiSn57U4JKk5VnOEf31wJkHbbsDeGtV\nvQ34FnA5QJItwPnAj3X3+XSSo8Y2rSTpiA0NfVXdDTx70La/qaoD3dU9wIbu8lbgpqr6n6p6HNgL\nnDrGeSVJR2gc5+h/AfhSd3k98NSi2+a7bZKkKekV+iS/DRwAbnh50xLL6hD33ZFkLsncwsJCnzEk\nSYcxcuiTbAPeD1xYVS/HfB44edGyDcDTS92/qnZW1WxVzc7MzIw6hiRpiJFCn+RM4FLgA1X14qKb\ndgPnJ3ltkk3AZuAf+o8pSRrV0M+jT3IjcDqwLsk8cAWDV9m8FrgjCcCeqvqlqnowyc3AQwxO6Vxc\nVS9NanhJ0nBDQ19VFyyx+drDrP848PE+Q0mSxsd3xkpS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO\n0EtS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS\n4wy9JDXO0EtS4wy9JDXO0EtS44aGPsl1SfYneWDRtuOS3JHk0e77sd32JPlkkr1J7k9yyiSHlyQN\nt5wj+uuBMw/adhlwZ1VtBu7srgOcBWzuvnYAnxnPmJKkUQ0NfVXdDTx70OatwK7u8i7gnEXbP1cD\ne4C1SU4a17CSpCM36jn6E6tqH0D3/YRu+3rgqUXr5rttkqQpGfcvY7PEtlpyYbIjyVySuYWFhTGP\nIUl62aihf+blUzLd9/3d9nng5EXrNgBPL/UAVbWzqmaranZmZmbEMSRJw4wa+t3Atu7yNuC2Rds/\n1L365jTghZdP8UiSpmPNsAVJbgROB9YlmQeuAK4Ebk6yHXgSOK9bfjtwNrAXeBG4aAIzS5KOwNDQ\nV9UFh7jpjCXWFnBx36EkSePjO2MlqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+gl\nqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGG\nXpIaZ+glqXGGXpIa1yv0ST6S5MEkDyS5MckxSTYluSfJo0k+n+TocQ0rSTpyI4c+yXrg14DZqnor\ncBRwPnAV8Imq2gw8B2wfx6CSpNH0PXWzBvj+JGuA1wH7gPcCt3S37wLO6fkckqQeRg59Vf0r8AfA\nkwwC/wJwL/B8VR3ols0D65e6f5IdSeaSzC0sLIw6hiRpiD6nbo4FtgKbgB8CXg+ctcTSWur+VbWz\nqmaranZmZmbUMSRJQ/Q5dfOTwONVtVBV3wVuBd4JrO1O5QBsAJ7uOaMkqYc+oX8SOC3J65IEOAN4\nCLgLOLdbsw24rd+IkqQ++pyjv4fBL12/Bnyje6ydwKXAR5PsBY4Hrh3DnJKkEa0ZvuTQquoK4IqD\nNj8GnNrncSVJ4+M7YyWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn\n6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWp\ncb1Cn2RtkluSfDPJw0l+IslxSe5I8mj3/dhxDStJOnJ9j+j/CPirqvpR4O3Aw8BlwJ1VtRm4s7su\nSZqSkUOf5A3Au4FrAarqO1X1PLAV2NUt2wWc03dISdLo+hzRvwlYAP4kyX1JrknyeuDEqtoH0H0/\nYQxzSpJG1Cf0a4BTgM9U1TuA/+YITtMk2ZFkLsncwsJCjzEkSYfTJ/TzwHxV3dNdv4VB+J9JchJA\n933/Uneuqp1VNVtVszMzMz3GkCQdzsihr6p/A55K8iPdpjOAh4DdwLZu2zbgtl4TSpJ6WdPz/pcA\nNyQ5GngMuIjBXx43J9kOPAmc1/M5JEk99Ap9VX0dmF3ipjP6PK4kaXx8Z6wkNc7QS1LjDL0kNc7Q\nS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1Lj\nDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNa536JMcleS+JH/RXd+U5J4kjyb5fJKj+48p\nSRrVOI7oPww8vOj6VcAnqmoz8BywfQzPIUkaUa/QJ9kAvA+4prse4L3ALd2SXcA5fZ5DktRP3yP6\nq4HfBP63u3488HxVHeiuzwPrez6HJKmHkUOf5P3A/qq6d/HmJZbWIe6/I8lckrmFhYVRx5AkDdHn\niP5dwAeSPAHcxOCUzdXA2iRrujUbgKeXunNV7ayq2aqanZmZ6TGGJOlwRg59VV1eVRuqaiNwPvDl\nqroQuAs4t1u2Dbit95SSpJFN4nX0lwIfTbKXwTn7ayfwHJKkZVozfMlwVfUV4Cvd5ceAU8fxuJKk\n/nxnrCQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMM\nvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuNGDn2S\nk5PcleThJA8m+XC3/bgkdyR5tPt+7PjGlSQdqT5H9AeAj1XVW4DTgIuTbAEuA+6sqs3And11SdKU\njBz6qtpXVV/rLv8n8DCwHtgK7OqW7QLO6TukJGl0YzlHn2Qj8A7gHuDEqtoHg78MgBMOcZ8dSeaS\nzC0sLIxjDEnSEnqHPskPAF8Afr2q/mO596uqnVU1W1WzMzMzfceQJB1Cr9AneQ2DyN9QVbd2m59J\nclJ3+0nA/n4jSpL66POqmwDXAg9X1R8uumk3sK27vA24bfTxJEl9relx33cBHwS+keTr3bbfAq4E\nbk6yHXgSOK/fiJKkPkYOfVX9LZBD3HzGqI8rSRov3xkrSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLU\nOEMvSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLUOEMvSY0z9JLUOEMv\nSY0z9JLUOEMvSY0z9JLUOEMvSY2bWOiTnJnkkSR7k1w2qeeRJB3eREKf5CjgU8BZwBbggiRbJvFc\nkqTDm9QR/anA3qp6rKq+A9wEbJ3Qc0mSDmNSoV8PPLXo+ny3TZK0wtZM6HGzxLb6ngXJDmBHd/W/\nkjwyoVkmaR3w7WkPscLc5xWSq1b6Gf+f/41Xjx9ezqJJhX4eOHnR9Q3A04sXVNVOYOeEnn9FJJmr\nqtlpz7GS3Of2vdr2F9rf50mduvlHYHOSTUmOBs4Hdk/ouSRJhzGRI/qqOpDkV4G/Bo4CrquqByfx\nXJKkw5vUqRuq6nbg9kk9/ivEqj71NCL3uX2vtv2Fxvc5VTV8lSRp1fIjECSpcYZ+GZbzcQ5JfjbJ\nQ0keTPJnKz3jOA3b3yRvTHJXkvuS3J/k7GnMOU5JrkuyP8kDh7g9ST7Z/Zncn+SUlZ5x3Jaxzxd2\n+3p/kq8meftKzzhuw/Z50bofT/JSknNXaraJqiq/DvPF4JfJ/wy8CTga+Cdgy0FrNgP3Acd210+Y\n9twT3t+dwC93l7cAT0x77jHs97uBU4AHDnH72cCXGLxH5DTgnmnPvAL7/M5FP9NnvRr2uVtzFPBl\nBr9jPHfaM4/jyyP64ZbzcQ6/CHyqqp4DqKr9KzzjOC1nfwt4Q3f5BznoPRKrUVXdDTx7mCVbgc/V\nwB5gbZKTVma6yRi2z1X11Zd/poE9DN4Ps6ot478zwCXAF4DV/P/x9zD0wy3n4xzeDLw5yd8l2ZPk\nzBWbbvyWs7+/C/xcknkGRz2XrMxoU/Vq/1iP7Qz+RdO0JOuBnwE+O+1ZxsnQDzf04xwYvEx1M3A6\ncAFwTZK1E55rUpazvxcA11fVBganNP40Ses/S8v5c2lSkvcwCP2l055lBVwNXFpVL017kHGa2Ovo\nGzL04xy6NXuq6rvA493n9mxm8A7h1WY5+7sdOBOgqv4+yTEMPiukmX/qLmE5fy7NSfI24BrgrKr6\n92nPswJmgZuSwOBn+uwkB6rqi9Mdq5/Wj8LGYTkf5/BF4D0ASdYxOJXz2IpOOT7L2d8ngTMAkrwF\nOAZYWNEpV95u4EPdq29OA16oqn3THmqSkrwRuBX4YFV9a9rzrISq2lRVG6tqI3AL8CurPfLgEf1Q\ndYiPc0jye8BcVe3ubvupJA8BLwG/sVqPfpa5vx8D/jjJRxicvvj56l6usFoluZHBqbd13e8ergBe\nA1BVn2Xwu4izgb3Ai8BF05l0fJaxz78DHA98ujvCPVCr/IO/lrHPTfKdsZLUOE/dSFLjDL0kNc7Q\nS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNe7/APDkU3rN3fHLAAAAAElFTkSuQmCC\n", 187 | "text/plain": [ 188 | "" 189 | ] 190 | }, 191 | "metadata": {}, 192 | "output_type": "display_data" 193 | } 194 | ], 195 | "source": [ 196 | "_ = plt.hist(np.transpose(layer_output))" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": null, 202 | "metadata": { 203 | "collapsed": true 204 | }, 205 | "outputs": [], 206 | "source": [] 207 | } 208 | ], 209 | "metadata": { 210 | "kernelspec": { 211 | "display_name": "Python 3", 212 | "language": "python", 213 | "name": "python3" 214 | }, 215 | "language_info": { 216 | "codemirror_mode": { 217 | "name": "ipython", 218 | "version": 3 219 | }, 220 | "file_extension": ".py", 221 | "mimetype": "text/x-python", 222 | "name": "python", 223 | "nbconvert_exporter": "python", 224 | "pygments_lexer": "ipython3", 225 | "version": "3.6.1" 226 | } 227 | }, 228 | "nbformat": 4, 229 | "nbformat_minor": 2 230 | } 231 | -------------------------------------------------------------------------------- /notebooks/gru_in_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# GRU in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we use a GRU to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stderr", 31 | "output_type": "stream", 32 | "text": [ 33 | "Using TensorFlow backend.\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "import keras\n", 39 | "from keras.datasets import imdb\n", 40 | "from keras.preprocessing.sequence import pad_sequences\n", 41 | "from keras.models import Sequential\n", 42 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D\n", 43 | "from keras.layers import GRU # new! \n", 44 | "from keras.callbacks import ModelCheckpoint\n", 45 | "import os\n", 46 | "from sklearn.metrics import roc_auc_score \n", 47 | "import matplotlib.pyplot as plt \n", 48 | "%matplotlib inline" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "#### Set hyperparameters" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 2, 61 | "metadata": { 62 | "collapsed": true 63 | }, 64 | "outputs": [], 65 | "source": [ 66 | "# output directory name:\n", 67 | "output_dir = 'model_output/gru'\n", 68 | "\n", 69 | "# training:\n", 70 | "epochs = 4\n", 71 | "batch_size = 128\n", 72 | "\n", 73 | "# vector-space embedding: \n", 74 | "n_dim = 64 \n", 75 | "n_unique_words = 10000 \n", 76 | "max_review_length = 100 # lowered due to vanishing gradient over time\n", 77 | "pad_type = trunc_type = 'pre'\n", 78 | "drop_embed = 0.2 \n", 79 | "\n", 80 | "# GRU layer architecture:\n", 81 | "n_gru = 256 \n", 82 | "drop_gru = 0.2\n", 83 | "\n", 84 | "# dense layer architecture: \n", 85 | "# n_dense = 256\n", 86 | "# dropout = 0.2" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "#### Load data" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 3, 99 | "metadata": { 100 | "collapsed": true 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "#### Preprocess data" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": 4, 117 | "metadata": { 118 | "collapsed": true 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 123 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": { 129 | "collapsed": true 130 | }, 131 | "source": [ 132 | "#### Design neural network architecture" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 5, 138 | "metadata": { 139 | "collapsed": true 140 | }, 141 | "outputs": [], 142 | "source": [ 143 | "model = Sequential()\n", 144 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n", 145 | "model.add(SpatialDropout1D(drop_embed))\n", 146 | "model.add(GRU(n_gru, dropout=drop_gru))\n", 147 | "# model.add(Dense(n_dense, activation='relu')) # typically don't see top dense layer in NLP like in \n", 148 | "# model.add(Dropout(dropout))\n", 149 | "model.add(Dense(1, activation='sigmoid'))" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 6, 155 | "metadata": {}, 156 | "outputs": [ 157 | { 158 | "name": "stdout", 159 | "output_type": "stream", 160 | "text": [ 161 | "_________________________________________________________________\n", 162 | "Layer (type) Output Shape Param # \n", 163 | "=================================================================\n", 164 | "embedding_1 (Embedding) (None, 100, 64) 640000 \n", 165 | "_________________________________________________________________\n", 166 | "spatial_dropout1d_1 (Spatial (None, 100, 64) 0 \n", 167 | "_________________________________________________________________\n", 168 | "gru_1 (GRU) (None, 256) 246528 \n", 169 | "_________________________________________________________________\n", 170 | "dense_1 (Dense) (None, 1) 257 \n", 171 | "=================================================================\n", 172 | "Total params: 886,785\n", 173 | "Trainable params: 886,785\n", 174 | "Non-trainable params: 0\n", 175 | "_________________________________________________________________\n" 176 | ] 177 | } 178 | ], 179 | "source": [ 180 | "model.summary() " 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "#### Configure model" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 7, 193 | "metadata": { 194 | "collapsed": true 195 | }, 196 | "outputs": [], 197 | "source": [ 198 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": 8, 204 | "metadata": { 205 | "collapsed": true 206 | }, 207 | "outputs": [], 208 | "source": [ 209 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 210 | "if not os.path.exists(output_dir):\n", 211 | " os.makedirs(output_dir)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": {}, 217 | "source": [ 218 | "#### Train!" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 9, 224 | "metadata": {}, 225 | "outputs": [ 226 | { 227 | "name": "stdout", 228 | "output_type": "stream", 229 | "text": [ 230 | "Train on 25000 samples, validate on 25000 samples\n", 231 | "Epoch 1/4\n", 232 | "25000/25000 [==============================] - 31s - loss: 0.5238 - acc: 0.7241 - val_loss: 0.3771 - val_acc: 0.8350\n", 233 | "Epoch 2/4\n", 234 | "25000/25000 [==============================] - 29s - loss: 0.3062 - acc: 0.8731 - val_loss: 0.3390 - val_acc: 0.8504\n", 235 | "Epoch 3/4\n", 236 | "25000/25000 [==============================] - 29s - loss: 0.2484 - acc: 0.9004 - val_loss: 0.3680 - val_acc: 0.8452\n", 237 | "Epoch 4/4\n", 238 | "25000/25000 [==============================] - 28s - loss: 0.2123 - acc: 0.9176 - val_loss: 0.3837 - val_acc: 0.8396\n" 239 | ] 240 | }, 241 | { 242 | "data": { 243 | "text/plain": [ 244 | "" 245 | ] 246 | }, 247 | "execution_count": 9, 248 | "metadata": {}, 249 | "output_type": "execute_result" 250 | } 251 | ], 252 | "source": [ 253 | "# XXX% validation accuracy in epoch X\n", 254 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": { 260 | "collapsed": true 261 | }, 262 | "source": [ 263 | "#### Evaluate" 264 | ] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": 10, 269 | "metadata": { 270 | "collapsed": true 271 | }, 272 | "outputs": [], 273 | "source": [ 274 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": 11, 280 | "metadata": {}, 281 | "outputs": [ 282 | { 283 | "name": "stdout", 284 | "output_type": "stream", 285 | "text": [ 286 | "25000/25000 [==============================] - 24s \n" 287 | ] 288 | } 289 | ], 290 | "source": [ 291 | "y_hat = model.predict_proba(x_valid)" 292 | ] 293 | }, 294 | { 295 | "cell_type": "code", 296 | "execution_count": 12, 297 | "metadata": {}, 298 | "outputs": [ 299 | { 300 | "data": { 301 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE5NJREFUeJzt3X+s3fV93/HnKzgka5vEJlwQss1MVDcLrRTCLCCK1NE4\nM8apMH+EydE6XGTNU0erdqu2kO0Pb9BMZNPGhtTSucWridoQypZhJazMc0DZpplgCqUBinxDKFyZ\nYReDsw4lHel7f5yPkwO51/dc+95zcvk8H9LR9/t9fz/f8/18fO37Ot9fx6kqJEn9edukOyBJmgwD\nQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktSpFZPuwKmce+65tW7dukl3Q/pB33pm\nMH33+yfbD2kWjz766J9V1dR87X6oA2DdunUcOnRo0t2QftB/u3Iw/dhDk+yFNKskfzpKO08BSVKn\nDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUqXkDIMn7kzw+9PpWkl9Jck6S/UkOt+mq1j5Jbk8y\nneSJJJcOvdf21v5wku1LOTBJ0qnNGwBV9UxVXVJVlwB/HXgN+CJwE3CgqtYDB9oywNXA+vbaCdwB\nkOQcYBdwOXAZsOtkaEiSxm+hTwJvBL5RVX+aZCtwZavvBR4CPgVsBe6qwf82fzDJyiQXtLb7q+o4\nQJL9wGbg82c6CElaCutu+vLE9v3crR9f8n0s9BrANr7/C/v8qnoRoE3Pa/XVwAtD28y02lz1N0iy\nM8mhJIeOHTu2wO5JkkY1cgAkORu4Bvj9+ZrOUqtT1N9YqNpdVRuqasPU1LzfZSRJOk0LOQK4GvjD\nqnqpLb/UTu3QpkdbfQZYO7TdGuDIKeqSpAlYSAB8kjeer98HnLyTZztw31D9+nY30BXAiXaK6AFg\nU5JV7eLvplaTJE3ASBeBk/wI8DeBvzdUvhW4J8kO4Hngula/H9gCTDO4Y+gGgKo6nuQW4JHW7uaT\nF4QlSeM3UgBU1WvAe99Ue5nBXUFvblvAjXO8zx5gz8K7eXomdQV/HFfvJelM+SSwJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJ\nnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6NVIAJFmZ5N4kf5Lk6SQfTnJOkv1JDrfpqtY2SW5PMp3k\niSSXDr3P9tb+cJLtSzUoSdL8Rj0C+HfAH1TVXwM+CDwN3AQcqKr1wIG2DHA1sL69dgJ3ACQ5B9gF\nXA5cBuw6GRqSpPGbNwCSvBv4aeBOgKr6i6p6FdgK7G3N9gLXtvmtwF01cBBYmeQC4Cpgf1Udr6pX\ngP3A5kUdjSRpZKMcAbwPOAb8hySPJfntJD8KnF9VLwK06Xmt/WrghaHtZ1ptrrokaQJGCYAVwKXA\nHVX1IeD/8v3TPbPJLLU6Rf2NGyc7kxxKcujYsWMjdE+SdDpGCYAZYKaqHm7L9zIIhJfaqR3a9OhQ\n+7VD268Bjpyi/gZVtbuqNlTVhqmpqYWMRZK0APMGQFX9b+CFJO9vpY3AU8A+4OSdPNuB+9r8PuD6\ndjfQFcCJdoroAWBTklXt4u+mVpMkTcCKEdv9EvC7Sc4GngVuYBAe9yTZATwPXNfa3g9sAaaB11pb\nqup4kluAR1q7m6vq+KKMQpK0YCMFQFU9DmyYZdXGWdoWcOMc77MH2LOQDkqSloZPAktSpwwASeqU\nASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMjBUCS55L8cZLHkxxqtXOS7E9yuE1XtXqS3J5kOskT\nSS4dep/trf3hJNuXZkiSpFEs5AjgZ6rqkqra0JZvAg5U1XrgQFsGuBpY3147gTtgEBjALuBy4DJg\n18nQkCSN35mcAtoK7G3ze4Frh+p31cBBYGWSC4CrgP1VdbyqXgH2A5vPYP+SpDMwagAU8F+TPJpk\nZ6udX1UvArTpea2+GnhhaNuZVpurLkmagBUjtvtIVR1Jch6wP8mfnKJtZqnVKepv3HgQMDsBLrzw\nwhG7J0laqJGOAKrqSJseBb7I4Bz+S+3UDm16tDWfAdYObb4GOHKK+pv3tbuqNlTVhqmpqYWNRpI0\nsnkDIMmPJnnXyXlgE/B1YB9w8k6e7cB9bX4fcH27G+gK4EQ7RfQAsCnJqnbxd1OrSZImYJRTQOcD\nX0xysv3vVdUfJHkEuCfJDuB54LrW/n5gCzANvAbcAFBVx5PcAjzS2t1cVccXbSSSpAWZNwCq6lng\ng7PUXwY2zlIv4MY53msPsGfh3ZQkLTafBJakThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMG\ngCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBI\nUqcMAEnq1MgBkOSsJI8l+VJbvijJw0kOJ/lCkrNb/R1tebqtXzf0Hp9u9WeSXLXYg5EkjW4hRwC/\nDDw9tPxZ4LaqWg+8Auxo9R3AK1X148BtrR1JLga2AT8JbAZ+I8lZZ9Z9SdLpGikAkqwBPg78dlsO\n8FHg3tZkL3Btm9/almnrN7b2W4G7q+o7VfVNYBq4bDEGIUlauFGPAP4t8I+Bv2zL7wVerarX2/IM\nsLrNrwZeAGjrT7T236vPss33JNmZ5FCSQ8eOHVvAUCRJCzFvACT5WeBoVT06XJ6lac2z7lTbfL9Q\ntbuqNlTVhqmpqfm6J0k6TStGaPMR4JokW4B3Au9mcESwMsmK9il/DXCktZ8B1gIzSVYA7wGOD9VP\nGt5GkjRm8x4BVNWnq2pNVa1jcBH3K1X1t4EHgU+0ZtuB+9r8vrZMW/+VqqpW39buEroIWA98bdFG\nIklakFGOAObyKeDuJL8GPAbc2ep3Ap9LMs3gk/82gKp6Msk9wFPA68CNVfXdM9i/JOkMLCgAquoh\n4KE2/yyz3MVTVd8Grptj+88An1loJyVJi88ngSWpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ\n6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKRO\nGQCS1Kl5AyDJO5N8LckfJXkyyT9v9YuSPJzkcJIvJDm71d/Rlqfb+nVD7/XpVn8myVVLNShJ0vxG\nOQL4DvDRqvogcAmwOckVwGeB26pqPfAKsKO13wG8UlU/DtzW2pHkYmAb8JPAZuA3kpy1mIORJI1u\n3gCogT9vi29vrwI+Ctzb6nuBa9v81rZMW78xSVr97qr6TlV9E5gGLluUUUiSFmykawBJzkryOHAU\n2A98A3i1ql5vTWaA1W1+NfACQFt/AnjvcH2WbSRJYzZSAFTVd6vqEmANg0/tH5itWZtmjnVz1d8g\nyc4kh5IcOnbs2CjdkySdhgXdBVRVrwIPAVcAK5OsaKvWAEfa/AywFqCtfw9wfLg+yzbD+9hdVRuq\nasPU1NRCuidJWoBR7gKaSrKyzf8V4GPA08CDwCdas+3AfW1+X1umrf9KVVWrb2t3CV0ErAe+tlgD\nkSQtzIr5m3ABsLfdsfM24J6q+lKSp4C7k/wa8BhwZ2t/J/C5JNMMPvlvA6iqJ5PcAzwFvA7cWFXf\nXdzhSJJGNW8AVNUTwIdmqT/LLHfxVNW3gevmeK/PAJ9ZeDclSYvNJ4ElqVMGgCR1ygCQpE4ZAJLU\nKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0y\nACSpUwaAJHVqlP8TWAu07qYvT2S/z9368YnsV1pqk/o39VbnEYAkdWreAEiyNsmDSZ5O8mSSX271\nc5LsT3K4TVe1epLcnmQ6yRNJLh16r+2t/eEk25duWJKk+YxyBPA68KtV9QHgCuDGJBcDNwEHqmo9\ncKAtA1wNrG+vncAdMAgMYBdwOXAZsOtkaEiSxm/eAKiqF6vqD9v8/wGeBlYDW4G9rdle4No2vxW4\nqwYOAiuTXABcBeyvquNV9QqwH9i8qKORJI1sQdcAkqwDPgQ8DJxfVS/CICSA81qz1cALQ5vNtNpc\ndUnSBIwcAEl+DPiPwK9U1bdO1XSWWp2i/ub97ExyKMmhY8eOjdo9SdICjRQASd7O4Jf/71bVf2rl\nl9qpHdr0aKvPAGuHNl8DHDlF/Q2qandVbaiqDVNTUwsZiyRpAUa5CyjAncDTVfVvhlbtA07eybMd\nuG+ofn27G+gK4EQ7RfQAsCnJqnbxd1OrSZImYJQHwT4C/B3gj5M83mr/BLgVuCfJDuB54Lq27n5g\nCzANvAbcAFBVx5PcAjzS2t1cVccXZRSSpAWbNwCq6n8w+/l7gI2ztC/gxjneaw+wZyEdlCQtDZ8E\nlqROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ\n6tQoXwctSQCsu+nLk+6CFpEB8BYyyX+cz9368YntW9Lp8RSQJHXKAJCkThkAktQpA0CSOmUASFKn\nDABJ6tS8AZBkT5KjSb4+VDsnyf4kh9t0Vasnye1JppM8keTSoW22t/aHk2xfmuFIkkY1yhHA7wCb\n31S7CThQVeuBA20Z4GpgfXvtBO6AQWAAu4DLgcuAXSdDQ5I0GfM+CFZVX02y7k3lrcCVbX4v8BDw\nqVa/q6oKOJhkZZILWtv9VXUcIMl+BqHy+TMegdQZn8bVYjndJ4HPr6oXAarqxSTntfpq4IWhdjOt\nNlf9ByTZyeDogQsvvPA0u6dxm9QvJZ9Alk7fYl8Eziy1OkX9B4tVu6tqQ1VtmJqaWtTOSZK+73SP\nAF5KckH79H8BcLTVZ4C1Q+3WAEda/co31R86zX1LPxQOPvsy2zwdo2XsdANgH7AduLVN7xuq/2KS\nuxlc8D3RQuIB4F8MXfjdBHz69LstDUzq1NPd73t5IvuVFtO8AZDk8ww+vZ+bZIbB3Ty3Avck2QE8\nD1zXmt8PbAGmgdeAGwCq6niSW4BHWrubT14QliRNxih3AX1yjlUbZ2lbwI1zvM8eYM+CeidJWjI+\nCSxJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE6NPQCSbE7yTJLpJDeNe/+SpIGxBkCS\ns4BfB64GLgY+meTicfZBkjQw7iOAy4Dpqnq2qv4CuBvYOuY+SJIYfwCsBl4YWp5pNUnSmK0Y8/4y\nS63e0CDZCexsi3+e5JnT3Ne5wJ+d5rbLlWMekw9/b+5nx71r8OfchXz2jMb8V0dpNO4AmAHWDi2v\nAY4MN6iq3cDuM91RkkNVteFM32c5ccx9cMx9GMeYx30K6BFgfZKLkpwNbAP2jbkPkiTGfARQVa8n\n+UXgAeAsYE9VPTnOPkiSBsZ9Coiquh+4fwy7OuPTSMuQY+6DY+7Dko85VTV/K0nSW45fBSFJnVr2\nATDfV0skeUeSL7T1DydZN/5eLq4RxvwPkzyV5IkkB5KMdEvYD7NRv0IkySeSVJJlf8fIKGNO8rfa\nz/rJJL837j4uthH+bl+Y5MEkj7W/31sm0c/FkmRPkqNJvj7H+iS5vf15PJHk0kXtQFUt2xeDC8nf\nAN4HnA38EXDxm9r8feA32/w24AuT7vcYxvwzwI+0+V/oYcyt3buArwIHgQ2T7vcYfs7rgceAVW35\nvEn3ewxj3g38Qpu/GHhu0v0+wzH/NHAp8PU51m8B/guDZ6iuAB5ezP0v9yOAUb5aYiuwt83fC2xM\nMtsDacvFvGOuqger6rW2eJDB8xbL2ahfIXIL8C+Bb4+zc0tklDH/XeDXq+oVgKo6OuY+LrZRxlzA\nu9v8e3jTc0TLTVV9FTh+iiZbgbtq4CCwMskFi7X/5R4Ao3y1xPfaVNXrwAngvWPp3dJY6Ndp7GDw\nCWI5m3fMST4ErK2qL42zY0tolJ/zTwA/keR/JjmYZPPYerc0RhnzPwN+LskMg7sJf2k8XZuYJf36\nnLHfBrrI5v1qiRHbLCcjjyfJzwEbgL+xpD1aeqccc5K3AbcBPz+uDo3BKD/nFQxOA13J4Cjvvyf5\nqap6dYn7tlRGGfMngd+pqn+d5MPA59qY/3LpuzcRS/r7a7kfAcz71RLDbZKsYHDYeKpDrh92o4yZ\nJB8D/ilwTVV9Z0x9WyrzjfldwE8BDyV5jsG50n3L/ELwqH+376uq/1dV3wSeYRAIy9UoY94B3ANQ\nVf8LeCeD7wl6qxrp3/vpWu4BMMpXS+wDtrf5TwBfqXZ1ZZmad8ztdMi/Z/DLf7mfF4Z5xlxVJ6rq\n3KpaV1XrGFz3uKaqDk2mu4tilL/b/5nBBX+SnMvglNCzY+3l4hplzM8DGwGSfIBBABwbay/Hax9w\nfbsb6ArgRFW9uFhvvqxPAdUcXy2R5GbgUFXtA+5kcJg4zeCT/7bJ9fjMjTjmfwX8GPD77Xr381V1\nzcQ6fYZGHPNbyohjfgDYlOQp4LvAP6qqlyfX6zMz4ph/FfitJP+AwamQn1/OH+iSfJ7BKbxz23WN\nXcDbAarqNxlc59gCTAOvATcs6v6X8Z+dJOkMLPdTQJKk02QASFKnDABJ6pQBIEmdMgAkqVMGgCR1\nygCQpE4ZAJLUqf8PwRa4eRxq3rIAAAAASUVORK5CYII=\n", 302 | "text/plain": [ 303 | "" 304 | ] 305 | }, 306 | "metadata": {}, 307 | "output_type": "display_data" 308 | } 309 | ], 310 | "source": [ 311 | "plt.hist(y_hat)\n", 312 | "_ = plt.axvline(x=0.5, color='orange')" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": 13, 318 | "metadata": {}, 319 | "outputs": [ 320 | { 321 | "data": { 322 | "text/plain": [ 323 | "'92.98'" 324 | ] 325 | }, 326 | "execution_count": 13, 327 | "metadata": {}, 328 | "output_type": "execute_result" 329 | } 330 | ], 331 | "source": [ 332 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": null, 338 | "metadata": { 339 | "collapsed": true 340 | }, 341 | "outputs": [], 342 | "source": [] 343 | } 344 | ], 345 | "metadata": { 346 | "kernelspec": { 347 | "display_name": "Python 3", 348 | "language": "python", 349 | "name": "python3" 350 | }, 351 | "language_info": { 352 | "codemirror_mode": { 353 | "name": "ipython", 354 | "version": 3 355 | }, 356 | "file_extension": ".py", 357 | "mimetype": "text/x-python", 358 | "name": "python", 359 | "nbconvert_exporter": "python", 360 | "pygments_lexer": "ipython3", 361 | "version": "3.6.2" 362 | } 363 | }, 364 | "nbformat": 4, 365 | "nbformat_minor": 2 366 | } 367 | -------------------------------------------------------------------------------- /notebooks/intermediate_net_in_tensorflow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Intermediate Neural Network in TensorFlow" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we convert our [intermediate-depth MNIST-classifying neural network](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/intermediate_net_in_keras.ipynb) from Keras to TensorFlow (compare them side by side) following Aymeric Damien's [Multi-Layer Perceptron Notebook](https://github.com/aymericdamien/TensorFlow-Examples/blob/master/notebooks/3_NeuralNetworks/multilayer_perceptron.ipynb) style." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "np.random.seed(42)\n", 34 | "import tensorflow as tf\n", 35 | "tf.set_random_seed(42)" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": {}, 41 | "source": [ 42 | "#### Load data" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 2, 48 | "metadata": {}, 49 | "outputs": [ 50 | { 51 | "name": "stdout", 52 | "output_type": "stream", 53 | "text": [ 54 | "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", 55 | "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", 56 | "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", 57 | "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n" 58 | ] 59 | } 60 | ], 61 | "source": [ 62 | "from tensorflow.examples.tutorials.mnist import input_data\n", 63 | "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "#### Set neural network hyperparameters (tidier at top of file!)" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 3, 76 | "metadata": { 77 | "collapsed": true 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "lr = 0.1\n", 82 | "epochs = 20\n", 83 | "batch_size = 128\n", 84 | "weight_initializer = tf.contrib.layers.xavier_initializer()" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "#### Set number of neurons for each layer" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 4, 97 | "metadata": { 98 | "collapsed": true 99 | }, 100 | "outputs": [], 101 | "source": [ 102 | "n_input = 784\n", 103 | "n_dense_1 = 64\n", 104 | "n_dense_2 = 64\n", 105 | "n_classes = 10" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "#### Define placeholders Tensors for inputs and labels" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 5, 118 | "metadata": { 119 | "collapsed": true 120 | }, 121 | "outputs": [], 122 | "source": [ 123 | "x = tf.placeholder(tf.float32, [None, n_input])\n", 124 | "y = tf.placeholder(tf.float32, [None, n_classes])" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "metadata": {}, 130 | "source": [ 131 | "#### Define types of layers" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 6, 137 | "metadata": { 138 | "collapsed": true 139 | }, 140 | "outputs": [], 141 | "source": [ 142 | "# dense layer with ReLU activation:\n", 143 | "def dense(x, W, b):\n", 144 | " z = tf.add(tf.matmul(x, W), b)\n", 145 | " a = tf.nn.relu(z)\n", 146 | " return a" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "metadata": {}, 152 | "source": [ 153 | "#### Design neural network architecture" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 7, 159 | "metadata": { 160 | "collapsed": true 161 | }, 162 | "outputs": [], 163 | "source": [ 164 | "def network(x, weights, biases):\n", 165 | " \n", 166 | " # two dense hidden layers: \n", 167 | " dense_1 = dense(x, weights['W1'], biases['b1'])\n", 168 | " dense_2 = dense(dense_1, weights['W2'], biases['b2'])\n", 169 | " \n", 170 | " # linear output layer (softmax)\n", 171 | " out_layer_z = tf.add(tf.matmul(dense_2, weights['W_out']), biases['b_out'])\n", 172 | " \n", 173 | " return out_layer_z" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "metadata": {}, 179 | "source": [ 180 | "#### Define dictionaries for storing weights and biases for each layer -- and initialize" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 8, 186 | "metadata": { 187 | "collapsed": true 188 | }, 189 | "outputs": [], 190 | "source": [ 191 | "bias_dict = {\n", 192 | " 'b1': tf.Variable(tf.zeros([n_dense_1])), \n", 193 | " 'b2': tf.Variable(tf.zeros([n_dense_2])),\n", 194 | " 'b_out': tf.Variable(tf.zeros([n_classes]))\n", 195 | "}\n", 196 | "\n", 197 | "weight_dict = {\n", 198 | " 'W1': tf.get_variable('W1', [n_input, n_dense_1], initializer=weight_initializer),\n", 199 | " 'W2': tf.get_variable('W2', [n_dense_1, n_dense_2], initializer=weight_initializer),\n", 200 | " 'W_out': tf.get_variable('W_out', [n_dense_2, n_classes], initializer=weight_initializer)\n", 201 | "}" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "metadata": {}, 207 | "source": [ 208 | "#### Build model" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 9, 214 | "metadata": { 215 | "collapsed": true 216 | }, 217 | "outputs": [], 218 | "source": [ 219 | "predictions = network(x, weights=weight_dict, biases=bias_dict)" 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "metadata": {}, 225 | "source": [ 226 | "#### Define model's loss and its optimizer" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": 10, 232 | "metadata": { 233 | "collapsed": true 234 | }, 235 | "outputs": [], 236 | "source": [ 237 | "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predictions, labels=y))\n", 238 | "optimizer = tf.train.GradientDescentOptimizer(learning_rate=lr).minimize(cost)" 239 | ] 240 | }, 241 | { 242 | "cell_type": "markdown", 243 | "metadata": {}, 244 | "source": [ 245 | "#### Define evaluation metrics" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": 11, 251 | "metadata": { 252 | "collapsed": true 253 | }, 254 | "outputs": [], 255 | "source": [ 256 | "# calculate accuracy by identifying test cases where the model's highest-probability class matches the true y label: \n", 257 | "correct_prediction = tf.equal(tf.argmax(predictions, 1), tf.argmax(y, 1))\n", 258 | "accuracy_pct = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) * 100" 259 | ] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": {}, 264 | "source": [ 265 | "#### Create op for variable initialization" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": 12, 271 | "metadata": { 272 | "collapsed": true 273 | }, 274 | "outputs": [], 275 | "source": [ 276 | "initializer_op = tf.global_variables_initializer()" 277 | ] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "metadata": {}, 282 | "source": [ 283 | "#### Train the network in a session" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": 13, 289 | "metadata": {}, 290 | "outputs": [ 291 | { 292 | "name": "stdout", 293 | "output_type": "stream", 294 | "text": [ 295 | "Training for 20 epochs.\n", 296 | "Epoch 001: cost = 0.500, accuracy = 85.40%\n", 297 | "Epoch 002: cost = 0.250, accuracy = 92.61%\n", 298 | "Epoch 003: cost = 0.192, accuracy = 94.40%\n", 299 | "Epoch 004: cost = 0.158, accuracy = 95.44%\n", 300 | "Epoch 005: cost = 0.136, accuracy = 95.99%\n", 301 | "Epoch 006: cost = 0.120, accuracy = 96.43%\n", 302 | "Epoch 007: cost = 0.106, accuracy = 96.96%\n", 303 | "Epoch 008: cost = 0.095, accuracy = 97.18%\n", 304 | "Epoch 009: cost = 0.086, accuracy = 97.48%\n", 305 | "Epoch 010: cost = 0.078, accuracy = 97.67%\n", 306 | "Epoch 011: cost = 0.071, accuracy = 97.90%\n", 307 | "Epoch 012: cost = 0.066, accuracy = 98.02%\n", 308 | "Epoch 013: cost = 0.060, accuracy = 98.28%\n", 309 | "Epoch 014: cost = 0.056, accuracy = 98.38%\n", 310 | "Epoch 015: cost = 0.052, accuracy = 98.48%\n", 311 | "Epoch 016: cost = 0.047, accuracy = 98.66%\n", 312 | "Epoch 017: cost = 0.043, accuracy = 98.74%\n", 313 | "Epoch 018: cost = 0.041, accuracy = 98.79%\n", 314 | "Epoch 019: cost = 0.038, accuracy = 98.94%\n", 315 | "Epoch 020: cost = 0.035, accuracy = 98.98%\n", 316 | "Training Complete. Testing Model.\n", 317 | "\n", 318 | "Test Cost: 0.081\n", 319 | "Test Accuracy: 97.37%\n" 320 | ] 321 | } 322 | ], 323 | "source": [ 324 | "with tf.Session() as session:\n", 325 | " session.run(initializer_op)\n", 326 | " \n", 327 | " print(\"Training for\", epochs, \"epochs.\")\n", 328 | " \n", 329 | " # loop over epochs: \n", 330 | " for epoch in range(epochs):\n", 331 | " \n", 332 | " avg_cost = 0.0 # track cost to monitor performance during training\n", 333 | " avg_accuracy_pct = 0.0\n", 334 | " \n", 335 | " # loop over all batches of the epoch:\n", 336 | " n_batches = int(mnist.train.num_examples / batch_size)\n", 337 | " for i in range(n_batches):\n", 338 | " \n", 339 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 340 | " \n", 341 | " # feed batch data to run optimization and fetching cost and accuracy: \n", 342 | " _, batch_cost, batch_acc = session.run([optimizer, cost, accuracy_pct], \n", 343 | " feed_dict={x: batch_x, y: batch_y})\n", 344 | " \n", 345 | " # accumulate mean loss and accuracy over epoch: \n", 346 | " avg_cost += batch_cost / n_batches\n", 347 | " avg_accuracy_pct += batch_acc / n_batches\n", 348 | " \n", 349 | " # output logs at end of each epoch of training:\n", 350 | " print(\"Epoch \", '%03d' % (epoch+1), \n", 351 | " \": cost = \", '{:.3f}'.format(avg_cost), \n", 352 | " \", accuracy = \", '{:.2f}'.format(avg_accuracy_pct), \"%\", \n", 353 | " sep='')\n", 354 | " \n", 355 | " print(\"Training Complete. Testing Model.\\n\")\n", 356 | " \n", 357 | " test_cost = cost.eval({x: mnist.test.images, y: mnist.test.labels})\n", 358 | " test_accuracy_pct = accuracy_pct.eval({x: mnist.test.images, y: mnist.test.labels})\n", 359 | " \n", 360 | " print(\"Test Cost:\", '{:.3f}'.format(test_cost))\n", 361 | " print(\"Test Accuracy: \", '{:.2f}'.format(test_accuracy_pct), \"%\", sep='')" 362 | ] 363 | } 364 | ], 365 | "metadata": { 366 | "kernelspec": { 367 | "display_name": "Python 3", 368 | "language": "python", 369 | "name": "python3" 370 | }, 371 | "language_info": { 372 | "codemirror_mode": { 373 | "name": "ipython", 374 | "version": 3 375 | }, 376 | "file_extension": ".py", 377 | "mimetype": "text/x-python", 378 | "name": "python", 379 | "nbconvert_exporter": "python", 380 | "pygments_lexer": "ipython3", 381 | "version": "3.5.2" 382 | } 383 | }, 384 | "nbformat": 4, 385 | "nbformat_minor": 2 386 | } 387 | -------------------------------------------------------------------------------- /notebooks/lenet_in_keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Deep Convolutional Neural Network in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we build a deep, convolutional, MNIST-classifying network inspired by [LeNet-5](http://yann.lecun.com/exdb/publis/pdf/lecun-01a.pdf) and [this example code](https://github.com/fchollet/keras/blob/master/examples/mnist_cnn.py)." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Set seed for reproducibility" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "np.random.seed(42)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Load dependencies" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "name": "stderr", 50 | "output_type": "stream", 51 | "text": [ 52 | "Using TensorFlow backend.\n" 53 | ] 54 | } 55 | ], 56 | "source": [ 57 | "import keras\n", 58 | "from keras.datasets import mnist\n", 59 | "from keras.models import Sequential\n", 60 | "from keras.layers import Dense, Dropout\n", 61 | "from keras.layers import Flatten, Conv2D, MaxPooling2D # new!" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "#### Load data" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": { 75 | "collapsed": true 76 | }, 77 | "outputs": [], 78 | "source": [ 79 | "(X_train, y_train), (X_test, y_test) = mnist.load_data()" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "#### Preprocess data" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 4, 92 | "metadata": { 93 | "collapsed": true 94 | }, 95 | "outputs": [], 96 | "source": [ 97 | "X_train = X_train.reshape(60000, 28, 28, 1).astype('float32')\n", 98 | "X_test = X_test.reshape(10000, 28, 28, 1).astype('float32')" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 5, 104 | "metadata": { 105 | "collapsed": true 106 | }, 107 | "outputs": [], 108 | "source": [ 109 | "X_train /= 255\n", 110 | "X_test /= 255" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 6, 116 | "metadata": { 117 | "collapsed": true 118 | }, 119 | "outputs": [], 120 | "source": [ 121 | "n_classes = 10\n", 122 | "y_train = keras.utils.to_categorical(y_train, n_classes)\n", 123 | "y_test = keras.utils.to_categorical(y_test, n_classes)" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "#### Design neural network architecture" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 7, 136 | "metadata": { 137 | "collapsed": true 138 | }, 139 | "outputs": [], 140 | "source": [ 141 | "model = Sequential()\n", 142 | "model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1)))\n", 143 | "model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))\n", 144 | "model.add(MaxPooling2D(pool_size=(2, 2)))\n", 145 | "model.add(Dropout(0.25))\n", 146 | "model.add(Flatten())\n", 147 | "model.add(Dense(128, activation='relu'))\n", 148 | "model.add(Dropout(0.5))\n", 149 | "model.add(Dense(n_classes, activation='softmax'))" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 8, 155 | "metadata": {}, 156 | "outputs": [ 157 | { 158 | "name": "stdout", 159 | "output_type": "stream", 160 | "text": [ 161 | "_________________________________________________________________\n", 162 | "Layer (type) Output Shape Param # \n", 163 | "=================================================================\n", 164 | "conv2d_1 (Conv2D) (None, 26, 26, 32) 320 \n", 165 | "_________________________________________________________________\n", 166 | "conv2d_2 (Conv2D) (None, 24, 24, 64) 18496 \n", 167 | "_________________________________________________________________\n", 168 | "max_pooling2d_1 (MaxPooling2 (None, 12, 12, 64) 0 \n", 169 | "_________________________________________________________________\n", 170 | "dropout_1 (Dropout) (None, 12, 12, 64) 0 \n", 171 | "_________________________________________________________________\n", 172 | "flatten_1 (Flatten) (None, 9216) 0 \n", 173 | "_________________________________________________________________\n", 174 | "dense_1 (Dense) (None, 128) 1179776 \n", 175 | "_________________________________________________________________\n", 176 | "dropout_2 (Dropout) (None, 128) 0 \n", 177 | "_________________________________________________________________\n", 178 | "dense_2 (Dense) (None, 10) 1290 \n", 179 | "=================================================================\n", 180 | "Total params: 1,199,882\n", 181 | "Trainable params: 1,199,882\n", 182 | "Non-trainable params: 0\n", 183 | "_________________________________________________________________\n" 184 | ] 185 | } 186 | ], 187 | "source": [ 188 | "model.summary()" 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": {}, 194 | "source": [ 195 | "#### Configure model" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 8, 201 | "metadata": { 202 | "collapsed": true 203 | }, 204 | "outputs": [], 205 | "source": [ 206 | "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])" 207 | ] 208 | }, 209 | { 210 | "cell_type": "markdown", 211 | "metadata": {}, 212 | "source": [ 213 | "#### Train!" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": 9, 219 | "metadata": {}, 220 | "outputs": [ 221 | { 222 | "name": "stdout", 223 | "output_type": "stream", 224 | "text": [ 225 | "Train on 60000 samples, validate on 10000 samples\n", 226 | "Epoch 1/20\n", 227 | "60000/60000 [==============================] - 66s - loss: 0.2421 - acc: 0.9264 - val_loss: 0.0510 - val_acc: 0.9842\n", 228 | "Epoch 2/20\n", 229 | "60000/60000 [==============================] - 65s - loss: 0.0853 - acc: 0.9743 - val_loss: 0.0453 - val_acc: 0.9853\n", 230 | "Epoch 3/20\n", 231 | "60000/60000 [==============================] - 64s - loss: 0.0631 - acc: 0.9809 - val_loss: 0.0372 - val_acc: 0.9878\n", 232 | "Epoch 4/20\n", 233 | "60000/60000 [==============================] - 66s - loss: 0.0549 - acc: 0.9834 - val_loss: 0.0345 - val_acc: 0.9876\n", 234 | "Epoch 5/20\n", 235 | "60000/60000 [==============================] - 64s - loss: 0.0452 - acc: 0.9859 - val_loss: 0.0282 - val_acc: 0.9905\n", 236 | "Epoch 6/20\n", 237 | "60000/60000 [==============================] - 64s - loss: 0.0399 - acc: 0.9873 - val_loss: 0.0293 - val_acc: 0.9910\n", 238 | "Epoch 7/20\n", 239 | "60000/60000 [==============================] - 64s - loss: 0.0350 - acc: 0.9889 - val_loss: 0.0271 - val_acc: 0.9914\n", 240 | "Epoch 8/20\n", 241 | "60000/60000 [==============================] - 64s - loss: 0.0303 - acc: 0.9897 - val_loss: 0.0283 - val_acc: 0.9920\n", 242 | "Epoch 9/20\n", 243 | "60000/60000 [==============================] - 66s - loss: 0.0306 - acc: 0.9902 - val_loss: 0.0316 - val_acc: 0.9909\n", 244 | "Epoch 10/20\n", 245 | "60000/60000 [==============================] - 63s - loss: 0.0259 - acc: 0.9916 - val_loss: 0.0304 - val_acc: 0.9907\n", 246 | "Epoch 11/20\n", 247 | "60000/60000 [==============================] - 64s - loss: 0.0227 - acc: 0.9929 - val_loss: 0.0272 - val_acc: 0.9921\n", 248 | "Epoch 12/20\n", 249 | "60000/60000 [==============================] - 64s - loss: 0.0230 - acc: 0.9925 - val_loss: 0.0326 - val_acc: 0.9913\n", 250 | "Epoch 13/20\n", 251 | "60000/60000 [==============================] - 62s - loss: 0.0212 - acc: 0.9927 - val_loss: 0.0326 - val_acc: 0.9913\n", 252 | "Epoch 14/20\n", 253 | "60000/60000 [==============================] - 55s - loss: 0.0202 - acc: 0.9934 - val_loss: 0.0284 - val_acc: 0.9924\n", 254 | "Epoch 15/20\n", 255 | "60000/60000 [==============================] - 56s - loss: 0.0181 - acc: 0.9939 - val_loss: 0.0302 - val_acc: 0.9927\n", 256 | "Epoch 16/20\n", 257 | "60000/60000 [==============================] - 63s - loss: 0.0176 - acc: 0.9940 - val_loss: 0.0320 - val_acc: 0.9919\n", 258 | "Epoch 17/20\n", 259 | "60000/60000 [==============================] - 63s - loss: 0.0174 - acc: 0.9943 - val_loss: 0.0275 - val_acc: 0.9926\n", 260 | "Epoch 18/20\n", 261 | "60000/60000 [==============================] - 66s - loss: 0.0166 - acc: 0.9945 - val_loss: 0.0297 - val_acc: 0.9928\n", 262 | "Epoch 19/20\n", 263 | "60000/60000 [==============================] - 64s - loss: 0.0147 - acc: 0.9953 - val_loss: 0.0326 - val_acc: 0.9925\n", 264 | "Epoch 20/20\n", 265 | "60000/60000 [==============================] - 64s - loss: 0.0149 - acc: 0.9949 - val_loss: 0.0300 - val_acc: 0.9930\n" 266 | ] 267 | }, 268 | { 269 | "data": { 270 | "text/plain": [ 271 | "" 272 | ] 273 | }, 274 | "execution_count": 9, 275 | "metadata": {}, 276 | "output_type": "execute_result" 277 | } 278 | ], 279 | "source": [ 280 | "model.fit(X_train, y_train, batch_size=128, epochs=20, verbose=1, validation_data=(X_test, y_test))" 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": null, 286 | "metadata": { 287 | "collapsed": true 288 | }, 289 | "outputs": [], 290 | "source": [] 291 | } 292 | ], 293 | "metadata": { 294 | "kernelspec": { 295 | "display_name": "Python 3", 296 | "language": "python", 297 | "name": "python3" 298 | }, 299 | "language_info": { 300 | "codemirror_mode": { 301 | "name": "ipython", 302 | "version": 3 303 | }, 304 | "file_extension": ".py", 305 | "mimetype": "text/x-python", 306 | "name": "python", 307 | "nbconvert_exporter": "python", 308 | "pygments_lexer": "ipython3", 309 | "version": "3.5.2" 310 | } 311 | }, 312 | "nbformat": 4, 313 | "nbformat_minor": 2 314 | } 315 | -------------------------------------------------------------------------------- /notebooks/live_training/convolutional_sentiment_classifier_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Convolutional Sentiment Classifier" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we build a *convolutional* neural net to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import keras\n", 31 | "from keras.datasets import imdb\n", 32 | "from keras.preprocessing.sequence import pad_sequences\n", 33 | "from keras.models import Sequential\n", 34 | "from keras.layers import Dense, Dropout, Embedding\n", 35 | "from keras.layers import SpatialDropout1D, Conv1D, GlobalMaxPooling1D # new! \n", 36 | "from keras.callbacks import ModelCheckpoint\n", 37 | "import os\n", 38 | "from sklearn.metrics import roc_auc_score \n", 39 | "import matplotlib.pyplot as plt \n", 40 | "%matplotlib inline" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "#### Set hyperparameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": { 54 | "collapsed": true 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "# output directory name:\n", 59 | "output_dir = 'model_output/conv'\n", 60 | "\n", 61 | "# training:\n", 62 | "epochs = 4\n", 63 | "batch_size = 128\n", 64 | "\n", 65 | "# vector-space embedding: \n", 66 | "n_dim = 64\n", 67 | "n_unique_words = 5000 \n", 68 | "max_review_length = 400\n", 69 | "pad_type = trunc_type = 'pre'\n", 70 | "drop_embed = 0.2 # new!\n", 71 | "\n", 72 | "# convolutional layer architecture:\n", 73 | "n_conv = 256 # filters, a.k.a. kernels\n", 74 | "k_conv = 3 # kernel length\n", 75 | "\n", 76 | "# dense layer architecture: \n", 77 | "n_dense = 256\n", 78 | "dropout = 0.2" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "#### Load data" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": { 92 | "collapsed": true 93 | }, 94 | "outputs": [], 95 | "source": [ 96 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "#### Preprocess data" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": { 110 | "collapsed": true 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 115 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": { 121 | "collapsed": true 122 | }, 123 | "source": [ 124 | "#### Design neural network architecture" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": { 131 | "collapsed": true 132 | }, 133 | "outputs": [], 134 | "source": [ 135 | "model = Sequential()\n", 136 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n", 137 | "\n", 138 | "# CODE HERE\n", 139 | "\n", 140 | "model.add(Dense(n_dense, activation='relu'))\n", 141 | "model.add(Dropout(dropout))\n", 142 | "model.add(Dense(1, activation='sigmoid'))" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "model.summary() " 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "#### Configure model" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": { 165 | "collapsed": true 166 | }, 167 | "outputs": [], 168 | "source": [ 169 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "metadata": { 176 | "collapsed": true 177 | }, 178 | "outputs": [], 179 | "source": [ 180 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 181 | "if not os.path.exists(output_dir):\n", 182 | " os.makedirs(output_dir)" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": {}, 188 | "source": [ 189 | "#### Train!" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "# 89.1% validation accuracy in epoch 2\n", 199 | "# ...with second convolutional layer is essentially the same at 89.0%\n", 200 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "metadata": { 206 | "collapsed": true 207 | }, 208 | "source": [ 209 | "#### Evaluate" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": { 216 | "collapsed": true 217 | }, 218 | "outputs": [], 219 | "source": [ 220 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": {}, 227 | "outputs": [], 228 | "source": [ 229 | "y_hat = model.predict_proba(x_valid)" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "metadata": { 236 | "scrolled": true 237 | }, 238 | "outputs": [], 239 | "source": [ 240 | "plt.hist(y_hat)\n", 241 | "_ = plt.axvline(x=0.5, color='orange')" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [ 250 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": null, 256 | "metadata": { 257 | "collapsed": true 258 | }, 259 | "outputs": [], 260 | "source": [] 261 | } 262 | ], 263 | "metadata": { 264 | "kernelspec": { 265 | "display_name": "Python 3", 266 | "language": "python", 267 | "name": "python3" 268 | }, 269 | "language_info": { 270 | "codemirror_mode": { 271 | "name": "ipython", 272 | "version": 3 273 | }, 274 | "file_extension": ".py", 275 | "mimetype": "text/x-python", 276 | "name": "python", 277 | "nbconvert_exporter": "python", 278 | "pygments_lexer": "ipython3", 279 | "version": "3.6.1" 280 | } 281 | }, 282 | "nbformat": 4, 283 | "nbformat_minor": 2 284 | } 285 | -------------------------------------------------------------------------------- /notebooks/live_training/deep_net_in_keras_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Deep Neural Network in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we build a Deep Neural Network while discussing the theory we've learnt thus far." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Set seed for reproducibility" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "np.random.seed(42)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Load dependencies" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": { 47 | "collapsed": true 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "import keras\n", 52 | "from keras.datasets import mnist\n", 53 | "from keras.models import Sequential\n", 54 | "from keras.layers import Dense\n", 55 | "from keras.layers import Dropout # new!\n", 56 | "from keras.optimizers import SGD" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "#### Load data" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": { 70 | "collapsed": true 71 | }, 72 | "outputs": [], 73 | "source": [ 74 | "(X_train, y_train), (X_test, y_test) = mnist.load_data()" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "#### Preprocess data" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": { 88 | "collapsed": true 89 | }, 90 | "outputs": [], 91 | "source": [ 92 | "X_train = X_train.reshape(60000, 784).astype('float32')\n", 93 | "X_test = X_test.reshape(10000, 784).astype('float32')" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": { 100 | "collapsed": true 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "X_train /= 255\n", 105 | "X_test /= 255" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": { 112 | "collapsed": true 113 | }, 114 | "outputs": [], 115 | "source": [ 116 | "n_classes = 10\n", 117 | "y_train = keras.utils.to_categorical(y_train, n_classes)\n", 118 | "y_test = keras.utils.to_categorical(y_test, n_classes)" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": {}, 124 | "source": [ 125 | "#### Design neural network architecture" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": { 132 | "collapsed": true 133 | }, 134 | "outputs": [], 135 | "source": [ 136 | "model = Sequential()\n", 137 | "model.add(Dense(64, activation='sigmoid', input_shape=(784,)))\n", 138 | "model.add(Dense(10, activation='softmax'))" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": { 145 | "collapsed": true 146 | }, 147 | "outputs": [], 148 | "source": [ 149 | "model.summary()" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "#### Configure model" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": { 163 | "collapsed": true 164 | }, 165 | "outputs": [], 166 | "source": [ 167 | "model.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])" 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": {}, 173 | "source": [ 174 | "#### Train!" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "metadata": { 181 | "collapsed": true 182 | }, 183 | "outputs": [], 184 | "source": [ 185 | "model.fit(X_train, y_train, batch_size=128, epochs=200, verbose=1, validation_data=(X_test, y_test))" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": { 192 | "collapsed": true 193 | }, 194 | "outputs": [], 195 | "source": [] 196 | } 197 | ], 198 | "metadata": { 199 | "kernelspec": { 200 | "display_name": "Python 3", 201 | "language": "python", 202 | "name": "python3" 203 | }, 204 | "language_info": { 205 | "codemirror_mode": { 206 | "name": "ipython", 207 | "version": 3 208 | }, 209 | "file_extension": ".py", 210 | "mimetype": "text/x-python", 211 | "name": "python", 212 | "nbconvert_exporter": "python", 213 | "pygments_lexer": "ipython3", 214 | "version": "3.6.1" 215 | } 216 | }, 217 | "nbformat": 4, 218 | "nbformat_minor": 2 219 | } 220 | -------------------------------------------------------------------------------- /notebooks/live_training/deep_net_in_tensorflow_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Deep Neural Network in TensorFlow" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we convert our [intermediate-depth MNIST-classifying neural network](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/intermediate_net_in_keras.ipynb) from Keras to TensorFlow (compare them side by side) following Aymeric Damien's [Multi-Layer Perceptron Notebook](https://github.com/aymericdamien/TensorFlow-Examples/blob/master/notebooks/3_NeuralNetworks/multilayer_perceptron.ipynb) style. \n", 15 | "\n", 16 | "Subsequently, we add a layer to make it deep! " 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "#### Load dependencies" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": { 30 | "collapsed": true 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "import numpy as np\n", 35 | "np.random.seed(42)\n", 36 | "import tensorflow as tf\n", 37 | "tf.set_random_seed(42)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "#### Load data" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": { 51 | "collapsed": true 52 | }, 53 | "outputs": [], 54 | "source": [ 55 | "from tensorflow.examples.tutorials.mnist import input_data\n", 56 | "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "#### Set neural network hyperparameters (tidier at top of file!)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": { 70 | "collapsed": true 71 | }, 72 | "outputs": [], 73 | "source": [ 74 | "lr = 0.1\n", 75 | "epochs = 1\n", 76 | "batch_size = 128\n", 77 | "weight_initializer = tf.contrib.layers.xavier_initializer()" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": {}, 83 | "source": [ 84 | "#### Set number of neurons for each layer" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "metadata": { 91 | "collapsed": true 92 | }, 93 | "outputs": [], 94 | "source": [ 95 | "n_input = 784\n", 96 | "n_dense_1 = 64\n", 97 | "n_dense_2 = 64\n", 98 | "n_classes = 10" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "#### Define placeholders Tensors for inputs and labels" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": { 112 | "collapsed": true 113 | }, 114 | "outputs": [], 115 | "source": [ 116 | "x = tf.placeholder(tf.float32, [None, n_input])\n", 117 | "y = tf.placeholder(tf.float32, [None, n_classes])" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "#### Define types of layers" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": { 131 | "collapsed": true 132 | }, 133 | "outputs": [], 134 | "source": [ 135 | "# dense layer with ReLU activation:\n", 136 | "def dense(x, W, b):\n", 137 | " # DEFINE\n", 138 | " # DEFINE\n", 139 | " return a" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "#### Define dictionaries for storing weights and biases for each layer -- and initialize" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": { 153 | "collapsed": true 154 | }, 155 | "outputs": [], 156 | "source": [ 157 | "bias_dict = {\n", 158 | " 'b1': tf.Variable(tf.zeros([n_dense_1])), \n", 159 | " 'b2': tf.Variable(tf.zeros([n_dense_2])),\n", 160 | " 'b_out': tf.Variable(tf.zeros([n_classes]))\n", 161 | "}\n", 162 | "\n", 163 | "weight_dict = {\n", 164 | " 'W1': tf.get_variable('W1', [n_input, n_dense_1], initializer=weight_initializer),\n", 165 | " 'W2': tf.get_variable('W2', [n_dense_1, n_dense_2], initializer=weight_initializer),\n", 166 | " 'W_out': tf.get_variable('W_out', [n_dense_2, n_classes], initializer=weight_initializer)\n", 167 | "}" 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": {}, 173 | "source": [ 174 | "#### Design neural network architecture" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "metadata": { 181 | "collapsed": true 182 | }, 183 | "outputs": [], 184 | "source": [ 185 | "def network(x, weights, biases):\n", 186 | " \n", 187 | " # DEFINE\n", 188 | " \n", 189 | " return out_layer_z" 190 | ] 191 | }, 192 | { 193 | "cell_type": "markdown", 194 | "metadata": {}, 195 | "source": [ 196 | "#### Build model" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": null, 202 | "metadata": { 203 | "collapsed": true 204 | }, 205 | "outputs": [], 206 | "source": [ 207 | "predictions = network(x, weights=weight_dict, biases=bias_dict)" 208 | ] 209 | }, 210 | { 211 | "cell_type": "markdown", 212 | "metadata": {}, 213 | "source": [ 214 | "#### Define model's loss and its optimizer" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": null, 220 | "metadata": { 221 | "collapsed": true 222 | }, 223 | "outputs": [], 224 | "source": [ 225 | "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predictions, labels=y))\n", 226 | "optimizer = tf.train.GradientDescentOptimizer(learning_rate=lr).minimize(cost)" 227 | ] 228 | }, 229 | { 230 | "cell_type": "markdown", 231 | "metadata": {}, 232 | "source": [ 233 | "#### Define evaluation metrics" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": null, 239 | "metadata": { 240 | "collapsed": true 241 | }, 242 | "outputs": [], 243 | "source": [ 244 | "# calculate accuracy by identifying test cases where the model's highest-probability class matches the true y label: \n", 245 | "correct_prediction = tf.equal(tf.argmax(predictions, 1), tf.argmax(y, 1))\n", 246 | "accuracy_pct = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) * 100" 247 | ] 248 | }, 249 | { 250 | "cell_type": "markdown", 251 | "metadata": {}, 252 | "source": [ 253 | "#### Create op for variable initialization" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": null, 259 | "metadata": { 260 | "collapsed": true 261 | }, 262 | "outputs": [], 263 | "source": [ 264 | "initializer_op = tf.global_variables_initializer()" 265 | ] 266 | }, 267 | { 268 | "cell_type": "markdown", 269 | "metadata": {}, 270 | "source": [ 271 | "#### Train the network in a session" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": null, 277 | "metadata": { 278 | "collapsed": true 279 | }, 280 | "outputs": [], 281 | "source": [ 282 | "with tf.Session() as session:\n", 283 | " session.run(initializer_op)\n", 284 | " \n", 285 | " print(\"Training for\", epochs, \"epochs.\")\n", 286 | " \n", 287 | " # loop over epochs: \n", 288 | " for epoch in range(epochs):\n", 289 | " \n", 290 | " avg_cost = 0.0 # track cost to monitor performance during training\n", 291 | " avg_accuracy_pct = 0.0\n", 292 | " \n", 293 | " # loop over all batches of the epoch:\n", 294 | " n_batches = int(mnist.train.num_examples / batch_size)\n", 295 | " for i in range(n_batches):\n", 296 | " \n", 297 | " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", 298 | " \n", 299 | " # feed batch data to run optimization and fetching cost and accuracy: \n", 300 | " _, batch_cost, batch_acc = session.run([optimizer, cost, accuracy_pct], \n", 301 | " feed_dict={x: batch_x, y: batch_y})\n", 302 | " \n", 303 | " # accumulate mean loss and accuracy over epoch: \n", 304 | " avg_cost += batch_cost / n_batches\n", 305 | " avg_accuracy_pct += batch_acc / n_batches\n", 306 | " \n", 307 | " # output logs at end of each epoch of training:\n", 308 | " print(\"Epoch \", '%03d' % (epoch+1), \n", 309 | " \": cost = \", '{:.3f}'.format(avg_cost), \n", 310 | " \", accuracy = \", '{:.2f}'.format(avg_accuracy_pct), \"%\", \n", 311 | " sep='')\n", 312 | " \n", 313 | " print(\"Training Complete. Testing Model.\\n\")\n", 314 | " \n", 315 | " test_cost = cost.eval({x: mnist.test.images, y: mnist.test.labels})\n", 316 | " test_accuracy_pct = accuracy_pct.eval({x: mnist.test.images, y: mnist.test.labels})\n", 317 | " \n", 318 | " print(\"Test Cost:\", '{:.3f}'.format(test_cost))\n", 319 | " print(\"Test Accuracy: \", '{:.2f}'.format(test_accuracy_pct), \"%\", sep='')" 320 | ] 321 | } 322 | ], 323 | "metadata": { 324 | "kernelspec": { 325 | "display_name": "Python 3", 326 | "language": "python", 327 | "name": "python3" 328 | }, 329 | "language_info": { 330 | "codemirror_mode": { 331 | "name": "ipython", 332 | "version": 3 333 | }, 334 | "file_extension": ".py", 335 | "mimetype": "text/x-python", 336 | "name": "python", 337 | "nbconvert_exporter": "python", 338 | "pygments_lexer": "ipython3", 339 | "version": "3.6.2" 340 | } 341 | }, 342 | "nbformat": 4, 343 | "nbformat_minor": 2 344 | } 345 | -------------------------------------------------------------------------------- /notebooks/live_training/dense_sentiment_classifier_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Dense Sentiment Classifier" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we build a dense neural net to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import keras\n", 31 | "from keras.datasets import imdb\n", 32 | "from keras.preprocessing.sequence import pad_sequences\n", 33 | "from keras.models import Sequential\n", 34 | "from keras.layers import Dense, Flatten, Dropout\n", 35 | "from keras.layers import Embedding # new!\n", 36 | "from keras.callbacks import ModelCheckpoint # new! \n", 37 | "import os # new! \n", 38 | "from sklearn.metrics import roc_auc_score, roc_curve # new!\n", 39 | "import pandas as pd\n", 40 | "import matplotlib.pyplot as plt # new!\n", 41 | "%matplotlib inline" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "#### Set hyperparameters" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": { 55 | "collapsed": true 56 | }, 57 | "outputs": [], 58 | "source": [ 59 | "# output directory name:\n", 60 | "output_dir = 'model_output/dense'\n", 61 | "\n", 62 | "# training:\n", 63 | "epochs = 4\n", 64 | "batch_size = 128\n", 65 | "\n", 66 | "# vector-space embedding: \n", 67 | "n_dim = 64\n", 68 | "n_unique_words = 5000 # as per Maas et al. (2011); may not be optimal\n", 69 | "n_words_to_skip = 50 # ditto\n", 70 | "max_review_length = 100\n", 71 | "pad_type = trunc_type = 'pre'\n", 72 | "\n", 73 | "# neural network architecture: \n", 74 | "n_dense = 64\n", 75 | "dropout = 0.5" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": {}, 81 | "source": [ 82 | "#### Load data" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "For a given data set: \n", 90 | "\n", 91 | "* the Keras text utilities [here](https://keras.io/preprocessing/text/) quickly preprocess natural language and convert it into an index\n", 92 | "* the `keras.preprocessing.text.Tokenizer` class may do everything you need in one line:\n", 93 | " * tokenize into words or characters\n", 94 | " * `num_words`: maximum unique tokens\n", 95 | " * filter out punctuation\n", 96 | " * lower case\n", 97 | " * convert words to an integer index" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": null, 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words, skip_top=n_words_to_skip) " 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "x_train[0:6] # 0 reserved for padding; 1 would be starting character; 2 is unknown; 3 is most common word, etc." 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "for x in x_train[0:6]:\n", 125 | " print(len(x))" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "y_train[0:6]" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "len(x_train), len(x_valid)" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "#### Restoring words from index" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "word_index = keras.datasets.imdb.get_word_index()\n", 160 | "word_index = {k:(v+3) for k,v in word_index.items()}\n", 161 | "word_index[\"PAD\"] = 0\n", 162 | "word_index[\"START\"] = 1\n", 163 | "word_index[\"UNK\"] = 2" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "word_index" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "metadata": { 179 | "collapsed": true 180 | }, 181 | "outputs": [], 182 | "source": [ 183 | "index_word = {v:k for k,v in word_index.items()}" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [ 192 | "x_train[0]" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "metadata": {}, 199 | "outputs": [], 200 | "source": [ 201 | "' '.join(index_word[id] for id in x_train[0])" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "metadata": { 208 | "collapsed": true 209 | }, 210 | "outputs": [], 211 | "source": [ 212 | "(all_x_train,_),(all_x_valid,_) = imdb.load_data() " 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": null, 218 | "metadata": {}, 219 | "outputs": [], 220 | "source": [ 221 | "' '.join(index_word[id] for id in all_x_train[0])" 222 | ] 223 | }, 224 | { 225 | "cell_type": "markdown", 226 | "metadata": {}, 227 | "source": [ 228 | "#### Preprocess data" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": { 235 | "collapsed": true 236 | }, 237 | "outputs": [], 238 | "source": [ 239 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 240 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "x_train[0:6]" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": null, 255 | "metadata": {}, 256 | "outputs": [], 257 | "source": [ 258 | "for x in x_train[0:6]:\n", 259 | " print(len(x))" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": null, 265 | "metadata": {}, 266 | "outputs": [], 267 | "source": [ 268 | "' '.join(index_word[id] for id in x_train[0])" 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "execution_count": null, 274 | "metadata": {}, 275 | "outputs": [], 276 | "source": [ 277 | "' '.join(index_word[id] for id in x_train[5])" 278 | ] 279 | }, 280 | { 281 | "cell_type": "markdown", 282 | "metadata": { 283 | "collapsed": true 284 | }, 285 | "source": [ 286 | "#### Design neural network architecture" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": null, 292 | "metadata": { 293 | "collapsed": true 294 | }, 295 | "outputs": [], 296 | "source": [ 297 | "# CODE HERE" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": null, 303 | "metadata": {}, 304 | "outputs": [], 305 | "source": [ 306 | "model.summary() # so many parameters!" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": null, 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [ 315 | "# embedding layer dimensions and parameters: \n", 316 | "n_dim, n_unique_words, n_dim*n_unique_words" 317 | ] 318 | }, 319 | { 320 | "cell_type": "code", 321 | "execution_count": null, 322 | "metadata": {}, 323 | "outputs": [], 324 | "source": [ 325 | "# ...flatten:\n", 326 | "max_review_length, n_dim, n_dim*max_review_length" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [ 335 | "# ...dense:\n", 336 | "n_dense, n_dim*max_review_length*n_dense + n_dense # weights + biases" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": null, 342 | "metadata": {}, 343 | "outputs": [], 344 | "source": [ 345 | "# ...and output:\n", 346 | "n_dense + 1 " 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "metadata": {}, 352 | "source": [ 353 | "#### Configure model" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": null, 359 | "metadata": { 360 | "collapsed": true 361 | }, 362 | "outputs": [], 363 | "source": [ 364 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": null, 370 | "metadata": { 371 | "collapsed": true 372 | }, 373 | "outputs": [], 374 | "source": [ 375 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")" 376 | ] 377 | }, 378 | { 379 | "cell_type": "code", 380 | "execution_count": null, 381 | "metadata": { 382 | "collapsed": true 383 | }, 384 | "outputs": [], 385 | "source": [ 386 | "if not os.path.exists(output_dir):\n", 387 | " os.makedirs(output_dir)" 388 | ] 389 | }, 390 | { 391 | "cell_type": "markdown", 392 | "metadata": {}, 393 | "source": [ 394 | "#### Train!" 395 | ] 396 | }, 397 | { 398 | "cell_type": "code", 399 | "execution_count": null, 400 | "metadata": {}, 401 | "outputs": [], 402 | "source": [ 403 | "# 84.7% validation accuracy in epoch 2\n", 404 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 405 | ] 406 | }, 407 | { 408 | "cell_type": "markdown", 409 | "metadata": {}, 410 | "source": [ 411 | "#### Evaluate" 412 | ] 413 | }, 414 | { 415 | "cell_type": "code", 416 | "execution_count": null, 417 | "metadata": { 418 | "collapsed": true 419 | }, 420 | "outputs": [], 421 | "source": [ 422 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 423 | ] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": null, 428 | "metadata": {}, 429 | "outputs": [], 430 | "source": [ 431 | "y_hat = model.predict_proba(x_valid)" 432 | ] 433 | }, 434 | { 435 | "cell_type": "code", 436 | "execution_count": null, 437 | "metadata": {}, 438 | "outputs": [], 439 | "source": [ 440 | "len(y_hat)" 441 | ] 442 | }, 443 | { 444 | "cell_type": "code", 445 | "execution_count": null, 446 | "metadata": {}, 447 | "outputs": [], 448 | "source": [ 449 | "y_hat[0]" 450 | ] 451 | }, 452 | { 453 | "cell_type": "code", 454 | "execution_count": null, 455 | "metadata": {}, 456 | "outputs": [], 457 | "source": [ 458 | "plt.hist(y_hat)\n", 459 | "_ = plt.axvline(x=0.5, color='orange')" 460 | ] 461 | }, 462 | { 463 | "cell_type": "code", 464 | "execution_count": null, 465 | "metadata": { 466 | "collapsed": true 467 | }, 468 | "outputs": [], 469 | "source": [ 470 | "pct_auc = roc_auc_score(y_valid, y_hat)*100.0" 471 | ] 472 | }, 473 | { 474 | "cell_type": "code", 475 | "execution_count": null, 476 | "metadata": {}, 477 | "outputs": [], 478 | "source": [ 479 | "\"{:0.2f}\".format(pct_auc)" 480 | ] 481 | }, 482 | { 483 | "cell_type": "code", 484 | "execution_count": null, 485 | "metadata": { 486 | "collapsed": true 487 | }, 488 | "outputs": [], 489 | "source": [ 490 | "float_y_hat = []\n", 491 | "for y in y_hat:\n", 492 | " float_y_hat.append(y[0])" 493 | ] 494 | }, 495 | { 496 | "cell_type": "code", 497 | "execution_count": null, 498 | "metadata": { 499 | "collapsed": true 500 | }, 501 | "outputs": [], 502 | "source": [ 503 | "ydf = pd.DataFrame(list(zip(float_y_hat, y_valid)), columns=['y_hat', 'y'])" 504 | ] 505 | }, 506 | { 507 | "cell_type": "code", 508 | "execution_count": null, 509 | "metadata": {}, 510 | "outputs": [], 511 | "source": [ 512 | "ydf.head(10)" 513 | ] 514 | }, 515 | { 516 | "cell_type": "code", 517 | "execution_count": null, 518 | "metadata": {}, 519 | "outputs": [], 520 | "source": [ 521 | "' '.join(index_word[id] for id in all_x_valid[0])" 522 | ] 523 | }, 524 | { 525 | "cell_type": "code", 526 | "execution_count": null, 527 | "metadata": {}, 528 | "outputs": [], 529 | "source": [ 530 | "' '.join(index_word[id] for id in all_x_valid[6]) " 531 | ] 532 | }, 533 | { 534 | "cell_type": "code", 535 | "execution_count": null, 536 | "metadata": {}, 537 | "outputs": [], 538 | "source": [ 539 | "ydf[(ydf.y == 0) & (ydf.y_hat > 0.9)].head(10)" 540 | ] 541 | }, 542 | { 543 | "cell_type": "code", 544 | "execution_count": null, 545 | "metadata": {}, 546 | "outputs": [], 547 | "source": [ 548 | "' '.join(index_word[id] for id in all_x_valid[489]) " 549 | ] 550 | }, 551 | { 552 | "cell_type": "code", 553 | "execution_count": null, 554 | "metadata": {}, 555 | "outputs": [], 556 | "source": [ 557 | "ydf[(ydf.y == 1) & (ydf.y_hat < 0.1)].head(10)" 558 | ] 559 | }, 560 | { 561 | "cell_type": "code", 562 | "execution_count": null, 563 | "metadata": {}, 564 | "outputs": [], 565 | "source": [ 566 | "' '.join(index_word[id] for id in all_x_valid[927]) " 567 | ] 568 | }, 569 | { 570 | "cell_type": "code", 571 | "execution_count": null, 572 | "metadata": { 573 | "collapsed": true 574 | }, 575 | "outputs": [], 576 | "source": [] 577 | } 578 | ], 579 | "metadata": { 580 | "kernelspec": { 581 | "display_name": "Python 3", 582 | "language": "python", 583 | "name": "python3" 584 | }, 585 | "language_info": { 586 | "codemirror_mode": { 587 | "name": "ipython", 588 | "version": 3 589 | }, 590 | "file_extension": ".py", 591 | "mimetype": "text/x-python", 592 | "name": "python", 593 | "nbconvert_exporter": "python", 594 | "pygments_lexer": "ipython3", 595 | "version": "3.6.1" 596 | } 597 | }, 598 | "nbformat": 4, 599 | "nbformat_minor": 2 600 | } 601 | -------------------------------------------------------------------------------- /notebooks/live_training/first_tensorflow_graphs_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# First TensorFlow Graphs" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we execute elementary TensorFlow computational graphs." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "import tensorflow as tf" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Simple arithmetic" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": { 47 | "collapsed": true 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "x1 = tf.placeholder(tf.float32)\n", 52 | "x2 = tf.placeholder(tf.float32)" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": { 59 | "collapsed": true 60 | }, 61 | "outputs": [], 62 | "source": [ 63 | "sum_op = tf.add(x1, x2)\n", 64 | "product_op = tf.multiply(x1, x2)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": { 71 | "collapsed": true 72 | }, 73 | "outputs": [], 74 | "source": [ 75 | "with tf.Session() as session:\n", 76 | " sum_result = session.run(sum_op, feed_dict={x1: , x2: }) \n", 77 | " product_result = session.run(product_op, feed_dict={x1: , x2: }) " 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "sum_result" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "product_result" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "#### Simple array arithmetic" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": { 109 | "collapsed": true 110 | }, 111 | "outputs": [], 112 | "source": [ 113 | "with tf.Session() as session:\n", 114 | " sum_result = session.run(sum_op, feed_dict={x1: , x2: })\n", 115 | " product_result = session.run(product_op, feed_dict={x1: , x2: })" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "sum_result" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "product_result" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": null, 139 | "metadata": { 140 | "collapsed": true 141 | }, 142 | "outputs": [], 143 | "source": [] 144 | } 145 | ], 146 | "metadata": { 147 | "kernelspec": { 148 | "display_name": "Python 3", 149 | "language": "python", 150 | "name": "python3" 151 | }, 152 | "language_info": { 153 | "codemirror_mode": { 154 | "name": "ipython", 155 | "version": 3 156 | }, 157 | "file_extension": ".py", 158 | "mimetype": "text/x-python", 159 | "name": "python", 160 | "nbconvert_exporter": "python", 161 | "pygments_lexer": "ipython3", 162 | "version": "3.6.1" 163 | } 164 | }, 165 | "nbformat": 4, 166 | "nbformat_minor": 2 167 | } 168 | -------------------------------------------------------------------------------- /notebooks/live_training/first_tensorflow_neurons_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# First TensorFlow Neurons" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "#### Load dependencies" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": { 21 | "collapsed": true 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "import numpy as np\n", 26 | "np.random.seed(42)\n", 27 | "import tensorflow as tf\n", 28 | "tf.set_random_seed(42)\n", 29 | "import matplotlib.pyplot as plt\n", 30 | "%matplotlib inline" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "metadata": {}, 36 | "source": [ 37 | "#### Set number of neurons" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": { 44 | "collapsed": true 45 | }, 46 | "outputs": [], 47 | "source": [ 48 | "n_input = 784\n", 49 | "n_dense = 128" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "#### Define placeholder Tensor for simulated MNIST digits" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": { 63 | "collapsed": true 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "x = tf.placeholder(tf.float32, [None, n_input])" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "#### Create Variable Tensors for neuron biases `b` and weight matrix `W`" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": { 81 | "collapsed": true 82 | }, 83 | "outputs": [], 84 | "source": [ 85 | "b = tf.Variable(tf.zeros([n_dense]))\n", 86 | "W = tf.Variable(tf.random_uniform([n_input, n_dense]))" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "#### Design the computational graph" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": { 100 | "collapsed": true 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "z = \n", 105 | "a = " 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "#### Create op for variable initialization" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": null, 118 | "metadata": { 119 | "collapsed": true 120 | }, 121 | "outputs": [], 122 | "source": [ 123 | "initializer_op = tf.global_variables_initializer()" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "#### Execute the graph in a session" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": { 137 | "collapsed": true 138 | }, 139 | "outputs": [], 140 | "source": [ 141 | "with tf.Session() as session:\n", 142 | " session.run(initializer_op)\n", 143 | " \n", 144 | " layer_output = session.run(a, {x: np.random.random([1, n_input])})" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "layer_output" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "_ = plt.hist(np.transpose(layer_output))" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": null, 168 | "metadata": { 169 | "collapsed": true 170 | }, 171 | "outputs": [], 172 | "source": [] 173 | } 174 | ], 175 | "metadata": { 176 | "kernelspec": { 177 | "display_name": "Python 3", 178 | "language": "python", 179 | "name": "python3" 180 | }, 181 | "language_info": { 182 | "codemirror_mode": { 183 | "name": "ipython", 184 | "version": 3 185 | }, 186 | "file_extension": ".py", 187 | "mimetype": "text/x-python", 188 | "name": "python", 189 | "nbconvert_exporter": "python", 190 | "pygments_lexer": "ipython3", 191 | "version": "3.6.1" 192 | } 193 | }, 194 | "nbformat": 4, 195 | "nbformat_minor": 2 196 | } 197 | -------------------------------------------------------------------------------- /notebooks/live_training/multi_convnet_architectures_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Multi-ConvNet Architectures" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we concatenate *multiple parallel convolutional nets together* to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import keras\n", 31 | "from keras.datasets import imdb\n", 32 | "from keras.preprocessing.sequence import pad_sequences\n", 33 | "from keras.models import Model # new!\n", 34 | "from keras.layers import Input, concatenate # new! \n", 35 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, Conv1D, GlobalMaxPooling1D\n", 36 | "from keras.callbacks import ModelCheckpoint\n", 37 | "import os\n", 38 | "from sklearn.metrics import roc_auc_score \n", 39 | "import matplotlib.pyplot as plt \n", 40 | "%matplotlib inline" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "#### Set hyperparameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": { 54 | "collapsed": true 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "# output directory name:\n", 59 | "output_dir = 'model_output/multiconv'\n", 60 | "\n", 61 | "# training:\n", 62 | "epochs = 4\n", 63 | "batch_size = 128\n", 64 | "\n", 65 | "# vector-space embedding: \n", 66 | "n_dim = 64\n", 67 | "n_unique_words = 5000 \n", 68 | "max_review_length = 400\n", 69 | "pad_type = trunc_type = 'pre'\n", 70 | "drop_embed = 0.2 \n", 71 | "\n", 72 | "# convolutional layer architecture:\n", 73 | "n_conv_1 = n_conv_2 = n_conv_3 = 256 \n", 74 | "k_conv_1 = 3\n", 75 | "k_conv_2 = 2\n", 76 | "k_conv_3 = 4\n", 77 | "\n", 78 | "# dense layer architecture: \n", 79 | "n_dense = 256\n", 80 | "dropout = 0.2" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "#### Load data" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": { 94 | "collapsed": true 95 | }, 96 | "outputs": [], 97 | "source": [ 98 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) " 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "#### Preprocess data" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": { 112 | "collapsed": true 113 | }, 114 | "outputs": [], 115 | "source": [ 116 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 117 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": { 123 | "collapsed": true 124 | }, 125 | "source": [ 126 | "#### Design neural network architecture" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": { 133 | "collapsed": true 134 | }, 135 | "outputs": [], 136 | "source": [ 137 | "input_layer = Input(shape=(max_review_length,), dtype='int16', name='input') # supports integers +/- 32.7k \n", 138 | "embedding_layer = Embedding(n_unique_words, n_dim, input_length=max_review_length, name='embedding')(input_layer)\n", 139 | "drop_embed_layer = SpatialDropout1D(drop_embed, name='drop_embed')(embedding_layer)\n", 140 | "\n", 141 | "# CODE HERE\n", 142 | "\n", 143 | "# start with conv_1 only and no concat\n", 144 | "# add conv_2\n", 145 | "# add conv_3\n", 146 | "\n", 147 | "dense_layer = Dense(n_dense, activation='relu', name='dense')(concat)\n", 148 | "drop_dense_layer = Dropout(dropout, name='drop_dense')(dense_layer)\n", 149 | "dense_2 = Dense(64, activation='relu', name='dense_2')(drop_dense_layer)\n", 150 | "dropout_2 = Dropout(dropout, name='drop_dense_2')(dense_2)\n", 151 | "\n", 152 | "predictions = Dense(1, activation='sigmoid', name='output')(dropout_2)\n", 153 | "\n", 154 | "model = Model(input_layer, predictions)" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "model.summary() " 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "#### Configure model" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": null, 176 | "metadata": { 177 | "collapsed": true 178 | }, 179 | "outputs": [], 180 | "source": [ 181 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": { 188 | "collapsed": true 189 | }, 190 | "outputs": [], 191 | "source": [ 192 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 193 | "if not os.path.exists(output_dir):\n", 194 | " os.makedirs(output_dir)" 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": {}, 200 | "source": [ 201 | "#### Train!" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "# start with conv_1 only and no concat: 89.1% validation accuracy in epoch 2, as earlier notebook\n", 211 | "# add conv_2: 89.5% in epoch 3\n", 212 | "# add conv_3: ditto\n", 213 | "# add dense_2: ditto in epoch 2\n", 214 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 215 | ] 216 | }, 217 | { 218 | "cell_type": "markdown", 219 | "metadata": { 220 | "collapsed": true 221 | }, 222 | "source": [ 223 | "#### Evaluate" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": null, 229 | "metadata": { 230 | "collapsed": true 231 | }, 232 | "outputs": [], 233 | "source": [ 234 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": null, 240 | "metadata": { 241 | "collapsed": true 242 | }, 243 | "outputs": [], 244 | "source": [ 245 | "y_hat = model.predict(x_valid)" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": null, 251 | "metadata": {}, 252 | "outputs": [], 253 | "source": [ 254 | "plt.hist(y_hat)\n", 255 | "_ = plt.axvline(x=0.5, color='orange')" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": null, 270 | "metadata": { 271 | "collapsed": true 272 | }, 273 | "outputs": [], 274 | "source": [] 275 | } 276 | ], 277 | "metadata": { 278 | "kernelspec": { 279 | "display_name": "Python 3", 280 | "language": "python", 281 | "name": "python3" 282 | }, 283 | "language_info": { 284 | "codemirror_mode": { 285 | "name": "ipython", 286 | "version": 3 287 | }, 288 | "file_extension": ".py", 289 | "mimetype": "text/x-python", 290 | "name": "python", 291 | "nbconvert_exporter": "python", 292 | "pygments_lexer": "ipython3", 293 | "version": "3.6.1" 294 | } 295 | }, 296 | "nbformat": 4, 297 | "nbformat_minor": 2 298 | } 299 | -------------------------------------------------------------------------------- /notebooks/live_training/point_by_point_intro_to_tensorflow_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Introduction to TensorFlow, fitting point by point" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we introduce TensorFlow by fitting a line of the form y=m\\*x+b point by point. This is a derivation of Jared Ostmeyer's [Naked Tensor](https://github.com/jostmey/NakedTensor/) code. " 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies and set seeds for reproducibility" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "np.random.seed(42)\n", 34 | "import pandas as pd\n", 35 | "import matplotlib.pyplot as plt\n", 36 | "%matplotlib inline\n", 37 | "import tensorflow as tf\n", 38 | "tf.set_random_seed(42)" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "#### Create a very small data set" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": { 52 | "collapsed": true 53 | }, 54 | "outputs": [], 55 | "source": [ 56 | "xs = [0., 1., 2., 3., 4., 5., 6., 7.] \n", 57 | "ys = [-.82, -.94, -.12, .26, .39, .64, 1.02, 1.] " 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "metadata": { 64 | "collapsed": true 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "fig, ax = plt.subplots()\n", 69 | "_ = ax.scatter(xs, ys)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "#### Define *variables* -- the model parameters we'll learn -- and initialize them with \"random\" values" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "metadata": { 83 | "collapsed": true 84 | }, 85 | "outputs": [], 86 | "source": [ 87 | "m = tf.Variable(-0.5)\n", 88 | "b = tf.Variable(1.0)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "#### One single point at a time, define the error between the true label and the model's prediction of the label" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "metadata": { 102 | "collapsed": true 103 | }, 104 | "outputs": [], 105 | "source": [ 106 | "total_error = # DEFINE\n", 107 | "for x,y in zip(xs, ys):\n", 108 | " y_model = # DEFINE\n", 109 | " total_error += # DEFINE" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "#### Define optimizer as SSE-minimizing gradient descent" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": { 123 | "collapsed": true 124 | }, 125 | "outputs": [], 126 | "source": [ 127 | "optimizer_operation = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(total_error)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "metadata": {}, 133 | "source": [ 134 | "#### Define an operator that will initialize the graph with all available global variables" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": { 141 | "collapsed": true 142 | }, 143 | "outputs": [], 144 | "source": [ 145 | "initializer_operation = tf.global_variables_initializer()" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "metadata": {}, 151 | "source": [ 152 | "#### With the computational graph designed, we initialize a session to execute it" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": { 159 | "collapsed": true 160 | }, 161 | "outputs": [], 162 | "source": [ 163 | "with tf.Session() as session:\n", 164 | " \n", 165 | " session.run(initializer_operation) \n", 166 | " \n", 167 | " n_epochs = # DEFINE\n", 168 | " for iteration in range(n_epochs):\n", 169 | " session.run(optimizer_operation) \n", 170 | " \n", 171 | " slope, intercept = session.run([m, b]) " 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": null, 177 | "metadata": { 178 | "collapsed": true 179 | }, 180 | "outputs": [], 181 | "source": [ 182 | "slope" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "metadata": { 189 | "collapsed": true 190 | }, 191 | "outputs": [], 192 | "source": [ 193 | "intercept" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": {}, 199 | "source": [ 200 | "#### Calculate the predicted model outputs given the inputs xs" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": { 207 | "collapsed": true 208 | }, 209 | "outputs": [], 210 | "source": [ 211 | "y_hat = intercept + slope*np.array(xs)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": null, 217 | "metadata": { 218 | "collapsed": true 219 | }, 220 | "outputs": [], 221 | "source": [ 222 | "pd.DataFrame(list(zip(ys, y_hat)), columns=['y', 'y_hat'])" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "metadata": { 229 | "collapsed": true 230 | }, 231 | "outputs": [], 232 | "source": [ 233 | "fig, ax = plt.subplots()\n", 234 | "\n", 235 | "ax.scatter(xs, ys)\n", 236 | "x_min, x_max = ax.get_xlim()\n", 237 | "y_min, y_max = intercept, intercept + slope*(x_max-x_min)\n", 238 | "\n", 239 | "ax.plot([x_min, x_max], [y_min, y_max])\n", 240 | "_ = ax.set_xlim([x_min, x_max])" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "metadata": { 247 | "collapsed": true 248 | }, 249 | "outputs": [], 250 | "source": [] 251 | } 252 | ], 253 | "metadata": { 254 | "kernelspec": { 255 | "display_name": "Python 3", 256 | "language": "python", 257 | "name": "python3" 258 | }, 259 | "language_info": { 260 | "codemirror_mode": { 261 | "name": "ipython", 262 | "version": 3 263 | }, 264 | "file_extension": ".py", 265 | "mimetype": "text/x-python", 266 | "name": "python", 267 | "nbconvert_exporter": "python", 268 | "pygments_lexer": "ipython3", 269 | "version": "3.6.1" 270 | } 271 | }, 272 | "nbformat": 4, 273 | "nbformat_minor": 2 274 | } 275 | -------------------------------------------------------------------------------- /notebooks/live_training/rnn_in_keras_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Recurrent Neural Network in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we use an RNN to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import keras\n", 31 | "from keras.datasets import imdb\n", 32 | "from keras.preprocessing.sequence import pad_sequences\n", 33 | "from keras.models import Sequential\n", 34 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D\n", 35 | "from keras.layers import SimpleRNN # new! \n", 36 | "from keras.callbacks import ModelCheckpoint\n", 37 | "import os\n", 38 | "from sklearn.metrics import roc_auc_score \n", 39 | "import matplotlib.pyplot as plt \n", 40 | "%matplotlib inline" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "#### Set hyperparameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": { 54 | "collapsed": true 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "# output directory name:\n", 59 | "output_dir = 'model_output/rnn'\n", 60 | "\n", 61 | "# training:\n", 62 | "epochs = 16 # way more!\n", 63 | "batch_size = 128\n", 64 | "\n", 65 | "# vector-space embedding: \n", 66 | "n_dim = 64 \n", 67 | "n_unique_words = 10000 \n", 68 | "max_review_length = 100 # lowered due to vanishing gradient over time\n", 69 | "pad_type = trunc_type = 'pre'\n", 70 | "drop_embed = 0.2 \n", 71 | "\n", 72 | "# RNN layer architecture:\n", 73 | "n_rnn = 256 \n", 74 | "drop_rnn = 0.2\n", 75 | "\n", 76 | "# dense layer architecture: \n", 77 | "# n_dense = 256\n", 78 | "# dropout = 0.2" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "#### Load data" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": { 92 | "collapsed": true 93 | }, 94 | "outputs": [], 95 | "source": [ 96 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "#### Preprocess data" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": { 110 | "collapsed": true 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 115 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": { 121 | "collapsed": true 122 | }, 123 | "source": [ 124 | "#### Design neural network architecture" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": { 131 | "collapsed": true 132 | }, 133 | "outputs": [], 134 | "source": [ 135 | "model = Sequential()\n", 136 | "\n", 137 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length))\n", 138 | "model.add(SpatialDropout1D(drop_embed))\n", 139 | "\n", 140 | "# CODE HERE\n", 141 | "\n", 142 | "model.add(Dense(1, activation='sigmoid'))" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "model.summary() " 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "#### Configure model" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": { 165 | "collapsed": true 166 | }, 167 | "outputs": [], 168 | "source": [ 169 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "metadata": { 176 | "collapsed": true 177 | }, 178 | "outputs": [], 179 | "source": [ 180 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 181 | "if not os.path.exists(output_dir):\n", 182 | " os.makedirs(output_dir)" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": {}, 188 | "source": [ 189 | "#### Train!" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "# 80.6% validation accuracy in epoch 4\n", 199 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": { 205 | "collapsed": true 206 | }, 207 | "source": [ 208 | "#### Evaluate" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": null, 214 | "metadata": { 215 | "collapsed": true 216 | }, 217 | "outputs": [], 218 | "source": [ 219 | "model.load_weights(output_dir+\"/weights.03.hdf5\") # zero-indexed" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "metadata": { 226 | "collapsed": true 227 | }, 228 | "outputs": [], 229 | "source": [ 230 | "y_hat = model.predict_proba(x_valid)" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": { 237 | "collapsed": true 238 | }, 239 | "outputs": [], 240 | "source": [ 241 | "plt.hist(y_hat)\n", 242 | "_ = plt.axvline(x=0.5, color='orange')" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "metadata": { 249 | "collapsed": true 250 | }, 251 | "outputs": [], 252 | "source": [ 253 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": null, 259 | "metadata": { 260 | "collapsed": true 261 | }, 262 | "outputs": [], 263 | "source": [] 264 | } 265 | ], 266 | "metadata": { 267 | "kernelspec": { 268 | "display_name": "Python 3", 269 | "language": "python", 270 | "name": "python3" 271 | }, 272 | "language_info": { 273 | "codemirror_mode": { 274 | "name": "ipython", 275 | "version": 3 276 | }, 277 | "file_extension": ".py", 278 | "mimetype": "text/x-python", 279 | "name": "python", 280 | "nbconvert_exporter": "python", 281 | "pygments_lexer": "ipython3", 282 | "version": "3.6.1" 283 | } 284 | }, 285 | "nbformat": 4, 286 | "nbformat_minor": 2 287 | } 288 | -------------------------------------------------------------------------------- /notebooks/live_training/shallow_net_in_keras_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Shallow Neural Network in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Build a shallow neural network to classify MNIST digits" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Set seed for reproducibility" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "np.random.seed(42)" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "#### Load dependencies" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "import keras\n", 50 | "from keras.datasets import mnist\n", 51 | "from keras.models import Sequential\n", 52 | "from keras.layers import Dense\n", 53 | "from keras.optimizers import SGD" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "#### Load data" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": { 67 | "collapsed": true 68 | }, 69 | "outputs": [], 70 | "source": [ 71 | "(X_train, y_train), (X_test, y_test) = mnist.load_data()" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "X_train.shape" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "y_train.shape" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "y_train[0:99]" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "X_test.shape" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "X_test[0]" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "y_test.shape" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "y_test[0]" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "#### Preprocess data" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "metadata": { 148 | "collapsed": true 149 | }, 150 | "outputs": [], 151 | "source": [ 152 | "X_train = X_train.reshape(60000, 784).astype('float32')\n", 153 | "X_test = X_test.reshape(10000, 784).astype('float32')" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": { 160 | "collapsed": true 161 | }, 162 | "outputs": [], 163 | "source": [ 164 | "X_train /= 255\n", 165 | "X_test /= 255" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "X_test[0]" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "metadata": { 181 | "collapsed": true 182 | }, 183 | "outputs": [], 184 | "source": [ 185 | "n_classes = 10\n", 186 | "y_train = keras.utils.to_categorical(y_train, n_classes)\n", 187 | "y_test = keras.utils.to_categorical(y_test, n_classes)" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [ 196 | "y_test[0]" 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": {}, 202 | "source": [ 203 | "#### Design neural network architecture" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": { 210 | "collapsed": true 211 | }, 212 | "outputs": [], 213 | "source": [ 214 | "# add architecture here" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": null, 220 | "metadata": {}, 221 | "outputs": [], 222 | "source": [ 223 | "model.summary()" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": null, 229 | "metadata": {}, 230 | "outputs": [], 231 | "source": [ 232 | "(64*784)" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [ 241 | "(64*784)+64" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [ 250 | "(10*64)+10" 251 | ] 252 | }, 253 | { 254 | "cell_type": "markdown", 255 | "metadata": {}, 256 | "source": [ 257 | "#### Configure model" 258 | ] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": null, 263 | "metadata": { 264 | "collapsed": true 265 | }, 266 | "outputs": [], 267 | "source": [ 268 | "# compile model here" 269 | ] 270 | }, 271 | { 272 | "cell_type": "markdown", 273 | "metadata": {}, 274 | "source": [ 275 | "#### Train!" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": null, 281 | "metadata": {}, 282 | "outputs": [], 283 | "source": [ 284 | "# train model here" 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": null, 290 | "metadata": {}, 291 | "outputs": [], 292 | "source": [ 293 | "model.evaluate(X_test, y_test)" 294 | ] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "execution_count": null, 299 | "metadata": { 300 | "collapsed": true 301 | }, 302 | "outputs": [], 303 | "source": [] 304 | } 305 | ], 306 | "metadata": { 307 | "kernelspec": { 308 | "display_name": "Python 3", 309 | "language": "python", 310 | "name": "python3" 311 | }, 312 | "language_info": { 313 | "codemirror_mode": { 314 | "name": "ipython", 315 | "version": 3 316 | }, 317 | "file_extension": ".py", 318 | "mimetype": "text/x-python", 319 | "name": "python", 320 | "nbconvert_exporter": "python", 321 | "pygments_lexer": "ipython3", 322 | "version": "3.6.1" 323 | } 324 | }, 325 | "nbformat": 4, 326 | "nbformat_minor": 2 327 | } 328 | -------------------------------------------------------------------------------- /notebooks/live_training/tensor-fied_intro_to_tensorflow_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Introduction to TensorFlow, now leveraging tensors!" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we modify our [intro to TensorFlow notebook](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/point_by_point_intro_to_tensorflow.ipynb) to use tensors in place of our *for* loop. This is a derivation of Jared Ostmeyer's [Naked Tensor](https://github.com/jostmey/NakedTensor/) code. " 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### The initial steps are identical to the earlier notebook" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": { 28 | "collapsed": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "np.random.seed(42)\n", 34 | "import pandas as pd\n", 35 | "import matplotlib.pyplot as plt\n", 36 | "%matplotlib inline\n", 37 | "import tensorflow as tf\n", 38 | "tf.set_random_seed(42)" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": { 45 | "collapsed": true 46 | }, 47 | "outputs": [], 48 | "source": [ 49 | "xs = [0., 1., 2., 3., 4., 5., 6., 7.] \n", 50 | "ys = [-.82, -.94, -.12, .26, .39, .64, 1.02, 1.] " 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "metadata": { 57 | "collapsed": true 58 | }, 59 | "outputs": [], 60 | "source": [ 61 | "fig, ax = plt.subplots()\n", 62 | "_ = ax.scatter(xs, ys)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": { 69 | "collapsed": true 70 | }, 71 | "outputs": [], 72 | "source": [ 73 | "m = tf.Variable(-0.5)\n", 74 | "b = tf.Variable(1.0)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "#### Define the cost as a tensor -- more elegant than a *for* loop and enables distributed computing in TensorFlow" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": { 88 | "collapsed": true 89 | }, 90 | "outputs": [], 91 | "source": [ 92 | "ys_model = m*xs+b\n", 93 | "total_error = # DEFINE" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "#### The remaining steps are also identical to the earlier notebook!" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": { 107 | "collapsed": true 108 | }, 109 | "outputs": [], 110 | "source": [ 111 | "optimizer_operation = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(total_error)" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": null, 117 | "metadata": { 118 | "collapsed": true 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "initializer_operation = tf.global_variables_initializer()" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "metadata": { 129 | "collapsed": true 130 | }, 131 | "outputs": [], 132 | "source": [ 133 | "with tf.Session() as session:\n", 134 | " \n", 135 | " session.run(initializer_operation)\n", 136 | " \n", 137 | " n_epochs = 1000 \n", 138 | " for iteration in range(n_epochs):\n", 139 | " session.run(optimizer_operation)\n", 140 | " \n", 141 | " slope, intercept = session.run([m, b])" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "metadata": { 148 | "collapsed": true 149 | }, 150 | "outputs": [], 151 | "source": [ 152 | "slope" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": { 159 | "collapsed": true 160 | }, 161 | "outputs": [], 162 | "source": [ 163 | "intercept" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": { 170 | "collapsed": true 171 | }, 172 | "outputs": [], 173 | "source": [ 174 | "y_hat = intercept + slope*np.array(xs)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "metadata": { 181 | "collapsed": true 182 | }, 183 | "outputs": [], 184 | "source": [ 185 | "pd.DataFrame(list(zip(ys, y_hat)), columns=['y', 'y_hat'])" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": { 192 | "collapsed": true 193 | }, 194 | "outputs": [], 195 | "source": [ 196 | "fig, ax = plt.subplots()\n", 197 | "\n", 198 | "ax.scatter(xs, ys)\n", 199 | "x_min, x_max = ax.get_xlim()\n", 200 | "y_min, y_max = intercept, intercept + slope*(x_max-x_min)\n", 201 | "\n", 202 | "ax.plot([x_min, x_max], [y_min, y_max])\n", 203 | "_ = ax.set_xlim([x_min, x_max])" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": { 210 | "collapsed": true 211 | }, 212 | "outputs": [], 213 | "source": [] 214 | } 215 | ], 216 | "metadata": { 217 | "kernelspec": { 218 | "display_name": "Python 3", 219 | "language": "python", 220 | "name": "python3" 221 | }, 222 | "language_info": { 223 | "codemirror_mode": { 224 | "name": "ipython", 225 | "version": 3 226 | }, 227 | "file_extension": ".py", 228 | "mimetype": "text/x-python", 229 | "name": "python", 230 | "nbconvert_exporter": "python", 231 | "pygments_lexer": "ipython3", 232 | "version": "3.6.1" 233 | } 234 | }, 235 | "nbformat": 4, 236 | "nbformat_minor": 2 237 | } 238 | -------------------------------------------------------------------------------- /notebooks/live_training/vanilla_lstm_in_keras_LT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Vanilla LSTM in Keras" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook, we use an LSTM to classify IMDB movie reviews by their sentiment." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Load dependencies" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import keras\n", 31 | "from keras.datasets import imdb\n", 32 | "from keras.preprocessing.sequence import pad_sequences\n", 33 | "from keras.models import Sequential\n", 34 | "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D\n", 35 | "from keras.layers import LSTM # new! \n", 36 | "from keras.callbacks import ModelCheckpoint\n", 37 | "import os\n", 38 | "from sklearn.metrics import roc_auc_score \n", 39 | "import matplotlib.pyplot as plt \n", 40 | "%matplotlib inline" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "#### Set hyperparameters" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": { 54 | "collapsed": true 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "# output directory name:\n", 59 | "output_dir = 'model_output/vanillaLSTM'\n", 60 | "\n", 61 | "# training:\n", 62 | "epochs = 4\n", 63 | "batch_size = 128\n", 64 | "\n", 65 | "# vector-space embedding: \n", 66 | "n_dim = 64 \n", 67 | "n_unique_words = 10000 \n", 68 | "max_review_length = 100 # lowered due to vanishing gradient over time\n", 69 | "pad_type = trunc_type = 'pre'\n", 70 | "drop_embed = 0.2 \n", 71 | "\n", 72 | "# LSTM layer architecture:\n", 73 | "n_lstm = 256 \n", 74 | "drop_lstm = 0.2\n", 75 | "\n", 76 | "# dense layer architecture: \n", 77 | "# n_dense = 256\n", 78 | "# dropout = 0.2" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "#### Load data" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": { 92 | "collapsed": true 93 | }, 94 | "outputs": [], 95 | "source": [ 96 | "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "#### Preprocess data" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": { 110 | "collapsed": true 111 | }, 112 | "outputs": [], 113 | "source": [ 114 | "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n", 115 | "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": { 121 | "collapsed": true 122 | }, 123 | "source": [ 124 | "#### Design neural network architecture" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": { 131 | "collapsed": true 132 | }, 133 | "outputs": [], 134 | "source": [ 135 | "model = Sequential()\n", 136 | "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n", 137 | "model.add(SpatialDropout1D(drop_embed))\n", 138 | "\n", 139 | "# CODE HERE\n", 140 | "\n", 141 | "model.add(Dense(1, activation='sigmoid'))" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "model.summary() " 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "#### Configure model" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": null, 163 | "metadata": { 164 | "collapsed": true 165 | }, 166 | "outputs": [], 167 | "source": [ 168 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "metadata": { 175 | "collapsed": true 176 | }, 177 | "outputs": [], 178 | "source": [ 179 | "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n", 180 | "if not os.path.exists(output_dir):\n", 181 | " os.makedirs(output_dir)" 182 | ] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": {}, 187 | "source": [ 188 | "#### Train!" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": null, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "# go have a gander at nvidia-smi\n", 198 | "# 85.2% validation accuracy in epoch 2\n", 199 | "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])" 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": { 205 | "collapsed": true 206 | }, 207 | "source": [ 208 | "#### Evaluate" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": null, 214 | "metadata": { 215 | "collapsed": true 216 | }, 217 | "outputs": [], 218 | "source": [ 219 | "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "metadata": {}, 226 | "outputs": [], 227 | "source": [ 228 | "y_hat = model.predict_proba(x_valid)" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": {}, 235 | "outputs": [], 236 | "source": [ 237 | "plt.hist(y_hat)\n", 238 | "_ = plt.axvline(x=0.5, color='orange')" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [ 247 | "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": null, 253 | "metadata": { 254 | "collapsed": true 255 | }, 256 | "outputs": [], 257 | "source": [] 258 | } 259 | ], 260 | "metadata": { 261 | "kernelspec": { 262 | "display_name": "Python 3", 263 | "language": "python", 264 | "name": "python3" 265 | }, 266 | "language_info": { 267 | "codemirror_mode": { 268 | "name": "ipython", 269 | "version": 3 270 | }, 271 | "file_extension": ".py", 272 | "mimetype": "text/x-python", 273 | "name": "python", 274 | "nbconvert_exporter": "python", 275 | "pygments_lexer": "ipython3", 276 | "version": "3.6.1" 277 | } 278 | }, 279 | "nbformat": 4, 280 | "nbformat_minor": 2 281 | } 282 | -------------------------------------------------------------------------------- /notebooks/sigmoid_function.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "from math import e" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 3, 17 | "metadata": { 18 | "collapsed": true 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "def sigmoid(x):\n", 23 | " return 1/(1+e**-x)" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 4, 29 | "metadata": {}, 30 | "outputs": [ 31 | { 32 | "data": { 33 | "text/plain": [ 34 | "0.5000024999999999" 35 | ] 36 | }, 37 | "execution_count": 4, 38 | "metadata": {}, 39 | "output_type": "execute_result" 40 | } 41 | ], 42 | "source": [ 43 | "sigmoid(.00001)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 5, 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "data": { 53 | "text/plain": [ 54 | "1.0" 55 | ] 56 | }, 57 | "execution_count": 5, 58 | "metadata": {}, 59 | "output_type": "execute_result" 60 | } 61 | ], 62 | "source": [ 63 | "sigmoid(10000)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 7, 69 | "metadata": {}, 70 | "outputs": [ 71 | { 72 | "data": { 73 | "text/plain": [ 74 | "0.2689414213699951" 75 | ] 76 | }, 77 | "execution_count": 7, 78 | "metadata": {}, 79 | "output_type": "execute_result" 80 | } 81 | ], 82 | "source": [ 83 | "sigmoid(-1)" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 8, 89 | "metadata": {}, 90 | "outputs": [ 91 | { 92 | "data": { 93 | "text/plain": [ 94 | "4.539786870243442e-05" 95 | ] 96 | }, 97 | "execution_count": 8, 98 | "metadata": {}, 99 | "output_type": "execute_result" 100 | } 101 | ], 102 | "source": [ 103 | "sigmoid(-10)" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": { 110 | "collapsed": true 111 | }, 112 | "outputs": [], 113 | "source": [] 114 | } 115 | ], 116 | "metadata": { 117 | "kernelspec": { 118 | "display_name": "Python 3", 119 | "language": "python", 120 | "name": "python3" 121 | }, 122 | "language_info": { 123 | "codemirror_mode": { 124 | "name": "ipython", 125 | "version": 3 126 | }, 127 | "file_extension": ".py", 128 | "mimetype": "text/x-python", 129 | "name": "python", 130 | "nbconvert_exporter": "python", 131 | "pygments_lexer": "ipython3", 132 | "version": "3.5.2" 133 | } 134 | }, 135 | "nbformat": 4, 136 | "nbformat_minor": 2 137 | } 138 | -------------------------------------------------------------------------------- /notebooks/softmax_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "from math import exp" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 13, 17 | "metadata": { 18 | "collapsed": true 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "a = [-1.0, 1.0, 5.0]" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 18, 28 | "metadata": {}, 29 | "outputs": [ 30 | { 31 | "data": { 32 | "text/plain": [ 33 | "0.36787944117144233" 34 | ] 35 | }, 36 | "execution_count": 18, 37 | "metadata": {}, 38 | "output_type": "execute_result" 39 | } 40 | ], 41 | "source": [ 42 | "exp(a[0])" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 19, 48 | "metadata": {}, 49 | "outputs": [ 50 | { 51 | "data": { 52 | "text/plain": [ 53 | "2.718281828459045" 54 | ] 55 | }, 56 | "execution_count": 19, 57 | "metadata": {}, 58 | "output_type": "execute_result" 59 | } 60 | ], 61 | "source": [ 62 | "exp(a[1])" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 20, 68 | "metadata": {}, 69 | "outputs": [ 70 | { 71 | "data": { 72 | "text/plain": [ 73 | "148.4131591025766" 74 | ] 75 | }, 76 | "execution_count": 20, 77 | "metadata": {}, 78 | "output_type": "execute_result" 79 | } 80 | ], 81 | "source": [ 82 | "exp(a[2])" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 21, 88 | "metadata": { 89 | "collapsed": true 90 | }, 91 | "outputs": [], 92 | "source": [ 93 | "sigma = exp(a[0]) + exp(a[1]) + exp(a[2])" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 22, 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "data": { 103 | "text/plain": [ 104 | "151.49932037220708" 105 | ] 106 | }, 107 | "execution_count": 22, 108 | "metadata": {}, 109 | "output_type": "execute_result" 110 | } 111 | ], 112 | "source": [ 113 | "sigma" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 23, 119 | "metadata": {}, 120 | "outputs": [ 121 | { 122 | "data": { 123 | "text/plain": [ 124 | "0.0024282580295913376" 125 | ] 126 | }, 127 | "execution_count": 23, 128 | "metadata": {}, 129 | "output_type": "execute_result" 130 | } 131 | ], 132 | "source": [ 133 | "exp(a[0])/sigma" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 24, 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "data": { 143 | "text/plain": [ 144 | "0.9796292071670795" 145 | ] 146 | }, 147 | "execution_count": 24, 148 | "metadata": {}, 149 | "output_type": "execute_result" 150 | } 151 | ], 152 | "source": [ 153 | "exp(a[2])/sigma" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": { 160 | "collapsed": true 161 | }, 162 | "outputs": [], 163 | "source": [] 164 | } 165 | ], 166 | "metadata": { 167 | "kernelspec": { 168 | "display_name": "Python 3", 169 | "language": "python", 170 | "name": "python3" 171 | }, 172 | "language_info": { 173 | "codemirror_mode": { 174 | "name": "ipython", 175 | "version": 3 176 | }, 177 | "file_extension": ".py", 178 | "mimetype": "text/x-python", 179 | "name": "python", 180 | "nbconvert_exporter": "python", 181 | "pygments_lexer": "ipython3", 182 | "version": "3.5.2" 183 | } 184 | }, 185 | "nbformat": 4, 186 | "nbformat_minor": 2 187 | } 188 | --------------------------------------------------------------------------------