├── .gitignore ├── LICENSE ├── README.md ├── notebooks ├── chapter_02 │ └── 01 - A first look at a neural network.ipynb ├── chapter_03 │ ├── 01 - Binary classifier.ipynb │ ├── 02 - Multiclass classifier.ipynb │ └── 03 - Logistic regression.ipynb ├── chapter_04 │ ├── 01 - Binary classifier - mitigate overfitting.ipynb │ └── resources │ │ └── binary_class_model_validation_loss.png ├── chapter_05 │ ├── 01 - Introduction to CNN.ipynb │ ├── 02 - Using convnets with small datasets.ipynb │ ├── 03 - Using a pretrained CNN.ipynb │ ├── 04 - Visualizing what CNN learn.ipynb │ ├── datasets │ │ └── create_cats_dogs_datasets.py │ └── resources │ │ ├── creative_commons_elephant.jpg │ │ └── elephant_cam.jpg ├── chapter_06 │ ├── 01 - One-hot encoding of words or characters.ipynb │ ├── 02 - Using word embeddings.ipynb │ ├── 03 - Understanding RNNs.ipynb │ ├── 04 - Advanced use of RNNs.ipynb │ ├── 05 - Sequence processing with convnets.ipynb │ └── resources │ │ ├── RNN-basic-schema.png │ │ ├── bidirectional-RNN.png │ │ └── onehot-vs-embedding.png ├── chapter_07 │ ├── 01 - The Keras functional API.ipynb │ └── resources │ │ ├── inception-module.png │ │ ├── question-answering-model.png │ │ └── three-outputs-model.png └── chapter_08 │ ├── 01 - Text generation with LSTM.ipynb │ ├── 02 - Deep Dream.ipynb │ ├── 03 - Neural style transfer.ipynb │ ├── 04 - Variational autoencoders.ipynb │ ├── 05 - Introduction to GANs.ipynb │ ├── resources │ ├── kandinsky_style.jpg │ ├── riva.jpg │ └── vae.png │ └── results │ ├── dream_final.jpg │ └── gan │ ├── gan.h5 │ ├── generated_frog0.png │ ├── generated_frog100.png │ ├── generated_frog1000.png │ ├── generated_frog1100.png │ ├── generated_frog1200.png │ ├── generated_frog1300.png │ ├── generated_frog1400.png │ ├── generated_frog1500.png │ ├── generated_frog1600.png │ ├── generated_frog1700.png │ ├── generated_frog1800.png │ ├── generated_frog1900.png │ ├── generated_frog200.png │ ├── generated_frog2000.png │ ├── generated_frog2100.png │ ├── generated_frog2200.png │ ├── generated_frog2300.png │ ├── generated_frog2400.png │ ├── generated_frog2500.png │ ├── generated_frog2600.png │ ├── generated_frog2700.png │ ├── generated_frog2800.png │ ├── generated_frog2900.png │ ├── generated_frog300.png │ ├── generated_frog3000.png │ ├── generated_frog3100.png │ ├── generated_frog3200.png │ ├── generated_frog3300.png │ ├── generated_frog3400.png │ ├── generated_frog3500.png │ ├── generated_frog3600.png │ ├── generated_frog3700.png │ ├── generated_frog3800.png │ ├── generated_frog3900.png │ ├── generated_frog400.png │ ├── generated_frog4000.png │ ├── generated_frog4100.png │ ├── generated_frog4200.png │ ├── generated_frog4300.png │ ├── generated_frog4400.png │ ├── generated_frog4500.png │ ├── generated_frog4600.png │ ├── generated_frog4700.png │ ├── generated_frog4800.png │ ├── generated_frog4900.png │ ├── generated_frog500.png │ ├── generated_frog5000.png │ ├── generated_frog5100.png │ ├── generated_frog5200.png │ ├── generated_frog5300.png │ ├── generated_frog5400.png │ ├── generated_frog5500.png │ ├── generated_frog5600.png │ ├── generated_frog5700.png │ ├── generated_frog5800.png │ ├── generated_frog5900.png │ ├── generated_frog600.png │ ├── generated_frog6000.png │ ├── generated_frog6100.png │ ├── generated_frog6200.png │ ├── generated_frog6300.png │ ├── generated_frog6400.png │ ├── generated_frog6500.png │ ├── generated_frog6600.png │ ├── generated_frog6700.png │ ├── generated_frog6800.png │ ├── generated_frog6900.png │ ├── generated_frog700.png │ ├── generated_frog7000.png │ ├── generated_frog7100.png │ ├── generated_frog7200.png │ ├── generated_frog7300.png │ ├── generated_frog7400.png │ ├── generated_frog7500.png │ ├── generated_frog7600.png │ ├── generated_frog7700.png │ ├── generated_frog7800.png │ ├── generated_frog7900.png │ ├── generated_frog800.png │ ├── generated_frog8000.png │ ├── generated_frog8100.png │ ├── generated_frog8200.png │ ├── generated_frog8300.png │ ├── generated_frog8400.png │ ├── generated_frog8500.png │ ├── generated_frog8600.png │ ├── generated_frog8700.png │ ├── generated_frog8800.png │ ├── generated_frog8900.png │ ├── generated_frog900.png │ ├── generated_frog9000.png │ ├── generated_frog9100.png │ ├── generated_frog9200.png │ ├── generated_frog9300.png │ ├── generated_frog9400.png │ ├── generated_frog9500.png │ ├── generated_frog9600.png │ ├── generated_frog9700.png │ ├── generated_frog9800.png │ ├── generated_frog9900.png │ ├── real_frog0.png │ ├── real_frog100.png │ ├── real_frog1000.png │ ├── real_frog1100.png │ ├── real_frog1200.png │ ├── real_frog1300.png │ ├── real_frog1400.png │ ├── real_frog1500.png │ ├── real_frog1600.png │ ├── real_frog1700.png │ ├── real_frog1800.png │ ├── real_frog1900.png │ ├── real_frog200.png │ ├── real_frog2000.png │ ├── real_frog2100.png │ ├── real_frog2200.png │ ├── real_frog2300.png │ ├── real_frog2400.png │ ├── real_frog2500.png │ ├── real_frog2600.png │ ├── real_frog2700.png │ ├── real_frog2800.png │ ├── real_frog2900.png │ ├── real_frog300.png │ ├── real_frog3000.png │ ├── real_frog3100.png │ ├── real_frog3200.png │ ├── real_frog3300.png │ ├── real_frog3400.png │ ├── real_frog3500.png │ ├── real_frog3600.png │ ├── real_frog3700.png │ ├── real_frog3800.png │ ├── real_frog3900.png │ ├── real_frog400.png │ ├── real_frog4000.png │ ├── real_frog4100.png │ ├── real_frog4200.png │ ├── real_frog4300.png │ ├── real_frog4400.png │ ├── real_frog4500.png │ ├── real_frog4600.png │ ├── real_frog4700.png │ ├── real_frog4800.png │ ├── real_frog4900.png │ ├── real_frog500.png │ ├── real_frog5000.png │ ├── real_frog5100.png │ ├── real_frog5200.png │ ├── real_frog5300.png │ ├── real_frog5400.png │ ├── real_frog5500.png │ ├── real_frog5600.png │ ├── real_frog5700.png │ ├── real_frog5800.png │ ├── real_frog5900.png │ ├── real_frog600.png │ ├── real_frog6000.png │ ├── real_frog6100.png │ ├── real_frog6200.png │ ├── real_frog6300.png │ ├── real_frog6400.png │ ├── real_frog6500.png │ ├── real_frog6600.png │ ├── real_frog6700.png │ ├── real_frog6800.png │ ├── real_frog6900.png │ ├── real_frog700.png │ ├── real_frog7000.png │ ├── real_frog7100.png │ ├── real_frog7200.png │ ├── real_frog7300.png │ ├── real_frog7400.png │ ├── real_frog7500.png │ ├── real_frog7600.png │ ├── real_frog7700.png │ ├── real_frog7800.png │ ├── real_frog7900.png │ ├── real_frog800.png │ ├── real_frog8000.png │ ├── real_frog8100.png │ ├── real_frog8200.png │ ├── real_frog8300.png │ ├── real_frog8400.png │ ├── real_frog8500.png │ ├── real_frog8600.png │ ├── real_frog8700.png │ ├── real_frog8800.png │ ├── real_frog8900.png │ ├── real_frog900.png │ ├── real_frog9000.png │ ├── real_frog9100.png │ ├── real_frog9200.png │ ├── real_frog9300.png │ ├── real_frog9400.png │ ├── real_frog9500.png │ ├── real_frog9600.png │ ├── real_frog9700.png │ ├── real_frog9800.png │ └── real_frog9900.png └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Models 2 | models/ 3 | 4 | # Datasets 5 | cats_and_dogs*/ 6 | aclImdb* 7 | glove*6B* 8 | jena* 9 | Kaggle* 10 | 11 | # IDE and tools 12 | .ipynb_checkpoints 13 | .vscode 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Luca Ballore 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow 2.* exercises for the book "Deep Learning with Python" by François Chollet. 2 | This repository contains Jupyter notebooks implementing the code samples found in the book [Deep Learning with Python (Manning Pubblications)](https://www.manning.com/books/deep-learning-with-python?a_aid=keras&a_bid=76564dff) by [François Chollet](https://github.com/fchollet), gathered by chapter. 3 | 4 | ## Disclaimer 5 | The author [already published notebooks with the exercises in his Github account](https://github.com/fchollet/deep-learning-with-python-notebooks). Those you can find in this repository are made by myself for my own learning, with no intention so steal any intellectual property. **Exercises that worked flawlessly in TF 2 have been simply copied**. 6 | 7 | ## Requirements 8 | - Create a virtual environment with `conda` or `virtualenv`. Recommended `conda` with Python 3.6. 9 | ``` 10 | conda create --name python=3.6 11 | ``` 12 | - Install dependencies: 13 | ``` 14 | pip install -r requirements.txt 15 | ``` 16 | 17 | ## Contents 18 | The exercises in the book are written for `tensorflow 1.*` and `Keras 2.0.8`. All the code in this repo have been rewritten to work with `tensorflow 2.2.*` and the corresponding Keras version `2.2.4-tf`. 19 | 20 | --- 21 | 22 | ### Chapter 1. - What is deep learning? 23 | No relevant exercises 24 | 25 | ### Chapter 2. - Before we begin: the mathematical building blocks of neural networks 26 | - 01: a first look at a neural network - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_02/01%20-%20A%20first%20look%20at%20a%20neural%20network.ipynb) 27 | 28 | ### Chapter 3. - Getting started with neural networks 29 | - 01: binary classification - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_03/01%20-%20Binary%20classifier.ipynb) 30 | - 02: multiclass classification - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_03/02%20-%20Multiclass%20classifier.ipynb) 31 | - 03: logistic regression - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_03/03%20-%20Logistic%20regression.ipynb) 32 | 33 | ### Chapter 4. - Fundamentals of machine learning 34 | - 01: binary classification: mitigate overfitting and underfitting - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_04/01%20-%20Binary%20classifier%20-%20mitigate%20overfitting.ipynb) 35 | 36 | ### Chapter 5. - Deep learning for computer vision 37 | - 01: introduction to CNN - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_05/01%20-%20Introduction%20to%20CNN.ipynb) 38 | - 02: using CNNs with small datasets - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_05/02%20-%20Using%20convnets%20with%20small%20datasets.ipynb) 39 | - 03: using a pretrained CNN - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_05/03%20-%20Using%20a%20pretrained%20CNN.ipynb) 40 | - 04: visualizing what a CNN learn - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_05/04%20-%20Visualizing%20what%20CNN%20learn.ipynb) 41 | 42 | ### Chapter 6. - Deep learning for text and sequences 43 | - 01: one-hot encoding of words or characters - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_06/01%20-%20One-hot%20encoding%20of%20words%20or%20characters.ipynb) 44 | - 02: using word embeddings - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_06/02%20-%20Using%20word%20embeddings.ipynb) 45 | - 03: understanding RNNs - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_06/03%20-%20Understanding%20RNNs.ipynb) 46 | - 04: advanced use of RNNs - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_06/04%20-%20Advanced%20use%20of%20RNNs.ipynb) 47 | - 05: sequence processing with convnets - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_06/05%20-%20Sequence%20processing%20with%20convnets.ipynb) 48 | 49 | ### Chapter 7. - Advanced deep-learning best practices 50 | - 01: the Keras functional API - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_07/01%20-%20The%20Keras%20functional%20API.ipynb) 51 | 52 | ### Chapter 8. - Generative deep learning 53 | - 01: text generation with LSTM - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_08/01%20-%20Text%20generation%20with%20LSTM.ipynb) 54 | - 02: Deep Dream - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_08/02%20-%20Deep%20Dream.ipynb) 55 | - 03: neural style transfer - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_08/03%20-%20Neural%20style%20transfer.ipynb) 56 | - 04: variational autoencoders (VAEs) - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_08/04%20-%20Variational%20autoencoders.ipynb) 57 | - 05: introduction to GANs - [notebook](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_08/05%20-%20Introduction%20to%20GANs.ipynb) 58 | -------------------------------------------------------------------------------- /notebooks/chapter_02/01 - A first look at a neural network.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# A first look at a neural network\n", 8 | "\n", 9 | "The problem is about **classifying grayscale images of handwritten digits** (28 pixels by 28 pixels), into their 10 categories (0 to 9). The dataset we will use is the MNIST dataset. It's a set of 60,000 training images, plus 10,000 test images, assembled by the National Institute of Standards and Technology (the NIST in MNIST) in the 1980s.\n", 10 | "\n", 11 | "The MNIST dataset comes pre-loaded in Keras, in the form of a set of four Numpy arrays." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 12, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import numpy as np\n", 21 | "import matplotlib.pyplot as plt\n", 22 | "import tensorflow as tf\n", 23 | "\n", 24 | "from tensorflow.keras import models, layers, optimizers, losses, metrics\n", 25 | "from tensorflow.keras.datasets import mnist\n", 26 | "from tensorflow.keras.utils import to_categorical" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "## Load dataset\n", 34 | "\n", 35 | "`train_images` and `train_labels` form the \"training set\", the data that the model will learn from. The model will then be tested on the \"test set\", `test_images` and `test_labels`. The images are encoded as Numpy arrays, and the labels are simply an array of digits, ranging from 0 to 9. There is a one-to-one correspondence between the images and the labels." 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 2, 41 | "metadata": {}, 42 | "outputs": [ 43 | { 44 | "name": "stdout", 45 | "output_type": "stream", 46 | "text": [ 47 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n", 48 | "11493376/11490434 [==============================] - 1s 0us/step\n" 49 | ] 50 | } 51 | ], 52 | "source": [ 53 | "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 3, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "text/plain": [ 64 | "(60000, 28, 28)" 65 | ] 66 | }, 67 | "execution_count": 3, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | } 71 | ], 72 | "source": [ 73 | "train_images.shape" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 4, 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "data": { 83 | "text/plain": [ 84 | "60000" 85 | ] 86 | }, 87 | "execution_count": 4, 88 | "metadata": {}, 89 | "output_type": "execute_result" 90 | } 91 | ], 92 | "source": [ 93 | "len(train_labels)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 5, 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "data": { 103 | "text/plain": [ 104 | "array([5, 0, 4, ..., 5, 6, 8], dtype=uint8)" 105 | ] 106 | }, 107 | "execution_count": 5, 108 | "metadata": {}, 109 | "output_type": "execute_result" 110 | } 111 | ], 112 | "source": [ 113 | "train_labels" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 6, 119 | "metadata": {}, 120 | "outputs": [ 121 | { 122 | "data": { 123 | "text/plain": [ 124 | "(10000, 28, 28)" 125 | ] 126 | }, 127 | "execution_count": 6, 128 | "metadata": {}, 129 | "output_type": "execute_result" 130 | } 131 | ], 132 | "source": [ 133 | "test_images.shape" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 7, 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "data": { 143 | "text/plain": [ 144 | "10000" 145 | ] 146 | }, 147 | "execution_count": 7, 148 | "metadata": {}, 149 | "output_type": "execute_result" 150 | } 151 | ], 152 | "source": [ 153 | "len(test_labels)" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 8, 159 | "metadata": {}, 160 | "outputs": [ 161 | { 162 | "data": { 163 | "text/plain": [ 164 | "array([7, 2, 1, ..., 4, 5, 6], dtype=uint8)" 165 | ] 166 | }, 167 | "execution_count": 8, 168 | "metadata": {}, 169 | "output_type": "execute_result" 170 | } 171 | ], 172 | "source": [ 173 | "test_labels" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "metadata": {}, 179 | "source": [ 180 | "## Build the network\n", 181 | "\n", 182 | "- One dense layer with 512 units;\n", 183 | "- A final layer with `softmax` activation function;\n", 184 | "\n", 185 | "The core building block of neural networks is the \"layer\", a data-processing module which you can conceive as a \"filter\" for data. Some data comes in, and comes out in a more useful form. Precisely, layers extract representations out of the data fed into them -- hopefully representations that are more meaningful for the problem at hand. Most of deep learning really consists of chaining together simple layers which will implement a form of progressive \"data distillation\". A deep learning model is like a sieve for data processing, made of a succession of increasingly refined data filters -- the \"layers\".\n", 186 | "\n", 187 | "Here our network consists of a sequence of two Dense layers, which are densely-connected (also called \"fully-connected\") neural layers. The second (and last) layer is a 10-way \"softmax\" layer, which means it will return an array of 10 probability scores (summing to 1). Each score will be the probability that the current digit image belongs to one of our 10 digit classes.\n", 188 | "\n", 189 | "To make our network ready for training, we need to pick three more things, as part of \"compilation\" step:\n", 190 | "\n", 191 | "A loss function: the is how the network will be able to measure how good a job it is doing on its training data, and thus how it will be able to steer itself in the right direction.\n", 192 | "An optimizer: this is the mechanism through which the network will update itself based on the data it sees and its loss function.\n", 193 | "Metrics to monitor during training and testing. Here we will only care about accuracy (the fraction of the images that were correctly classified).\n", 194 | "The exact purpose of the loss function and the optimizer will be made clear throughout the next two chapters." 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 10, 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "model = models.Sequential()\n", 204 | "model.add(layers.Dense(512, activation='relu', input_shape=(28 * 28,)))\n", 205 | "model.add(layers.Dense(10, activation='softmax'))" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "#### Compile model" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": 11, 218 | "metadata": {}, 219 | "outputs": [], 220 | "source": [ 221 | "model.compile(\n", 222 | " optimizer='rmsprop',\n", 223 | " loss='categorical_crossentropy',\n", 224 | " metrics=['accuracy']\n", 225 | ")" 226 | ] 227 | }, 228 | { 229 | "cell_type": "markdown", 230 | "metadata": {}, 231 | "source": [ 232 | "## Preprocess data\n", 233 | "\n", 234 | "Before training, we will preprocess our data by reshaping it into the shape that the network expects, and scaling it so that all values are in the [0, 1] interval. Previously, our training images for instance were stored in an array of shape (60000, 28, 28) of type uint8 with values in the [0, 255] interval. We transform it into a float32 array of shape (60000, 28 * 28) with values between 0 and 1.\n", 235 | "\n", 236 | "We do also need to categorically encode the labels (one-hot)" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": 13, 242 | "metadata": {}, 243 | "outputs": [], 244 | "source": [ 245 | "train_images = train_images.reshape((60000, 28 * 28))\n", 246 | "train_images = train_images.astype('float32') / 255\n", 247 | "\n", 248 | "test_images = test_images.reshape((10000, 28 * 28))\n", 249 | "test_images = test_images.astype('float32') / 255\n", 250 | "\n", 251 | "train_labels = to_categorical(train_labels)\n", 252 | "test_labels = to_categorical(test_labels)" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": {}, 258 | "source": [ 259 | "## Train the model\n", 260 | "\n", 261 | "Two quantities are being displayed during training: the \"loss\" of the network over the training data, and the accuracy of the network over the training data" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 14, 267 | "metadata": {}, 268 | "outputs": [ 269 | { 270 | "name": "stdout", 271 | "output_type": "stream", 272 | "text": [ 273 | "Train on 60000 samples\n", 274 | "Epoch 1/5\n", 275 | "60000/60000 [==============================] - 2s 30us/sample - loss: 0.2571 - accuracy: 0.9253\n", 276 | "Epoch 2/5\n", 277 | "60000/60000 [==============================] - 1s 23us/sample - loss: 0.1039 - accuracy: 0.9686\n", 278 | "Epoch 3/5\n", 279 | "60000/60000 [==============================] - 1s 25us/sample - loss: 0.0671 - accuracy: 0.9797\n", 280 | "Epoch 4/5\n", 281 | "60000/60000 [==============================] - 1s 23us/sample - loss: 0.0498 - accuracy: 0.9849\n", 282 | "Epoch 5/5\n", 283 | "60000/60000 [==============================] - 1s 23us/sample - loss: 0.0367 - accuracy: 0.9890\n" 284 | ] 285 | }, 286 | { 287 | "data": { 288 | "text/plain": [ 289 | "" 290 | ] 291 | }, 292 | "execution_count": 14, 293 | "metadata": {}, 294 | "output_type": "execute_result" 295 | } 296 | ], 297 | "source": [ 298 | "model.fit(train_images, train_labels, epochs=5, batch_size=128)" 299 | ] 300 | }, 301 | { 302 | "cell_type": "markdown", 303 | "metadata": {}, 304 | "source": [ 305 | "## Evaluation" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 15, 311 | "metadata": {}, 312 | "outputs": [ 313 | { 314 | "name": "stdout", 315 | "output_type": "stream", 316 | "text": [ 317 | "10000/10000 [==============================] - 0s 38us/sample - loss: 0.0687 - accuracy: 0.9792\n" 318 | ] 319 | } 320 | ], 321 | "source": [ 322 | "test_loss, test_acc = model.evaluate(test_images, test_labels)" 323 | ] 324 | }, 325 | { 326 | "cell_type": "code", 327 | "execution_count": 16, 328 | "metadata": {}, 329 | "outputs": [ 330 | { 331 | "name": "stdout", 332 | "output_type": "stream", 333 | "text": [ 334 | "test_acc: 0.9792\n" 335 | ] 336 | } 337 | ], 338 | "source": [ 339 | "print('test_acc:', test_acc)" 340 | ] 341 | }, 342 | { 343 | "cell_type": "markdown", 344 | "metadata": {}, 345 | "source": [ 346 | "The test set accuracy turns out to be *97.8%* -- that's quite a bit lower than the training set accuracy. This gap between training accuracy and test accuracy is an example of \"overfitting\", the fact that machine learning models tend to perform worse on new data than on their training data." 347 | ] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "execution_count": null, 352 | "metadata": {}, 353 | "outputs": [], 354 | "source": [] 355 | } 356 | ], 357 | "metadata": { 358 | "kernelspec": { 359 | "display_name": "Python 3", 360 | "language": "python", 361 | "name": "python3" 362 | }, 363 | "language_info": { 364 | "codemirror_mode": { 365 | "name": "ipython", 366 | "version": 3 367 | }, 368 | "file_extension": ".py", 369 | "mimetype": "text/x-python", 370 | "name": "python", 371 | "nbconvert_exporter": "python", 372 | "pygments_lexer": "ipython3", 373 | "version": "3.6.10" 374 | } 375 | }, 376 | "nbformat": 4, 377 | "nbformat_minor": 4 378 | } 379 | -------------------------------------------------------------------------------- /notebooks/chapter_03/02 - Multiclass classifier.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Multiclass classifier\n", 8 | "\n", 9 | "Classify articles from a Reuters dataset having 46 different categories" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "### Import libraries" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 1, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "import numpy as np\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "import tensorflow as tf\n", 28 | "\n", 29 | "from tensorflow.keras import models, layers, optimizers, losses, metrics\n", 30 | "from tensorflow.keras.datasets import reuters\n", 31 | "from tensorflow.keras.utils import to_categorical" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "### Get datasets" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 2, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "LIMIT_WORD = 10000\n", 48 | "\n", 49 | "(train_data, train_labels), (test_data, test_labels) = reuters.load_data(num_words=LIMIT_WORD)" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "### Data exploring (convert sequences to original review)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 3, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "def convert_to_english(sequence):\n", 66 | " word_index = reuters.get_word_index()\n", 67 | " reverse_word_index = dict(\n", 68 | " [(value, key) for (key, value) in word_index.items()]\n", 69 | " )\n", 70 | " decoded_review = \" \".join(\n", 71 | " [reverse_word_index.get(i - 3, '?') for i in sequence]\n", 72 | " )\n", 73 | " \n", 74 | " return decoded_review" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 4, 80 | "metadata": {}, 81 | "outputs": [ 82 | { 83 | "name": "stdout", 84 | "output_type": "stream", 85 | "text": [ 86 | "? ? ? said as a result of its december acquisition of space co it expects earnings per share in 1987 of 1 15 to 1 30 dlrs per share up from 70 cts in 1986 the company said pretax net should rise to nine to 10 mln dlrs from six mln dlrs in 1986 and rental operation revenues to 19 to 22 mln dlrs from 12 5 mln dlrs it said cash flow per share this year should be 2 50 to three dlrs reuter 3\n" 87 | ] 88 | } 89 | ], 90 | "source": [ 91 | "print(convert_to_english(train_data[0]))" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "### Pre-process the data (convert sequences into tensors)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 5, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "def vectorize_sequences(sequences, dimension=LIMIT_WORD):\n", 108 | " results = np.zeros((len(sequences), dimension))\n", 109 | " for i, sequence in enumerate(sequences):\n", 110 | " results[i, sequence] = 1\n", 111 | " \n", 112 | " return results" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "#### Vectorize examples" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 6, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "x_train = vectorize_sequences(train_data)\n", 129 | "x_test = vectorize_sequences(test_data)" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 7, 135 | "metadata": {}, 136 | "outputs": [ 137 | { 138 | "data": { 139 | "text/plain": [ 140 | "array([0., 1., 1., ..., 0., 0., 0.])" 141 | ] 142 | }, 143 | "execution_count": 7, 144 | "metadata": {}, 145 | "output_type": "execute_result" 146 | } 147 | ], 148 | "source": [ 149 | "x_train[0]" 150 | ] 151 | }, 152 | { 153 | "cell_type": "markdown", 154 | "metadata": {}, 155 | "source": [ 156 | "#### One-hot encoding of labels (categorical encoding)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 8, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "one_hot_train_labels = to_categorical(train_labels)\n", 166 | "one_hot_test_labels = to_categorical(test_labels)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 9, 172 | "metadata": {}, 173 | "outputs": [ 174 | { 175 | "data": { 176 | "text/plain": [ 177 | "array([0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", 178 | " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", 179 | " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32)" 180 | ] 181 | }, 182 | "execution_count": 9, 183 | "metadata": {}, 184 | "output_type": "execute_result" 185 | } 186 | ], 187 | "source": [ 188 | "one_hot_train_labels[0]" 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": {}, 194 | "source": [ 195 | "### Building the network\n", 196 | "\n", 197 | "#### Architecture\n", 198 | "\n", 199 | "- 2 intermediate Dense layers with 16 hidden unites per each (relu actovation function)\n", 200 | "- 1 output layer (with sigmoid activation function)" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 10, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [ 209 | "model = models.Sequential()\n", 210 | "model.add(layers.Dense(64, activation='relu', input_shape=(LIMIT_WORD,)))\n", 211 | "model.add(layers.Dense(64, activation='relu'))\n", 212 | "model.add(layers.Dense(46, activation='softmax'))" 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "metadata": {}, 218 | "source": [ 219 | "#### Compile\n", 220 | "\n", 221 | "**Loss function:** _binary crossentropy_\n", 222 | "**Optimizer:** _rmsprop_" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 11, 228 | "metadata": {}, 229 | "outputs": [], 230 | "source": [ 231 | "model.compile(\n", 232 | " optimizer=optimizers.RMSprop(lr=0.001),\n", 233 | " loss=losses.categorical_crossentropy,\n", 234 | " metrics=[metrics.categorical_accuracy]\n", 235 | ")" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "metadata": {}, 241 | "source": [ 242 | "#### Training\n", 243 | "- Define validation data\n", 244 | "- Define epochs and batch size\n", 245 | "- Fit the model" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": 12, 251 | "metadata": {}, 252 | "outputs": [], 253 | "source": [ 254 | "x_val = x_train[:1000]\n", 255 | "partial_x_train = x_train[1000:]\n", 256 | "y_val = one_hot_train_labels[:1000]\n", 257 | "partial_y_train = one_hot_train_labels[1000:]" 258 | ] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": 13, 263 | "metadata": {}, 264 | "outputs": [ 265 | { 266 | "name": "stdout", 267 | "output_type": "stream", 268 | "text": [ 269 | "Train on 7982 samples, validate on 1000 samples\n", 270 | "Epoch 1/20\n", 271 | "7982/7982 [==============================] - 1s 116us/sample - loss: 2.5877 - categorical_accuracy: 0.4866 - val_loss: 1.7296 - val_categorical_accuracy: 0.6190\n", 272 | "Epoch 2/20\n", 273 | "7982/7982 [==============================] - 1s 67us/sample - loss: 1.3991 - categorical_accuracy: 0.7038 - val_loss: 1.3148 - val_categorical_accuracy: 0.7090\n", 274 | "Epoch 3/20\n", 275 | "7982/7982 [==============================] - 1s 65us/sample - loss: 1.0326 - categorical_accuracy: 0.7841 - val_loss: 1.1677 - val_categorical_accuracy: 0.7300\n", 276 | "Epoch 4/20\n", 277 | "7982/7982 [==============================] - 0s 61us/sample - loss: 0.8048 - categorical_accuracy: 0.8334 - val_loss: 1.0171 - val_categorical_accuracy: 0.7800\n", 278 | "Epoch 5/20\n", 279 | "7982/7982 [==============================] - 1s 65us/sample - loss: 0.6392 - categorical_accuracy: 0.8678 - val_loss: 0.9491 - val_categorical_accuracy: 0.8020\n", 280 | "Epoch 6/20\n", 281 | "7982/7982 [==============================] - 0s 61us/sample - loss: 0.5050 - categorical_accuracy: 0.8958 - val_loss: 0.9167 - val_categorical_accuracy: 0.7980\n", 282 | "Epoch 7/20\n", 283 | "7982/7982 [==============================] - 0s 59us/sample - loss: 0.4044 - categorical_accuracy: 0.9154 - val_loss: 0.8861 - val_categorical_accuracy: 0.8130\n", 284 | "Epoch 8/20\n", 285 | "7982/7982 [==============================] - 1s 64us/sample - loss: 0.3292 - categorical_accuracy: 0.9301 - val_loss: 0.8768 - val_categorical_accuracy: 0.8130\n", 286 | "Epoch 9/20\n", 287 | "7982/7982 [==============================] - 1s 69us/sample - loss: 0.2738 - categorical_accuracy: 0.9390 - val_loss: 0.8838 - val_categorical_accuracy: 0.8140\n", 288 | "Epoch 10/20\n", 289 | "7982/7982 [==============================] - 0s 63us/sample - loss: 0.2331 - categorical_accuracy: 0.9448 - val_loss: 0.9033 - val_categorical_accuracy: 0.8150\n", 290 | "Epoch 11/20\n", 291 | "7982/7982 [==============================] - 1s 64us/sample - loss: 0.2009 - categorical_accuracy: 0.9500 - val_loss: 0.8862 - val_categorical_accuracy: 0.8150\n", 292 | "Epoch 12/20\n", 293 | "7982/7982 [==============================] - 0s 61us/sample - loss: 0.1801 - categorical_accuracy: 0.9516 - val_loss: 0.9070 - val_categorical_accuracy: 0.8090\n", 294 | "Epoch 13/20\n", 295 | "7982/7982 [==============================] - 0s 60us/sample - loss: 0.1626 - categorical_accuracy: 0.9520 - val_loss: 0.9447 - val_categorical_accuracy: 0.8050\n", 296 | "Epoch 14/20\n", 297 | "7982/7982 [==============================] - 1s 68us/sample - loss: 0.1490 - categorical_accuracy: 0.9549 - val_loss: 0.9549 - val_categorical_accuracy: 0.8060\n", 298 | "Epoch 15/20\n", 299 | "7982/7982 [==============================] - 1s 69us/sample - loss: 0.1421 - categorical_accuracy: 0.9539 - val_loss: 0.9170 - val_categorical_accuracy: 0.8140\n", 300 | "Epoch 16/20\n", 301 | "7982/7982 [==============================] - 1s 63us/sample - loss: 0.1286 - categorical_accuracy: 0.9568 - val_loss: 0.9971 - val_categorical_accuracy: 0.8050\n", 302 | "Epoch 17/20\n", 303 | "7982/7982 [==============================] - 1s 68us/sample - loss: 0.1250 - categorical_accuracy: 0.9560 - val_loss: 1.0387 - val_categorical_accuracy: 0.8000\n", 304 | "Epoch 18/20\n", 305 | "7982/7982 [==============================] - 1s 65us/sample - loss: 0.1236 - categorical_accuracy: 0.9563 - val_loss: 0.9871 - val_categorical_accuracy: 0.8170\n", 306 | "Epoch 19/20\n", 307 | "7982/7982 [==============================] - 1s 66us/sample - loss: 0.1178 - categorical_accuracy: 0.9558 - val_loss: 1.0047 - val_categorical_accuracy: 0.8130\n", 308 | "Epoch 20/20\n", 309 | "7982/7982 [==============================] - 1s 77us/sample - loss: 0.1135 - categorical_accuracy: 0.9565 - val_loss: 1.0720 - val_categorical_accuracy: 0.7960\n" 310 | ] 311 | } 312 | ], 313 | "source": [ 314 | "history = model.fit(\n", 315 | " partial_x_train,\n", 316 | " partial_y_train,\n", 317 | " epochs=20,\n", 318 | " batch_size=512,\n", 319 | " validation_data=(x_val, y_val)\n", 320 | ")" 321 | ] 322 | }, 323 | { 324 | "cell_type": "markdown", 325 | "metadata": {}, 326 | "source": [ 327 | "### Plot results\n", 328 | "\n", 329 | "#### Training and validation loss" 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": 14, 335 | "metadata": {}, 336 | "outputs": [ 337 | { 338 | "data": { 339 | "text/plain": [ 340 | "" 341 | ] 342 | }, 343 | "execution_count": 14, 344 | "metadata": {}, 345 | "output_type": "execute_result" 346 | }, 347 | { 348 | "data": { 349 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZwU1bn/8c8DjsIAomzKIosJgqIwwAAqSlCzCCq44EK4KhJFuC5RExUlUX4m3HsTSa4hrsQ1hghqEqICV6OIuEVZRBTEKGZQFBFRYBBQluf3x6mBnqF7Fmaqu2f6+3696tXVtfXTNT311Dmn6pS5OyIikrvqZToAERHJLCUCEZEcp0QgIpLjlAhERHKcEoGISI5TIhARyXFKBFKjzGy2mV1Y08tmkpkVmdl3Y9ium9m3o/G7zeznlVl2Lz5nhJk9s7dxlrPdgWa2qqa3K+m3T6YDkMwzs00Jb/OBr4Ed0ftL3X1qZbfl7oPiWLauc/cxNbEdM+sI/BvIc/ft0banApX+G0ruUSIQ3L1xybiZFQEXu/uzZZczs31KDi4iUneoakhSKin6m9n1ZvYp8ICZHWhmT5nZWjP7Mhpvl7DOXDO7OBofaWYvmdmkaNl/m9mgvVy2k5nNM7NiM3vWzO4wsz+liLsyMf7CzF6OtveMmbVImH++ma00s3VmNr6c/dPPzD41s/oJ084wsyXReF8ze9XM1pvZajO73cz2TbGtB83slwnvr43W+cTMRpVZ9hQze8PMNprZR2Y2IWH2vOh1vZltMrNjSvZtwvrHmtl8M9sQvR5b2X1THjM7PFp/vZktNbMhCfMGm9myaJsfm9lPo+ktor/PejP7wsxeNDMdl9JMO1wqcjDQDOgAjCb8Zh6I3rcHtgC3l7N+P+BdoAXwa+A+M7O9WPbPwOtAc2ACcH45n1mZGH8IXAS0AvYFSg5MRwB3RdtvE31eO5Jw99eAr4ATy2z3z9H4DuDq6PscA5wE/Gc5cRPFcHIUz/eAzkDZ9omvgAuAA4BTgLFmdno0b0D0eoC7N3b3V8tsuxkwE5gcfbffAjPNrHmZ77DHvqkg5jzgSeCZaL0rgKlm1iVa5D5CNWMT4EhgTjT9J8AqoCVwEHAjoH5v0kyJQCqyE7jZ3b929y3uvs7d/+Lum929GJgIfKec9Ve6+x/cfQfwENCa8A9f6WXNrD3QB7jJ3b9x95eAJ1J9YCVjfMDd/+XuW4BHgYJo+jDgKXef5+5fAz+P9kEqjwDDAcysCTA4moa7L3T3f7r7dncvAu5JEkcy50Txve3uXxESX+L3m+vub7n7TndfEn1eZbYLIXG85+4PR3E9AiwHTktYJtW+Kc/RQGPgf6K/0RzgKaJ9A2wDjjCz/d39S3dflDC9NdDB3be5+4uuDtDSTolAKrLW3beWvDGzfDO7J6o62UioijggsXqkjE9LRtx9czTauIrLtgG+SJgG8FGqgCsZ46cJ45sTYmqTuO3oQLwu1WcRzv7PNLP9gDOBRe6+MorjsKja49Mojv8ilA4qUioGYGWZ79fPzJ6Pqr42AGMqud2Sba8sM20l0Dbhfap9U2HM7p6YNBO3exYhSa40sxfM7Jho+q3A+8AzZvaBmY2r3NeQmqREIBUpe3b2E6AL0M/d92d3VUSq6p6asBpoZmb5CdMOKWf56sS4OnHb0Wc2T7Wwuy8jHPAGUbpaCEIV03KgcxTHjXsTA6F6K9GfCSWiQ9y9KXB3wnYrOpv+hFBllqg98HEl4qpou4eUqd/ftV13n+/uQwnVRjMIJQ3cvdjdf+LuhwJDgGvM7KRqxiJVpEQgVdWEUOe+PqpvvjnuD4zOsBcAE8xs3+hs8rRyVqlOjI8Dp5rZcVHD7i1U/H/yZ+DHhITzWJk4NgKbzKwrMLaSMTwKjDSzI6JEVDb+JoQS0lYz60tIQCXWEqqyDk2x7VnAYWb2QzPbx8zOBY4gVONUx2uE0sN1ZpZnZgMJf6Np0d9shJk1dfdthH2yE8DMTjWzb0dtQRsI7SrlVcVJDJQIpKpuAxoCnwP/BP4vTZ87gtDgug74JTCdcL9DMnsdo7svBS4jHNxXA18SGjPLU1JHP8fdP0+Y/lPCQboY+EMUc2VimB19hzmEapM5ZRb5T+AWMysGbiI6u47W3UxoE3k5uhLn6DLbXgecSig1rQOuA04tE3eVufs3hAP/IMJ+vxO4wN2XR4ucDxRFVWRjCH9PCI3hzwKbgFeBO939+erEIlVnapeR2sjMpgPL3T32EolIXacSgdQKZtbHzL5lZvWiyyuHEuqaRaSadGex1BYHA38lNNyuAsa6+xuZDUmkblDVkIhIjlPVkIhIjqt1VUMtWrTwjh07ZjoMEZFaZeHChZ+7e8tk82pdIujYsSMLFizIdBgiIrWKmZW9o3wXVQ2JiOQ4JQIRkRynRCAikuNqXRuBiKTftm3bWLVqFVu3bq14YcmoBg0a0K5dO/Ly8iq9jhKBiFRo1apVNGnShI4dO5L6uUKSae7OunXrWLVqFZ06dar0ejlRNTR1KnTsCPXqhdepeoy3SJVs3bqV5s2bKwlkOTOjefPmVS651fkSwdSpMHo0bI4eabJyZXgPMGJE6vVEpDQlgdphb/5Odb5EMH787iRQYvPmMF1ERHIgEXz4YdWmi0j2WbduHQUFBRQUFHDwwQfTtm3bXe+/+eabctddsGABV155ZYWfceyxx9ZIrHPnzuXUU0+tkW2lS51PBO3LPuSvgukiUn013S7XvHlzFi9ezOLFixkzZgxXX331rvf77rsv27dvT7luYWEhkydPrvAzXnnlleoFWYvV+UQwcSLk55eelp8fpotIzStpl1u5Etx3t8vV9EUaI0eOZMyYMfTr14/rrruO119/nWOOOYaePXty7LHH8u677wKlz9AnTJjAqFGjGDhwIIceemipBNG4ceNdyw8cOJBhw4bRtWtXRowYQUkvzbNmzaJr16707t2bK6+8ssIz/y+++ILTTz+d7t27c/TRR7NkyRIAXnjhhV0lmp49e1JcXMzq1asZMGAABQUFHHnkkbz44os1u8PKUecbi0sahMePD9VB7duHJKCGYpF4lNcuV9P/d6tWreKVV16hfv36bNy4kRdffJF99tmHZ599lhtvvJG//OUve6yzfPlynn/+eYqLi+nSpQtjx47d45r7N954g6VLl9KmTRv69+/Pyy+/TGFhIZdeeinz5s2jU6dODB8+vML4br75Znr27MmMGTOYM2cOF1xwAYsXL2bSpEnccccd9O/fn02bNtGgQQOmTJnCD37wA8aPH8+OHTvYXHYnxii2RGBmhwB/BA4CHJji7r8rs8xA4O/Av6NJf3X3W2o6lhEjdOAXSZd0tsudffbZ1K9fH4ANGzZw4YUX8t5772FmbNu2Lek6p5xyCvvttx/77bcfrVq1Ys2aNbRr167UMn379t01raCggKKiIho3bsyhhx666/r84cOHM2XKlHLje+mll3YloxNPPJF169axceNG+vfvzzXXXMOIESM488wzadeuHX369GHUqFFs27aN008/nYKCgmrtm6qIs2poO/ATdz8COBq4zMyOSLLci+5eEA01ngREJL3S2S7XqFGjXeM///nPOeGEE3j77bd58sknU15Lv99+++0ar1+/ftL2hcosUx3jxo3j3nvvZcuWLfTv35/ly5czYMAA5s2bR9u2bRk5ciR//OMfa/QzyxNbInD31e6+KBovBt4B2sb1eSKSHTLVLrdhwwbatg2HmAcffLDGt9+lSxc++OADioqKAJg+fXqF6xx//PFMjRpH5s6dS4sWLdh///1ZsWIFRx11FNdffz19+vRh+fLlrFy5koMOOohLLrmEiy++mEWLFtX4d0glLY3FZtYR6Am8lmT2MWb2ppnNNrNuKdYfbWYLzGzB2rVrY4xURKprxAiYMgU6dACz8DplSvzVs9dddx033HADPXv2rPEzeICGDRty5513cvLJJ9O7d2+aNGlC06ZNy11nwoQJLFy4kO7duzNu3DgeeughAG677TaOPPJIunfvTl5eHoMGDWLu3Ln06NGDnj17Mn36dH784x/X+HdIJfZnFptZY+AFYKK7/7XMvP2Bne6+ycwGA79z987lba+wsND1YBqR9HrnnXc4/PDDMx1Gxm3atInGjRvj7lx22WV07tyZq6++OtNh7SHZ38vMFrp7YbLlYy0RmFke8BdgatkkAODuG919UzQ+C8gzsxZxxiQisrf+8Ic/UFBQQLdu3diwYQOXXnpppkOqEXFeNWTAfcA77v7bFMscDKxxdzezvoTEtC6umEREquPqq6/OyhJAdcV5H0F/4HzgLTNbHE27EWgP4O53A8OAsWa2HdgCnOdx11WJiEgpsSUCd38JKLcbPHe/Hbg9rhhERKRidb6LCRERKZ8SgYhIjlMiEJGsd8IJJ/D000+XmnbbbbcxduzYlOsMHDiQkkvNBw8ezPr16/dYZsKECUyaNKncz54xYwbLli3b9f6mm27i2WefrUr4SWVTd9VKBCKS9YYPH860adNKTZs2bVqlOn6D0GvoAQccsFefXTYR3HLLLXz3u9/dq21lKyUCEcl6w4YNY+bMmbseQlNUVMQnn3zC8ccfz9ixYyksLKRbt27cfPPNSdfv2LEjn3/+OQATJ07ksMMO47jjjtvVVTWEewT69OlDjx49OOuss9i8eTOvvPIKTzzxBNdeey0FBQWsWLGCkSNH8vjjjwPw3HPP0bNnT4466ihGjRrF119/vevzbr75Znr16sVRRx3F8uXLy/1+me6uus53Qy0iNeuqq2Dx4oqXq4qCArjtttTzmzVrRt++fZk9ezZDhw5l2rRpnHPOOZgZEydOpFmzZuzYsYOTTjqJJUuW0L1796TbWbhwIdOmTWPx4sVs376dXr160bt3bwDOPPNMLrnkEgB+9rOfcd9993HFFVcwZMgQTj31VIYNG1ZqW1u3bmXkyJE899xzHHbYYVxwwQXcddddXHXVVQC0aNGCRYsWceeddzJp0iTuvffelN8v091Vq0QgIrVCYvVQYrXQo48+Sq9evejZsydLly4tVY1T1osvvsgZZ5xBfn4++++/P0OGDNk17+233+b444/nqKOOYurUqSxdurTceN599106derEYYcdBsCFF17IvHnzds0/88wzAejdu/eujupSeemllzj//POB5N1VT548mfXr17PPPvvQp08fHnjgASZMmMBbb71FkyZNyt12ZahEICJVUt6Ze5yGDh3K1VdfzaJFi9i8eTO9e/fm3//+N5MmTWL+/PkceOCBjBw5MmX30xUZOXIkM2bMoEePHjz44IPMnTu3WvGWdGVdnW6sx40bxymnnMKsWbPo378/Tz/99K7uqmfOnMnIkSO55ppruOCCC6oVq0oEIlIrNG7cmBNOOIFRo0btKg1s3LiRRo0a0bRpU9asWcPs2bPL3caAAQOYMWMGW7Zsobi4mCeffHLXvOLiYlq3bs22bdt2dR0N0KRJE4qLi/fYVpcuXSgqKuL9998H4OGHH+Y73/nOXn23THdXrRKBiNQaw4cP54wzzthVRVTSbXPXrl055JBD6N+/f7nr9+rVi3PPPZcePXrQqlUr+vTps2veL37xC/r160fLli3p16/froP/eeedxyWXXMLkyZN3NRIDNGjQgAceeICzzz6b7du306dPH8aMGbNX36vkWcrdu3cnPz+/VHfVzz//PPXq1aNbt24MGjSIadOmceutt5KXl0fjxo1r5AE2sXdDXdPUDbVI+qkb6tolq7qhFhGR7KdEICKS45QIRKRSals1cq7am7+TEoGIVKhBgwasW7dOySDLuTvr1q2jQYMGVVpPVw2JSIXatWvHqlWrWLt2baZDkQo0aNCAdu3aVWkdJQIRqVBeXh6dOnXKdBgSE1UNiYjkOCUCEZEcp0QgIpLjlAhERHKcEoGISI5TIhARyXFKBCIiOU6JQEQkxykRiIjkOCUCEZEcp0QgIpLjlAhERHKcEoGISI5TIhARyXGxJQIzO8TMnjezZWa21Mx+nGQZM7PJZva+mS0xs15xxSMiIsnF+TyC7cBP3H2RmTUBFprZP9x9WcIyg4DO0dAPuCt6FRGRNImtRODuq919UTReDLwDtC2z2FDgjx78EzjAzFrHFZOIiOwpLW0EZtYR6Am8VmZWW+CjhPer2DNZYGajzWyBmS3Qo/JERGpW7InAzBoDfwGucveNe7MNd5/i7oXuXtiyZcuaDVBEJMfFmgjMLI+QBKa6+1+TLPIxcEjC+3bRNBERSZM4rxoy4D7gHXf/bYrFngAuiK4eOhrY4O6r44pJRET2FOdVQ/2B84G3zGxxNO1GoD2Au98NzAIGA+8Dm4GLYoxHRESSiC0RuPtLgFWwjAOXxRWDiIhUTHcWi4jkOCUCEZEcp0QgIpLjlAhERHKcEoGISI5TIhARyXFKBCIiOU6JQEQkxykRiIjkOCUCEZEcp0QgIpLjlAhERHKcEoGISI5TIhARyXE5lQg27tWDMkVE6racSQSPPQZt2kBRUaYjERHJLjmTCI45BrZtg1//OtORiIhkl5xJBO3awUUXwX33wSefZDoaEZHskTOJAOD662HHDpg0KdORiIhkj5xKBJ06wYgRcPfdsHZtpqMREckOOZUIAG64AbZuhdtuy3QkIiLZIecSQdeucPbZ8Pvfw5dfZjoaEZHMy7lEAHDjjVBcDLffnulIREQyLycTQY8ecNppoXpo06ZMRyMiklk5mQgAxo+HL74IDcciIrksZxNBv37wve+FS0m3bMl0NCIimZOziQBCqWDNmnCTmYhIrsrpRDBgABx3HPzqV/DNN5mORkQkM3I6EZjBz34Gq1bBww9nOhoRkczI6UQA8P3vQ2Eh/Pd/w/btmY5GRCT9cj4RmIW2ghUrYPr0TEcjIpJ+sSUCM7vfzD4zs7dTzB9oZhvMbHE03BRXLBUZMgSOPBL+679g585MRSEikhlxlggeBE6uYJkX3b0gGm6JMZZy1asXSgXLlsGMGZmKQkQkM2JLBO4+D/giru3XtLPPhs6d4Ze/BPdMRyMikj6ZbiM4xszeNLPZZtYt1UJmNtrMFpjZgrUx9R9dv37omfSNN2D27Fg+QkQkK2UyESwCOrh7D+D3QMpKGXef4u6F7l7YsmXL2AL6j/+A9u1VKhCR3JKxRODuG919UzQ+C8gzsxaZigcgLw/GjYNXX4W5czMZiYhI+mQsEZjZwWZm0XjfKJZ1mYqnxEUXQevWoVQgIpILKpUIzKyRmdWLxg8zsyFmllfBOo8ArwJdzGyVmf3IzMaY2ZhokWHA22b2JjAZOM898xUyDRrAT38Kc+bAK69kOhoRkfhZZY69ZrYQOB44EHgZmA984+4j4g1vT4WFhb5gwYJYP+Orr6BDh9BD6cyZMHVquLz0ww9DG8LEieHZxyIitYWZLXT3wmTzKls1ZO6+GTgTuNPdzwZSXuVT2zVqBNdcA7NmhYP+6NGwcmVoQF65MryfOjXTUYqI1IxKJwIzOwYYAcyMptWPJ6TscNll0LRpuNt48+bS8zZvDiUEEZG6oLKJ4CrgBuBv7r7UzA4Fno8vrMxr2hSuuGLPJFDiww/TG4+ISFwq1UZQaoXQaNzY3TfGE1L50tFGUOLzz6FVq+T3FHToAEVFaQlDRKTaqt1GYGZ/NrP9zawR8DawzMyurckgs1GLFjBo0J7T8/ND24GISF1Q2aqhI6ISwOnAbKATcH5sUWWRe+8NN5o1bhy6rO7QAaZM0VVDIlJ3VDYR5EX3DZwOPOHu24CMX/OfDq1bh6uEvv46VAUVFSkJiEjdUtlEcA9QBDQC5plZByAjbQSZcN11oZ3g1lszHYmISM2rVCJw98nu3tbdB3uwEjgh5tiyRvv2cOGFoZro5ZczHY2ISM2qbGNxUzP7bUlX0Gb2G0LpIGfcdBO0aQPf+U64t2DHjkxHJCJSMypbNXQ/UAycEw0bgQfiCiobtW8PixbBsGHhZrLvfx9Wr850VCIi1VfZRPAtd7/Z3T+Ihv8HHBpnYNmoaVN45JFQRfTqq9Cjhx5iIyK1X2UTwRYzO67kjZn1B7bEE1J2M4Mf/QgWLICDDoLBg+Haa+GbbzIdmYjI3qlsIhgD3GFmRWZWBNwOXBpbVLXAEUfA66/DmDEwaRIcdxx88EGmoxIRqbrKXjX0ZvRIye5Ad3fvCZwYa2S1QMOGcNdd8Pjj8K9/QUEBTJuW6ahERKqmSk8oix4vWXL/wDUxxFMrnXUWLF4MRx4Jw4fDxReHZxqIiNQG1XlUpdVYFHVAx47wwgtwww1w//3Qpw+89VamoxIRqVh1EkFOdDFRFXl54R6DZ56BL76Avn3h7ruT914qIpItyk0EZlZsZhuTDMVAmzTFWOt897vw5pvh5rOxY+Hss+HLLzMdlYhIcuUmAndv4u77JxmauPs+6QqyNjrooPCoy1//Gv7+d+jZM9x7ICKSbapTNSQVqFcv3GPw0kvh/oPjj4df/Qp27sx0ZCIiuykRpEG/fuGqojPOgHHj4LTTwtPPRESygRJBmjRtCo8+CrffDs8+G6qK1JOpiGQDJYI0MoPLLoNXXoF99w2NybfeqqoiEcksJYIM6N079GR6+unhoTdDhsC6dZmOSkRylRJBhjRtCo89Br//PfzjH7qqSEQyR4kgg8zg8stDW8E++8CAAaEDO1UViUg6KRFkgcLCUFU0dGi43HToUFUVichuGzbAww+HHo/joESQBlOnhr6I6tULr1On7rnMAQeEqqLJk+Hpp1VVJJLrNm4Mx4qhQ6FVK7jgguTHjpqgRBCzqVNh9GhYuTL0ObRyZXif7A9qBldcUbqq6De/UV9FIrmiuDg8BfGMM8LB/z/+I9QWXHZZODH83/+N53PNa9lRprCw0BcsWJDpMCqtY8dw8C+rQwcoKkq93vr1MGoU/O1v4Qa0Bx+EZs1iClKkjvr4Y5g/Pwxvvhn+7048EQYOhObNMx1dsGkTPPVUuM9o9mzYuhXatAl9lJ1zDhx9dKhNqC4zW+juhUnnxZUIzOx+4FTgM3c/Msl8A34HDAY2AyPdfVFF261tiaBeveRn9GYVNwq7h6uKfvpTaN0apk8PPwoR2dMXX4RHyM6fH+rS58+H1avDvPr1oWvXcPL11Vfh/69Hj5AUTjwxdP+y//7pi/Wrr2DmzHDwnzkzHPxbt4Zhw8LB/9hja+bgnyhTiWAAsAn4Y4pEMBi4gpAI+gG/c/d+FW23tiWCvS0RJJo/P/w4PvwQTjoJzj03FB1VQpBc9dVXocqk5Gx//nxYsWL3/C5dwjNBSoaCgvBEwW3bwrJz5sDzz4dq2K+/DomiTx844YSQGI49FvLzaz7m2bPDwf+pp2DLltA5ZcnBv3//EEdcMpIIog/uCDyVIhHcA8x190ei9+8CA919dXnbrG2JoKSNYPPm3dPy82HKFBgxovLbWb8+3IU8bVp4NnJeHnzveyEpDB0a7ksQqas2bgyPhH355XAgX7p0d4n6kENKH/R79w4XX1TG1q2h7n3OnDC8/jps3x7u/D/mmN0lhr59w7RE7qFOf80a+OyzPV/LTlu/PqzXqlV4quE554SSSJwH/0TZmgieAv7H3V+K3j8HXO/uexzlzWw0MBqgffv2vVcmO8XOYlOnwvjx4Yy+fXuYOLFqSSCROyxcGM4qHn00lDb23RdOPjkkhdNOgyZNajZ+kUxwD1U999wTGlA3b4YWLUof9Pv0CWfVNaW4OPQWXFJiWLQoxJGfH87Y99239MH966+Tb6dZs3DAP+ig8Foy3r9/6FomXQf/RLU+ESSqbSWCOLnDa6+FtoPHHgsNYw0awODBISmccgo0apTpKEWqZsMG+POfQ6l58eLwGx4+HC65JBz4LY0Pyf3yy/AI2jlzYN68UG+feIBP9tqixZ6lh2yQrYkgJ6qG0mXnztCZXUlSWLMmnMWcdloogg4aFOpIRfbGzp2hGqVhw3gOxO6hymfKlN1n/wUFcOml8MMfprcht64qLxFk8iljTwCXm9k0QmPxhoqSgKRWrx4cd1wYbrstnL08+mioV50+HRo3Dkmhd2844gg4/PBQTVXTVyZIdiouDvXq69aF8cRh06Y9pyVbBsJZb0FBuOGxoCAMnTvvfVVHydn/PfeEyzsbNQoH/tGjwx336Tz7z2VxXjX0CDAQaAGsAW4G8gDc/e7o8tHbgZMJl49eVFG1EKhEUFXbt4e6zunTw5UKa9bsnpefHy6pO/zw3cnh8MPhW98KjdFS+7iHSyYXLw7DG2+E1/ffT71Ow4ahXamioUGDsJ033oC33w5X4ED4HXXvXjpBHHlk6qtu3EOj7JQp4eIHnf2nR8aqhuKgRFA9n38O77wThmXLdo9/9NHuZfLywlle2QTRpYuql7LJjh3wr3/tedBfu3b3Moceuvvg3L17qMNOPLg3bhzuYq+qb74Jv5vEz128OJzhQyhpdu26u9TQs2f4Tc2cGRJAydn/8OE6+08XJQKpUHExLF9eOjksWxYuVS25TM8s3P/QpUv4J+/SZffQpo3+keOybVs4y//oI3jrrd0H3SVLwrXoEBonu3UrXWXTvXt6Lyt2D/fGlE0OiScZoLP/TFEikL22dSu8997uBPHuu7uHxHsjGjcunRhKhsMOq/kbc2qae7hO/dNPwwE31fD55+HA1aoVtGy5+7LAZO9btKj4THvnzlBn//HH8MknqV8/+6z0egccUPqAX1AQEnM2XqkCYb+9+Wb4DR19tM7+M0WJQGrczp3hQJWYGEqGDz8s3a1G+/YhKXTuDAceGJJGydCoUen3iUN+fsWNkO6hmmLLloqHzZtDG0myg3zJmXWi/fYLt/2XDC1bhpJTyc1Cn30WqmF27EgeW8m15CWJolmzcDliyQH+k09217MnatUK2rYNpayS15Lxbt3C/tSBVKpKiUDSavPmUIoomyBWrAh1yFV58E7DhqUTQ7KDflV/wk2b7j64H3xw6YN94vQDDqj4gLtzZ7hjNDExlE0UJePr1oVkkHiAL/t68MHZe2YvtVu2Xj4qdVR+fujQq0ePPee5h7sxN20qPXz11Z7Tys7bvDkcJBs2DFewNGxYtSE/P/4P5OYAAAvXSURBVJyZ12RVVb164eDerFmonhGpjZQIJK3MwkG8QYNQjy4imafbiUREcpwSQS1QmUddiojsLVUNZbmy3ViXPOoS9r4HUxGRRCoRZLnx40tfrw/h/fjxmYlHROoeJYIs9+GHVZsuIlJVSgRZrn37qk0XEakqJYIsN3Hinte95+eH6SIiNUGJIMuNGBF6a+zQYXenb1V93rGISHl01VAtMGKEDvwiEh+VCEREcpwSgYhIjlMiEBHJcUoEIiI5TolARCTHKRHkAHVaJyLl0eWjdZw6rRORiqhEUMep0zoRqYgSQR2nTutEpCJKBHWcOq0TkYooEdRx6rRORCqiRFDHqdM6EamIrhrKAeq0TkTKoxKBiEiOUyIQEclxSgQiIjku1kRgZieb2btm9r6ZjUsyf6SZrTWzxdFwcZzxyN5TNxUidVdsjcVmVh+4A/gesAqYb2ZPuPuyMotOd/fL44pDqk/dVIjUbXGWCPoC77v7B+7+DTANGBrj50lM1E2FSN0WZyJoC3yU8H5VNK2ss8xsiZk9bmaHJNuQmY02swVmtmDt2rVxxCrlUDcVInVbphuLnwQ6unt34B/AQ8kWcvcp7l7o7oUtW7ZMa4CibipE6ro4E8HHQOIZfrto2i7uvs7dv47e3gv0jjEe2UvqpkKkboszEcwHOptZJzPbFzgPeCJxATNrnfB2CPBOjPHIXlI3FSJ1W2xXDbn7djO7HHgaqA/c7+5LzewWYIG7PwFcaWZDgO3AF8DIuOKR6lE3FSJ1V6xtBO4+y90Pc/dvufvEaNpNURLA3W9w927u3sPdT3D35XHGI5mj+xBEspc6nZPY6T4EkeyW6auGJAfoPgSR7KZEILHTfQgi2U2JQGKn+xBEspsSgcRO9yGIZDclAoldTdyHoKuOROKjq4YkLapzH4KuOhKJl0oEkvV01ZFIvJQIJOvpqiOReCkRSNariauO1MYgkpoSgWS96l51VNLGsHIluO9uY1AyEAmUCCTrVfeqI7UxiJRPiUBqhREjoKgIdu4Mr1W5Wqgm2hhUtSR1mRKB1HnVbWNQ1ZLUdUoEUudVt41BVUtS1ykRSJ1X3TYGVS1JXac7iyUnVOfO5vbtQ3VQsumVoTujJdupRCBSgWyoWlKJQuKkRCBSgUxXLdVEY7USiZRHiUCkEqpz+Wp1r1qqbolCiUQqokQgErPqVi1Vt0ShRCIVUSIQiVl1q5aqW6JQIsn8+lnP3WvV0Lt3bxfJJX/6k3t+vns4jIYhPz9Mr4wOHUqvWzJ06FC59c2Sr2+Wns+v7vfP9Pol2+jQIeyzDh2qtm5NrO/uDizwFMfVjB/YqzooEUguqs6BINcTSabXz4ZE5K5EIJLzcjmRZHr9TCeiEuUlArURiOSA6lz1VN02juo2lle3jSTT61e3jSYdD2ZSIhCRCtXmRJLp9TOdiColVVEhWwdVDYnknkw3tmayai0dbQQW5tcehYWFvmDBgkyHISJSaVOnhsttP/wwnMlPnFi1UlV11wcws4XuXph0nhKBiEjdV14iUBuBiEiOizURmNnJZvaumb1vZuOSzN/PzKZH818zs45xxiMiInuKLRGYWX3gDmAQcAQw3MyOKLPYj4Av3f3bwP8Cv4orHhERSS7OEkFf4H13/8DdvwGmAUPLLDMUeCgafxw4ycwsxphERKSMOBNBW+CjhPeromlJl3H37cAGoHnZDZnZaDNbYGYL1q5dG1O4IiK5qVY8qtLdpwBTAMxsrZkleXBgVmgBfJ7pIMqR7fFB9seo+KpH8VVPdeLrkGpGnIngY+CQhPftomnJllllZvsATYF15W3U3VvWZJA1ycwWpLo8Kxtke3yQ/TEqvupRfNUTV3xxVg3NBzqbWScz2xc4D3iizDJPABdG48OAOV7bbmwQEanlYisRuPt2M7sceBqoD9zv7kvN7BbCrc5PAPcBD5vZ+8AXhGQhIiJpFGsbgbvPAmaVmXZTwvhW4Ow4Y0izKZkOoALZHh9kf4yKr3oUX/XEEl+t62JCRERqlrqYEBHJcUoEIiI5TomgiszsEDN73syWmdlSM/txkmUGmtkGM1scDTcl21aMMRaZ2VvRZ+/RVasFk6M+npaYWa80xtYlYb8sNrONZnZVmWXSvv/M7H4z+8zM3k6Y1szM/mFm70WvB6ZY98JomffM7MJky8QU361mtjz6G/7NzA5IsW65v4cY45tgZh8n/B0Hp1i33D7JYoxvekJsRWa2OMW6se6/VMeUtP7+Uj2oQEPyAWgN9IrGmwD/Ao4os8xA4KkMxlgEtChn/mBgNmDA0cBrGYqzPvAp0CHT+w8YAPQC3k6Y9mtgXDQ+DvhVkvWaAR9ErwdG4wemKb7vA/tE479KFl9lfg8xxjcB+GklfgMrgEOBfYE3y/4/xRVfmfm/AW7KxP5LdUxJ5+9PJYIqcvfV7r4oGi8G3mHPrjOy3VDgjx78EzjAzFpnII6TgBXunvE7xd19HuES5kSJfWE9BJyeZNUfAP9w9y/c/UvgH8DJ6YjP3Z/x0DULwD8JN21mRIr9VxmV6ZOs2sqLL+rf7BzgkZr+3Moo55iStt+fEkE1RN1m9wReSzL7GDN708xmm1m3tAYGDjxjZgvNbHSS+ZXpByodziP1P18m91+Jg9x9dTT+KXBQkmWyZV+OIpTykqno9xCny6Oqq/tTVG1kw/47Hljj7u+lmJ+2/VfmmJK2358SwV4ys8bAX4Cr3H1jmdmLCNUdPYDfAzPSHN5x7t6L0AX4ZWY2IM2fX6HobvMhwGNJZmd6/+3BQzk8K6+1NrPxwHZgaopFMvV7uAv4FlAArCZUv2Sj4ZRfGkjL/ivvmBL370+JYC+YWR7hDzbV3f9adr67b3T3TdH4LCDPzFqkKz53/zh6/Qz4G6H4nagy/UDFbRCwyN3XlJ2R6f2XYE1JlVn0+lmSZTK6L81sJHAqMCI6WOyhEr+HWLj7Gnff4e47gT+k+NxM7799gDOB6amWScf+S3FMSdvvT4mgiqL6xPuAd9z9tymWOThaDjPrS9jP5XamV4PxNTKzJiXjhAbFt8ss9gRwQXT10NHAhoQiaLqkPAvL5P4rI7EvrAuBvydZ5mng+2Z2YFT18f1oWuzM7GTgOmCIu29OsUxlfg9xxZfY7nRGis+tTJ9kcfousNzdVyWbmY79V84xJX2/v7hawuvqABxHKKItARZHw2BgDDAmWuZyYCnhCoh/AsemMb5Do899M4phfDQ9MT4jPD1uBfAWUJjmfdiIcGBvmjAto/uPkJRWA9sI9aw/Ijwb4zngPeBZoFm0bCFwb8K6o4D3o+GiNMb3PqF+uOR3eHe0bBtgVnm/hzTF93D0+1pCOKi1Lhtf9H4w4UqZFemML5r+YMnvLmHZtO6/co4pafv9qYsJEZEcp6ohEZEcp0QgIpLjlAhERHKcEoGISI5TIhARyXFKBCIRM9thpXtGrbGeMM2sY2LPlyLZJNZHVYrUMlvcvSDTQYikm0oEIhWI+qP/ddQn/etm9u1oekczmxN1qvacmbWPph9k4fkAb0bDsdGm6pvZH6I+558xs4bR8ldGfdEvMbNpGfqaksOUCER2a1imaujchHkb3P0o4Hbgtmja74GH3L07ocO3ydH0ycALHjrN60W4IxWgM3CHu3cD1gNnRdPHAT2j7YyJ68uJpKI7i0UiZrbJ3RsnmV4EnOjuH0Sdg33q7s3N7HNCtwnboumr3b2Fma0F2rn71wnb6EjoN75z9P56IM/df2lm/wdsIvSyOsOjDvdE0kUlApHK8RTjVfF1wvgOdrfRnULo+6kXMD/qEVMkbZQIRCrn3ITXV6PxVwi9ZQKMAF6Mxp8DxgKYWX0za5pqo2ZWDzjE3Z8HrgeaAnuUSkTipDMPkd0aWukHmP+fu5dcQnqgmS0hnNUPj6ZdATxgZtcCa4GLouk/BqaY2Y8IZ/5jCT1fJlMf+FOULAyY7O7ra+wbiVSC2ghEKhC1ERS6++eZjkUkDqoaEhHJcSoRiIjkOJUIRERynBKBiEiOUyIQEclxSgQiIjlOiUBEJMf9f+lkFY/swag8AAAAAElFTkSuQmCC\n", 350 | "text/plain": [ 351 | "
" 352 | ] 353 | }, 354 | "metadata": { 355 | "needs_background": "light" 356 | }, 357 | "output_type": "display_data" 358 | } 359 | ], 360 | "source": [ 361 | "history_dict = history.history\n", 362 | "loss_values = history_dict['loss']\n", 363 | "val_loss_values = history_dict['val_loss']\n", 364 | "acc = history_dict['categorical_accuracy']\n", 365 | "\n", 366 | "epochs = range(1, len(acc) + 1)\n", 367 | "plt.plot(epochs, loss_values, 'bo', label='Training loss')\n", 368 | "plt.plot(epochs, val_loss_values, 'b', label='Validation loss')\n", 369 | "plt.title('Training and validation loss')\n", 370 | "plt.xlabel('Epochs')\n", 371 | "plt.ylabel('Loss')\n", 372 | "plt.legend()\n", 373 | "\n", 374 | "plt.show" 375 | ] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": 15, 380 | "metadata": {}, 381 | "outputs": [ 382 | { 383 | "data": { 384 | "text/plain": [ 385 | "dict_keys(['loss', 'categorical_accuracy', 'val_loss', 'val_categorical_accuracy'])" 386 | ] 387 | }, 388 | "execution_count": 15, 389 | "metadata": {}, 390 | "output_type": "execute_result" 391 | } 392 | ], 393 | "source": [ 394 | "history_dict.keys()" 395 | ] 396 | }, 397 | { 398 | "cell_type": "markdown", 399 | "metadata": {}, 400 | "source": [ 401 | "#### Training and validation accuracy" 402 | ] 403 | }, 404 | { 405 | "cell_type": "code", 406 | "execution_count": 16, 407 | "metadata": { 408 | "scrolled": true 409 | }, 410 | "outputs": [ 411 | { 412 | "data": { 413 | "text/plain": [ 414 | "" 415 | ] 416 | }, 417 | "execution_count": 16, 418 | "metadata": {}, 419 | "output_type": "execute_result" 420 | }, 421 | { 422 | "data": { 423 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZgU1bnH8e/LICBCQMCNRUCDoASHZcSIG0a9wSUYcAMxEXFF0eCNURM3rgm516hxNxH3BQNqrgSvuETUaMSFAQEVQQEHHUREdkTWee8fp2Zohp6ZnqW6Z6Z/n+fpp6uqa3m7pqfeOqeqzjF3R0REsleDTAcgIiKZpUQgIpLllAhERLKcEoGISJZTIhARyXJKBCIiWU6JQHZiZi+a2Tk1PW8mmVmBmR0Xw3rdzH4YDf/VzK5PZd4qbGeYmb1S1ThFymN6jqB+MLP1CaNNgU3Atmj8Incfn/6oag8zKwDOd/dXa3i9DnRx9wU1Na+ZdQI+B3Zx9601EadIeRpmOgCpGe7erHi4vIOemTXUwUVqC/0eawdVDdVzZtbfzArN7Goz+xp4xMx2N7P/M7PlZrYqGm6fsMwbZnZ+NDzczP5tZrdG835uZidUcd7OZvamma0zs1fN7F4ze7KMuFOJ8fdm9na0vlfMrE3C578ws8VmtsLMri1n/xxqZl+bWU7CtEFmNica7mtm75jZajNbamb3mFmjMtb1qJn9IWH8N9EyX5nZiFLznmRmH5jZWjP70szGJHz8ZvS+2szWm9lhxfs2Yfl+ZjbdzNZE7/1S3TeV3M+tzOyR6DusMrNJCZ+dYmazou+w0MwGRNN3qIYzszHFf2cz6xRVkZ1nZl8Ar0XTn4n+Dmui30j3hOV3NbPbor/nmug3tquZvWBml5X6PnPMbFCy7yplUyLIDnsDrYCOwIWEv/sj0fi+wPfAPeUsfygwH2gD/Al4yMysCvM+BbwPtAbGAL8oZ5upxHgWcC6wJ9AIuBLAzA4C/hKtv220vfYk4e7vAd8BPym13qei4W3AFdH3OQw4FriknLiJYhgQxXM80AUofX3iO+CXQEvgJGCkmf08+uyo6L2luzdz93dKrbsV8AJwV/Td/gy8YGatS32HnfZNEhXt5ycIVY3do3XdHsXQF3gc+E30HY4CCsraH0kcDRwI/DQaf5Gwn/YEZgKJVZm3An2AfoTf8VVAEfAYcHbxTGaWC7Qj7BupDHfXq569CP+Qx0XD/YHNQJNy5u8JrEoYf4NQtQQwHFiQ8FlTwIG9KzMv4SCzFWia8PmTwJMpfqdkMV6XMH4J8FI0fAMwIeGz3aJ9cFwZ6/4D8HA03JxwkO5YxryjgecSxh34YTT8KPCHaPhh4H8S5jsgcd4k670DuD0a7hTN2zDh8+HAv6PhXwDvl1r+HWB4RfumMvsZ2IdwwN09yXz3F8db3u8vGh9T/HdO+G77lRNDy2ieFoRE9T2Qm2S+JsAqwnUXCAnjvnT/v9WHl0oE2WG5u28sHjGzpmZ2f1TUXkuoimiZWD1SytfFA+6+IRpsVsl52wIrE6YBfFlWwCnG+HXC8IaEmNomrtvdvwNWlLUtwtn/YDNrDAwGZrr74iiOA6Lqkq+jOP5IKB1UZIcYgMWlvt+hZvZ6VCWzBrg4xfUWr3txqWmLCWfDxcraNzuoYD93IPzNViVZtAOwMMV4kynZN2aWY2b/E1UvrWV7yaJN9GqSbFvRb3oicLaZNQCGEkowUklKBNmh9K1hvwa6Aoe6+w/YXhVRVnVPTVgKtDKzpgnTOpQzf3ViXJq47mibrcua2d3nEg6kJ7BjtRCEKqZ5hLPOHwC/q0oMhBJRoqeAyUAHd28B/DVhvRXdyvcVoSon0b7AkhTiKq28/fwl4W/WMslyXwL7l7HO7wilwWJ7J5kn8TueBZxCqD5rQSg1FMfwLbCxnG09BgwjVNlt8FLVaJIaJYLs1JxQ3F4d1TffGPcGozPsfGCMmTUys8OAn8UU47PAyWZ2RHRh9yYq/q0/BfyKcCB8plQca4H1ZtYNGJliDE8Dw83soCgRlY6/OeFse2NU335WwmfLCVUy+5Wx7inAAWZ2lpk1NLMzgYOA/0sxttJxJN3P7r6UUHd/X3RReRczK04UDwHnmtmxZtbAzNpF+wdgFjAkmj8POC2FGDYRSm1NCaWu4hiKCNVsfzaztlHp4bCo9EZ04C8CbkOlgSpTIshOdwC7Es623gVeStN2hxEuuK4g1MtPJBwAkqlyjO7+MXAp4eC+lFCPXFjBYn8jXMB8zd2/TZh+JeEgvQ54IIo5lRhejL7Da8CC6D3RJcBNZraOcE3j6YRlNwBjgbct3K3041LrXgGcTDibX0G4eHpyqbhTVdF+/gWwhVAq+oZwjQR3f59wMfp2YA3wL7aXUq4nnMGvAv6LHUtYyTxOKJEtAeZGcSS6EvgQmA6sBG5mx2PX40APwjUnqQI9UCYZY2YTgXnuHnuJROovM/slcKG7H5HpWOoqlQgkbczsEDPbP6pKGECoF55U0XIiZYmq3S4BxmU6lrpMiUDSaW/CrY3rCffAj3T3DzIakdRZZvZTwvWUZVRc/STlUNWQiEiWU4lARCTL1blG59q0aeOdOnXKdBgiInXKjBkzvnX3PZJ9VucSQadOncjPz890GCIidYqZlX4avYSqhkREspwSgYhIllMiEBHJckoEIiJZTolARCTLKRGISK03fjx06gQNGoT38eMrWkLbrwwlAhGpUHUPRNVZfvx4uPBCWLwY3MP7hRdWfh3ZvP0KZbqLtMq++vTp4yLZ5skn3Tt2dDcL708+mb7ln3zSvWlT93AYCq+mTVNfR3WX79hxx2WLXx07avuVAeR7GcfVjB/YK/tSIpC6piYO4nX5QFzd5c2SL2+m7VdGeYmgzjU6l5eX53qyWOqK4mL9hoSemps2hXHjYNiw1NbRqVOoDiitY0coKIh/+QYNwqGnNDMoKop/+UzHX9e3v317NsPd85LGmPpqRLJTdep3r712xyQAYfzaa1NfxxdfVG56TS+/b+neliuYXtPLjx0bkmeipk3DdG2/hpRVVKitL1UNSTpVt1qlJor1ma6ayXTVVPE6MnWNo65vvxi6RiBSNZk+CLvX/QNxTSxfXdm+fXclAsly1fknrO4ZfU0chKv7HWpiean7yksEulgs9Vp1L9bWxIW68ePDNYEvvgj1wmPHpn6hWKSmlHexWIlA6rXqHshr4q4fkdpAdw1JnVadu3aqe8fMsGHhoN+xY7jdr2NHJQGpf+pcD2WSXUqfkRc/Xg+pHYz33Td5iSDVW/eKt6MDv9RnKhFIrVbd+/DTcg+2SB2nRCC1mqp2ROKnqiGp1VS1IxI/lQikVlPVjkj8lAikVlPVjkj8lAgkdtXt1GTYsHDPf1FReFcSEKlZukYgsaru7Z8iEj+VCCRWNdEMs4jES4lAYlXd2z9FJH5KBBKr6nbKISLxUyKQWOn2T5HaL9ZEYGYDzGy+mS0ws2uSfN7RzKaa2Rwze8PM2scZj6Sfbv8Uqf1ia4bazHKAT4HjgUJgOjDU3ecmzPMM8H/u/piZ/QQ4191/Ud561Qy1iEjlZaoZ6r7AAndf5O6bgQnAKaXmOQh4LRp+PcnnUgtU9zkAEand4kwE7YAvE8YLo2mJZgODo+FBQHMza116RWZ2oZnlm1n+8uXLYwlWkit+DmDx4tDRYvFzAEoGIvVHpi8WXwkcbWYfAEcDS4BtpWdy93HunufueXvssUe6Y8xqeg5ApP6L88niJUCHhPH20bQS7v4VUYnAzJoBp7r76hhjkkrScwAi9V+cJYLpQBcz62xmjYAhwOTEGcysjZkVx/Bb4OEY45Eq0HMAIvVfbInA3bcCo4CXgU+Ap939YzO7ycwGRrP1B+ab2afAXoDuLq9l9ByASP0X2+2jcdHto+k3fny4JvDFF6EkMHasngMQqWvKu31UrY9KhdTDl0j9lum7hkREJMOUCEREspwSgYhIllMiEBHJckoEWUBtBYlIeXTXUD2nPoNFpCIqEdRzaitIRCqiRFDPqa0gEamIEkE9p7aCRKQiSgT1nNoKEpGKKBHUc+ozWEQqoruGsoDaChKR8qhEICKS5ZQIRESynBKBiEiWUyIQEclySgQiIllOiUBEJMspEdQBaj1UROKk5whqObUeKiJxU4mgllProSISNyWCWk6th4pI3JQIajm1HioicVMiqOXUeqiIxE2JoJZT66EiEjfdNVQHqPVQEYmTSgQiIllOJQIRkXJs2waTJ8Pjj0PjxtCuXXi1bbvjcJMmmY606pQIJG3Wrw//VFVVVASbN8OmTeGVOFzT48k+27y5et+/YUPYf3/o1i28DjwwvHfoEJ4ar69WroTp02Hu3PCdDzsMWrTIdFQVW78eHnkE7rgDFi0KB/xddw1J4fvvd56/devtiSHxVZwwOnQI89RGSgQSqw0b4Jln4MEH4d//znQ04YJ748bh1ajR9uHS482aQatWO36+yy5h+aratAkWLAj7Y+XK7dObNoWuXbcniOIk0aVL3TvL/P57+OADeP/98Jo+PXznRA0awMEHwxFHwJFHhve2bTMTbzKFhXDPPXD//bB6Nfz4x3DzzfDzn4dk7h6mL1kSXl99tX24+PXBB7BsWZg30dFHw69/DSedVLuSv3npSGu5vLw8z8/Pz3QYUoFZs+CBB0ITGWvWhIPa0KHQsmXV12lW/sG7vM+Kxxs2rN7BvKYsXw7z5sEnn4T34uHFi7cfPMygc+ftJYeuXcNZZfGZ5u67Z/a7bNsWzvITD/pz5mwv9bVvD4ccAn37hteBB4b533ornBS88872p+b322/HxNC1a/q/28yZ8Oc/w8SJofQ5eDD853+GEkxVbNkCX3+9PTnMmxfu+Pvii/D/cMUVcM45O98eHhczm+HueUk/UyKQmrJ2Lfztb+HsPz8/HHxPOw0uuACOOqp2HIBruw0b4LPPdk4Q8+eHEkWiJk12rqsuXSXRtm34O1TFtm07Vo2tXbvj2f7MmfDdd2Heli23H/QPOSS8KjrL37IlnDAUJ4Z//zskSIA2bXZMDL16hRJZTSsqghdeCAngjTdCSfD88+Hyy0MSrmlbt8Lf/w633RYSZ6tWcPHFMGoU7LNPzW8vUcYSgZkNAO4EcoAH3f1/Sn2+L/AY0DKa5xp3n1LeOpUIahd3eO+9cPY/YUI4kP3oR+Hgf/bZ4Ycu1bdtW6iyKCzcuRoi8VU6WUA4qLZrB3vvHQ585V0PSRwuKkoeS+PG4cBcfKZ/yCHwwx9Wv6rDHT79dHtieOutUDcP4az50ENDqaJz51CCKH6vyvWGDRvgscfg9ttD4u3QIRz8L7ggPdcv3OHtt0MCmjQplFTPOiuUEnJz49lmRhKBmeUAnwLHA4XAdGCou89NmGcc8IG7/8XMDgKmuHun8tarRFA1X30Vznxatdp+xrj33lU/y1q5Ep54Ipz9f/QR7LYbDBkS/pH69tXZfya4w6pVZSeJb76BnJzUqtCSDTdtCj16hETfqFF6vtNXX4UD5ltvwbvvhusNq1btOM/uu++YGBKH9913x1iXLg31/3/9a/gN5+WFOvtTT42nxJGKhQvhzjvh4YdDCevYY0OV1IABNXsdIVOJ4DBgjLv/NBr/LYC7/3fCPPcDi9z95mj+29y9X3nrVSKonI0bw1nHH/+4vRhfzAz22qvsqoXi6oWWLcO87qH4/OCDoXi7aVM4Gzz//FD/37x5Rr6iZJnVq+Hzz0NpofR7QcGOd3c1aBCuVXTuDD/4Abz0Uqie+fnPw8H28MNrz0nLqlXhGsJdd4UEeOCBoYRw9tnhbqXqylQiOA0Y4O7nR+O/AA5191EJ8+wDvALsDuwGHOfuM5Ks60LgQoB99923z+LFi2OJuT5xh+eeC2c7BQUwaBCMGROK++Xd7bBixc7r2nXXkBS2bAkXM1u0CD/OCy6IrxgrUhVFReF3nZgcioeXLoUTT4Rf/SpUZdVWmzeHO8tuuy1ck2nTBi65JLz22qvq663NieA/oxhui0oEDwE/cvcyaidVIkjFnDkwejS8/nooxt9xRyhupmLjxu0JonSi+P77UIQ+7bT03ekgkq3c4V//CiX6558P1XP33QcjRlRtfeUlgjifI1gCdEgYbx9NS3QeMADA3d8xsyZAG+CbGOOqt779Fq6/PhQvW7aEe+8NvZk1rMRfuUmT7fWsIpI5ZtC/f3jNnx+uI/TpE8+24nykYTrQxcw6m1kjYAgwudQ8XwDHApjZgUATYHmMMdVLW7aEH0mXLuHunVGjwp0Ql1xSuSQgIrVT166hNBBXVWxsicDdtwKjgJeBT4Cn3f1jM7vJzAZGs/0auMDMZgN/A4Z7XXuwIQVxdj7/0kvhKc3Ro8PdOnPmhKSg2zZFJFWxni9GzwRMKTXthoThucDhccaQaXF1Pv/pp+GuhxdeCBe+nn8+PLZeW+6AEJG6oxa1dlE/1XTn82vWwJVXhovAb74Jt9wS7uM/+WQlARGpGtUgx6ymOp8vKgoPnPzud+Gi8IgRobvK6txOJiICKhHErqY6n7/66nDffteuoR2fBx9UEhCRmqFEELOa6Hx+0iS49VYYOTJUB/XuXbMxikh2UyKIWXU7n1+0CIYPD0053H67rgOISM3TNYI0qGrn8xs3hqd4GzSAp5+uenPCIiLlUSKoxUaPDm2NPP98eP5ARCQOFVYNmdnPzExVSGn25JOhq7yrrw63hoqIxCWVA/yZwGdm9icz6xZ3QAIffwwXXRR69frDHzIdjYjUdxUmAnc/G+gFLAQeNbN3zOxCM1Pr8zFYvx5OPz10mTdhgtoKEpH4pVTl4+5rgWeBCcA+wCBgppldFmNsWcc9lATmzw99/8bdh6mICKR2jWCgmT0HvAHsAvR19xOAXEKjcVJD7r8fnnoK/uu/4Cc/yXQ0IpItUql4OBW43d3fTJzo7hvM7Lx4wso+M2aEnpMGDAjNSIiIpEsqiWAMsLR4xMx2BfZy9wJ3nxpXYNlk1apwXWDPPUOH8DXZYbWISEVSOeQ8AyR2HbktmiY1wB3OPRe+/DI8NNamTaYjEpFsk0qJoKG7by4ecffNUY9jUgNuuw3+8Y/QfMRhh2U6GhHJRqmUCJYn9CiGmZ0CfBtfSNnj3/+Ga64JHcL/6leZjkZEslUqJYKLgfFmdg9gwJfAL2ONKgt88w2ceSZ07gwPPaTG5EQkcypMBO6+EPixmTWLxtfHHlU9t21baIRu5UqYMgVatMh0RCKSzVJ6btXMTgK6A00sOnV195tijKte+/3v4dVXQ+cyubmZjkZEsl0qD5T9ldDe0GWEqqHTgY4xx1VvvfIK3HQTnHNO6G5SRCTTUrlY3M/dfwmscvf/Ag4DDog3rPqpsDBUCXXvDvfdp+sCIlI7pJIINkbvG8ysLbCF0N6QVMKWLeHi8MaN8OyzO3dfKSKSKakkgufNrCVwCzATKACeijOo+ubVV6FvX5g2LVwX6No10xGJiGxXbiKIOqSZ6u6r3f3vhGsD3dz9hrREV8d9+CGccAIcfzysXg0TJ4ZSgYhIbVJuInD3IuDehPFN7r4m9qjquCVL4LzzoGdPePdduPVWmDcPzjgj05GJiOwslaqhqWZ2qpkubVZk3Tq4/nro0iV0NTl6NCxcCL/+tTqeF5HaK5XnCC4C/hPYamYbCbeQurv/INbI6pCtW+GBB2DMmO1PDP/xj7DffpmOTESkYqk8WawuKcvgDpMnhw7m58+HI4+E558PF4ZFROqKChOBmR2VbHrpjmqyzfTpcOWV8Oab4S6gSZNg4EA9GyAidU8qVUO/SRhuAvQFZgBZ2Zni55+HHsQmTAgdydx3H5x/PuyyS6YjExGpmlSqhn6WOG5mHYA7YouolvruO7jhBrjnHsjJgeuug6uuguaqOBOROi6lRudKKQQOrOlAarPx4+GSS2DtWthtN/jv/4bLLst0VCIiNSOVawR3Ax6NNgB6Ep4wrpCZDQDuBHKAB939f0p9fjtwTDTaFNjT3VumFnp6jB8PF1wA338fxr/7LnQm06pVaDdIRKSuM3cvfwazcxJGtwIF7v52hSs2ywE+BY4nlCKmA0PdfW4Z818G9HL3ctvkzMvL8/z8/Io2X2M6dYLFi3ee3rEjFBSkLQwRkWoxsxnunpfss1Sqhp4FNrr7tmhlOWbW1N03VLBcX2CBuy+KlpsAnAIkTQTAUODGFOJJq2RJAOCLL9Ibh4hIXFJ6shjYNWF8V+DVFJZrR+jWslhhNG0nZtYR6Ay8VsbnF5pZvpnlL1++PIVN15y99ko+fd990xqGiEhsUkkETRK7p4yGa7oR5SHAs8WljtLcfZy757l73h577FHDmy5fp047T2vaFMaOTWsYIiKxSSURfGdmvYtHzKwP8H0Kyy0BOiSMt4+mJTME+FsK60yrJUtgxozQgmjHjuFhsY4dYdw4XSgWkfojlWsEo4FnzOwrQjtDexO6rqzIdKCLmXUmJIAhwFmlZzKzbsDuwDupBp0u48aFjubvvhv23z/T0YiIxCOVB8qmRwfr4u5U5rv7lhSW22pmo4CXCbePPuzuH5vZTUC+u0+OZh0CTPCKbl9Ks82b4f774cQTlQREpH5L5TmCS4Hx7v5RNL67mQ119/sqWtbdpwBTSk27odT4mEpFnCbPPgvLlunBMRGp/1K5RnCBu68uHnH3VcAF8YVUO9x9d+hX4PjjMx2JiEi8UkkEOYmd0kQPijWKL6TMy88PPYtdeik0SGUPiYjUYalcLH4JmGhm90fjFwEvxhdS5t17b2hTaPjwTEciIhK/VBLB1cCFwMXR+BzCnUP10vLl8Le/wYgR0KJFpqMREYlfhRUfUQf27wEFhGYjfgJ8Em9YmfPQQ7BpU6gWEhHJBmWWCMzsAEL7P0OBb4GJAO5+TFnL1HVbt4aOZn7yE+jePdPRiIikR3lVQ/OAt4CT3X0BgJldkZaoMuT55+HLL+HOOzMdiYhI+pRXNTQYWAq8bmYPmNmxhCeL66177gmNyf3sZxXPKyJSX5SZCNx9krsPAboBrxOamtjTzP5iZv+RrgDT5eOP4bXXYORIaFiVfttEROqoVC4Wf+fuT0V9F7cHPiDcSVSv3HsvNG4cOqIXEckmlXpcyt1XRU1CHxtXQJmwZg08/jgMHQpt2mQ6GhGR9NJzs8Cjj4a+iEeNynQkIiLpl/WJoKgoVAsddhj06ZPpaERE0i/rE8Err8Bnn6k0ICLZK+sTwT33hH6JTzst05GIiGRGVieChQthyhS46CJoVK/bUxURKVtWJ4L77oOcnJAIRESyVdYmgu++g4cfhsGDoW3bTEcjIpI5WZsIxo+H1avVFaWISFYmAvdwkTg3Fw4/PNPRiIhkVla2qvPWW/Dhh/Dgg2D1uhk9EZGKZWWJ4O67YffdQ5MSIiLZLusSQWEhPPccnHceNG2a6WhERDIv6xLBX/8ampW45JJMRyIiUjtkVSLYtAnGjYOTT4bOnTMdjYhI7ZBVieCZZ2D5ct0yKiKSKKsSwd13Q9eucGy96k1BRKR6siYRvP9+eF16KTTImm8tIlKxrDkkvv46NG8O55yT6UhERGqXrEkEV18NixbBD36Q6UhERGqXrEkEoP6IRUSSyapEICIiO1MiEBHJcrEmAjMbYGbzzWyBmV1TxjxnmNlcM/vYzJ6KMx4REdlZbK2PmlkOcC9wPFAITDezye4+N2GeLsBvgcPdfZWZ7RlXPCIiklycJYK+wAJ3X+Tum4EJwCml5rkAuNfdVwG4+zcxxiMiIknEmQjaAV8mjBdG0xIdABxgZm+b2btmNiDZiszsQjPLN7P85cuXxxSuiEh2yvTF4oZAF6A/MBR4wMxalp7J3ce5e5675+2xxx5pDlFEpH6LMxEsATokjLePpiUqBCa7+xZ3/xz4lJAYREQkTeJMBNOBLmbW2cwaAUOAyaXmmUQoDWBmbQhVRYtijElEREqJLRG4+1ZgFPAy8AnwtLt/bGY3mdnAaLaXgRVmNhd4HfiNu6+IKyYREdmZuXumY6iUvLw8z8/Pz3QYIiJ1ipnNcPe8ZJ9l+mKxiIhkmBKBiEiWUyIQEclySgQiIllOiUBEJMspEYiIZDklAhGRLKdEICKS5ZQIRESynBKBiEiWUyIQEclySgQiIllOiUBEJMspEYiIZLmGmQ5ARKpuy5YtFBYWsnHjxkyHIrVEkyZNaN++PbvsskvKyygRiNRhhYWFNG/enE6dOmFmmQ5HMszdWbFiBYWFhXTu3Dnl5VQ1JFKHbdy4kdatWysJCABmRuvWrStdQlQiEKnjlAQkUVV+D0oEIiJZTolAJIuMHw+dOkGDBuF9/PjqrW/FihX07NmTnj17svfee9OuXbuS8c2bN5e7bH5+PpdffnmF2+jXr1/1gpQK6WKxSJYYPx4uvBA2bAjjixeHcYBhw6q2ztatWzNr1iwAxowZQ7NmzbjyyitLPt+6dSsNGyY/zOTl5ZGXl7Qv9R1MmzatasFl0LZt28jJycl0GClTiUAkS1x77fYkUGzDhjC9Jg0fPpyLL76YQw89lKuuuor333+fww47jF69etGvXz/mz58PwBtvvMHJJ58MhCQyYsQI+vfvz3777cddd91Vsr5mzZqVzN+/f39OO+00unXrxrBhw3B3AKZMmUK3bt3o06cPl19+ecl6ExUUFHDkkUfSu3dvevfuvUOCufnmm+nRowe5ublcc801ACxYsIDjjjuO3NxcevfuzcKFC3eIGWDUqFE8+uijAHTq1Imrr76a3r1788wzz/DAAw9wyCGHkJuby6mnnsqGaOcvW7aMQYMGkZubS25uLtOmTeOGG27gjjvuKFnvtddey5133lntv0WqVCIQyRJffFG56dVRWFjItGnTyMnJYe3atbz11ls0bNiQV199ld/97nf8/e9/32mZefPm8frrr7Nu3Tq6du3KyJEjd7oX/oMPPuDjjz+mbdu2HH744bz99tvk5eVx0UUX8eabb9K5c2eGDh2aNKY999yTf/7znzRp0oTPPvuMoUOHkp+fz4svvsg//vEP3nvvPZo2bcrKlSsBGDZsGNdccw2DBg1i48aNFBUV8eWXX5b7vVu3bs3MmTOBULlByCwAAA98SURBVG12wQUXAHDdddfx0EMPcdlll3H55Zdz9NFH89xzz7Ft2zbWr19P27ZtGTx4MKNHj6aoqIgJEybw/vvvV3q/V5USgUiW2HffUB2UbHpNO/3000uqRtasWcM555zDZ599hpmxZcuWpMucdNJJNG7cmMaNG7PnnnuybNky2rdvv8M8ffv2LZnWs2dPCgoKaNasGfvtt1/JffNDhw5l3LhxO61/y5YtjBo1ilmzZpGTk8Onn34KwKuvvsq5555L06ZNAWjVqhXr1q1jyZIlDBo0CAgPaaXizDPPLBn+6KOPuO6661i9ejXr16/npz/9KQCvvfYajz/+OAA5OTm0aNGCFi1a0Lp1az744AOWLVtGr169aN26dUrbrAlKBCJZYuzYHa8RADRtGqbXtN12261k+Prrr+eYY47hueeeo6CggP79+yddpnHjxiXDOTk5bN26tUrzlOX2229nr732Yvbs2RQVFaV8cE/UsGFDioqKSsZL36+f+L2HDx/OpEmTyM3N5dFHH+WNN94od93nn38+jz76KF9//TUjRoyodGzVoWsEIlli2DAYNw46dgSz8D5uXNUvFKdqzZo1tGvXDqCkPr0mde3alUWLFlFQUADAxIkTy4xjn332oUGDBjzxxBNs27YNgOOPP55HHnmkpA5/5cqVNG/enPbt2zNp0iQANm3axIYNG+jYsSNz585l06ZNrF69mqlTp5YZ17p169hnn33YsmUL4xNuzzr22GP5y1/+AoSLymvWrAFg0KBBvPTSS0yfPr2k9JAuSgQiWWTYMCgogKKi8B53EgC46qqr+O1vf0uvXr0qdQafql133ZX77ruPAQMG0KdPH5o3b06LFi12mu+SSy7hscceIzc3l3nz5pWcvQ8YMICBAweSl5dHz549ufXWWwF44oknuOuuuzj44IPp168fX3/9NR06dOCMM87gRz/6EWeccQa9evUqM67f//73HHrooRx++OF069atZPqdd97J66+/To8ePejTpw9z584FoFGjRhxzzDGcccYZab/jyIqvutcVeXl5np+fn+kwRGqFTz75hAMPPDDTYWTc+vXradasGe7OpZdeSpcuXbjiiisyHValFBUVldxx1KVLl2qtK9nvwsxmuHvS+3VVIhCROu+BBx6gZ8+edO/enTVr1nDRRRdlOqRKmTt3Lj/84Q859thjq50EqkIXi0WkzrviiivqXAkg0UEHHcSiRYsytn2VCEREspwSgYhIllMiEBHJcrEmAjMbYGbzzWyBmV2T5PPhZrbczGZFr/PjjEdERHYWWyIwsxzgXuAE4CBgqJkdlGTWie7eM3o9GFc8IlLzjjnmGF5++eUdpt1xxx2MHDmyzGX69+9P8S3gJ554IqtXr95pnjFjxpTcz1+WSZMmldyDD3DDDTfw6quvViZ8icRZIugLLHD3Re6+GZgAnBLj9kQkzYYOHcqECRN2mDZhwoQyG34rbcqUKbRs2bJK2y6dCG666SaOO+64Kq0rU4qfbs60OBNBOyCxqb7CaFppp5rZHDN71sw6JFuRmV1oZvlmlr98+fI4YhWp80aPhv79a/Y1enT52zzttNN44YUXSjqhKSgo4KuvvuLII49k5MiR5OXl0b17d2688caky3fq1Ilvv/0WgLFjx3LAAQdwxBFHlDRVDSRtznnatGlMnjyZ3/zmN/Ts2ZOFCxcyfPhwnn32WQCmTp1Kr1696NGjByNGjGDTpk0l27vxxhvp3bs3PXr0YN68eTvFlI3NVWf6YvHzQCd3Pxj4J/BYspncfZy757l73h577JHWAEWkbK1ataJv3768+OKLQCgNnHHGGZgZY8eOJT8/nzlz5vCvf/2LOXPmlLmeGTNmMGHCBGbNmsWUKVOYPn16yWeDBw9m+vTpzJ49mwMPPJCHHnqIfv36MXDgQG655RZmzZrF/vvvXzL/xo0bGT58OBMnTuTDDz9k69atJW37ALRp04aZM2cycuTIpNVPxc1Vz5w5k4kTJ5b0opbYXPXs2bO56qqrgNBc9aWXXsrs2bOZNm0a++yzT4X7rbi56iFDhiT9fkBJc9WzZ89m5syZdO/enREjRpS0XFrcXPXZZ59d4fYqEucDZUuAxDP89tG0Eu6+ImH0QeBPMcYjUq8lnCimVXH10CmnnMKECRNKDmRPP/0048aNY+vWrSxdupS5c+dy8MEHJ13HW2+9xaBBg0qagh44cGDJZ2U151yW+fPn07lzZw444AAAzjnnHO69915GR8WbwYMHA9CnTx/+93//d6fls7G56jhLBNOBLmbW2cwaAUOAyYkzmFli6hwIfBJHIDXdT6uIbHfKKacwdepUZs6cyYYNG+jTpw+ff/45t956K1OnTmXOnDmcdNJJOzXZnKrhw4dzzz338OGHH3LjjTdWeT3FipuyLqsZ68TmqvPz8yvsezmZyjZXXZnvV9xc9SOPPFJjzVXHlgjcfSswCniZcIB/2t0/NrObzKw43V9uZh+b2WzgcmB4TcdR3E/r4sXgvr2fViUDkZrRrFkzjjnmGEaMGFFykXjt2rXstttutGjRgmXLlpVUHZXlqKOOYtKkSXz//fesW7eO559/vuSzsppzbt68OevWrdtpXV27dqWgoIAFCxYAoRXRo48+OuXvk43NVcd6jcDdp7j7Ae6+v7uPjabd4O6To+Hfunt3d89192PcfecrN9WUrn5aRbLZ0KFDmT17dkkiyM3NpVevXnTr1o2zzjqLww8/vNzle/fuzZlnnklubi4nnHAChxxySMlnZTXnPGTIEG655RZ69erFwoULS6Y3adKERx55hNNPP50ePXrQoEEDLr744pS/SzY2V13vm6Fu0CCUBEozC22yi9RlaoY6+6TSXLWaoS6lrP5Y4+inVUQkTnE1V13vm6FOZz+tIiJxiqu56npfIshUP60i6VLXqnclXlX5PdT7EgGEg74O/FIfNWnShBUrVtC6dWvMLNPhSIa5OytWrEj5eYZiWZEIROqr9u3bU1hYiJpekWJNmjShffv2lVpGiUCkDttll13o3LlzpsOQOq7eXyMQEZHyKRGIiGQ5JQIRkSxX554sNrPlwOJMx1GGNsC3mQ6iHIqvemp7fFD7Y1R81VOd+Dq6e9J2/OtcIqjNzCy/rEe4awPFVz21PT6o/TEqvuqJKz5VDYmIZDklAhGRLKdEULPGZTqACii+6qnt8UHtj1HxVU8s8ekagYhIllOJQEQkyykRiIhkOSWCSjKzDmb2upnNjfpb/lWSefqb2RozmxW9bkhzjAVm9mG07Z26c7PgLjNbYGZzzKx3GmPrmrBfZpnZWjMbXWqetO8/M3vYzL4xs48SprUys3+a2WfR++5lLHtONM9nZnZOmmK7xczmRX+/58ysZRnLlvtbiDnGMWa2JOHveGIZyw4ws/nR7/GaNMY3MSG2AjObVcayse7Dso4paf39ubtelXgB+wC9o+HmwKfAQaXm6Q/8XwZjLADalPP5icCLgAE/Bt7LUJw5wNeEB10yuv+Ao4DewEcJ0/4EXBMNXwPcnGS5VsCi6H33aHj3NMT2H0DDaPjmZLGl8luIOcYxwJUp/AYWAvsBjYDZpf+f4oqv1Oe3ATdkYh+WdUxJ5+9PJYJKcvel7j4zGl4HfAK0y2xUlXYK8LgH7wItzWyfDMRxLLDQ3TP+pLi7vwmsLDX5FOCxaPgx4OdJFv0p8E93X+nuq4B/AgPijs3dX3H3rdHou0Dl2h2uYWXsv1T0BRa4+yJ33wxMIOz3GlVefBY6cjgD+FtNbzcV5RxT0vb7UyKoBjPrBPQC3kvy8WFmNtvMXjSz7mkNDBx4xcxmmNmFST5vB3yZMF5IZpLZEMr+58vk/iu2l7svjYa/BvZKMk9t2JcjCCW8ZCr6LcRtVFR99XAZVRu1Yf8dCSxz98/K+Dxt+7DUMSVtvz8lgioys2bA34HR7r621MczCdUducDdwKQ0h3eEu/cGTgAuNbOj0rz9CplZI2Ag8EySjzO9/3bioRxe6+61NrNrga3A+DJmyeRv4S/A/kBPYCmh+qU2Gkr5pYG07MPyjilx//6UCKrAzHYh/MHGu/v/lv7c3de6+/poeAqwi5m1SVd87r4kev8GeI5Q/E60BOiQMN4+mpZOJwAz3X1Z6Q8yvf8SLCuuMovev0kyT8b2pZkNB04GhkUHip2k8FuIjbsvc/dt7l4EPFDGtjP6WzSzhsBgYGJZ86RjH5ZxTEnb70+JoJKi+sSHgE/c/c9lzLN3NB9m1pewn1ekKb7dzKx58TDhouJHpWabDPwyunvox8CahCJoupR5FpbJ/VfKZKD4LoxzgH8kmedl4D/MbPeo6uM/ommxMrMBwFXAQHffUMY8qfwW4owx8brToDK2PR3oYmado1LiEMJ+T5fjgHnuXpjsw3Tsw3KOKen7/cV1Jby+voAjCEW0OcCs6HUicDFwcTTPKOBjwh0Q7wL90hjfftF2Z0cxXBtNT4zPgHsJd2t8COSleR/uRjiwt0iYltH9R0hKS4EthHrW84DWwFTgM+BVoFU0bx7wYMKyI4AF0evcNMW2gFA3XPwb/Gs0b1tgSnm/hTTuvyei39ccwkFtn9IxRuMnEu6UWRhXjMnii6Y/Wvy7S5g3rfuwnGNK2n5/amJCRCTLqWpIRCTLKRGIiGQ5JQIRkSynRCAikuWUCEREspwSgUjEzLbZji2j1lhLmGbWKbHlS5HapGGmAxCpRb53956ZDkIk3VQiEKlA1B79n6I26d83sx9G0zuZ2WtRo2pTzWzfaPpeFvoImB29+kWryjGzB6I2518xs12j+S+P2qKfY2YTMvQ1JYspEYhst2upqqEzEz5b4+49gHuAO6JpdwOPufvBhEbf7oqm3wX8y0Ojeb0JT6QCdAHudffuwGrg1Gj6NUCvaD0Xx/XlRMqiJ4tFIma23t2bJZleAPzE3RdFjYN97e6tzexbQrMJW6LpS929jZktB9q7+6aEdXQitBvfJRq/GtjF3f9gZi8B6wmtrE7yqME9kXRRiUAkNV7GcGVsShjexvZrdCcR2n7qDUyPWsQUSRslApHUnJnw/k40PI3QWibAMOCtaHgqMBLAzHLMrEVZKzWzBkAHd38duBpoAexUKhGJk848RLbb1XbswPwldy++hXR3M5tDOKsfGk27DHjEzH4DLAfOjab/ChhnZucRzvxHElq+TCYHeDJKFgbc5e6ra+wbiaRA1whEKhBdI8hz928zHYtIHFQ1JCKS5VQiEBHJcioRiIhkOSUCEZEsp0QgIpLllAhERLKcEoGISJb7f4+BXwidUAHfAAAAAElFTkSuQmCC\n", 424 | "text/plain": [ 425 | "
" 426 | ] 427 | }, 428 | "metadata": { 429 | "needs_background": "light" 430 | }, 431 | "output_type": "display_data" 432 | } 433 | ], 434 | "source": [ 435 | "plt.clf()\n", 436 | "\n", 437 | "acc_values = history_dict['categorical_accuracy']\n", 438 | "val_acc_values = history_dict['val_categorical_accuracy']\n", 439 | "\n", 440 | "plt.plot(epochs, acc_values, 'bo', label='Training accuracy')\n", 441 | "plt.plot(epochs, val_acc_values, 'b', label='Validation accuracy')\n", 442 | "plt.title('Training and validation accuracy')\n", 443 | "plt.xlabel('Epochs')\n", 444 | "plt.ylabel('Accuracy')\n", 445 | "plt.legend()\n", 446 | "\n", 447 | "plt.show" 448 | ] 449 | }, 450 | { 451 | "cell_type": "markdown", 452 | "metadata": {}, 453 | "source": [ 454 | "The graphs show that the model starts overfitting after ~10 epochs. Let's train it again with better hyperparameters" 455 | ] 456 | }, 457 | { 458 | "cell_type": "code", 459 | "execution_count": 32, 460 | "metadata": {}, 461 | "outputs": [ 462 | { 463 | "name": "stdout", 464 | "output_type": "stream", 465 | "text": [ 466 | "Train on 7982 samples, validate on 1000 samples\n", 467 | "Epoch 1/10\n", 468 | "7982/7982 [==============================] - 1s 110us/sample - loss: 2.5890 - categorical_accuracy: 0.5153 - val_loss: 1.6957 - val_categorical_accuracy: 0.6260\n", 469 | "Epoch 2/10\n", 470 | "7982/7982 [==============================] - 1s 70us/sample - loss: 1.4276 - categorical_accuracy: 0.7008 - val_loss: 1.3144 - val_categorical_accuracy: 0.6970\n", 471 | "Epoch 3/10\n", 472 | "7982/7982 [==============================] - 1s 65us/sample - loss: 1.0681 - categorical_accuracy: 0.7681 - val_loss: 1.1524 - val_categorical_accuracy: 0.7470\n", 473 | "Epoch 4/10\n", 474 | "7982/7982 [==============================] - 1s 67us/sample - loss: 0.8389 - categorical_accuracy: 0.8191 - val_loss: 1.0426 - val_categorical_accuracy: 0.7780\n", 475 | "Epoch 5/10\n", 476 | "7982/7982 [==============================] - 1s 69us/sample - loss: 0.6735 - categorical_accuracy: 0.8577 - val_loss: 0.9675 - val_categorical_accuracy: 0.8000\n", 477 | "Epoch 6/10\n", 478 | "7982/7982 [==============================] - 1s 67us/sample - loss: 0.5400 - categorical_accuracy: 0.8865 - val_loss: 0.9544 - val_categorical_accuracy: 0.8130\n", 479 | "Epoch 7/10\n", 480 | "7982/7982 [==============================] - 1s 68us/sample - loss: 0.4338 - categorical_accuracy: 0.9098 - val_loss: 0.9292 - val_categorical_accuracy: 0.8030\n", 481 | "Epoch 8/10\n", 482 | "7982/7982 [==============================] - 1s 68us/sample - loss: 0.3550 - categorical_accuracy: 0.9266 - val_loss: 0.8958 - val_categorical_accuracy: 0.8180\n", 483 | "Epoch 9/10\n", 484 | "7982/7982 [==============================] - 1s 68us/sample - loss: 0.2919 - categorical_accuracy: 0.9385 - val_loss: 0.9145 - val_categorical_accuracy: 0.8030\n", 485 | "Epoch 10/10\n", 486 | "7982/7982 [==============================] - 1s 64us/sample - loss: 0.2514 - categorical_accuracy: 0.9444 - val_loss: 0.9362 - val_categorical_accuracy: 0.8040\n", 487 | "2246/2246 [==============================] - 0s 55us/sample - loss: 1.0099 - categorical_accuracy: 0.7903\n" 488 | ] 489 | } 490 | ], 491 | "source": [ 492 | "model = models.Sequential()\n", 493 | "model.add(layers.Dense(64, activation='relu', input_shape=(LIMIT_WORD,)))\n", 494 | "model.add(layers.Dense(64, activation='relu'))\n", 495 | "model.add(layers.Dense(46, activation='softmax'))\n", 496 | "\n", 497 | "model.compile(\n", 498 | " optimizer=optimizers.RMSprop(lr=0.001),\n", 499 | " loss=losses.categorical_crossentropy,\n", 500 | " metrics=[metrics.categorical_accuracy]\n", 501 | ")\n", 502 | "model.fit(\n", 503 | " partial_x_train,\n", 504 | " partial_y_train,\n", 505 | " epochs=10,\n", 506 | " batch_size=512,\n", 507 | " validation_data=(x_val, y_val)\n", 508 | ")\n", 509 | "results = model.evaluate(x_test, one_hot_test_labels)" 510 | ] 511 | }, 512 | { 513 | "cell_type": "code", 514 | "execution_count": 33, 515 | "metadata": {}, 516 | "outputs": [ 517 | { 518 | "name": "stdout", 519 | "output_type": "stream", 520 | "text": [ 521 | "[1.009892533957056, 0.7902939]\n" 522 | ] 523 | } 524 | ], 525 | "source": [ 526 | "print(results)" 527 | ] 528 | }, 529 | { 530 | "cell_type": "code", 531 | "execution_count": 34, 532 | "metadata": {}, 533 | "outputs": [ 534 | { 535 | "data": { 536 | "text/plain": [ 537 | "array([[9.5555675e-05, 4.1223677e-05, 5.9732270e-06, ..., 8.4474605e-06,\n", 538 | " 2.8842246e-06, 3.9834860e-05],\n", 539 | " [5.2923879e-05, 4.8729219e-03, 3.6130776e-04, ..., 2.7193574e-03,\n", 540 | " 7.5042913e-06, 1.2056468e-06],\n", 541 | " [1.4567733e-04, 8.8359046e-01, 4.8508430e-03, ..., 3.2348607e-03,\n", 542 | " 7.0754027e-06, 3.8999908e-05],\n", 543 | " ...,\n", 544 | " [6.5655986e-05, 1.3337305e-04, 2.7083925e-05, ..., 3.2714066e-05,\n", 545 | " 8.6744922e-06, 6.2861247e-05],\n", 546 | " [1.5852891e-03, 9.2632450e-02, 2.5552192e-03, ..., 6.8909382e-03,\n", 547 | " 6.1394251e-04, 9.0326834e-04],\n", 548 | " [1.1718799e-04, 7.2575665e-01, 1.6222894e-02, ..., 1.9016492e-03,\n", 549 | " 7.5688044e-06, 3.5573521e-05]], dtype=float32)" 550 | ] 551 | }, 552 | "execution_count": 34, 553 | "metadata": {}, 554 | "output_type": "execute_result" 555 | } 556 | ], 557 | "source": [ 558 | "model.predict(x_test)" 559 | ] 560 | }, 561 | { 562 | "cell_type": "markdown", 563 | "metadata": {}, 564 | "source": [ 565 | "With this approach we obtained an accuracy of ~80%." 566 | ] 567 | }, 568 | { 569 | "cell_type": "code", 570 | "execution_count": null, 571 | "metadata": {}, 572 | "outputs": [], 573 | "source": [] 574 | } 575 | ], 576 | "metadata": { 577 | "kernelspec": { 578 | "display_name": "Python 3", 579 | "language": "python", 580 | "name": "python3" 581 | }, 582 | "language_info": { 583 | "codemirror_mode": { 584 | "name": "ipython", 585 | "version": 3 586 | }, 587 | "file_extension": ".py", 588 | "mimetype": "text/x-python", 589 | "name": "python", 590 | "nbconvert_exporter": "python", 591 | "pygments_lexer": "ipython3", 592 | "version": "3.6.10" 593 | } 594 | }, 595 | "nbformat": 4, 596 | "nbformat_minor": 4 597 | } 598 | -------------------------------------------------------------------------------- /notebooks/chapter_04/resources/binary_class_model_validation_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_04/resources/binary_class_model_validation_loss.png -------------------------------------------------------------------------------- /notebooks/chapter_05/01 - Introduction to CNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Introduction to CNN\n", 8 | "\n", 9 | "The problem is about **classifying grayscale images of handwritten digits** (28 pixels by 28 pixels), into their 10 categories (0 to 9), exactly like the one in chapter 2. This time will try to solve the problem through a convolutional neural network (CNN) and see if performance improves." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import numpy as np\n", 19 | "import matplotlib.pyplot as plt\n", 20 | "import tensorflow as tf\n", 21 | "\n", 22 | "from tensorflow.keras import models, layers, optimizers, losses, metrics\n", 23 | "from tensorflow.keras.datasets import mnist\n", 24 | "from tensorflow.keras.utils import to_categorical" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "## Build the network\n", 32 | "\n", 33 | "A basic convnet will be used: a stack of `Conv2D` and `MaxPooling2D` layers.\n", 34 | "Importantly, a CNN takes as input tensors of shape (`image_height, image_width, image_channels`), not including the batch dimensions. In this case, we have to configure the CNN to process images of a size compatible with the MNIST database, so it will be of size (28, 28, 1). This will be the input shape to pass to the first layer." 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 2, 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "name": "stdout", 44 | "output_type": "stream", 45 | "text": [ 46 | "Model: \"sequential\"\n", 47 | "_________________________________________________________________\n", 48 | "Layer (type) Output Shape Param # \n", 49 | "=================================================================\n", 50 | "conv2d (Conv2D) (None, 26, 26, 32) 320 \n", 51 | "_________________________________________________________________\n", 52 | "max_pooling2d (MaxPooling2D) (None, 13, 13, 32) 0 \n", 53 | "_________________________________________________________________\n", 54 | "conv2d_1 (Conv2D) (None, 11, 11, 64) 18496 \n", 55 | "_________________________________________________________________\n", 56 | "max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64) 0 \n", 57 | "_________________________________________________________________\n", 58 | "conv2d_2 (Conv2D) (None, 3, 3, 64) 36928 \n", 59 | "=================================================================\n", 60 | "Total params: 55,744\n", 61 | "Trainable params: 55,744\n", 62 | "Non-trainable params: 0\n", 63 | "_________________________________________________________________\n" 64 | ] 65 | } 66 | ], 67 | "source": [ 68 | "model = models.Sequential()\n", 69 | "model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n", 70 | "model.add(layers.MaxPooling2D((2, 2)))\n", 71 | "model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n", 72 | "model.add(layers.MaxPooling2D((2, 2)))\n", 73 | "model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n", 74 | "\n", 75 | "model.summary()" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": {}, 81 | "source": [ 82 | "The output of every `Conv2D` and `MaxPooling2D` layer is also a tensor of shape (`height, width, channels`). Interesting to note how the dimensions tend to shrink as going deeper in the network. The channel is controlled by the first parameter in `Conv2D` layers.\n", 83 | "\n", 84 | "While the the 3rd parameter of the input shape indicates the color channels, in the output of a layer it indicates the number of filters (features) over its input. So, every dimension in the depth axis is a feature (filter), and the 2D tensor is a 2D spatial map of the response of this filter over the input." 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "#### Insert a classifier on top of the CNN\n", 92 | "\n", 93 | "This is a classification problem, so we need to put the last output tensor of the CNN as input to a densely connected network, similar to the one in chapter 2. We have a class of 10 categories, so the solution will be a 10-way classification, using a final layer of 10 outputs and a `softmax` activation." 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 3, 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "name": "stdout", 103 | "output_type": "stream", 104 | "text": [ 105 | "Model: \"sequential\"\n", 106 | "_________________________________________________________________\n", 107 | "Layer (type) Output Shape Param # \n", 108 | "=================================================================\n", 109 | "conv2d (Conv2D) (None, 26, 26, 32) 320 \n", 110 | "_________________________________________________________________\n", 111 | "max_pooling2d (MaxPooling2D) (None, 13, 13, 32) 0 \n", 112 | "_________________________________________________________________\n", 113 | "conv2d_1 (Conv2D) (None, 11, 11, 64) 18496 \n", 114 | "_________________________________________________________________\n", 115 | "max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64) 0 \n", 116 | "_________________________________________________________________\n", 117 | "conv2d_2 (Conv2D) (None, 3, 3, 64) 36928 \n", 118 | "_________________________________________________________________\n", 119 | "flatten (Flatten) (None, 576) 0 \n", 120 | "_________________________________________________________________\n", 121 | "dense (Dense) (None, 64) 36928 \n", 122 | "_________________________________________________________________\n", 123 | "dense_1 (Dense) (None, 10) 650 \n", 124 | "=================================================================\n", 125 | "Total params: 93,322\n", 126 | "Trainable params: 93,322\n", 127 | "Non-trainable params: 0\n", 128 | "_________________________________________________________________\n" 129 | ] 130 | } 131 | ], 132 | "source": [ 133 | "model.add(layers.Flatten())\n", 134 | "model.add(layers.Dense(64, activation='relu'))\n", 135 | "model.add(layers.Dense(10, activation='softmax'))\n", 136 | "\n", 137 | "model.summary()" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": {}, 143 | "source": [ 144 | "Note that the `Flatten` layer produces a vector of shape (`3 * 3 * 64`), which is the output of the last `Conv2D` layer. Operation needed before going through the 2 dense layers" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": {}, 150 | "source": [ 151 | "## Load dataset and preprocess data\n", 152 | "\n", 153 | "`train_images` and `train_labels` form the \"training set\", the data that the model will learn from. The model will then be tested on the \"test set\", `test_images` and `test_labels`. The images are encoded as Numpy arrays, and the labels are simply an array of digits, ranging from 0 to 9. There is a one-to-one correspondence between the images and the labels.\n", 154 | "\n", 155 | "Before training, we will preprocess our data by reshaping it into the shape that the CNN expects. We do also need to categorically encode the labels (one-hot)" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": 10, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 11, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "train_images = train_images.reshape((60000, 28, 28, 1))\n", 174 | "train_images = train_images.astype('float32') / 255" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 12, 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [ 183 | "test_images = test_images.reshape((10000, 28, 28, 1))\n", 184 | "test_images = test_images.astype('float32') / 255" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 13, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "train_labels = to_categorical(train_labels)\n", 194 | "test_labels = to_categorical(test_labels)" 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": {}, 200 | "source": [ 201 | "## Compile and train model" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": 14, 207 | "metadata": {}, 208 | "outputs": [ 209 | { 210 | "name": "stdout", 211 | "output_type": "stream", 212 | "text": [ 213 | "Train on 60000 samples\n", 214 | "Epoch 1/5\n", 215 | "60000/60000 [==============================] - 18s 305us/sample - loss: 0.1729 - accuracy: 0.9465\n", 216 | "Epoch 2/5\n", 217 | "60000/60000 [==============================] - 18s 296us/sample - loss: 0.0486 - accuracy: 0.9852\n", 218 | "Epoch 3/5\n", 219 | "60000/60000 [==============================] - 18s 295us/sample - loss: 0.0331 - accuracy: 0.9900\n", 220 | "Epoch 4/5\n", 221 | "60000/60000 [==============================] - 18s 306us/sample - loss: 0.0254 - accuracy: 0.9923\n", 222 | "Epoch 5/5\n", 223 | "60000/60000 [==============================] - 18s 305us/sample - loss: 0.0194 - accuracy: 0.9938\n" 224 | ] 225 | }, 226 | { 227 | "data": { 228 | "text/plain": [ 229 | "" 230 | ] 231 | }, 232 | "execution_count": 14, 233 | "metadata": {}, 234 | "output_type": "execute_result" 235 | } 236 | ], 237 | "source": [ 238 | "model.compile(\n", 239 | " optimizer='rmsprop',\n", 240 | " loss='categorical_crossentropy',\n", 241 | " metrics=['accuracy']\n", 242 | ")\n", 243 | "model.fit(train_images, train_labels, epochs=5, batch_size=64)" 244 | ] 245 | }, 246 | { 247 | "cell_type": "markdown", 248 | "metadata": {}, 249 | "source": [ 250 | "## Evaluation" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 15, 256 | "metadata": {}, 257 | "outputs": [ 258 | { 259 | "name": "stdout", 260 | "output_type": "stream", 261 | "text": [ 262 | "10000/10000 [==============================] - 1s 88us/sample - loss: 0.0287 - accuracy: 0.9909\n" 263 | ] 264 | } 265 | ], 266 | "source": [ 267 | "test_loss, test_acc = model.evaluate(test_images, test_labels)" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 16, 273 | "metadata": {}, 274 | "outputs": [ 275 | { 276 | "name": "stdout", 277 | "output_type": "stream", 278 | "text": [ 279 | "test_acc: 0.9909\n" 280 | ] 281 | } 282 | ], 283 | "source": [ 284 | "print('test_acc:', test_acc)" 285 | ] 286 | }, 287 | { 288 | "cell_type": "markdown", 289 | "metadata": {}, 290 | "source": [ 291 | "The test set accuracy turns out to be *99.1%*, which is quite an improvement from the fully connected network in chapter 2, which reached *97.8%* accuracy." 292 | ] 293 | }, 294 | { 295 | "cell_type": "code", 296 | "execution_count": null, 297 | "metadata": {}, 298 | "outputs": [], 299 | "source": [] 300 | } 301 | ], 302 | "metadata": { 303 | "kernelspec": { 304 | "display_name": "Python 3", 305 | "language": "python", 306 | "name": "python3" 307 | }, 308 | "language_info": { 309 | "codemirror_mode": { 310 | "name": "ipython", 311 | "version": 3 312 | }, 313 | "file_extension": ".py", 314 | "mimetype": "text/x-python", 315 | "name": "python", 316 | "nbconvert_exporter": "python", 317 | "pygments_lexer": "ipython3", 318 | "version": "3.6.10" 319 | } 320 | }, 321 | "nbformat": 4, 322 | "nbformat_minor": 4 323 | } 324 | -------------------------------------------------------------------------------- /notebooks/chapter_05/datasets/create_cats_dogs_datasets.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | # The path to the directory where the original dataset was uncompressed 5 | ORIGINAL_DATASET_DIR = f"{os.path.dirname(os.path.realpath(__file__))}/Kaggle-cats-dogs" 6 | 7 | # The directory where we will store our smaller dataset 8 | BASE_DIR = f"{os.path.dirname(os.path.realpath(__file__))}/cats_and_dogs_small" 9 | 10 | 11 | def main(): 12 | os.mkdir(BASE_DIR) 13 | 14 | # Directories for our training, validation and test splits 15 | train_dir = os.path.join(BASE_DIR, 'train') 16 | os.mkdir(train_dir) 17 | validation_dir = os.path.join(BASE_DIR, 'validation') 18 | os.mkdir(validation_dir) 19 | test_dir = os.path.join(BASE_DIR, 'test') 20 | os.mkdir(test_dir) 21 | 22 | # Directory with our training cat pictures 23 | train_cats_dir = os.path.join(train_dir, 'cats') 24 | os.mkdir(train_cats_dir) 25 | 26 | # Directory with our training dog pictures 27 | train_dogs_dir = os.path.join(train_dir, 'dogs') 28 | os.mkdir(train_dogs_dir) 29 | 30 | # Directory with our validation cat pictures 31 | validation_cats_dir = os.path.join(validation_dir, 'cats') 32 | os.mkdir(validation_cats_dir) 33 | 34 | # Directory with our validation dog pictures 35 | validation_dogs_dir = os.path.join(validation_dir, 'dogs') 36 | os.mkdir(validation_dogs_dir) 37 | 38 | # Directory with our validation cat pictures 39 | test_cats_dir = os.path.join(test_dir, 'cats') 40 | os.mkdir(test_cats_dir) 41 | 42 | # Directory with our validation dog pictures 43 | test_dogs_dir = os.path.join(test_dir, 'dogs') 44 | os.mkdir(test_dogs_dir) 45 | 46 | # Copy first 1000 cat images to train_cats_dir 47 | fnames = ['cat.{}.jpg'.format(i) for i in range(1000)] 48 | for fname in fnames: 49 | src = os.path.join(ORIGINAL_DATASET_DIR, fname) 50 | dst = os.path.join(train_cats_dir, fname) 51 | shutil.copyfile(src, dst) 52 | 53 | # Copy next 500 cat images to validation_cats_dir 54 | fnames = ['cat.{}.jpg'.format(i) for i in range(1000, 1500)] 55 | for fname in fnames: 56 | src = os.path.join(ORIGINAL_DATASET_DIR, fname) 57 | dst = os.path.join(validation_cats_dir, fname) 58 | shutil.copyfile(src, dst) 59 | 60 | # Copy next 500 cat images to test_cats_dir 61 | fnames = ['cat.{}.jpg'.format(i) for i in range(1500, 2000)] 62 | for fname in fnames: 63 | src = os.path.join(ORIGINAL_DATASET_DIR, fname) 64 | dst = os.path.join(test_cats_dir, fname) 65 | shutil.copyfile(src, dst) 66 | 67 | # Copy first 1000 dog images to train_dogs_dir 68 | fnames = ['dog.{}.jpg'.format(i) for i in range(1000)] 69 | for fname in fnames: 70 | src = os.path.join(ORIGINAL_DATASET_DIR, fname) 71 | dst = os.path.join(train_dogs_dir, fname) 72 | shutil.copyfile(src, dst) 73 | 74 | # Copy next 500 dog images to validation_dogs_dir 75 | fnames = ['dog.{}.jpg'.format(i) for i in range(1000, 1500)] 76 | for fname in fnames: 77 | src = os.path.join(ORIGINAL_DATASET_DIR, fname) 78 | dst = os.path.join(validation_dogs_dir, fname) 79 | shutil.copyfile(src, dst) 80 | 81 | # Copy next 500 dog images to test_dogs_dir 82 | fnames = ['dog.{}.jpg'.format(i) for i in range(1500, 2000)] 83 | for fname in fnames: 84 | src = os.path.join(ORIGINAL_DATASET_DIR, fname) 85 | dst = os.path.join(test_dogs_dir, fname) 86 | shutil.copyfile(src, dst) 87 | 88 | # Sanity checks 89 | print('total training cat images:', len(os.listdir(train_cats_dir))) 90 | print('total training dog images:', len(os.listdir(train_dogs_dir))) 91 | print('total validation cat images:', len(os.listdir(validation_cats_dir))) 92 | print('total validation dog images:', len(os.listdir(validation_dogs_dir))) 93 | print('total test cat images:', len(os.listdir(test_cats_dir))) 94 | print('total test dog images:', len(os.listdir(test_dogs_dir))) 95 | 96 | 97 | if __name__ == "__main__": 98 | main() 99 | -------------------------------------------------------------------------------- /notebooks/chapter_05/resources/creative_commons_elephant.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_05/resources/creative_commons_elephant.jpg -------------------------------------------------------------------------------- /notebooks/chapter_05/resources/elephant_cam.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_05/resources/elephant_cam.jpg -------------------------------------------------------------------------------- /notebooks/chapter_06/01 - One-hot encoding of words or characters.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## One-hot encoding of words or characters\n", 8 | "\n", 9 | "One-hot encoding is **the most common, most basic way to turn a token into a vector** (already used in champetr 3 for [notebook 01](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_03/01%20-%20Binary%20classifier.ipynb) and [02](https://github.com/lucone83/deep-learning-with-python/blob/master/notebooks/chapter_03/02%20-%20Multiclass%20classifier.ipynb)). \n", 10 | "It consists in:\n", 11 | "- associating a unique integer index to every word;\n", 12 | "- turning this integer index i into a binary vector of size _N_, the size of the vocabulary, that would be all-zeros except for the i-th entry, which would be 1.\n", 13 | "\n", 14 | "Of course, one-hot encoding can be done at the character level as well. To unambiguously drive home what one-hot encoding is and how to implement it, here are two toy examples of one-hot encoding: one for words, the other for characters." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "#### Word level one-hot encoding (toy example)" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 14, 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stdout", 31 | "output_type": "stream", 32 | "text": [ 33 | "(2, 10, 11)\n", 34 | "[[0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", 35 | " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n", 36 | " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", 37 | " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", 38 | " [0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", 39 | " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n", 40 | " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", 41 | " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", 42 | " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", 43 | " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]]\n" 44 | ] 45 | } 46 | ], 47 | "source": [ 48 | "import numpy as np\n", 49 | "\n", 50 | "samples = ['The cat sat on the mat.', 'The dog ate my homework.']\n", 51 | "\n", 52 | "# Build an index of all tokens in the data.\n", 53 | "token_index = {}\n", 54 | "for sample in samples:\n", 55 | " # Tokenize the samples via the `split` method.\n", 56 | " # In real life, we would also strip punctuation and special characters from the samples.\n", 57 | " for word in sample.split():\n", 58 | " if word not in token_index:\n", 59 | " # Assign a unique index to each unique word. Note that we don't attribute index 0 to anything.\n", 60 | " token_index[word] = len(token_index) + 1\n", 61 | "\n", 62 | "# Vectorize the samples. We will only consider the first `max_length` words in each sample.\n", 63 | "max_length = 10\n", 64 | "\n", 65 | "# This is where we store our results:\n", 66 | "results = np.zeros((len(samples), max_length, max(token_index.values()) + 1))\n", 67 | "for i, sample in enumerate(samples):\n", 68 | " for j, word in list(enumerate(sample.split()))[:max_length]:\n", 69 | " index = token_index.get(word)\n", 70 | " results[i, j, index] = 1.\n", 71 | " \n", 72 | "print(results.shape)\n", 73 | "print(results[0])" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "#### Character level one-hot encoding (toy example)" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 4, 86 | "metadata": {}, 87 | "outputs": [ 88 | { 89 | "name": "stdout", 90 | "output_type": "stream", 91 | "text": [ 92 | "(2, 50, 101)\n" 93 | ] 94 | } 95 | ], 96 | "source": [ 97 | "import string\n", 98 | "\n", 99 | "samples = ['The cat sat on the mat.', 'The dog ate my homework.']\n", 100 | "characters = string.printable # All printable ASCII characters.\n", 101 | "token_index = dict(zip(characters, range(1, len(characters) + 1)))\n", 102 | "\n", 103 | "max_length = 50\n", 104 | "results = np.zeros((len(samples), max_length, max(token_index.values()) + 1))\n", 105 | "for i, sample in enumerate(samples):\n", 106 | " for j, character in enumerate(sample[:max_length]):\n", 107 | " index = token_index.get(character)\n", 108 | " results[i, j, index] = 1.\n", 109 | " \n", 110 | "print(results.shape)" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | "#### Using Keras for word-level one-hot encoding (toy example)\n", 118 | "\n", 119 | "Keras has built-in utilities for doing one-hot encoding text at the word level or character level, starting from raw text data. It will take care of a number of important features, such as stripping special characters from strings, or only taking into the top N most common words in your dataset (a common restriction to avoid dealing with very large input vector spaces)." 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 10, 125 | "metadata": {}, 126 | "outputs": [ 127 | { 128 | "name": "stdout", 129 | "output_type": "stream", 130 | "text": [ 131 | "Found 9 unique tokens: {'the': 1, 'cat': 2, 'sat': 3, 'on': 4, 'mat': 5, 'dog': 6, 'ate': 7, 'my': 8, 'homework': 9}\n" 132 | ] 133 | } 134 | ], 135 | "source": [ 136 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 137 | "\n", 138 | "samples = ['The cat sat on the mat.', 'The dog ate my homework.']\n", 139 | "\n", 140 | "# Create a tokenizer, configured to only take into account the top-1000 most common words\n", 141 | "tokenizer = Tokenizer(num_words=1000)\n", 142 | "\n", 143 | "# Build the word index\n", 144 | "tokenizer.fit_on_texts(samples)\n", 145 | "\n", 146 | "# Turn strings into lists of integer indices.\n", 147 | "sequences = tokenizer.texts_to_sequences(samples)\n", 148 | "\n", 149 | "# You could also directly get the one-hot binary representations.\n", 150 | "# Note that other vectorization modes than one-hot encoding are supported!\n", 151 | "one_hot_results = tokenizer.texts_to_matrix(samples, mode='binary')\n", 152 | "\n", 153 | "# This is how you can recover the word index that was computed\n", 154 | "word_index = tokenizer.word_index\n", 155 | "\n", 156 | "print(f\"Found {len(word_index)} unique tokens: {word_index}\")" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "#### Word-level one-hot encoding with hashing trick (toy example)\n", 164 | "\n", 165 | "A variant of one-hot encoding is the so-called \"_one-hot hashing trick_\", which can be used when the number of unique tokens in your vocabulary is too large to handle explicitly. Instead of explicitly assigning an index to each word and keeping a reference of these indices in a dictionary, one may **hash words into vectors of fixed size**. This is typically done with a very lightweight hashing function. The main advantage of this method is that it does away with maintaining an explicit word index, which saves memory and allows online encoding of the data (starting to generate token vectors right away, before having seen all of the available data). The one drawback of this method is that it is susceptible to \"hash collisions\": two different words may end up with the same hash, and subsequently any machine learning model looking at these hashes won't be able to tell the difference between these words. The likelihood of hash collisions decreases when the dimensionality of the hashing space is much larger than the total number of unique tokens being hashed." 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 13, 171 | "metadata": {}, 172 | "outputs": [ 173 | { 174 | "name": "stdout", 175 | "output_type": "stream", 176 | "text": [ 177 | "(2, 10, 1000)\n", 178 | "[[0. 0. 0. ... 0. 0. 0.]\n", 179 | " [0. 0. 0. ... 0. 0. 0.]\n", 180 | " [0. 0. 0. ... 0. 0. 0.]\n", 181 | " ...\n", 182 | " [0. 0. 0. ... 0. 0. 0.]\n", 183 | " [0. 0. 0. ... 0. 0. 0.]\n", 184 | " [0. 0. 0. ... 0. 0. 0.]]\n" 185 | ] 186 | } 187 | ], 188 | "source": [ 189 | "samples = ['The cat sat on the mat.', 'The dog ate my homework.']\n", 190 | "\n", 191 | "# Store our words as vectors of size 1000. Note that if you have close to 1000 words (or more)\n", 192 | "# you will start seeing many hash collisions, which will decrease the accuracy of this encoding method.\n", 193 | "dimensionality = 1000\n", 194 | "max_length = 10\n", 195 | "\n", 196 | "results = np.zeros((len(samples), max_length, dimensionality))\n", 197 | "for i, sample in enumerate(samples):\n", 198 | " for j, word in list(enumerate(sample.split()))[:max_length]:\n", 199 | " # Hash the word into a \"random\" integer index that is between 0 and 1000\n", 200 | " index = abs(hash(word)) % dimensionality\n", 201 | " results[i, j, index] = 1.\n", 202 | " \n", 203 | "print(results.shape)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [] 212 | } 213 | ], 214 | "metadata": { 215 | "kernelspec": { 216 | "display_name": "Python 3", 217 | "language": "python", 218 | "name": "python3" 219 | }, 220 | "language_info": { 221 | "codemirror_mode": { 222 | "name": "ipython", 223 | "version": 3 224 | }, 225 | "file_extension": ".py", 226 | "mimetype": "text/x-python", 227 | "name": "python", 228 | "nbconvert_exporter": "python", 229 | "pygments_lexer": "ipython3", 230 | "version": "3.6.10" 231 | } 232 | }, 233 | "nbformat": 4, 234 | "nbformat_minor": 4 235 | } 236 | -------------------------------------------------------------------------------- /notebooks/chapter_06/resources/RNN-basic-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_06/resources/RNN-basic-schema.png -------------------------------------------------------------------------------- /notebooks/chapter_06/resources/bidirectional-RNN.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_06/resources/bidirectional-RNN.png -------------------------------------------------------------------------------- /notebooks/chapter_06/resources/onehot-vs-embedding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_06/resources/onehot-vs-embedding.png -------------------------------------------------------------------------------- /notebooks/chapter_07/01 - The Keras functional API.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# The Keras functional API\n", 8 | "\n", 9 | "In the functional API, you directly manipulate tensors, and you use layers as _functions_ that take tensors and return tensors (hence, the name _functional API_ )." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Introdiction to functional API\n", 17 | "\n", 18 | "Let’s start with a minimal example that shows side by side a simple Sequential model and its equivalent in the functional API." 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": {}, 25 | "outputs": [ 26 | { 27 | "name": "stdout", 28 | "output_type": "stream", 29 | "text": [ 30 | "Model: \"sequential\"\n", 31 | "_________________________________________________________________\n", 32 | "Layer (type) Output Shape Param # \n", 33 | "=================================================================\n", 34 | "dense (Dense) (None, 32) 2080 \n", 35 | "_________________________________________________________________\n", 36 | "dense_1 (Dense) (None, 32) 1056 \n", 37 | "_________________________________________________________________\n", 38 | "dense_2 (Dense) (None, 10) 330 \n", 39 | "=================================================================\n", 40 | "Total params: 3,466\n", 41 | "Trainable params: 3,466\n", 42 | "Non-trainable params: 0\n", 43 | "_________________________________________________________________\n" 44 | ] 45 | } 46 | ], 47 | "source": [ 48 | "from tensorflow.keras.models import Sequential, Model\n", 49 | "from tensorflow.keras import layers\n", 50 | "from tensorflow.keras import Input\n", 51 | "\n", 52 | "seq_model = Sequential()\n", 53 | "seq_model.add(layers.Dense(32, activation='relu', input_shape=(64,)))\n", 54 | "seq_model.add(layers.Dense(32, activation='relu'))\n", 55 | "seq_model.add(layers.Dense(10, activation='softmax'))\n", 56 | "\n", 57 | "seq_model.summary()" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 2, 63 | "metadata": {}, 64 | "outputs": [ 65 | { 66 | "name": "stdout", 67 | "output_type": "stream", 68 | "text": [ 69 | "Model: \"model\"\n", 70 | "_________________________________________________________________\n", 71 | "Layer (type) Output Shape Param # \n", 72 | "=================================================================\n", 73 | "input_1 (InputLayer) [(None, 64)] 0 \n", 74 | "_________________________________________________________________\n", 75 | "dense_3 (Dense) (None, 32) 2080 \n", 76 | "_________________________________________________________________\n", 77 | "dense_4 (Dense) (None, 32) 1056 \n", 78 | "_________________________________________________________________\n", 79 | "dense_5 (Dense) (None, 10) 330 \n", 80 | "=================================================================\n", 81 | "Total params: 3,466\n", 82 | "Trainable params: 3,466\n", 83 | "Non-trainable params: 0\n", 84 | "_________________________________________________________________\n" 85 | ] 86 | } 87 | ], 88 | "source": [ 89 | "# The functional equivalent\n", 90 | "input_tensor = Input(shape=(64,))\n", 91 | "x = layers.Dense(32, activation='relu')(input_tensor)\n", 92 | "x = layers.Dense(32, activation='relu')(x)\n", 93 | "output_tensor = layers.Dense(10, activation='softmax')(x)\n", 94 | "\n", 95 | "model = Model(input_tensor, output_tensor) \n", 96 | "\n", 97 | "model.summary()" 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": {}, 103 | "source": [ 104 | "_Keras_ retrieves every layer involved in going from input_tensor to output_tensor, bringing them together into a graph-like data structure (a `Model`). Of course, the reason it works is that `output_tensor` was obtained by repeatedly transforming `input_tensor`. If we try to build a model from inputs and outputs that weren’t related, we’d get a `ValueError`:" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 3, 110 | "metadata": {}, 111 | "outputs": [ 112 | { 113 | "ename": "ValueError", 114 | "evalue": "Graph disconnected: cannot obtain value for tensor Tensor(\"input_1:0\", shape=(None, 64), dtype=float32) at layer \"input_1\". The following previous layers were accessed without issue: []", 115 | "output_type": "error", 116 | "traceback": [ 117 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 118 | "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", 119 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0munrelated_input\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mInput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mbad_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munrelated_input\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 120 | "\u001b[0;32m~/.anaconda3/envs/dl-python/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 145\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 147\u001b[0m \u001b[0m_keras_api_gauge\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'model'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0;31m# initializing _distribution_strategy here since it is possible to call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 121 | "\u001b[0;32m~/.anaconda3/envs/dl-python/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 167\u001b[0m 'inputs' in kwargs and 'outputs' in kwargs):\n\u001b[1;32m 168\u001b[0m \u001b[0;31m# Graph network\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_graph_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 170\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[0;31m# Subclassed network\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 122 | "\u001b[0;32m~/.anaconda3/envs/dl-python/lib/python3.6/site-packages/tensorflow_core/python/training/tracking/base.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 455\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_self_setattr_tracking\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 456\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 457\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 458\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 459\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_self_setattr_tracking\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprevious_value\u001b[0m \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 123 | "\u001b[0;32m~/.anaconda3/envs/dl-python/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py\u001b[0m in \u001b[0;36m_init_graph_network\u001b[0;34m(self, inputs, outputs, name, **kwargs)\u001b[0m\n\u001b[1;32m 322\u001b[0m \u001b[0;31m# Keep track of the network's nodes and layers.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 323\u001b[0m nodes, nodes_by_depth, layers, _ = _map_graph_network(\n\u001b[0;32m--> 324\u001b[0;31m self.inputs, self.outputs)\n\u001b[0m\u001b[1;32m 325\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_network_nodes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnodes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nodes_by_depth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnodes_by_depth\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 124 | "\u001b[0;32m~/.anaconda3/envs/dl-python/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py\u001b[0m in \u001b[0;36m_map_graph_network\u001b[0;34m(inputs, outputs)\u001b[0m\n\u001b[1;32m 1674\u001b[0m \u001b[0;34m'The following previous layers '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1675\u001b[0m \u001b[0;34m'were accessed without issue: '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1676\u001b[0;31m str(layers_with_complete_input))\n\u001b[0m\u001b[1;32m 1677\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_tensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1678\u001b[0m \u001b[0mcomputable_tensors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 125 | "\u001b[0;31mValueError\u001b[0m: Graph disconnected: cannot obtain value for tensor Tensor(\"input_1:0\", shape=(None, 64), dtype=float32) at layer \"input_1\". The following previous layers were accessed without issue: []" 126 | ] 127 | } 128 | ], 129 | "source": [ 130 | "unrelated_input = Input(shape=(32,))\n", 131 | "bad_model = model = Model(unrelated_input, output_tensor)" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "metadata": {}, 137 | "source": [ 138 | "This error tells, in essence, that _Keras_ couldn’t reach `input_1` from the provided output tensor.\n", 139 | "When it comes to compiling, training, or evaluating such an instance of `Model`, the API is the same as that of `Sequential`:" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 4, 145 | "metadata": {}, 146 | "outputs": [ 147 | { 148 | "name": "stdout", 149 | "output_type": "stream", 150 | "text": [ 151 | "Train on 1000 samples\n", 152 | "Epoch 1/10\n", 153 | "1000/1000 [==============================] - 0s 335us/sample - loss: 12.8591\n", 154 | "Epoch 2/10\n", 155 | "1000/1000 [==============================] - 0s 11us/sample - loss: 14.7077\n", 156 | "Epoch 3/10\n", 157 | "1000/1000 [==============================] - 0s 11us/sample - loss: 17.1382\n", 158 | "Epoch 4/10\n", 159 | "1000/1000 [==============================] - 0s 13us/sample - loss: 19.6675\n", 160 | "Epoch 5/10\n", 161 | "1000/1000 [==============================] - 0s 13us/sample - loss: 22.1098\n", 162 | "Epoch 6/10\n", 163 | "1000/1000 [==============================] - 0s 15us/sample - loss: 24.2099\n", 164 | "Epoch 7/10\n", 165 | "1000/1000 [==============================] - 0s 12us/sample - loss: 25.7467\n", 166 | "Epoch 8/10\n", 167 | "1000/1000 [==============================] - 0s 12us/sample - loss: 27.5806\n", 168 | "Epoch 9/10\n", 169 | "1000/1000 [==============================] - 0s 10us/sample - loss: 29.2062\n", 170 | "Epoch 10/10\n", 171 | "1000/1000 [==============================] - 0s 9us/sample - loss: 31.3158\n", 172 | "1000/1000 [==============================] - 0s 91us/sample - loss: 32.5658\n" 173 | ] 174 | } 175 | ], 176 | "source": [ 177 | "import numpy as np\n", 178 | "\n", 179 | "# Compiles the model\n", 180 | "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')\n", 181 | "\n", 182 | "# Generates dummy Numpy data to train on\n", 183 | "x_train = np.random.random((1000, 64))\n", 184 | "y_train = np.random.random((1000, 10))\n", 185 | "\n", 186 | "# Trains the model for 10 epochs\n", 187 | "model.fit(x_train, y_train, epochs=10, batch_size=128)\n", 188 | "\n", 189 | "# Evaluates the model\n", 190 | "score = model.evaluate(x_train, y_train)" 191 | ] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "metadata": {}, 196 | "source": [ 197 | "## Multi-input models\n", 198 | "\n", 199 | "The functional API can be used to build models that have multiple inputs. Typically, such models at some point merge their different input branches using a layer that can combine several tensors: by adding them, concatenating them, and so on. " 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": {}, 205 | "source": [ 206 | "### A question-answering model\n", 207 | "\n", 208 | "A typical question-answering model has two inputs: a natural-language question and a text snippet (such as a news article) providing information to be used for answering the question. The model must then produce an answer: in the simplest possible setup, this is a one-word answer obtained via a softmax over some predefined vocabulary:\n", 209 | "\n", 210 | "\"qa-model\"" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "Here is the implementation with the _Keras_ functional API:" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": 5, 223 | "metadata": {}, 224 | "outputs": [ 225 | { 226 | "name": "stdout", 227 | "output_type": "stream", 228 | "text": [ 229 | "Model: \"model_2\"\n", 230 | "__________________________________________________________________________________________________\n", 231 | "Layer (type) Output Shape Param # Connected to \n", 232 | "==================================================================================================\n", 233 | "text (InputLayer) [(None, None)] 0 \n", 234 | "__________________________________________________________________________________________________\n", 235 | "question (InputLayer) [(None, None)] 0 \n", 236 | "__________________________________________________________________________________________________\n", 237 | "embedding (Embedding) (None, None, 64) 640000 text[0][0] \n", 238 | "__________________________________________________________________________________________________\n", 239 | "embedding_1 (Embedding) (None, None, 32) 320000 question[0][0] \n", 240 | "__________________________________________________________________________________________________\n", 241 | "lstm (LSTM) (None, 32) 12416 embedding[0][0] \n", 242 | "__________________________________________________________________________________________________\n", 243 | "lstm_1 (LSTM) (None, 16) 3136 embedding_1[0][0] \n", 244 | "__________________________________________________________________________________________________\n", 245 | "concatenate (Concatenate) (None, 48) 0 lstm[0][0] \n", 246 | " lstm_1[0][0] \n", 247 | "__________________________________________________________________________________________________\n", 248 | "dense_6 (Dense) (None, 500) 24500 concatenate[0][0] \n", 249 | "==================================================================================================\n", 250 | "Total params: 1,000,052\n", 251 | "Trainable params: 1,000,052\n", 252 | "Non-trainable params: 0\n", 253 | "__________________________________________________________________________________________________\n" 254 | ] 255 | } 256 | ], 257 | "source": [ 258 | "from tensorflow.keras.models import Model\n", 259 | "from tensorflow.keras import layers\n", 260 | "from tensorflow.keras import Input\n", 261 | "\n", 262 | "text_vocabulary_size = 10000\n", 263 | "question_vocabulary_size = 10000\n", 264 | "answer_vocabulary_size = 500\n", 265 | "\n", 266 | "# The text input is a variable-length sequence of integers. Note that you can optionally name the inputs.\n", 267 | "text_input = Input(shape=(None,), dtype='int32', name='text')\n", 268 | "# Embeds the inputs into a sequence of vectors of size 64\n", 269 | "embedded_text = layers.Embedding(text_vocabulary_size, 64)(text_input)\n", 270 | "# Encodes the vectors in a single vector via an LSTM\n", 271 | "encoded_text = layers.LSTM(32)(embedded_text)\n", 272 | "\n", 273 | "# Same process (with different layer instances) for the question\n", 274 | "question_input = Input(shape=(None,), dtype='int32', name='question')\n", 275 | "embedded_question = layers.Embedding(question_vocabulary_size, 32)(question_input)\n", 276 | "encoded_question = layers.LSTM(16)(embedded_question)\n", 277 | "\n", 278 | "# Concatenates the encoded question and encoded text\n", 279 | "concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1)\n", 280 | "# Adds a softmax classifier on top\n", 281 | "answer = layers.Dense(answer_vocabulary_size, activation='softmax')(concatenated)\n", 282 | "\n", 283 | "# At model instantiation, you specify the two inputs and the output.\n", 284 | "model = Model([text_input, question_input], answer)\n", 285 | "\n", 286 | "model.compile(\n", 287 | " optimizer='rmsprop',\n", 288 | " loss='categorical_crossentropy',\n", 289 | " metrics=['acc']\n", 290 | ")\n", 291 | "model.summary()" 292 | ] 293 | }, 294 | { 295 | "cell_type": "markdown", 296 | "metadata": {}, 297 | "source": [ 298 | "Now, **how do you train this two-input model?** There are two possible APIs: you can **feed the model a list of Numpy arrays as inputs**, or you can **feed it a dictionary that maps input names to Numpy arrays**. Naturally, the latter option is available only if you give names to your inputs." 299 | ] 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": 6, 304 | "metadata": {}, 305 | "outputs": [ 306 | { 307 | "name": "stdout", 308 | "output_type": "stream", 309 | "text": [ 310 | "Train on 1000 samples\n", 311 | "Epoch 1/10\n", 312 | "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.0000e+00 - acc: 0.0160\n", 313 | "Epoch 2/10\n", 314 | "1000/1000 [==============================] - 1s 590us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 315 | "Epoch 3/10\n", 316 | "1000/1000 [==============================] - 1s 564us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 317 | "Epoch 4/10\n", 318 | "1000/1000 [==============================] - 1s 560us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 319 | "Epoch 5/10\n", 320 | "1000/1000 [==============================] - 1s 590us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 321 | "Epoch 6/10\n", 322 | "1000/1000 [==============================] - 1s 581us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 323 | "Epoch 7/10\n", 324 | "1000/1000 [==============================] - 1s 569us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 325 | "Epoch 8/10\n", 326 | "1000/1000 [==============================] - 1s 572us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 327 | "Epoch 9/10\n", 328 | "1000/1000 [==============================] - 1s 550us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 329 | "Epoch 10/10\n", 330 | "1000/1000 [==============================] - 1s 581us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 331 | "Train on 1000 samples\n", 332 | "Epoch 1/10\n", 333 | "1000/1000 [==============================] - 1s 567us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 334 | "Epoch 2/10\n", 335 | "1000/1000 [==============================] - 1s 572us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 336 | "Epoch 3/10\n", 337 | "1000/1000 [==============================] - 1s 571us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 338 | "Epoch 4/10\n", 339 | "1000/1000 [==============================] - 1s 577us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 340 | "Epoch 5/10\n", 341 | "1000/1000 [==============================] - 1s 578us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 342 | "Epoch 6/10\n", 343 | "1000/1000 [==============================] - 1s 569us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 344 | "Epoch 7/10\n", 345 | "1000/1000 [==============================] - 1s 563us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 346 | "Epoch 8/10\n", 347 | "1000/1000 [==============================] - 1s 542us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 348 | "Epoch 9/10\n", 349 | "1000/1000 [==============================] - 1s 563us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n", 350 | "Epoch 10/10\n", 351 | "1000/1000 [==============================] - 1s 563us/sample - loss: 0.0000e+00 - acc: 0.0000e+00\n" 352 | ] 353 | }, 354 | { 355 | "data": { 356 | "text/plain": [ 357 | "" 358 | ] 359 | }, 360 | "execution_count": 6, 361 | "metadata": {}, 362 | "output_type": "execute_result" 363 | } 364 | ], 365 | "source": [ 366 | "import numpy as np\n", 367 | "\n", 368 | "num_samples = 1000\n", 369 | "max_length = 100\n", 370 | "\n", 371 | "# Generates dummy Numpy data\n", 372 | "text = np.random.randint(1, text_vocabulary_size, size=(num_samples, max_length))\n", 373 | "\n", 374 | "question = np.random.randint(1, question_vocabulary_size, size=(num_samples, max_length))\n", 375 | "# Answers are one-hot encoded, not integers\n", 376 | "answers = np.random.randint(0, 1, size=(num_samples, answer_vocabulary_size))\n", 377 | "\n", 378 | "# Fitting using a list of inputs\n", 379 | "model.fit([text, question], answers, epochs=10, batch_size=128)\n", 380 | "\n", 381 | "# Fitting using a dictionary of inputs (only if inputs are named)\n", 382 | "model.fit({'text': text, 'question': question}, answers, epochs=10, batch_size=128)" 383 | ] 384 | }, 385 | { 386 | "cell_type": "markdown", 387 | "metadata": {}, 388 | "source": [ 389 | "## Multi-input models\n", 390 | "\n", 391 | "In the same way, you can use the functional API to build models with multiple outputs (or multiple _heads_ )." 392 | ] 393 | }, 394 | { 395 | "cell_type": "markdown", 396 | "metadata": {}, 397 | "source": [ 398 | "### A three-outputs model\n", 399 | "\n", 400 | "A simple example is a network that attempts to simultaneously predict different properties of the data, such as a network that takes as input a series of social media posts from a single anonymous person and tries to predict attributes of that person, such as age, gender, and income level\n", 401 | "\n", 402 | "\"qa-model\"" 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": 7, 408 | "metadata": {}, 409 | "outputs": [ 410 | { 411 | "name": "stdout", 412 | "output_type": "stream", 413 | "text": [ 414 | "Model: \"model_3\"\n", 415 | "__________________________________________________________________________________________________\n", 416 | "Layer (type) Output Shape Param # Connected to \n", 417 | "==================================================================================================\n", 418 | "posts (InputLayer) [(None, None)] 0 \n", 419 | "__________________________________________________________________________________________________\n", 420 | "embedding_2 (Embedding) (None, None, 50000) 12800000 posts[0][0] \n", 421 | "__________________________________________________________________________________________________\n", 422 | "conv1d (Conv1D) (None, None, 128) 32000128 embedding_2[0][0] \n", 423 | "__________________________________________________________________________________________________\n", 424 | "max_pooling1d (MaxPooling1D) (None, None, 128) 0 conv1d[0][0] \n", 425 | "__________________________________________________________________________________________________\n", 426 | "conv1d_1 (Conv1D) (None, None, 256) 164096 max_pooling1d[0][0] \n", 427 | "__________________________________________________________________________________________________\n", 428 | "conv1d_2 (Conv1D) (None, None, 256) 327936 conv1d_1[0][0] \n", 429 | "__________________________________________________________________________________________________\n", 430 | "max_pooling1d_1 (MaxPooling1D) (None, None, 256) 0 conv1d_2[0][0] \n", 431 | "__________________________________________________________________________________________________\n", 432 | "conv1d_3 (Conv1D) (None, None, 256) 327936 max_pooling1d_1[0][0] \n", 433 | "__________________________________________________________________________________________________\n", 434 | "conv1d_4 (Conv1D) (None, None, 256) 327936 conv1d_3[0][0] \n", 435 | "__________________________________________________________________________________________________\n", 436 | "global_max_pooling1d (GlobalMax (None, 256) 0 conv1d_4[0][0] \n", 437 | "__________________________________________________________________________________________________\n", 438 | "dense_7 (Dense) (None, 128) 32896 global_max_pooling1d[0][0] \n", 439 | "__________________________________________________________________________________________________\n", 440 | "age (Dense) (None, 1) 129 dense_7[0][0] \n", 441 | "__________________________________________________________________________________________________\n", 442 | "income (Dense) (None, 10) 1290 dense_7[0][0] \n", 443 | "__________________________________________________________________________________________________\n", 444 | "gender (Dense) (None, 1) 129 dense_7[0][0] \n", 445 | "==================================================================================================\n", 446 | "Total params: 45,982,476\n", 447 | "Trainable params: 45,982,476\n", 448 | "Non-trainable params: 0\n", 449 | "__________________________________________________________________________________________________\n" 450 | ] 451 | } 452 | ], 453 | "source": [ 454 | "vocabulary_size = 50000\n", 455 | "num_income_groups = 10\n", 456 | "\n", 457 | "posts_input = Input(shape=(None,), dtype='int32', name='posts')\n", 458 | "\n", 459 | "embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input)\n", 460 | "\n", 461 | "x = layers.Conv1D(128, 5, activation='relu')(embedded_posts)\n", 462 | "x = layers.MaxPooling1D(5)(x)\n", 463 | "x = layers.Conv1D(256, 5, activation='relu')(x)\n", 464 | "x = layers.Conv1D(256, 5, activation='relu')(x)\n", 465 | "x = layers.MaxPooling1D(5)(x)\n", 466 | "x = layers.Conv1D(256, 5, activation='relu')(x)\n", 467 | "x = layers.Conv1D(256, 5, activation='relu')(x)\n", 468 | "x = layers.GlobalMaxPooling1D()(x)\n", 469 | "x = layers.Dense(128, activation='relu')(x)\n", 470 | "\n", 471 | "# Note that the output layers are given names\n", 472 | "age_prediction = layers.Dense(1, name='age')(x)\n", 473 | "income_prediction = layers.Dense(num_income_groups, activation='softmax', name='income')(x)\n", 474 | "gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x)\n", 475 | "\n", 476 | "model = Model(posts_input, [age_prediction, income_prediction, gender_prediction])\n", 477 | "model.summary()" 478 | ] 479 | }, 480 | { 481 | "cell_type": "markdown", 482 | "metadata": {}, 483 | "source": [ 484 | "#### Compile with multiple losses\n", 485 | "\n", 486 | "Importantly, training such a model requires the ability to **specify different loss functions for different heads of the network**: for instance, age prediction is a scalar regression task, but gender prediction is a binary classification task, requiring a different training procedure. But because gradient descent requires you to minimize a _scalar_, you must **combine these losses into a single value** in order to train the model. The simplest way to combine different losses is to sum them all. In *Keras*, you can use either a list or a dictionary of losses in `compile` to specify different objects for different outputs; **the resulting loss values are summed into a global loss, which is minimized during training**." 487 | ] 488 | }, 489 | { 490 | "cell_type": "code", 491 | "execution_count": 8, 492 | "metadata": {}, 493 | "outputs": [], 494 | "source": [ 495 | "model.compile(\n", 496 | " optimizer='rmsprop', \n", 497 | " loss=['mse', 'categorical_crossentropy', 'binary_crossentropy']\n", 498 | ")\n", 499 | "\n", 500 | "# Equivalent (possible only if you give names to the output layers)\n", 501 | "model.compile(\n", 502 | " optimizer='rmsprop', \n", 503 | " loss={\n", 504 | " 'age': 'mse',\n", 505 | " 'income': 'categorical_crossentropy',\n", 506 | " 'gender': 'binary_crossentropy'\n", 507 | " }\n", 508 | ")" 509 | ] 510 | }, 511 | { 512 | "cell_type": "markdown", 513 | "metadata": {}, 514 | "source": [ 515 | "#### Loss weighting\n", 516 | "\n", 517 | "Note that very **imbalanced loss contributions will cause the model representations to be optimized preferentially for the task with the largest individual loss**, at the expense of the other tasks. To remedy this, you can assign **different levels of importance** to the loss values in their contribution to the final loss. This is useful in particular if the losses' values use different scales." 518 | ] 519 | }, 520 | { 521 | "cell_type": "code", 522 | "execution_count": 9, 523 | "metadata": {}, 524 | "outputs": [], 525 | "source": [ 526 | "model.compile(\n", 527 | " optimizer='rmsprop',\n", 528 | " loss=['mse', 'categorical_crossentropy', 'binary_crossentropy'], \n", 529 | " loss_weights=[0.25, 1., 10.]\n", 530 | ")\n", 531 | "\n", 532 | "# Equivalent (possible only if you give names to the output layers)\n", 533 | "model.compile(\n", 534 | " optimizer='rmsprop', \n", 535 | " loss={\n", 536 | " 'age': 'mse',\n", 537 | " 'income': 'categorical_crossentropy',\n", 538 | " 'gender': 'binary_crossentropy'\n", 539 | " },\n", 540 | " loss_weights={\n", 541 | " 'age': 0.25,\n", 542 | " 'income': 1.,\n", 543 | " 'gender': 10.\n", 544 | " }\n", 545 | ")" 546 | ] 547 | }, 548 | { 549 | "cell_type": "markdown", 550 | "metadata": {}, 551 | "source": [ 552 | "#### Feeding data to a multi-output model\n", 553 | "\n", 554 | "Much as in the case of multi-input models, you can pass Numpy data to the model for training either via a list of arrays or via a dictionary of arrays.\n", 555 | "\n", 556 | "```python\n", 557 | "# age_targets, income_targets, and gender_targets are assumed to be Numpy arrays.\n", 558 | "model.fit(\n", 559 | " posts, \n", 560 | " [age_targets, income_targets, gender_targets], \n", 561 | " epochs=10, \n", 562 | " batch_size=64\n", 563 | ")\n", 564 | "\n", 565 | "# Equivalent (possible only if you give names to the output layers)\n", 566 | "model.fit(\n", 567 | " posts, \n", 568 | " {'age': age_targets, 'income': income_targets, 'gender': gender_targets},\n", 569 | " epochs=10, \n", 570 | " batch_size=64\n", 571 | ")\n", 572 | "```" 573 | ] 574 | }, 575 | { 576 | "cell_type": "markdown", 577 | "metadata": {}, 578 | "source": [ 579 | "## Directed acyclic graphs of layers\n", 580 | "\n", 581 | "With the functional API, not only can you build models with multiple inputs and multiple outputs, but you can also implement networks with a complex internal topology. Neural networks in _Keras_ are allowed to be arbitrary _directed acyclic graphs_ of layers. The qualifier _acyclic_ is important: **these graphs can’t have cycles**. It’s impossible for a tensor x to become the input of one of the layers that generated x. The only processing loops that are allowed (that is, recurrent connections) are those internal to recurrent layers." 582 | ] 583 | }, 584 | { 585 | "cell_type": "markdown", 586 | "metadata": {}, 587 | "source": [ 588 | "### Inception modules\n", 589 | "\n", 590 | "_Inception_ is a popular type of network architecture for convolutional neural networks; it was developed by _Christian Szegedy_ and his colleagues at Google in 2013–2014, inspired by the earlier network-in-network architecture. It consists of a stack of modules that themselves look like small independent networks, split into several parallel branches. Here's an example, taken from Inception V3:\n", 591 | "\n", 592 | "\"qa-model\"" 593 | ] 594 | }, 595 | { 596 | "cell_type": "markdown", 597 | "metadata": {}, 598 | "source": [ 599 | "```python\n", 600 | "x = ...\n", 601 | "\n", 602 | "# Every branch has the same stride value (2), which is necessary to \n", 603 | "# keep all branch outputs the same size so you can concatenate them.\n", 604 | "branch_a = layers.Conv2D(128, 1, activation='relu', strides=2)(x)\n", 605 | "\n", 606 | "# In this branch, the striding occurs in the spatial convolution layer.\n", 607 | "branch_b = layers.Conv2D(128, 1, activation='relu')(x)\n", 608 | "branch_b = layers.Conv2D(128, 3, activation='relu', strides=2)(branch_b)\n", 609 | "\n", 610 | "# In this branch, the striding occurs in the average pooling layer.\n", 611 | "branch_c = layers.AveragePooling2D(3, strides=2)(x)\n", 612 | "branch_c = layers.Conv2D(128, 3, activation='relu')(branch_c)\n", 613 | "\n", 614 | "branch_d = layers.Conv2D(128, 1, activation='relu')(x)\n", 615 | "branch_d = layers.Conv2D(128, 3, activation='relu')(branch_d)\n", 616 | "branch_d = layers.Conv2D(128, 3, activation='relu', strides=2)(branch_d)\n", 617 | "\n", 618 | "# Concatenates the branch outputs to obtain the module output\n", 619 | "output = layers.concatenate([branch_a, branch_b, branch_c, branch_d], axis=-1)\n", 620 | "```" 621 | ] 622 | }, 623 | { 624 | "cell_type": "markdown", 625 | "metadata": {}, 626 | "source": [ 627 | "### Residual connections\n", 628 | "\n", 629 | "_Residual connections_ are a common graph-like network component found in many post- 2015 network architectures, including _Xception_.\n", 630 | "A residual connection consists of **making the output of an earlier layer available as input to a later layer**, effectively creating a shortcut in a sequential network. Rather than being concatenated to the later activation, **the earlier output is summed with the later activation**, which **assumes that both activations are the same size**. If they’re different sizes, you can use a linear transformation to reshape the earlier activation into the target shape (for example, a `Dense` layer without an activation or, for convolutional feature maps, a 1 × 1 convolution without an activation)." 631 | ] 632 | }, 633 | { 634 | "cell_type": "markdown", 635 | "metadata": {}, 636 | "source": [ 637 | "Here’s how to implement a residual connection in _Keras_ when the feature-map sizes are the same, using identity residual connections. This example assumes the exis- tence of a 4D input tensor `x`:\n", 638 | "\n", 639 | "```python\n", 640 | "from tensorflow.keras import layers Applies a transformation to x\n", 641 | "\n", 642 | "x = ...\n", 643 | "\n", 644 | "# Applies a transformation to x\n", 645 | "y = layers.Conv2D(128, 3, activation='relu', padding='same')(x)\n", 646 | "y = layers.Conv2D(128, 3, activation='relu', padding='same')(y)\n", 647 | "y = layers.Conv2D(128, 3, activation='relu', padding='same')(y)\n", 648 | "\n", 649 | "# Adds the original x back to the output features\n", 650 | "y = layers.add([y, x])\n", 651 | "```" 652 | ] 653 | }, 654 | { 655 | "cell_type": "markdown", 656 | "metadata": {}, 657 | "source": [ 658 | "The following implements a residual connection when the feature-map sizes differ, using a linear residual connection (again, assuming the existence of a 4D input tensor `x`)\n", 659 | "\n", 660 | "```python\n", 661 | "from keras import layers \n", 662 | "\n", 663 | "x = ...\n", 664 | "y = layers.Conv2D(128, 3, activation='relu', padding='same')(x)\n", 665 | "y = layers.Conv2D(128, 3, activation='relu', padding='same')(y)\n", 666 | "y = layers.MaxPooling2D(2, strides=2)(y)\n", 667 | "\n", 668 | "# Uses a 1 × 1 convolution to linearly downsample \n", 669 | "# the original x tensor to the same shape as y\n", 670 | "residual = layers.Conv2D(128, 1, strides=2, padding='same')(x)\n", 671 | "\n", 672 | "# Adds the residual tensor back to the output features\n", 673 | "y = layers.add([y, residual])\n", 674 | "```" 675 | ] 676 | }, 677 | { 678 | "cell_type": "markdown", 679 | "metadata": {}, 680 | "source": [ 681 | "## Layer weight sharing\n", 682 | "\n", 683 | "One more important feature of the functional API is the **ability to reuse a layer instance several times**. When you call a layer instance twice, instead of instantiating a new layer for each call, you reuse the same weights with every call. This allows you to build models that have branches that share the same knowledge and perform the same operations. That is, they share the same representations and learn these representations simultaneously for different sets of inputs." 684 | ] 685 | }, 686 | { 687 | "cell_type": "markdown", 688 | "metadata": {}, 689 | "source": [ 690 | "For example, consider a model that attempts to assess the semantic similarity between two sentences. The model has two inputs (the two sentences to compare) and outputs a score between 0 and 1, where 0 means unrelated sentences and 1 means sentences that are either identical or reformulations of each other.\n", 691 | "\n", 692 | "In this setup, the two input sentences are interchangeable, because semantic similarity is a symmetrical relationship: the similarity of A to B is identical to the similarity of B to A. For this reason, it wouldn’t make sense to learn two independent models for processing each input sentence. Rather, you want to process both with a single `LSTM` layer. The representations of this `LSTM` layer (its weights) are learned based on both inputs simultaneously. This is what we call a _Siamese LSTM model_ or a _shared LSTM_." 693 | ] 694 | }, 695 | { 696 | "cell_type": "markdown", 697 | "metadata": {}, 698 | "source": [ 699 | "```python\n", 700 | "# Instantiates a single LSTM layer, once\n", 701 | "lstm = layers.LSTM(32)\n", 702 | "\n", 703 | "# Building the left branch of the model: \n", 704 | "# inputs are variable-length sequences of vectors of size 128.\n", 705 | "left_input = Input(shape=(None, 128))\n", 706 | "left_output = lstm(left_input)\n", 707 | "\n", 708 | "# Building the right branch of the model: \n", 709 | "# when you call an existing layer instance, you reuse its weights.\n", 710 | "right_input = Input(shape=(None, 128))\n", 711 | "right_output = lstm(right_input)\n", 712 | "\n", 713 | "# Builds the classifier on top\n", 714 | "merged = layers.concatenate([left_output, right_output], axis=-1) \n", 715 | "predictions = layers.Dense(1, activation='sigmoid')(merged)\n", 716 | "\n", 717 | "# Instantiating and training the model: when you train such a model, \n", 718 | "# the weights of the LSTM layer are updated based on both inputs.\n", 719 | "model = Model([left_input, right_input], predictions)\n", 720 | "model.fit([left_data, right_data], targets)\n", 721 | "```" 722 | ] 723 | }, 724 | { 725 | "cell_type": "markdown", 726 | "metadata": {}, 727 | "source": [ 728 | "## Models as layers\n", 729 | "\n", 730 | "Importantly, in the functional API, models can be used as you’d use layers—effectively, you can think of a model as a “bigger layer.” This is true of both the `Sequential` and `Model` classes.\n", 731 | "When you call a model instance, **you’re reusing the weights of the model, exactly like what happens when you call a layer instance**. Calling an instance, whether it’s a layer instance or a model instance, will always **reuse the existing learned representations of the instance**, which is intuitive." 732 | ] 733 | }, 734 | { 735 | "cell_type": "markdown", 736 | "metadata": {}, 737 | "source": [ 738 | "One simple practical example of what you can build by reusing a model instance is a vision model that uses a dual camera as its input: two parallel cameras, a few centimeters (one inch) apart. \n", 739 | "Such a model can perceive depth, which can be useful in many applications. \n", 740 | "\n", 741 | "**You shouldn’t need two independent models** to extract visual features from the left camera and the right camera before merging the two feeds. Such low-level processing can be shared across the two inputs: that is, done via layers that use the same weights and thus share the same representations. Here's how you'd implement a Siamese vision model (shared convolutional base) in _Keras_:" 742 | ] 743 | }, 744 | { 745 | "cell_type": "code", 746 | "execution_count": 10, 747 | "metadata": {}, 748 | "outputs": [], 749 | "source": [ 750 | "from tensorflow.keras import layers, Model\n", 751 | "from tensorflow.keras import applications\n", 752 | "from tensorflow.keras import Input\n", 753 | "\n", 754 | "# The base image-processing model is the Xception network (convolutional base only).\n", 755 | "xception_base = applications.Xception(weights=None, include_top=False)\n", 756 | "\n", 757 | "# The inputs are 250 × 250 RGB images.\n", 758 | "left_input = Input(shape=(250, 250, 3))\n", 759 | "right_input = Input(shape=(250, 250, 3))\n", 760 | "\n", 761 | "# Calls the same vision model twice\n", 762 | "left_features = xception_base(left_input)\n", 763 | "right_input = xception_base(right_input)\n", 764 | "\n", 765 | "# The merged features contain information from the right visual feed and the left visual feed.\n", 766 | "merged_features = layers.concatenate([left_features, right_input], axis=-1)" 767 | ] 768 | }, 769 | { 770 | "cell_type": "code", 771 | "execution_count": null, 772 | "metadata": {}, 773 | "outputs": [], 774 | "source": [] 775 | } 776 | ], 777 | "metadata": { 778 | "kernelspec": { 779 | "display_name": "Python 3", 780 | "language": "python", 781 | "name": "python3" 782 | }, 783 | "language_info": { 784 | "codemirror_mode": { 785 | "name": "ipython", 786 | "version": 3 787 | }, 788 | "file_extension": ".py", 789 | "mimetype": "text/x-python", 790 | "name": "python", 791 | "nbconvert_exporter": "python", 792 | "pygments_lexer": "ipython3", 793 | "version": "3.6.10" 794 | } 795 | }, 796 | "nbformat": 4, 797 | "nbformat_minor": 4 798 | } 799 | -------------------------------------------------------------------------------- /notebooks/chapter_07/resources/inception-module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_07/resources/inception-module.png -------------------------------------------------------------------------------- /notebooks/chapter_07/resources/question-answering-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_07/resources/question-answering-model.png -------------------------------------------------------------------------------- /notebooks/chapter_07/resources/three-outputs-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_07/resources/three-outputs-model.png -------------------------------------------------------------------------------- /notebooks/chapter_08/resources/kandinsky_style.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/resources/kandinsky_style.jpg -------------------------------------------------------------------------------- /notebooks/chapter_08/resources/riva.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/resources/riva.jpg -------------------------------------------------------------------------------- /notebooks/chapter_08/resources/vae.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/resources/vae.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/dream_final.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/dream_final.jpg -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/gan.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/gan.h5 -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog0.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog1900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog1900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog2900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog2900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog3900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog3900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog4900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog4900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog5900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog5900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog6900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog6900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog7900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog7900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog8900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog8900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/generated_frog9900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/generated_frog9900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog0.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog1900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog1900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog2900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog2900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog3900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog3900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog4900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog4900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog5900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog5900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog6900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog6900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog7900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog7900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog8900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog8900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog900.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9000.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9100.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9200.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9300.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9400.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9500.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9500.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9600.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9700.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9700.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9800.png -------------------------------------------------------------------------------- /notebooks/chapter_08/results/gan/real_frog9900.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lballore/deep-learning-with-python/a1aa11aa0b493e862664222550fd0734c30afa11/notebooks/chapter_08/results/gan/real_frog9900.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | jupyter==1.* 2 | matplotlib==3.2.* 3 | opencv_python==4.2.* 4 | pillow==8.1.* 5 | tensorflow==2.4.* 6 | --------------------------------------------------------------------------------