├── Convolutional Neural Networks in TensorFlow ├── week1 │ ├── Course_2_Part_2_Lesson_2_Notebook.ipynb │ └── Exercise_1_Cats_vs_Dogs_Question-FINAL.html ├── week2 │ ├── Course_2_Part_4_Lesson_2_Notebook_(Cats_v_Dogs_Augmentation).ipynb │ └── Exercise_2_Cats_vs_Dogs_using_augmentation_Question-FINAL.html ├── week3 │ ├── Course_2_Part_6_Lesson_3_Notebook.ipynb │ └── Exercise_3_Horses_vs_humans_using_Transfer_Learning_Question-FINAL.html └── week4 │ ├── Course_2_Part_8_Lesson_2_Notebook_(RockPaperScissors).ipynb │ └── Exercise_4_Multi_class_classifier_Question-FINAL.html ├── Introduction to TensorFlow for Artificial Intelligence, Machine Learning, and Deep Learning ├── week1 │ └── Exercise_1_House_Prices_Question.html ├── week2 │ ├── Course_1_Part_4_Lesson_2_Notebook.ipynb │ └── Exercise2-Question.html ├── week3 │ ├── Course_1_Part_6_Lesson_2_Notebook.ipynb │ └── Excercise-3-Question.html └── week4 │ ├── Course_1_Part_8_Lesson_2_Notebook.ipynb │ ├── Course_1_Part_8_Lesson_3_Notebook.ipynb │ └── Exercise4-Question.html ├── Natural Language Processing in TensorFlow ├── week1 │ ├── Course_3_Week_1_Exercise_answer.ipynb │ ├── Course_3_Week_1_Lesson_1.ipynb │ ├── Course_3_Week_1_Lesson_2.ipynb │ └── Course_3_Week_1_Lesson_3.ipynb ├── week2 │ ├── Course_3_Week_2_Exercise_Answer.ipynb │ ├── Course_3_Week_2_Lesson_1.ipynb │ └── Course_3_Week_2_Lesson_2.ipynb ├── week3 │ ├── Course_3_Week_3_Lesson_1a.ipynb │ ├── Course_3_Week_3_Lesson_1b.ipynb │ ├── Course_3_Week_3_Lesson_2.ipynb │ ├── Course_3_Week_3_Lesson_2c.ipynb │ └── Course_3_Week_3_Lesson_2d.ipynb └── week4 │ ├── Course_3_Week_4_Lesson_1_Notebook.ipynb │ ├── Course_3_Week_4_Lesson_2_Notebook.ipynb │ └── NLP_Week4_Exercise_Shakespeare_Answer.ipynb ├── README.md ├── Screen Shot 2021-07-21 at 11.33.06 PM.png └── Sequences, Time Series, and Prediction ├── week1 ├── S+P_Week_1_Lesson_2.ipynb ├── S+P_Week_1_Lesson_3_Notebook.ipynb └── Week_1_Exercise_Answer.ipynb ├── week2 ├── S+P_Week_2_Exercise_Answer.ipynb ├── S+P_Week_2_Exercise_Question.ipynb ├── S+P_Week_2_Lesson_1.ipynb ├── S+P_Week_2_Lesson_2.ipynb └── S+P_Week_2_Lesson_3.ipynb └── week3 ├── S+P_Week_3_Exercise_Answer.ipynb ├── S+P_Week_3_Lesson_2_RNN.ipynb └── S+P_Week_3_Lesson_4_LSTM.ipynb /Convolutional Neural Networks in TensorFlow/week3/Course_2_Part_6_Lesson_3_Notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 2 - Part 6 - Lesson 3 - Notebook.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "name": "python3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "rX8mhOLljYeM" 22 | }, 23 | "source": [ 24 | "##### Copyright 2019 The TensorFlow Authors." 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "cellView": "form", 31 | "id": "BZSlp3DAjdYf" 32 | }, 33 | "source": [ 34 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 35 | "# you may not use this file except in compliance with the License.\n", 36 | "# You may obtain a copy of the License at\n", 37 | "#\n", 38 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 39 | "#\n", 40 | "# Unless required by applicable law or agreed to in writing, software\n", 41 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 42 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 43 | "# See the License for the specific language governing permissions and\n", 44 | "# limitations under the License." 45 | ], 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "metadata": { 52 | "colab": { 53 | "base_uri": "https://localhost:8080/" 54 | }, 55 | "id": "1xJZ5glPPCRz", 56 | "outputId": "2bab3146-78c6-48b9-ec74-b9cf72aa1e4d" 57 | }, 58 | "source": [ 59 | "import os\n", 60 | "\n", 61 | "from tensorflow.keras import layers\n", 62 | "from tensorflow.keras import Model\n", 63 | "!wget --no-check-certificate \\\n", 64 | " https://storage.googleapis.com/mledu-datasets/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5 \\\n", 65 | " -O /tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5\n", 66 | " \n", 67 | "from tensorflow.keras.applications.inception_v3 import InceptionV3\n", 68 | "\n", 69 | "local_weights_file = '/tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'\n", 70 | "\n", 71 | "pre_trained_model = InceptionV3(input_shape = (150, 150, 3), \n", 72 | " include_top = False, \n", 73 | " weights = None)\n", 74 | "\n", 75 | "pre_trained_model.load_weights(local_weights_file)\n", 76 | "\n", 77 | "for layer in pre_trained_model.layers:\n", 78 | " layer.trainable = False\n", 79 | " \n", 80 | "# pre_trained_model.summary()\n", 81 | "\n", 82 | "last_layer = pre_trained_model.get_layer('mixed7')\n", 83 | "print('last layer output shape: ', last_layer.output_shape)\n", 84 | "last_output = last_layer.output" 85 | ], 86 | "execution_count": 3, 87 | "outputs": [ 88 | { 89 | "output_type": "stream", 90 | "text": [ 91 | "--2021-05-24 10:29:34-- https://storage.googleapis.com/mledu-datasets/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5\n", 92 | "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.204.128, 64.233.188.128, 64.233.189.128, ...\n", 93 | "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.204.128|:443... connected.\n", 94 | "HTTP request sent, awaiting response... 200 OK\n", 95 | "Length: 87910968 (84M) [application/x-hdf]\n", 96 | "Saving to: ‘/tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5’\n", 97 | "\n", 98 | "/tmp/inception_v3_w 100%[===================>] 83.84M 148MB/s in 0.6s \n", 99 | "\n", 100 | "2021-05-24 10:29:35 (148 MB/s) - ‘/tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5’ saved [87910968/87910968]\n", 101 | "\n", 102 | "last layer output shape: (None, 7, 7, 768)\n" 103 | ], 104 | "name": "stdout" 105 | } 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "metadata": { 111 | "id": "BMXb913pbvFg" 112 | }, 113 | "source": [ 114 | "from tensorflow.keras.optimizers import RMSprop\n", 115 | "\n", 116 | "# Flatten the output layer to 1 dimension\n", 117 | "x = layers.Flatten()(last_output)\n", 118 | "# Add a fully connected layer with 1,024 hidden units and ReLU activation\n", 119 | "x = layers.Dense(1024, activation='relu')(x)\n", 120 | "# Add a dropout rate of 0.2\n", 121 | "x = layers.Dropout(0.2)(x) \n", 122 | "# Add a final sigmoid layer for classification\n", 123 | "x = layers.Dense (1, activation='sigmoid')(x) \n", 124 | "\n", 125 | "model = Model( pre_trained_model.input, x) \n", 126 | "\n", 127 | "model.compile(optimizer = RMSprop(lr=0.0001), \n", 128 | " loss = 'binary_crossentropy', \n", 129 | " metrics = ['accuracy'])\n" 130 | ], 131 | "execution_count": 4, 132 | "outputs": [] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "metadata": { 137 | "colab": { 138 | "base_uri": "https://localhost:8080/" 139 | }, 140 | "id": "O4s8HckqGlnb", 141 | "outputId": "6677590d-7aa2-470a-bf26-2226bce8ec2a" 142 | }, 143 | "source": [ 144 | "!wget --no-check-certificate \\\n", 145 | " https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip \\\n", 146 | " -O /tmp/cats_and_dogs_filtered.zip\n", 147 | "\n", 148 | "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", 149 | "\n", 150 | "import os\n", 151 | "import zipfile\n", 152 | "\n", 153 | "local_zip = '//tmp/cats_and_dogs_filtered.zip'\n", 154 | "\n", 155 | "zip_ref = zipfile.ZipFile(local_zip, 'r')\n", 156 | "\n", 157 | "zip_ref.extractall('/tmp')\n", 158 | "zip_ref.close()\n", 159 | "\n", 160 | "# Define our example directories and files\n", 161 | "base_dir = '/tmp/cats_and_dogs_filtered'\n", 162 | "\n", 163 | "train_dir = os.path.join( base_dir, 'train')\n", 164 | "validation_dir = os.path.join( base_dir, 'validation')\n", 165 | "\n", 166 | "\n", 167 | "train_cats_dir = os.path.join(train_dir, 'cats') # Directory with our training cat pictures\n", 168 | "train_dogs_dir = os.path.join(train_dir, 'dogs') # Directory with our training dog pictures\n", 169 | "validation_cats_dir = os.path.join(validation_dir, 'cats') # Directory with our validation cat pictures\n", 170 | "validation_dogs_dir = os.path.join(validation_dir, 'dogs')# Directory with our validation dog pictures\n", 171 | "\n", 172 | "train_cat_fnames = os.listdir(train_cats_dir)\n", 173 | "train_dog_fnames = os.listdir(train_dogs_dir)\n", 174 | "\n", 175 | "# Add our data-augmentation parameters to ImageDataGenerator\n", 176 | "train_datagen = ImageDataGenerator(rescale = 1./255.,\n", 177 | " rotation_range = 40,\n", 178 | " width_shift_range = 0.2,\n", 179 | " height_shift_range = 0.2,\n", 180 | " shear_range = 0.2,\n", 181 | " zoom_range = 0.2,\n", 182 | " horizontal_flip = True)\n", 183 | "\n", 184 | "# Note that the validation data should not be augmented!\n", 185 | "test_datagen = ImageDataGenerator( rescale = 1.0/255. )\n", 186 | "\n", 187 | "# Flow training images in batches of 20 using train_datagen generator\n", 188 | "train_generator = train_datagen.flow_from_directory(train_dir,\n", 189 | " batch_size = 20,\n", 190 | " class_mode = 'binary', \n", 191 | " target_size = (150, 150)) \n", 192 | "\n", 193 | "# Flow validation images in batches of 20 using test_datagen generator\n", 194 | "validation_generator = test_datagen.flow_from_directory( validation_dir,\n", 195 | " batch_size = 20,\n", 196 | " class_mode = 'binary', \n", 197 | " target_size = (150, 150))" 198 | ], 199 | "execution_count": 5, 200 | "outputs": [ 201 | { 202 | "output_type": "stream", 203 | "text": [ 204 | "--2021-05-24 10:36:33-- https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip\n", 205 | "Resolving storage.googleapis.com (storage.googleapis.com)... 108.177.97.128, 108.177.125.128, 142.250.157.128, ...\n", 206 | "Connecting to storage.googleapis.com (storage.googleapis.com)|108.177.97.128|:443... connected.\n", 207 | "HTTP request sent, awaiting response... 200 OK\n", 208 | "Length: 68606236 (65M) [application/zip]\n", 209 | "Saving to: ‘/tmp/cats_and_dogs_filtered.zip’\n", 210 | "\n", 211 | "/tmp/cats_and_dogs_ 100%[===================>] 65.43M 77.7MB/s in 0.8s \n", 212 | "\n", 213 | "2021-05-24 10:36:34 (77.7 MB/s) - ‘/tmp/cats_and_dogs_filtered.zip’ saved [68606236/68606236]\n", 214 | "\n", 215 | "Found 2000 images belonging to 2 classes.\n", 216 | "Found 1000 images belonging to 2 classes.\n" 217 | ], 218 | "name": "stdout" 219 | } 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "metadata": { 225 | "colab": { 226 | "base_uri": "https://localhost:8080/" 227 | }, 228 | "id": "Blhq2MAUeyGA", 229 | "outputId": "53b81fe2-600d-4c84-e3cd-4dfaa7f9f353" 230 | }, 231 | "source": [ 232 | "history = model.fit(\n", 233 | " train_generator,\n", 234 | " validation_data = validation_generator,\n", 235 | " steps_per_epoch = 100,\n", 236 | " epochs = 20,\n", 237 | " validation_steps = 50,\n", 238 | " verbose = 2)" 239 | ], 240 | "execution_count": 6, 241 | "outputs": [ 242 | { 243 | "output_type": "stream", 244 | "text": [ 245 | "Epoch 1/20\n", 246 | "100/100 - 57s - loss: 0.3708 - accuracy: 0.8580 - val_loss: 0.0966 - val_accuracy: 0.9590\n", 247 | "Epoch 2/20\n", 248 | "100/100 - 21s - loss: 0.2114 - accuracy: 0.9225 - val_loss: 0.1304 - val_accuracy: 0.9550\n", 249 | "Epoch 3/20\n", 250 | "100/100 - 22s - loss: 0.2224 - accuracy: 0.9230 - val_loss: 0.1620 - val_accuracy: 0.9440\n", 251 | "Epoch 4/20\n", 252 | "100/100 - 21s - loss: 0.1778 - accuracy: 0.9345 - val_loss: 0.0983 - val_accuracy: 0.9660\n", 253 | "Epoch 5/20\n", 254 | "100/100 - 22s - loss: 0.1987 - accuracy: 0.9280 - val_loss: 0.1240 - val_accuracy: 0.9630\n", 255 | "Epoch 6/20\n", 256 | "100/100 - 22s - loss: 0.1770 - accuracy: 0.9390 - val_loss: 0.1379 - val_accuracy: 0.9570\n", 257 | "Epoch 7/20\n", 258 | "100/100 - 22s - loss: 0.1728 - accuracy: 0.9400 - val_loss: 0.1210 - val_accuracy: 0.9650\n", 259 | "Epoch 8/20\n", 260 | "100/100 - 21s - loss: 0.1724 - accuracy: 0.9440 - val_loss: 0.1182 - val_accuracy: 0.9620\n", 261 | "Epoch 9/20\n", 262 | "100/100 - 21s - loss: 0.1623 - accuracy: 0.9435 - val_loss: 0.1444 - val_accuracy: 0.9600\n", 263 | "Epoch 10/20\n", 264 | "100/100 - 22s - loss: 0.1460 - accuracy: 0.9505 - val_loss: 0.1177 - val_accuracy: 0.9700\n", 265 | "Epoch 11/20\n", 266 | "100/100 - 22s - loss: 0.1615 - accuracy: 0.9455 - val_loss: 0.1356 - val_accuracy: 0.9620\n", 267 | "Epoch 12/20\n", 268 | "100/100 - 22s - loss: 0.1507 - accuracy: 0.9515 - val_loss: 0.3044 - val_accuracy: 0.9260\n", 269 | "Epoch 13/20\n", 270 | "100/100 - 22s - loss: 0.1589 - accuracy: 0.9410 - val_loss: 0.1785 - val_accuracy: 0.9550\n", 271 | "Epoch 14/20\n", 272 | "100/100 - 22s - loss: 0.1525 - accuracy: 0.9455 - val_loss: 0.1303 - val_accuracy: 0.9680\n", 273 | "Epoch 15/20\n", 274 | "100/100 - 22s - loss: 0.1609 - accuracy: 0.9465 - val_loss: 0.1460 - val_accuracy: 0.9600\n", 275 | "Epoch 16/20\n", 276 | "100/100 - 22s - loss: 0.1422 - accuracy: 0.9545 - val_loss: 0.1191 - val_accuracy: 0.9660\n", 277 | "Epoch 17/20\n", 278 | "100/100 - 22s - loss: 0.1466 - accuracy: 0.9545 - val_loss: 0.1268 - val_accuracy: 0.9660\n", 279 | "Epoch 18/20\n", 280 | "100/100 - 22s - loss: 0.1299 - accuracy: 0.9590 - val_loss: 0.1358 - val_accuracy: 0.9660\n", 281 | "Epoch 19/20\n", 282 | "100/100 - 23s - loss: 0.1314 - accuracy: 0.9585 - val_loss: 0.1232 - val_accuracy: 0.9700\n", 283 | "Epoch 20/20\n", 284 | "100/100 - 22s - loss: 0.1128 - accuracy: 0.9590 - val_loss: 0.1673 - val_accuracy: 0.9670\n" 285 | ], 286 | "name": "stdout" 287 | } 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "metadata": { 293 | "id": "C2Fp6Se9rKuL" 294 | }, 295 | "source": [ 296 | "import matplotlib.pyplot as plt\n", 297 | "acc = history.history['accuracy']\n", 298 | "val_acc = history.history['val_accuracy']\n", 299 | "loss = history.history['loss']\n", 300 | "val_loss = history.history['val_loss']\n", 301 | "\n", 302 | "epochs = range(len(acc))\n", 303 | "\n", 304 | "plt.plot(epochs, acc, 'r', label='Training accuracy')\n", 305 | "plt.plot(epochs, val_acc, 'b', label='Validation accuracy')\n", 306 | "plt.title('Training and validation accuracy')\n", 307 | "plt.legend(loc=0)\n", 308 | "plt.figure()\n", 309 | "\n", 310 | "\n", 311 | "plt.show()" 312 | ], 313 | "execution_count": null, 314 | "outputs": [] 315 | } 316 | ] 317 | } -------------------------------------------------------------------------------- /Introduction to TensorFlow for Artificial Intelligence, Machine Learning, and Deep Learning/week2/Course_1_Part_4_Lesson_2_Notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Course 1 - Part 4 - Lesson 2 - Notebook.ipynb", 7 | "private_outputs": true, 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "language": "python", 15 | "name": "python3" 16 | }, 17 | "language_info": { 18 | "codemirror_mode": { 19 | "name": "ipython", 20 | "version": 3 21 | }, 22 | "file_extension": ".py", 23 | "mimetype": "text/x-python", 24 | "name": "python", 25 | "nbconvert_exporter": "python", 26 | "pygments_lexer": "ipython3", 27 | "version": "3.7.5rc1" 28 | } 29 | }, 30 | "cells": [ 31 | { 32 | "cell_type": "markdown", 33 | "metadata": { 34 | "id": "rX8mhOLljYeM" 35 | }, 36 | "source": [ 37 | "##### Copyright 2019 The TensorFlow Authors." 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "metadata": { 43 | "cellView": "form", 44 | "id": "BZSlp3DAjdYf" 45 | }, 46 | "source": [ 47 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 48 | "# you may not use this file except in compliance with the License.\n", 49 | "# You may obtain a copy of the License at\n", 50 | "#\n", 51 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 52 | "#\n", 53 | "# Unless required by applicable law or agreed to in writing, software\n", 54 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 55 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 56 | "# See the License for the specific language governing permissions and\n", 57 | "# limitations under the License." 58 | ], 59 | "execution_count": null, 60 | "outputs": [] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": { 65 | "id": "qnyTxjK_GbOD" 66 | }, 67 | "source": [ 68 | "# Beyond Hello World, A Computer Vision Example\n", 69 | "In the previous exercise you saw how to create a neural network that figured out the problem you were trying to solve. This gave an explicit example of learned behavior. Of course, in that instance, it was a bit of overkill because it would have been easier to write the function Y=2x-1 directly, instead of bothering with using Machine Learning to learn the relationship between X and Y for a fixed set of values, and extending that for all values.\n", 70 | "\n", 71 | "But what about a scenario where writing rules like that is much more difficult -- for example a computer vision problem? Let's take a look at a scenario where we can recognize different items of clothing, trained from a dataset containing 10 different types." 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "metadata": { 77 | "id": "H41FYgtlHPjW" 78 | }, 79 | "source": [ 80 | "## Start Coding\n", 81 | "\n", 82 | "Let's start with our import of TensorFlow" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "metadata": { 88 | "id": "q3KzJyjv3rnA" 89 | }, 90 | "source": [ 91 | "import tensorflow as tf\n", 92 | "print(tf.__version__)" 93 | ], 94 | "execution_count": null, 95 | "outputs": [] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": { 100 | "id": "n_n1U5do3u_F" 101 | }, 102 | "source": [ 103 | "The Fashion MNIST data is available directly in the tf.keras datasets API. You load it like this:" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "metadata": { 109 | "id": "PmxkHFpt31bM" 110 | }, 111 | "source": [ 112 | "mnist = tf.keras.datasets.fashion_mnist" 113 | ], 114 | "execution_count": null, 115 | "outputs": [] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": { 120 | "id": "GuoLQQBT4E-_" 121 | }, 122 | "source": [ 123 | "Calling load_data on this object will give you two sets of two lists, these will be the training and testing values for the graphics that contain the clothing items and their labels.\n" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "metadata": { 129 | "id": "BTdRgExe4TRB" 130 | }, 131 | "source": [ 132 | "(training_images, training_labels), (test_images, test_labels) = mnist.load_data()" 133 | ], 134 | "execution_count": null, 135 | "outputs": [] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": { 140 | "id": "rw395ROx4f5Q" 141 | }, 142 | "source": [ 143 | "What does these values look like? Let's print a training image, and a training label to see...Experiment with different indices in the array. For example, also take a look at index 42...that's a a different boot than the one at index 0\n" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "metadata": { 149 | "id": "FPc9d3gJ3jWF" 150 | }, 151 | "source": [ 152 | "import numpy as np\n", 153 | "np.set_printoptions(linewidth=200)\n", 154 | "import matplotlib.pyplot as plt\n", 155 | "plt.imshow(training_images[0])\n", 156 | "print(training_labels[0])\n", 157 | "print(training_images[0])" 158 | ], 159 | "execution_count": null, 160 | "outputs": [] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": { 165 | "id": "3cbrdH225_nH" 166 | }, 167 | "source": [ 168 | "You'll notice that all of the values in the number are between 0 and 255. If we are training a neural network, for various reasons it's easier if we treat all values as between 0 and 1, a process called '**normalizing**'...and fortunately in Python it's easy to normalize a list like this without looping. You do it like this:" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "metadata": { 174 | "id": "kRH19pWs6ZDn" 175 | }, 176 | "source": [ 177 | "training_images = training_images / 255.0\n", 178 | "test_images = test_images / 255.0" 179 | ], 180 | "execution_count": null, 181 | "outputs": [] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": { 186 | "id": "3DkO0As46lRn" 187 | }, 188 | "source": [ 189 | "Now you might be wondering why there are 2 sets...training and testing -- remember we spoke about this in the intro? The idea is to have 1 set of data for training, and then another set of data...that the model hasn't yet seen...to see how good it would be at classifying values. After all, when you're done, you're going to want to try it out with data that it hadn't previously seen!" 190 | ] 191 | }, 192 | { 193 | "cell_type": "markdown", 194 | "metadata": { 195 | "id": "dIn7S9gf62ie" 196 | }, 197 | "source": [ 198 | "Let's now design the model. There's quite a few new concepts here, but don't worry, you'll get the hang of them. " 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "metadata": { 204 | "id": "7mAyndG3kVlK" 205 | }, 206 | "source": [ 207 | "model = tf.keras.models.Sequential([tf.keras.layers.Flatten(), \n", 208 | " tf.keras.layers.Dense(128, activation=tf.nn.relu), \n", 209 | " tf.keras.layers.Dense(10, activation=tf.nn.softmax)])" 210 | ], 211 | "execution_count": null, 212 | "outputs": [] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": { 217 | "id": "-lUcWaiX7MFj" 218 | }, 219 | "source": [ 220 | "**Sequential**: That defines a SEQUENCE of layers in the neural network\n", 221 | "\n", 222 | "**Flatten**: Remember earlier where our images were a square, when you printed them out? Flatten just takes that square and turns it into a 1 dimensional set.\n", 223 | "\n", 224 | "**Dense**: Adds a layer of neurons\n", 225 | "\n", 226 | "Each layer of neurons need an **activation function** to tell them what to do. There's lots of options, but just use these for now. \n", 227 | "\n", 228 | "**Relu** effectively means \"If X>0 return X, else return 0\" -- so what it does it it only passes values 0 or greater to the next layer in the network.\n", 229 | "\n", 230 | "**Softmax** takes a set of values, and effectively picks the biggest one, so, for example, if the output of the last layer looks like [0.1, 0.1, 0.05, 0.1, 9.5, 0.1, 0.05, 0.05, 0.05], it saves you from fishing through it looking for the biggest value, and turns it into [0,0,0,0,1,0,0,0,0] -- The goal is to save a lot of coding!\n" 231 | ] 232 | }, 233 | { 234 | "cell_type": "markdown", 235 | "metadata": { 236 | "id": "c8vbMCqb9Mh6" 237 | }, 238 | "source": [ 239 | "The next thing to do, now the model is defined, is to actually build it. You do this by compiling it with an optimizer and loss function as before -- and then you train it by calling **model.fit ** asking it to fit your training data to your training labels -- i.e. have it figure out the relationship between the training data and its actual labels, so in future if you have data that looks like the training data, then it can make a prediction for what that data would look like. " 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "metadata": { 245 | "id": "BLMdl9aP8nQ0" 246 | }, 247 | "source": [ 248 | "model.compile(optimizer = tf.optimizers.Adam(),\n", 249 | " loss = 'sparse_categorical_crossentropy',\n", 250 | " metrics=['accuracy'])\n", 251 | "\n", 252 | "model.fit(training_images, training_labels, epochs=5)" 253 | ], 254 | "execution_count": null, 255 | "outputs": [] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": { 260 | "id": "-JJMsvSB-1UY" 261 | }, 262 | "source": [ 263 | "Once it's done training -- you should see an accuracy value at the end of the final epoch. It might look something like 0.9098. This tells you that your neural network is about 91% accurate in classifying the training data. I.E., it figured out a pattern match between the image and the labels that worked 91% of the time. Not great, but not bad considering it was only trained for 5 epochs and done quite quickly.\n", 264 | "\n", 265 | "But how would it work with unseen data? That's why we have the test images. We can call model.evaluate, and pass in the two sets, and it will report back the loss for each. Let's give it a try:" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "metadata": { 271 | "id": "WzlqsEzX9s5P" 272 | }, 273 | "source": [ 274 | "model.evaluate(test_images, test_labels)" 275 | ], 276 | "execution_count": null, 277 | "outputs": [] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "metadata": { 282 | "id": "6tki-Aro_Uax" 283 | }, 284 | "source": [ 285 | "For me, that returned a accuracy of about .8838, which means it was about 88% accurate. As expected it probably would not do as well with *unseen* data as it did with data it was trained on! As you go through this course, you'll look at ways to improve this. \n", 286 | "\n", 287 | "To explore further, try the below exercises:\n" 288 | ] 289 | }, 290 | { 291 | "cell_type": "markdown", 292 | "metadata": { 293 | "id": "htldZNWcIPSN" 294 | }, 295 | "source": [ 296 | "# Exploration Exercises" 297 | ] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": { 302 | "id": "rquQqIx4AaGR" 303 | }, 304 | "source": [ 305 | "###Exercise 1:\n", 306 | "For this first exercise run the below code: It creates a set of classifications for each of the test images, and then prints the first entry in the classifications. The output, after you run it is a list of numbers. Why do you think this is, and what do those numbers represent? " 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "metadata": { 312 | "id": "RyEIki0z_hAD" 313 | }, 314 | "source": [ 315 | "classifications = model.predict(test_images)\n", 316 | "\n", 317 | "print(classifications[0])" 318 | ], 319 | "execution_count": null, 320 | "outputs": [] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "metadata": { 325 | "id": "MdzqbQhRArzm" 326 | }, 327 | "source": [ 328 | "Hint: try running print(test_labels[0]) -- and you'll get a 9. Does that help you understand why this list looks the way it does? " 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "metadata": { 334 | "id": "WnBGOrMiA1n5" 335 | }, 336 | "source": [ 337 | "print(test_labels[0])" 338 | ], 339 | "execution_count": null, 340 | "outputs": [] 341 | }, 342 | { 343 | "cell_type": "markdown", 344 | "metadata": { 345 | "id": "uUs7eqr7uSvs" 346 | }, 347 | "source": [ 348 | "### What does this list represent?\n", 349 | "\n", 350 | "\n", 351 | "1. It's 10 random meaningless values\n", 352 | "2. It's the first 10 classifications that the computer made\n", 353 | "3. It's the probability that this item is each of the 10 classes\n" 354 | ] 355 | }, 356 | { 357 | "cell_type": "markdown", 358 | "metadata": { 359 | "id": "wAbr92RTA67u" 360 | }, 361 | "source": [ 362 | "####Answer: \n", 363 | "The correct answer is (3)\n", 364 | "\n", 365 | "The output of the model is a list of 10 numbers. These numbers are a probability that the value being classified is the corresponding value (https://github.com/zalandoresearch/fashion-mnist#labels), i.e. the first value in the list is the probability that the image is of a '0' (T-shirt/top), the next is a '1' (Trouser) etc. Notice that they are all VERY LOW probabilities.\n", 366 | "\n", 367 | "For the 9 (Ankle boot), the probability was in the 90's, i.e. the neural network is telling us that it's almost certainly a 7." 368 | ] 369 | }, 370 | { 371 | "cell_type": "markdown", 372 | "metadata": { 373 | "id": "CD4kC6TBu-69" 374 | }, 375 | "source": [ 376 | "### How do you know that this list tells you that the item is an ankle boot?\n", 377 | "\n", 378 | "\n", 379 | "1. There's not enough information to answer that question\n", 380 | "2. The 10th element on the list is the biggest, and the ankle boot is labelled 9\n", 381 | "2. The ankle boot is label 9, and there are 0->9 elements in the list\n" 382 | ] 383 | }, 384 | { 385 | "cell_type": "markdown", 386 | "metadata": { 387 | "id": "I-haLncrva5L" 388 | }, 389 | "source": [ 390 | "####Answer\n", 391 | "The correct answer is (2). Both the list and the labels are 0 based, so the ankle boot having label 9 means that it is the 10th of the 10 classes. The list having the 10th element being the highest value means that the Neural Network has predicted that the item it is classifying is most likely an ankle boot" 392 | ] 393 | }, 394 | { 395 | "cell_type": "markdown", 396 | "metadata": { 397 | "id": "OgQSIfDSOWv6" 398 | }, 399 | "source": [ 400 | "##Exercise 2: \n", 401 | "Let's now look at the layers in your model. Experiment with different values for the dense layer with 512 neurons. What different results do you get for loss, training time etc? Why do you think that's the case? \n" 402 | ] 403 | }, 404 | { 405 | "cell_type": "code", 406 | "metadata": { 407 | "id": "GSZSwV5UObQP" 408 | }, 409 | "source": [ 410 | "import tensorflow as tf\n", 411 | "print(tf.__version__)\n", 412 | "\n", 413 | "mnist = tf.keras.datasets.mnist\n", 414 | "\n", 415 | "(training_images, training_labels) , (test_images, test_labels) = mnist.load_data()\n", 416 | "\n", 417 | "training_images = training_images/255.0\n", 418 | "test_images = test_images/255.0\n", 419 | "\n", 420 | "model = tf.keras.models.Sequential([tf.keras.layers.Flatten(),\n", 421 | " tf.keras.layers.Dense(1024, activation=tf.nn.relu),\n", 422 | " tf.keras.layers.Dense(10, activation=tf.nn.softmax)])\n", 423 | "\n", 424 | "model.compile(optimizer = 'adam',\n", 425 | " loss = 'sparse_categorical_crossentropy')\n", 426 | "\n", 427 | "model.fit(training_images, training_labels, epochs=5)\n", 428 | "\n", 429 | "model.evaluate(test_images, test_labels)\n", 430 | "\n", 431 | "classifications = model.predict(test_images)\n", 432 | "\n", 433 | "print(classifications[0])\n", 434 | "print(test_labels[0])" 435 | ], 436 | "execution_count": null, 437 | "outputs": [] 438 | }, 439 | { 440 | "cell_type": "markdown", 441 | "metadata": { 442 | "id": "bOOEnHZFv5cS" 443 | }, 444 | "source": [ 445 | "###Question 1. Increase to 1024 Neurons -- What's the impact?\n", 446 | "\n", 447 | "1. Training takes longer, but is more accurate\n", 448 | "2. Training takes longer, but no impact on accuracy\n", 449 | "3. Training takes the same time, but is more accurate\n" 450 | ] 451 | }, 452 | { 453 | "cell_type": "markdown", 454 | "metadata": { 455 | "id": "U73MUP2lwrI2" 456 | }, 457 | "source": [ 458 | "####Answer\n", 459 | "The correct answer is (1) by adding more Neurons we have to do more calculations, slowing down the process, but in this case they have a good impact -- we do get more accurate. That doesn't mean it's always a case of 'more is better', you can hit the law of diminishing returns very quickly!" 460 | ] 461 | }, 462 | { 463 | "cell_type": "markdown", 464 | "metadata": { 465 | "id": "WtWxK16hQxLN" 466 | }, 467 | "source": [ 468 | "##Exercise 3: \n", 469 | "\n", 470 | "What would happen if you remove the Flatten() layer. Why do you think that's the case? \n", 471 | "\n", 472 | "You get an error about the shape of the data. It may seem vague right now, but it reinforces the rule of thumb that the first layer in your network should be the same shape as your data. Right now our data is 28x28 images, and 28 layers of 28 neurons would be infeasible, so it makes more sense to 'flatten' that 28,28 into a 784x1. Instead of wriitng all the code to handle that ourselves, we add the Flatten() layer at the begining, and when the arrays are loaded into the model later, they'll automatically be flattened for us." 473 | ] 474 | }, 475 | { 476 | "cell_type": "code", 477 | "metadata": { 478 | "id": "ExNxCwhcQ18S" 479 | }, 480 | "source": [ 481 | "import tensorflow as tf\n", 482 | "print(tf.__version__)\n", 483 | "\n", 484 | "mnist = tf.keras.datasets.mnist\n", 485 | "\n", 486 | "(training_images, training_labels) , (test_images, test_labels) = mnist.load_data()\n", 487 | "\n", 488 | "training_images = training_images/255.0\n", 489 | "test_images = test_images/255.0\n", 490 | "\n", 491 | "model = tf.keras.models.Sequential([#tf.keras.layers.Flatten(),\n", 492 | " tf.keras.layers.Dense(64, activation=tf.nn.relu),\n", 493 | " tf.keras.layers.Dense(10, activation=tf.nn.softmax)])\n", 494 | "\n", 495 | "model.compile(optimizer = 'adam',\n", 496 | " loss = 'sparse_categorical_crossentropy')\n", 497 | "\n", 498 | "model.fit(training_images, training_labels, epochs=5)\n", 499 | "\n", 500 | "model.evaluate(test_images, test_labels)\n", 501 | "\n", 502 | "classifications = model.predict(test_images)\n", 503 | "\n", 504 | "print(classifications[0])\n", 505 | "print(test_labels[0])" 506 | ], 507 | "execution_count": null, 508 | "outputs": [] 509 | }, 510 | { 511 | "cell_type": "markdown", 512 | "metadata": { 513 | "id": "VqoCR-ieSGDg" 514 | }, 515 | "source": [ 516 | "##Exercise 4: \n", 517 | "\n", 518 | "Consider the final (output) layers. Why are there 10 of them? What would happen if you had a different amount than 10? For example, try training the network with 5\n", 519 | "\n", 520 | "You get an error as soon as it finds an unexpected value. Another rule of thumb -- the number of neurons in the last layer should match the number of classes you are classifying for. In this case it's the digits 0-9, so there are 10 of them, hence you should have 10 neurons in your final layer." 521 | ] 522 | }, 523 | { 524 | "cell_type": "code", 525 | "metadata": { 526 | "id": "MMckVntcSPvo" 527 | }, 528 | "source": [ 529 | "import tensorflow as tf\n", 530 | "print(tf.__version__)\n", 531 | "\n", 532 | "mnist = tf.keras.datasets.mnist\n", 533 | "\n", 534 | "(training_images, training_labels) , (test_images, test_labels) = mnist.load_data()\n", 535 | "\n", 536 | "training_images = training_images/255.0\n", 537 | "test_images = test_images/255.0\n", 538 | "\n", 539 | "model = tf.keras.models.Sequential([tf.keras.layers.Flatten(),\n", 540 | " tf.keras.layers.Dense(64, activation=tf.nn.relu),\n", 541 | " tf.keras.layers.Dense(5, activation=tf.nn.softmax)])\n", 542 | "\n", 543 | "model.compile(optimizer = 'adam',\n", 544 | " loss = 'sparse_categorical_crossentropy')\n", 545 | "\n", 546 | "model.fit(training_images, training_labels, epochs=5)\n", 547 | "\n", 548 | "model.evaluate(test_images, test_labels)\n", 549 | "\n", 550 | "classifications = model.predict(test_images)\n", 551 | "\n", 552 | "print(classifications[0])\n", 553 | "print(test_labels[0])" 554 | ], 555 | "execution_count": null, 556 | "outputs": [] 557 | }, 558 | { 559 | "cell_type": "markdown", 560 | "metadata": { 561 | "id": "-0lF5MuvSuZF" 562 | }, 563 | "source": [ 564 | "##Exercise 5: \n", 565 | "\n", 566 | "Consider the effects of additional layers in the network. What will happen if you add another layer between the one with 512 and the final layer with 10. \n", 567 | "\n", 568 | "Ans: There isn't a significant impact -- because this is relatively simple data. For far more complex data (including color images to be classified as flowers that you'll see in the next lesson), extra layers are often necessary. " 569 | ] 570 | }, 571 | { 572 | "cell_type": "code", 573 | "metadata": { 574 | "id": "b1YPa6UhS8Es" 575 | }, 576 | "source": [ 577 | "import tensorflow as tf\n", 578 | "print(tf.__version__)\n", 579 | "\n", 580 | "mnist = tf.keras.datasets.mnist\n", 581 | "\n", 582 | "(training_images, training_labels) , (test_images, test_labels) = mnist.load_data()\n", 583 | "\n", 584 | "training_images = training_images/255.0\n", 585 | "test_images = test_images/255.0\n", 586 | "\n", 587 | "model = tf.keras.models.Sequential([tf.keras.layers.Flatten(),\n", 588 | " tf.keras.layers.Dense(512, activation=tf.nn.relu),\n", 589 | " tf.keras.layers.Dense(256, activation=tf.nn.relu),\n", 590 | " tf.keras.layers.Dense(10, activation=tf.nn.softmax)])\n", 591 | "\n", 592 | "model.compile(optimizer = 'adam',\n", 593 | " loss = 'sparse_categorical_crossentropy')\n", 594 | "\n", 595 | "model.fit(training_images, training_labels, epochs=5)\n", 596 | "\n", 597 | "model.evaluate(test_images, test_labels)\n", 598 | "\n", 599 | "classifications = model.predict(test_images)\n", 600 | "\n", 601 | "print(classifications[0])\n", 602 | "print(test_labels[0])" 603 | ], 604 | "execution_count": null, 605 | "outputs": [] 606 | }, 607 | { 608 | "cell_type": "markdown", 609 | "metadata": { 610 | "id": "Bql9fyaNUSFy" 611 | }, 612 | "source": [ 613 | "#Exercise 6: \n", 614 | "\n", 615 | "Consider the impact of training for more or less epochs. Why do you think that would be the case? \n", 616 | "\n", 617 | "Try 15 epochs -- you'll probably get a model with a much better loss than the one with 5\n", 618 | "Try 30 epochs -- you might see the loss value stops decreasing, and sometimes increases. This is a side effect of something called 'overfitting' which you can learn about [somewhere] and it's something you need to keep an eye out for when training neural networks. There's no point in wasting your time training if you aren't improving your loss, right! :)" 619 | ] 620 | }, 621 | { 622 | "cell_type": "code", 623 | "metadata": { 624 | "id": "uE3esj9BURQe" 625 | }, 626 | "source": [ 627 | "import tensorflow as tf\n", 628 | "print(tf.__version__)\n", 629 | "\n", 630 | "mnist = tf.keras.datasets.mnist\n", 631 | "\n", 632 | "(training_images, training_labels) , (test_images, test_labels) = mnist.load_data()\n", 633 | "\n", 634 | "training_images = training_images/255.0\n", 635 | "test_images = test_images/255.0\n", 636 | "\n", 637 | "model = tf.keras.models.Sequential([tf.keras.layers.Flatten(),\n", 638 | " tf.keras.layers.Dense(128, activation=tf.nn.relu),\n", 639 | " tf.keras.layers.Dense(10, activation=tf.nn.softmax)])\n", 640 | "\n", 641 | "model.compile(optimizer = 'adam',\n", 642 | " loss = 'sparse_categorical_crossentropy')\n", 643 | "\n", 644 | "model.fit(training_images, training_labels, epochs=30)\n", 645 | "\n", 646 | "model.evaluate(test_images, test_labels)\n", 647 | "\n", 648 | "classifications = model.predict(test_images)\n", 649 | "\n", 650 | "print(classifications[34])\n", 651 | "print(test_labels[34])" 652 | ], 653 | "execution_count": null, 654 | "outputs": [] 655 | }, 656 | { 657 | "cell_type": "markdown", 658 | "metadata": { 659 | "id": "HS3vVkOgCDGZ" 660 | }, 661 | "source": [ 662 | "#Exercise 7: \n", 663 | "\n", 664 | "Before you trained, you normalized the data, going from values that were 0-255 to values that were 0-1. What would be the impact of removing that? Here's the complete code to give it a try. Why do you think you get different results? " 665 | ] 666 | }, 667 | { 668 | "cell_type": "code", 669 | "metadata": { 670 | "id": "JDqNAqrpCNg0" 671 | }, 672 | "source": [ 673 | "import tensorflow as tf\n", 674 | "print(tf.__version__)\n", 675 | "mnist = tf.keras.datasets.mnist\n", 676 | "(training_images, training_labels), (test_images, test_labels) = mnist.load_data()\n", 677 | "training_images=training_images/255.0\n", 678 | "test_images=test_images/255.0\n", 679 | "model = tf.keras.models.Sequential([\n", 680 | " tf.keras.layers.Flatten(),\n", 681 | " tf.keras.layers.Dense(512, activation=tf.nn.relu),\n", 682 | " tf.keras.layers.Dense(10, activation=tf.nn.softmax)\n", 683 | "])\n", 684 | "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')\n", 685 | "model.fit(training_images, training_labels, epochs=5)\n", 686 | "model.evaluate(test_images, test_labels)\n", 687 | "classifications = model.predict(test_images)\n", 688 | "print(classifications[0])\n", 689 | "print(test_labels[0])" 690 | ], 691 | "execution_count": null, 692 | "outputs": [] 693 | }, 694 | { 695 | "cell_type": "markdown", 696 | "metadata": { 697 | "id": "E7W2PT66ZBHQ" 698 | }, 699 | "source": [ 700 | "#Exercise 8: \n", 701 | "\n", 702 | "Earlier when you trained for extra epochs you had an issue where your loss might change. It might have taken a bit of time for you to wait for the training to do that, and you might have thought 'wouldn't it be nice if I could stop the training when I reach a desired value?' -- i.e. 95% accuracy might be enough for you, and if you reach that after 3 epochs, why sit around waiting for it to finish a lot more epochs....So how would you fix that? Like any other program...you have callbacks! Let's see them in action..." 703 | ] 704 | }, 705 | { 706 | "cell_type": "code", 707 | "metadata": { 708 | "id": "pkaEHHgqZbYv" 709 | }, 710 | "source": [ 711 | "import tensorflow as tf\n", 712 | "print(tf.__version__)\n", 713 | "\n", 714 | "class myCallback(tf.keras.callbacks.Callback):\n", 715 | " def on_epoch_end(self, epoch, logs={}):\n", 716 | " if(logs.get('loss')<0.4):\n", 717 | " print(\"\\nReached 60% accuracy so cancelling training!\")\n", 718 | " self.model.stop_training = True\n", 719 | "\n", 720 | "callbacks = myCallback()\n", 721 | "mnist = tf.keras.datasets.fashion_mnist\n", 722 | "(training_images, training_labels), (test_images, test_labels) = mnist.load_data()\n", 723 | "training_images=training_images/255.0\n", 724 | "test_images=test_images/255.0\n", 725 | "model = tf.keras.models.Sequential([\n", 726 | " tf.keras.layers.Flatten(),\n", 727 | " tf.keras.layers.Dense(512, activation=tf.nn.relu),\n", 728 | " tf.keras.layers.Dense(10, activation=tf.nn.softmax)\n", 729 | "])\n", 730 | "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')\n", 731 | "model.fit(training_images, training_labels, epochs=5, callbacks=[callbacks])\n" 732 | ], 733 | "execution_count": null, 734 | "outputs": [] 735 | } 736 | ] 737 | } -------------------------------------------------------------------------------- /Introduction to TensorFlow for Artificial Intelligence, Machine Learning, and Deep Learning/week4/Course_1_Part_8_Lesson_2_Notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 1 - Part 8 - Lesson 2 - Notebook.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "name": "python3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "rX8mhOLljYeM" 22 | }, 23 | "source": [ 24 | "##### Copyright 2019 The TensorFlow Authors." 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "cellView": "form", 31 | "id": "BZSlp3DAjdYf" 32 | }, 33 | "source": [ 34 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 35 | "# you may not use this file except in compliance with the License.\n", 36 | "# You may obtain a copy of the License at\n", 37 | "#\n", 38 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 39 | "#\n", 40 | "# Unless required by applicable law or agreed to in writing, software\n", 41 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 42 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 43 | "# See the License for the specific language governing permissions and\n", 44 | "# limitations under the License." 45 | ], 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "metadata": { 52 | "id": "RXZT2UsyIVe_" 53 | }, 54 | "source": [ 55 | "!wget --no-check-certificate \\\n", 56 | " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/horse-or-human.zip \\\n", 57 | " -O /tmp/horse-or-human.zip" 58 | ], 59 | "execution_count": null, 60 | "outputs": [] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": { 65 | "id": "9brUxyTpYZHy" 66 | }, 67 | "source": [ 68 | "The following python code will use the OS library to use Operating System libraries, giving you access to the file system, and the zipfile library allowing you to unzip the data. " 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "metadata": { 74 | "id": "PLy3pthUS0D2" 75 | }, 76 | "source": [ 77 | "import os\n", 78 | "import zipfile\n", 79 | "\n", 80 | "local_zip = '/tmp/horse-or-human.zip'\n", 81 | "zip_ref = zipfile.ZipFile(local_zip, 'r')\n", 82 | "zip_ref.extractall('/tmp/horse-or-human')\n", 83 | "zip_ref.close()" 84 | ], 85 | "execution_count": null, 86 | "outputs": [] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": { 91 | "id": "o-qUPyfO7Qr8" 92 | }, 93 | "source": [ 94 | "The contents of the .zip are extracted to the base directory `/tmp/horse-or-human`, which in turn each contain `horses` and `humans` subdirectories.\n", 95 | "\n", 96 | "In short: The training set is the data that is used to tell the neural network model that 'this is what a horse looks like', 'this is what a human looks like' etc. \n", 97 | "\n", 98 | "One thing to pay attention to in this sample: We do not explicitly label the images as horses or humans. If you remember with the handwriting example earlier, we had labelled 'this is a 1', 'this is a 7' etc. Later you'll see something called an ImageGenerator being used -- and this is coded to read images from subdirectories, and automatically label them from the name of that subdirectory. So, for example, you will have a 'training' directory containing a 'horses' directory and a 'humans' one. ImageGenerator will label the images appropriately for you, reducing a coding step. \n", 99 | "\n", 100 | "Let's define each of these directories:" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "metadata": { 106 | "id": "NR_M9nWN-K8B" 107 | }, 108 | "source": [ 109 | "# Directory with our training horse pictures\n", 110 | "train_horse_dir = os.path.join('/tmp/horse-or-human/horses')\n", 111 | "\n", 112 | "# Directory with our training human pictures\n", 113 | "train_human_dir = os.path.join('/tmp/horse-or-human/humans')" 114 | ], 115 | "execution_count": null, 116 | "outputs": [] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": { 121 | "id": "LuBYtA_Zd8_T" 122 | }, 123 | "source": [ 124 | "Now, let's see what the filenames look like in the `horses` and `humans` training directories:" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "metadata": { 130 | "id": "4PIP1rkmeAYS" 131 | }, 132 | "source": [ 133 | "train_horse_names = os.listdir(train_horse_dir)\n", 134 | "print(train_horse_names[:10])\n", 135 | "\n", 136 | "train_human_names = os.listdir(train_human_dir)\n", 137 | "print(train_human_names[:10])" 138 | ], 139 | "execution_count": null, 140 | "outputs": [] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": { 145 | "id": "HlqN5KbafhLI" 146 | }, 147 | "source": [ 148 | "Let's find out the total number of horse and human images in the directories:" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "metadata": { 154 | "id": "H4XHh2xSfgie" 155 | }, 156 | "source": [ 157 | "print('total training horse images:', len(os.listdir(train_horse_dir)))\n", 158 | "print('total training human images:', len(os.listdir(train_human_dir)))" 159 | ], 160 | "execution_count": null, 161 | "outputs": [] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "metadata": { 166 | "id": "C3WZABE9eX-8" 167 | }, 168 | "source": [ 169 | "Now let's take a look at a few pictures to get a better sense of what they look like. First, configure the matplot parameters:" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "metadata": { 175 | "id": "b2_Q0-_5UAv-" 176 | }, 177 | "source": [ 178 | "%matplotlib inline\n", 179 | "\n", 180 | "import matplotlib.pyplot as plt\n", 181 | "import matplotlib.image as mpimg\n", 182 | "\n", 183 | "# Parameters for our graph; we'll output images in a 4x4 configuration\n", 184 | "nrows = 4\n", 185 | "ncols = 4\n", 186 | "\n", 187 | "# Index for iterating over images\n", 188 | "pic_index = 0" 189 | ], 190 | "execution_count": null, 191 | "outputs": [] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "metadata": { 196 | "id": "xTvHzGCxXkqp" 197 | }, 198 | "source": [ 199 | "Now, display a batch of 8 horse and 8 human pictures. You can rerun the cell to see a fresh batch each time:" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "metadata": { 205 | "id": "Wpr8GxjOU8in" 206 | }, 207 | "source": [ 208 | "# Set up matplotlib fig, and size it to fit 4x4 pics\n", 209 | "fig = plt.gcf()\n", 210 | "fig.set_size_inches(ncols * 4, nrows * 4)\n", 211 | "\n", 212 | "pic_index += 8\n", 213 | "next_horse_pix = [os.path.join(train_horse_dir, fname) \n", 214 | " for fname in train_horse_names[pic_index-8:pic_index]]\n", 215 | "next_human_pix = [os.path.join(train_human_dir, fname) \n", 216 | " for fname in train_human_names[pic_index-8:pic_index]]\n", 217 | "\n", 218 | "for i, img_path in enumerate(next_horse_pix+next_human_pix):\n", 219 | " # Set up subplot; subplot indices start at 1\n", 220 | " sp = plt.subplot(nrows, ncols, i + 1)\n", 221 | " sp.axis('Off') # Don't show axes (or gridlines)\n", 222 | "\n", 223 | " img = mpimg.imread(img_path)\n", 224 | " plt.imshow(img)\n", 225 | "\n", 226 | "plt.show()\n" 227 | ], 228 | "execution_count": null, 229 | "outputs": [] 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "metadata": { 234 | "id": "5oqBkNBJmtUv" 235 | }, 236 | "source": [ 237 | "## Building a Small Model from Scratch\n", 238 | "\n", 239 | "But before we continue, let's start defining the model:\n", 240 | "\n", 241 | "Step 1 will be to import tensorflow." 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "metadata": { 247 | "id": "qvfZg3LQbD-5" 248 | }, 249 | "source": [ 250 | "import tensorflow as tf" 251 | ], 252 | "execution_count": null, 253 | "outputs": [] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": { 258 | "id": "BnhYCP4tdqjC" 259 | }, 260 | "source": [ 261 | "We then add convolutional layers as in the previous example, and flatten the final result to feed into the densely connected layers." 262 | ] 263 | }, 264 | { 265 | "cell_type": "markdown", 266 | "metadata": { 267 | "id": "gokG5HKpdtzm" 268 | }, 269 | "source": [ 270 | "Finally we add the densely connected layers. \n", 271 | "\n", 272 | "Note that because we are facing a two-class classification problem, i.e. a *binary classification problem*, we will end our network with a [*sigmoid* activation](https://wikipedia.org/wiki/Sigmoid_function), so that the output of our network will be a single scalar between 0 and 1, encoding the probability that the current image is class 1 (as opposed to class 0)." 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "metadata": { 278 | "id": "PixZ2s5QbYQ3" 279 | }, 280 | "source": [ 281 | "model = tf.keras.models.Sequential([\n", 282 | " # Note the input shape is the desired size of the image 300x300 with 3 bytes color\n", 283 | " # This is the first convolution\n", 284 | " tf.keras.layers.Conv2D(16, (3,3), activation='relu', input_shape=(300, 300, 3)),\n", 285 | " tf.keras.layers.MaxPooling2D(2, 2),\n", 286 | " # The second convolution\n", 287 | " tf.keras.layers.Conv2D(32, (3,3), activation='relu'),\n", 288 | " tf.keras.layers.MaxPooling2D(2,2),\n", 289 | " # The third convolution\n", 290 | " tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n", 291 | " tf.keras.layers.MaxPooling2D(2,2),\n", 292 | " # The fourth convolution\n", 293 | " tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n", 294 | " tf.keras.layers.MaxPooling2D(2,2),\n", 295 | " # The fifth convolution\n", 296 | " tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n", 297 | " tf.keras.layers.MaxPooling2D(2,2),\n", 298 | " # Flatten the results to feed into a DNN\n", 299 | " tf.keras.layers.Flatten(),\n", 300 | " # 512 neuron hidden layer\n", 301 | " tf.keras.layers.Dense(512, activation='relu'),\n", 302 | " # Only 1 output neuron. It will contain a value from 0-1 where 0 for 1 class ('horses') and 1 for the other ('humans')\n", 303 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 304 | "])" 305 | ], 306 | "execution_count": null, 307 | "outputs": [] 308 | }, 309 | { 310 | "cell_type": "markdown", 311 | "metadata": { 312 | "id": "s9EaFDP5srBa" 313 | }, 314 | "source": [ 315 | "The model.summary() method call prints a summary of the NN " 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "metadata": { 321 | "id": "7ZKj8392nbgP" 322 | }, 323 | "source": [ 324 | "model.summary()" 325 | ], 326 | "execution_count": null, 327 | "outputs": [] 328 | }, 329 | { 330 | "cell_type": "markdown", 331 | "metadata": { 332 | "id": "DmtkTn06pKxF" 333 | }, 334 | "source": [ 335 | "The \"output shape\" column shows how the size of your feature map evolves in each successive layer. The convolution layers reduce the size of the feature maps by a bit due to padding, and each pooling layer halves the dimensions." 336 | ] 337 | }, 338 | { 339 | "cell_type": "markdown", 340 | "metadata": { 341 | "id": "PEkKSpZlvJXA" 342 | }, 343 | "source": [ 344 | "Next, we'll configure the specifications for model training. We will train our model with the `binary_crossentropy` loss, because it's a binary classification problem and our final activation is a sigmoid. (For a refresher on loss metrics, see the [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/descending-into-ml/video-lecture).) We will use the `rmsprop` optimizer with a learning rate of `0.001`. During training, we will want to monitor classification accuracy.\n", 345 | "\n", 346 | "**NOTE**: In this case, using the [RMSprop optimization algorithm](https://wikipedia.org/wiki/Stochastic_gradient_descent#RMSProp) is preferable to [stochastic gradient descent](https://developers.google.com/machine-learning/glossary/#SGD) (SGD), because RMSprop automates learning-rate tuning for us. (Other optimizers, such as [Adam](https://wikipedia.org/wiki/Stochastic_gradient_descent#Adam) and [Adagrad](https://developers.google.com/machine-learning/glossary/#AdaGrad), also automatically adapt the learning rate during training, and would work equally well here.)" 347 | ] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "metadata": { 352 | "id": "8DHWhFP_uhq3" 353 | }, 354 | "source": [ 355 | "from tensorflow.keras.optimizers import RMSprop\n", 356 | "\n", 357 | "model.compile(loss='binary_crossentropy',\n", 358 | " optimizer=RMSprop(lr=0.001),\n", 359 | " metrics=['accuracy'])" 360 | ], 361 | "execution_count": null, 362 | "outputs": [] 363 | }, 364 | { 365 | "cell_type": "markdown", 366 | "metadata": { 367 | "id": "Sn9m9D3UimHM" 368 | }, 369 | "source": [ 370 | "### Data Preprocessing\n", 371 | "\n", 372 | "Let's set up data generators that will read pictures in our source folders, convert them to `float32` tensors, and feed them (with their labels) to our network. We'll have one generator for the training images and one for the validation images. Our generators will yield batches of images of size 300x300 and their labels (binary).\n", 373 | "\n", 374 | "As you may already know, data that goes into neural networks should usually be normalized in some way to make it more amenable to processing by the network. (It is uncommon to feed raw pixels into a convnet.) In our case, we will preprocess our images by normalizing the pixel values to be in the `[0, 1]` range (originally all values are in the `[0, 255]` range).\n", 375 | "\n", 376 | "In Keras this can be done via the `keras.preprocessing.image.ImageDataGenerator` class using the `rescale` parameter. This `ImageDataGenerator` class allows you to instantiate generators of augmented image batches (and their labels) via `.flow(data, labels)` or `.flow_from_directory(directory)`. These generators can then be used with the Keras model methods that accept data generators as inputs: `fit`, `evaluate_generator`, and `predict_generator`." 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "metadata": { 382 | "id": "ClebU9NJg99G" 383 | }, 384 | "source": [ 385 | "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", 386 | "\n", 387 | "# All images will be rescaled by 1./255\n", 388 | "train_datagen = ImageDataGenerator(rescale=1/255)\n", 389 | "\n", 390 | "# Flow training images in batches of 128 using train_datagen generator\n", 391 | "train_generator = train_datagen.flow_from_directory(\n", 392 | " '/tmp/horse-or-human/', # This is the source directory for training images\n", 393 | " target_size=(300, 300), # All images will be resized to 150x150\n", 394 | " batch_size=128,\n", 395 | " # Since we use binary_crossentropy loss, we need binary labels\n", 396 | " class_mode='binary')\n" 397 | ], 398 | "execution_count": null, 399 | "outputs": [] 400 | }, 401 | { 402 | "cell_type": "markdown", 403 | "metadata": { 404 | "id": "mu3Jdwkjwax4" 405 | }, 406 | "source": [ 407 | "### Training\n", 408 | "Let's train for 15 epochs -- this may take a few minutes to run.\n", 409 | "\n", 410 | "Do note the values per epoch.\n", 411 | "\n", 412 | "The Loss and Accuracy are a great indication of progress of training. It's making a guess as to the classification of the training data, and then measuring it against the known label, calculating the result. Accuracy is the portion of correct guesses. " 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "metadata": { 418 | "id": "Fb1_lgobv81m" 419 | }, 420 | "source": [ 421 | "history = model.fit(\n", 422 | " train_generator,\n", 423 | " steps_per_epoch=8, \n", 424 | " epochs=15,\n", 425 | " verbose=1)" 426 | ], 427 | "execution_count": null, 428 | "outputs": [] 429 | }, 430 | { 431 | "cell_type": "markdown", 432 | "metadata": { 433 | "id": "o6vSHzPR2ghH" 434 | }, 435 | "source": [ 436 | "###Running the Model\n", 437 | "\n", 438 | "Let's now take a look at actually running a prediction using the model. This code will allow you to choose 1 or more files from your file system, it will then upload them, and run them through the model, giving an indication of whether the object is a horse or a human." 439 | ] 440 | }, 441 | { 442 | "cell_type": "code", 443 | "metadata": { 444 | "id": "DoWp43WxJDNT" 445 | }, 446 | "source": [ 447 | "import numpy as np\n", 448 | "from google.colab import files\n", 449 | "from keras.preprocessing import image\n", 450 | "\n", 451 | "uploaded = files.upload()\n", 452 | "\n", 453 | "for fn in uploaded.keys():\n", 454 | " \n", 455 | " # predicting images\n", 456 | " path = '/content/' + fn\n", 457 | " img = image.load_img(path, target_size=(300, 300))\n", 458 | " x = image.img_to_array(img)\n", 459 | " x = np.expand_dims(x, axis=0)\n", 460 | "\n", 461 | " images = np.vstack([x])\n", 462 | " classes = model.predict(images, batch_size=10)\n", 463 | " print(classes[0])\n", 464 | " if classes[0]>0.5:\n", 465 | " print(fn + \" is a human\")\n", 466 | " else:\n", 467 | " print(fn + \" is a horse\")\n", 468 | " " 469 | ], 470 | "execution_count": null, 471 | "outputs": [] 472 | }, 473 | { 474 | "cell_type": "markdown", 475 | "metadata": { 476 | "id": "-8EHQyWGDvWz" 477 | }, 478 | "source": [ 479 | "### Visualizing Intermediate Representations\n", 480 | "\n", 481 | "To get a feel for what kind of features our convnet has learned, one fun thing to do is to visualize how an input gets transformed as it goes through the convnet.\n", 482 | "\n", 483 | "Let's pick a random image from the training set, and then generate a figure where each row is the output of a layer, and each image in the row is a specific filter in that output feature map. Rerun this cell to generate intermediate representations for a variety of training images." 484 | ] 485 | }, 486 | { 487 | "cell_type": "code", 488 | "metadata": { 489 | "id": "-5tES8rXFjux" 490 | }, 491 | "source": [ 492 | "import numpy as np\n", 493 | "import random\n", 494 | "from tensorflow.keras.preprocessing.image import img_to_array, load_img\n", 495 | "\n", 496 | "# Let's define a new Model that will take an image as input, and will output\n", 497 | "# intermediate representations for all layers in the previous model after\n", 498 | "# the first.\n", 499 | "successive_outputs = [layer.output for layer in model.layers[1:]]\n", 500 | "#visualization_model = Model(img_input, successive_outputs)\n", 501 | "visualization_model = tf.keras.models.Model(inputs = model.input, outputs = successive_outputs)\n", 502 | "# Let's prepare a random input image from the training set.\n", 503 | "horse_img_files = [os.path.join(train_horse_dir, f) for f in train_horse_names]\n", 504 | "human_img_files = [os.path.join(train_human_dir, f) for f in train_human_names]\n", 505 | "img_path = random.choice(horse_img_files + human_img_files)\n", 506 | "\n", 507 | "img = load_img(img_path, target_size=(300, 300)) # this is a PIL image\n", 508 | "x = img_to_array(img) # Numpy array with shape (150, 150, 3)\n", 509 | "x = x.reshape((1,) + x.shape) # Numpy array with shape (1, 150, 150, 3)\n", 510 | "\n", 511 | "# Rescale by 1/255\n", 512 | "x /= 255\n", 513 | "\n", 514 | "# Let's run our image through our network, thus obtaining all\n", 515 | "# intermediate representations for this image.\n", 516 | "successive_feature_maps = visualization_model.predict(x)\n", 517 | "\n", 518 | "# These are the names of the layers, so can have them as part of our plot\n", 519 | "layer_names = [layer.name for layer in model.layers[1:]]\n", 520 | "\n", 521 | "# Now let's display our representations\n", 522 | "for layer_name, feature_map in zip(layer_names, successive_feature_maps):\n", 523 | " if len(feature_map.shape) == 4:\n", 524 | " # Just do this for the conv / maxpool layers, not the fully-connected layers\n", 525 | " n_features = feature_map.shape[-1] # number of features in feature map\n", 526 | " # The feature map has shape (1, size, size, n_features)\n", 527 | " size = feature_map.shape[1]\n", 528 | " # We will tile our images in this matrix\n", 529 | " display_grid = np.zeros((size, size * n_features))\n", 530 | " for i in range(n_features):\n", 531 | " # Postprocess the feature to make it visually palatable\n", 532 | " x = feature_map[0, :, :, i]\n", 533 | " x -= x.mean()\n", 534 | " x /= x.std()\n", 535 | " x *= 64\n", 536 | " x += 128\n", 537 | " x = np.clip(x, 0, 255).astype('uint8')\n", 538 | " # We'll tile each filter into this big horizontal grid\n", 539 | " display_grid[:, i * size : (i + 1) * size] = x\n", 540 | " # Display the grid\n", 541 | " scale = 20. / n_features\n", 542 | " plt.figure(figsize=(scale * n_features, scale))\n", 543 | " plt.title(layer_name)\n", 544 | " plt.grid(False)\n", 545 | " plt.imshow(display_grid, aspect='auto', cmap='viridis')" 546 | ], 547 | "execution_count": null, 548 | "outputs": [] 549 | }, 550 | { 551 | "cell_type": "markdown", 552 | "metadata": { 553 | "id": "tuqK2arJL0wo" 554 | }, 555 | "source": [ 556 | "As you can see we go from the raw pixels of the images to increasingly abstract and compact representations. The representations downstream start highlighting what the network pays attention to, and they show fewer and fewer features being \"activated\"; most are set to zero. This is called \"sparsity.\" Representation sparsity is a key feature of deep learning.\n", 557 | "\n", 558 | "\n", 559 | "These representations carry increasingly less information about the original pixels of the image, but increasingly refined information about the class of the image. You can think of a convnet (or a deep network in general) as an information distillation pipeline." 560 | ] 561 | }, 562 | { 563 | "cell_type": "markdown", 564 | "metadata": { 565 | "id": "j4IBgYCYooGD" 566 | }, 567 | "source": [ 568 | "## Clean Up\n", 569 | "\n", 570 | "Before running the next exercise, run the following cell to terminate the kernel and free memory resources:" 571 | ] 572 | }, 573 | { 574 | "cell_type": "code", 575 | "metadata": { 576 | "id": "651IgjLyo-Jx" 577 | }, 578 | "source": [ 579 | "import os, signal\n", 580 | "os.kill(os.getpid(), signal.SIGKILL)" 581 | ], 582 | "execution_count": null, 583 | "outputs": [] 584 | } 585 | ] 586 | } -------------------------------------------------------------------------------- /Introduction to TensorFlow for Artificial Intelligence, Machine Learning, and Deep Learning/week4/Course_1_Part_8_Lesson_3_Notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 1 - Part 8 - Lesson 3 - Notebook.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "name": "python3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "rX8mhOLljYeM" 22 | }, 23 | "source": [ 24 | "##### Copyright 2019 The TensorFlow Authors." 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "cellView": "form", 31 | "id": "BZSlp3DAjdYf" 32 | }, 33 | "source": [ 34 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 35 | "# you may not use this file except in compliance with the License.\n", 36 | "# You may obtain a copy of the License at\n", 37 | "#\n", 38 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 39 | "#\n", 40 | "# Unless required by applicable law or agreed to in writing, software\n", 41 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 42 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 43 | "# See the License for the specific language governing permissions and\n", 44 | "# limitations under the License." 45 | ], 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "metadata": { 52 | "id": "RXZT2UsyIVe_" 53 | }, 54 | "source": [ 55 | "!wget --no-check-certificate \\\n", 56 | " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/horse-or-human.zip \\\n", 57 | " -O /tmp/horse-or-human.zip" 58 | ], 59 | "execution_count": null, 60 | "outputs": [] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "metadata": { 65 | "id": "0mLij6qde6Ox" 66 | }, 67 | "source": [ 68 | "!wget --no-check-certificate \\\n", 69 | " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/validation-horse-or-human.zip \\\n", 70 | " -O /tmp/validation-horse-or-human.zip" 71 | ], 72 | "execution_count": null, 73 | "outputs": [] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": { 78 | "id": "9brUxyTpYZHy" 79 | }, 80 | "source": [ 81 | "The following python code will use the OS library to use Operating System libraries, giving you access to the file system, and the zipfile library allowing you to unzip the data. " 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "metadata": { 87 | "id": "PLy3pthUS0D2" 88 | }, 89 | "source": [ 90 | "import os\n", 91 | "import zipfile\n", 92 | "\n", 93 | "local_zip = '/tmp/horse-or-human.zip'\n", 94 | "zip_ref = zipfile.ZipFile(local_zip, 'r')\n", 95 | "zip_ref.extractall('/tmp/horse-or-human')\n", 96 | "local_zip = '/tmp/validation-horse-or-human.zip'\n", 97 | "zip_ref = zipfile.ZipFile(local_zip, 'r')\n", 98 | "zip_ref.extractall('/tmp/validation-horse-or-human')\n", 99 | "zip_ref.close()" 100 | ], 101 | "execution_count": null, 102 | "outputs": [] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": { 107 | "id": "o-qUPyfO7Qr8" 108 | }, 109 | "source": [ 110 | "The contents of the .zip are extracted to the base directory `/tmp/horse-or-human`, which in turn each contain `horses` and `humans` subdirectories.\n", 111 | "\n", 112 | "In short: The training set is the data that is used to tell the neural network model that 'this is what a horse looks like', 'this is what a human looks like' etc. \n", 113 | "\n", 114 | "One thing to pay attention to in this sample: We do not explicitly label the images as horses or humans. If you remember with the handwriting example earlier, we had labelled 'this is a 1', 'this is a 7' etc. Later you'll see something called an ImageGenerator being used -- and this is coded to read images from subdirectories, and automatically label them from the name of that subdirectory. So, for example, you will have a 'training' directory containing a 'horses' directory and a 'humans' one. ImageGenerator will label the images appropriately for you, reducing a coding step. \n", 115 | "\n", 116 | "Let's define each of these directories:" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "metadata": { 122 | "id": "NR_M9nWN-K8B" 123 | }, 124 | "source": [ 125 | "# Directory with our training horse pictures\n", 126 | "train_horse_dir = os.path.join('/tmp/horse-or-human/horses')\n", 127 | "\n", 128 | "# Directory with our training human pictures\n", 129 | "train_human_dir = os.path.join('/tmp/horse-or-human/humans')\n", 130 | "\n", 131 | "# Directory with our training horse pictures\n", 132 | "validation_horse_dir = os.path.join('/tmp/validation-horse-or-human/horses')\n", 133 | "\n", 134 | "# Directory with our training human pictures\n", 135 | "validation_human_dir = os.path.join('/tmp/validation-horse-or-human/humans')" 136 | ], 137 | "execution_count": null, 138 | "outputs": [] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": { 143 | "id": "LuBYtA_Zd8_T" 144 | }, 145 | "source": [ 146 | "Now, let's see what the filenames look like in the `horses` and `humans` training directories:" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "metadata": { 152 | "id": "4PIP1rkmeAYS" 153 | }, 154 | "source": [ 155 | "train_horse_names = os.listdir(train_horse_dir)\n", 156 | "print(train_horse_names[:10])\n", 157 | "\n", 158 | "train_human_names = os.listdir(train_human_dir)\n", 159 | "print(train_human_names[:10])\n", 160 | "\n", 161 | "validation_horse_hames = os.listdir(validation_horse_dir)\n", 162 | "print(validation_horse_hames[:10])\n", 163 | "\n", 164 | "validation_human_names = os.listdir(validation_human_dir)\n", 165 | "print(validation_human_names[:10])" 166 | ], 167 | "execution_count": null, 168 | "outputs": [] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": { 173 | "id": "HlqN5KbafhLI" 174 | }, 175 | "source": [ 176 | "Let's find out the total number of horse and human images in the directories:" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "metadata": { 182 | "id": "H4XHh2xSfgie" 183 | }, 184 | "source": [ 185 | "print('total training horse images:', len(os.listdir(train_horse_dir)))\n", 186 | "print('total training human images:', len(os.listdir(train_human_dir)))\n", 187 | "print('total validation horse images:', len(os.listdir(validation_horse_dir)))\n", 188 | "print('total validation human images:', len(os.listdir(validation_human_dir)))" 189 | ], 190 | "execution_count": null, 191 | "outputs": [] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "metadata": { 196 | "id": "C3WZABE9eX-8" 197 | }, 198 | "source": [ 199 | "Now let's take a look at a few pictures to get a better sense of what they look like. First, configure the matplot parameters:" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "metadata": { 205 | "id": "b2_Q0-_5UAv-" 206 | }, 207 | "source": [ 208 | "%matplotlib inline\n", 209 | "\n", 210 | "import matplotlib.pyplot as plt\n", 211 | "import matplotlib.image as mpimg\n", 212 | "\n", 213 | "# Parameters for our graph; we'll output images in a 4x4 configuration\n", 214 | "nrows = 4\n", 215 | "ncols = 4\n", 216 | "\n", 217 | "# Index for iterating over images\n", 218 | "pic_index = 0" 219 | ], 220 | "execution_count": null, 221 | "outputs": [] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "metadata": { 226 | "id": "xTvHzGCxXkqp" 227 | }, 228 | "source": [ 229 | "Now, display a batch of 8 horse and 8 human pictures. You can rerun the cell to see a fresh batch each time:" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "metadata": { 235 | "id": "Wpr8GxjOU8in" 236 | }, 237 | "source": [ 238 | "# Set up matplotlib fig, and size it to fit 4x4 pics\n", 239 | "fig = plt.gcf()\n", 240 | "fig.set_size_inches(ncols * 4, nrows * 4)\n", 241 | "\n", 242 | "pic_index += 8\n", 243 | "next_horse_pix = [os.path.join(train_horse_dir, fname) \n", 244 | " for fname in train_horse_names[pic_index-8:pic_index]]\n", 245 | "next_human_pix = [os.path.join(train_human_dir, fname) \n", 246 | " for fname in train_human_names[pic_index-8:pic_index]]\n", 247 | "\n", 248 | "for i, img_path in enumerate(next_horse_pix+next_human_pix):\n", 249 | " # Set up subplot; subplot indices start at 1\n", 250 | " sp = plt.subplot(nrows, ncols, i + 1)\n", 251 | " sp.axis('Off') # Don't show axes (or gridlines)\n", 252 | "\n", 253 | " img = mpimg.imread(img_path)\n", 254 | " plt.imshow(img)\n", 255 | "\n", 256 | "plt.show()\n" 257 | ], 258 | "execution_count": null, 259 | "outputs": [] 260 | }, 261 | { 262 | "cell_type": "markdown", 263 | "metadata": { 264 | "id": "5oqBkNBJmtUv" 265 | }, 266 | "source": [ 267 | "## Building a Small Model from Scratch\n", 268 | "\n", 269 | "But before we continue, let's start defining the model:\n", 270 | "\n", 271 | "Step 1 will be to import tensorflow." 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "metadata": { 277 | "id": "qvfZg3LQbD-5" 278 | }, 279 | "source": [ 280 | "import tensorflow as tf" 281 | ], 282 | "execution_count": null, 283 | "outputs": [] 284 | }, 285 | { 286 | "cell_type": "markdown", 287 | "metadata": { 288 | "id": "BnhYCP4tdqjC" 289 | }, 290 | "source": [ 291 | "We then add convolutional layers as in the previous example, and flatten the final result to feed into the densely connected layers." 292 | ] 293 | }, 294 | { 295 | "cell_type": "markdown", 296 | "metadata": { 297 | "id": "gokG5HKpdtzm" 298 | }, 299 | "source": [ 300 | "Finally we add the densely connected layers. \n", 301 | "\n", 302 | "Note that because we are facing a two-class classification problem, i.e. a *binary classification problem*, we will end our network with a [*sigmoid* activation](https://wikipedia.org/wiki/Sigmoid_function), so that the output of our network will be a single scalar between 0 and 1, encoding the probability that the current image is class 1 (as opposed to class 0)." 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "metadata": { 308 | "id": "PixZ2s5QbYQ3" 309 | }, 310 | "source": [ 311 | "model = tf.keras.models.Sequential([\n", 312 | " # Note the input shape is the desired size of the image 300x300 with 3 bytes color\n", 313 | " # This is the first convolution\n", 314 | " tf.keras.layers.Conv2D(16, (3,3), activation='relu', input_shape=(300, 300, 3)),\n", 315 | " tf.keras.layers.MaxPooling2D(2, 2),\n", 316 | " # The second convolution\n", 317 | " tf.keras.layers.Conv2D(32, (3,3), activation='relu'),\n", 318 | " tf.keras.layers.MaxPooling2D(2,2),\n", 319 | " # The third convolution\n", 320 | " tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n", 321 | " tf.keras.layers.MaxPooling2D(2,2),\n", 322 | " # The fourth convolution\n", 323 | " tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n", 324 | " tf.keras.layers.MaxPooling2D(2,2),\n", 325 | " # The fifth convolution\n", 326 | " tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n", 327 | " tf.keras.layers.MaxPooling2D(2,2),\n", 328 | " # Flatten the results to feed into a DNN\n", 329 | " tf.keras.layers.Flatten(),\n", 330 | " # 512 neuron hidden layer\n", 331 | " tf.keras.layers.Dense(512, activation='relu'),\n", 332 | " # Only 1 output neuron. It will contain a value from 0-1 where 0 for 1 class ('horses') and 1 for the other ('humans')\n", 333 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 334 | "])" 335 | ], 336 | "execution_count": null, 337 | "outputs": [] 338 | }, 339 | { 340 | "cell_type": "markdown", 341 | "metadata": { 342 | "id": "s9EaFDP5srBa" 343 | }, 344 | "source": [ 345 | "The model.summary() method call prints a summary of the NN " 346 | ] 347 | }, 348 | { 349 | "cell_type": "code", 350 | "metadata": { 351 | "id": "7ZKj8392nbgP" 352 | }, 353 | "source": [ 354 | "model.summary()" 355 | ], 356 | "execution_count": null, 357 | "outputs": [] 358 | }, 359 | { 360 | "cell_type": "markdown", 361 | "metadata": { 362 | "id": "DmtkTn06pKxF" 363 | }, 364 | "source": [ 365 | "The \"output shape\" column shows how the size of your feature map evolves in each successive layer. The convolution layers reduce the size of the feature maps by a bit due to padding, and each pooling layer halves the dimensions." 366 | ] 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "metadata": { 371 | "id": "PEkKSpZlvJXA" 372 | }, 373 | "source": [ 374 | "Next, we'll configure the specifications for model training. We will train our model with the `binary_crossentropy` loss, because it's a binary classification problem and our final activation is a sigmoid. (For a refresher on loss metrics, see the [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/descending-into-ml/video-lecture).) We will use the `rmsprop` optimizer with a learning rate of `0.001`. During training, we will want to monitor classification accuracy.\n", 375 | "\n", 376 | "**NOTE**: In this case, using the [RMSprop optimization algorithm](https://wikipedia.org/wiki/Stochastic_gradient_descent#RMSProp) is preferable to [stochastic gradient descent](https://developers.google.com/machine-learning/glossary/#SGD) (SGD), because RMSprop automates learning-rate tuning for us. (Other optimizers, such as [Adam](https://wikipedia.org/wiki/Stochastic_gradient_descent#Adam) and [Adagrad](https://developers.google.com/machine-learning/glossary/#AdaGrad), also automatically adapt the learning rate during training, and would work equally well here.)" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "metadata": { 382 | "id": "8DHWhFP_uhq3" 383 | }, 384 | "source": [ 385 | "from tensorflow.keras.optimizers import RMSprop\n", 386 | "\n", 387 | "model.compile(loss='binary_crossentropy',\n", 388 | " optimizer=RMSprop(lr=0.001),\n", 389 | " metrics=['accuracy'])" 390 | ], 391 | "execution_count": null, 392 | "outputs": [] 393 | }, 394 | { 395 | "cell_type": "markdown", 396 | "metadata": { 397 | "id": "Sn9m9D3UimHM" 398 | }, 399 | "source": [ 400 | "### Data Preprocessing\n", 401 | "\n", 402 | "Let's set up data generators that will read pictures in our source folders, convert them to `float32` tensors, and feed them (with their labels) to our network. We'll have one generator for the training images and one for the validation images. Our generators will yield batches of images of size 300x300 and their labels (binary).\n", 403 | "\n", 404 | "As you may already know, data that goes into neural networks should usually be normalized in some way to make it more amenable to processing by the network. (It is uncommon to feed raw pixels into a convnet.) In our case, we will preprocess our images by normalizing the pixel values to be in the `[0, 1]` range (originally all values are in the `[0, 255]` range).\n", 405 | "\n", 406 | "In Keras this can be done via the `keras.preprocessing.image.ImageDataGenerator` class using the `rescale` parameter. This `ImageDataGenerator` class allows you to instantiate generators of augmented image batches (and their labels) via `.flow(data, labels)` or `.flow_from_directory(directory)`. These generators can then be used with the Keras model methods that accept data generators as inputs: `fit`, `evaluate_generator`, and `predict_generator`." 407 | ] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "metadata": { 412 | "id": "ClebU9NJg99G" 413 | }, 414 | "source": [ 415 | "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", 416 | "\n", 417 | "# All images will be rescaled by 1./255\n", 418 | "train_datagen = ImageDataGenerator(rescale=1/255)\n", 419 | "validation_datagen = ImageDataGenerator(rescale=1/255)\n", 420 | "\n", 421 | "# Flow training images in batches of 128 using train_datagen generator\n", 422 | "train_generator = train_datagen.flow_from_directory(\n", 423 | " '/tmp/horse-or-human/', # This is the source directory for training images\n", 424 | " target_size=(300, 300), # All images will be resized to 300x300\n", 425 | " batch_size=128,\n", 426 | " # Since we use binary_crossentropy loss, we need binary labels\n", 427 | " class_mode='binary')\n", 428 | "\n", 429 | "# Flow training images in batches of 128 using train_datagen generator\n", 430 | "validation_generator = validation_datagen.flow_from_directory(\n", 431 | " '/tmp/validation-horse-or-human/', # This is the source directory for training images\n", 432 | " target_size=(300, 300), # All images will be resized to 300x300\n", 433 | " batch_size=32,\n", 434 | " # Since we use binary_crossentropy loss, we need binary labels\n", 435 | " class_mode='binary')" 436 | ], 437 | "execution_count": null, 438 | "outputs": [] 439 | }, 440 | { 441 | "cell_type": "markdown", 442 | "metadata": { 443 | "id": "mu3Jdwkjwax4" 444 | }, 445 | "source": [ 446 | "### Training\n", 447 | "Let's train for 15 epochs -- this may take a few minutes to run.\n", 448 | "\n", 449 | "Do note the values per epoch.\n", 450 | "\n", 451 | "The Loss and Accuracy are a great indication of progress of training. It's making a guess as to the classification of the training data, and then measuring it against the known label, calculating the result. Accuracy is the portion of correct guesses. " 452 | ] 453 | }, 454 | { 455 | "cell_type": "code", 456 | "metadata": { 457 | "id": "Fb1_lgobv81m" 458 | }, 459 | "source": [ 460 | "history = model.fit(\n", 461 | " train_generator,\n", 462 | " steps_per_epoch=8, \n", 463 | " epochs=15,\n", 464 | " verbose=1,\n", 465 | " validation_data = validation_generator,\n", 466 | " validation_steps=8)" 467 | ], 468 | "execution_count": null, 469 | "outputs": [] 470 | }, 471 | { 472 | "cell_type": "markdown", 473 | "metadata": { 474 | "id": "o6vSHzPR2ghH" 475 | }, 476 | "source": [ 477 | "###Running the Model\n", 478 | "\n", 479 | "Let's now take a look at actually running a prediction using the model. This code will allow you to choose 1 or more files from your file system, it will then upload them, and run them through the model, giving an indication of whether the object is a horse or a human." 480 | ] 481 | }, 482 | { 483 | "cell_type": "code", 484 | "metadata": { 485 | "id": "DoWp43WxJDNT" 486 | }, 487 | "source": [ 488 | "import numpy as np\n", 489 | "from google.colab import files\n", 490 | "from keras.preprocessing import image\n", 491 | "\n", 492 | "uploaded = files.upload()\n", 493 | "\n", 494 | "for fn in uploaded.keys():\n", 495 | " \n", 496 | " # predicting images\n", 497 | " path = '/content/' + fn\n", 498 | " img = image.load_img(path, target_size=(300, 300))\n", 499 | " x = image.img_to_array(img)\n", 500 | " x = np.expand_dims(x, axis=0)\n", 501 | "\n", 502 | " images = np.vstack([x])\n", 503 | " classes = model.predict(images, batch_size=10)\n", 504 | " print(classes[0])\n", 505 | " if classes[0]>0.5:\n", 506 | " print(fn + \" is a human\")\n", 507 | " else:\n", 508 | " print(fn + \" is a horse\")\n", 509 | " " 510 | ], 511 | "execution_count": null, 512 | "outputs": [] 513 | }, 514 | { 515 | "cell_type": "markdown", 516 | "metadata": { 517 | "id": "-8EHQyWGDvWz" 518 | }, 519 | "source": [ 520 | "### Visualizing Intermediate Representations\n", 521 | "\n", 522 | "To get a feel for what kind of features our convnet has learned, one fun thing to do is to visualize how an input gets transformed as it goes through the convnet.\n", 523 | "\n", 524 | "Let's pick a random image from the training set, and then generate a figure where each row is the output of a layer, and each image in the row is a specific filter in that output feature map. Rerun this cell to generate intermediate representations for a variety of training images." 525 | ] 526 | }, 527 | { 528 | "cell_type": "code", 529 | "metadata": { 530 | "id": "-5tES8rXFjux" 531 | }, 532 | "source": [ 533 | "import numpy as np\n", 534 | "import random\n", 535 | "from tensorflow.keras.preprocessing.image import img_to_array, load_img\n", 536 | "\n", 537 | "# Let's define a new Model that will take an image as input, and will output\n", 538 | "# intermediate representations for all layers in the previous model after\n", 539 | "# the first.\n", 540 | "successive_outputs = [layer.output for layer in model.layers[1:]]\n", 541 | "#visualization_model = Model(img_input, successive_outputs)\n", 542 | "visualization_model = tf.keras.models.Model(inputs = model.input, outputs = successive_outputs)\n", 543 | "# Let's prepare a random input image from the training set.\n", 544 | "horse_img_files = [os.path.join(train_horse_dir, f) for f in train_horse_names]\n", 545 | "human_img_files = [os.path.join(train_human_dir, f) for f in train_human_names]\n", 546 | "img_path = random.choice(horse_img_files + human_img_files)\n", 547 | "\n", 548 | "img = load_img(img_path, target_size=(300, 300)) # this is a PIL image\n", 549 | "x = img_to_array(img) # Numpy array with shape (150, 150, 3)\n", 550 | "x = x.reshape((1,) + x.shape) # Numpy array with shape (1, 150, 150, 3)\n", 551 | "\n", 552 | "# Rescale by 1/255\n", 553 | "x /= 255\n", 554 | "\n", 555 | "# Let's run our image through our network, thus obtaining all\n", 556 | "# intermediate representations for this image.\n", 557 | "successive_feature_maps = visualization_model.predict(x)\n", 558 | "\n", 559 | "# These are the names of the layers, so can have them as part of our plot\n", 560 | "layer_names = [layer.name for layer in model.layers[1:]]\n", 561 | "\n", 562 | "# Now let's display our representations\n", 563 | "for layer_name, feature_map in zip(layer_names, successive_feature_maps):\n", 564 | " if len(feature_map.shape) == 4:\n", 565 | " # Just do this for the conv / maxpool layers, not the fully-connected layers\n", 566 | " n_features = feature_map.shape[-1] # number of features in feature map\n", 567 | " # The feature map has shape (1, size, size, n_features)\n", 568 | " size = feature_map.shape[1]\n", 569 | " # We will tile our images in this matrix\n", 570 | " display_grid = np.zeros((size, size * n_features))\n", 571 | " for i in range(n_features):\n", 572 | " # Postprocess the feature to make it visually palatable\n", 573 | " x = feature_map[0, :, :, i]\n", 574 | " x -= x.mean()\n", 575 | " x /= x.std()\n", 576 | " x *= 64\n", 577 | " x += 128\n", 578 | " x = np.clip(x, 0, 255).astype('uint8')\n", 579 | " # We'll tile each filter into this big horizontal grid\n", 580 | " display_grid[:, i * size : (i + 1) * size] = x\n", 581 | " # Display the grid\n", 582 | " scale = 20. / n_features\n", 583 | " plt.figure(figsize=(scale * n_features, scale))\n", 584 | " plt.title(layer_name)\n", 585 | " plt.grid(False)\n", 586 | " plt.imshow(display_grid, aspect='auto', cmap='viridis')" 587 | ], 588 | "execution_count": null, 589 | "outputs": [] 590 | }, 591 | { 592 | "cell_type": "markdown", 593 | "metadata": { 594 | "id": "tuqK2arJL0wo" 595 | }, 596 | "source": [ 597 | "As you can see we go from the raw pixels of the images to increasingly abstract and compact representations. The representations downstream start highlighting what the network pays attention to, and they show fewer and fewer features being \"activated\"; most are set to zero. This is called \"sparsity.\" Representation sparsity is a key feature of deep learning.\n", 598 | "\n", 599 | "\n", 600 | "These representations carry increasingly less information about the original pixels of the image, but increasingly refined information about the class of the image. You can think of a convnet (or a deep network in general) as an information distillation pipeline." 601 | ] 602 | }, 603 | { 604 | "cell_type": "markdown", 605 | "metadata": { 606 | "id": "j4IBgYCYooGD" 607 | }, 608 | "source": [ 609 | "## Clean Up\n", 610 | "\n", 611 | "Before running the next exercise, run the following cell to terminate the kernel and free memory resources:" 612 | ] 613 | }, 614 | { 615 | "cell_type": "code", 616 | "metadata": { 617 | "id": "651IgjLyo-Jx" 618 | }, 619 | "source": [ 620 | "import os, signal\n", 621 | "os.kill(os.getpid(), signal.SIGKILL)" 622 | ], 623 | "execution_count": null, 624 | "outputs": [] 625 | } 626 | ] 627 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week1/Course_3_Week_1_Lesson_1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Course 3 - Week 1 - Lesson 1.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "toc_visible": true 10 | }, 11 | "kernelspec": { 12 | "display_name": "Python 3", 13 | "name": "python3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "metadata": { 20 | "id": "zX4Kg8DUTKWO" 21 | }, 22 | "source": [ 23 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 24 | "# you may not use this file except in compliance with the License.\n", 25 | "# You may obtain a copy of the License at\n", 26 | "#\n", 27 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 28 | "#\n", 29 | "# Unless required by applicable law or agreed to in writing, software\n", 30 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 31 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 32 | "# See the License for the specific language governing permissions and\n", 33 | "# limitations under the License." 34 | ], 35 | "execution_count": 1, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": { 41 | "id": "view-in-github" 42 | }, 43 | "source": [ 44 | "\"Open" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "id": "rX8mhOLljYeM" 51 | }, 52 | "source": [ 53 | "##### Copyright 2019 The TensorFlow Authors." 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "metadata": { 59 | "cellView": "form", 60 | "id": "BZSlp3DAjdYf" 61 | }, 62 | "source": [ 63 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 64 | "# you may not use this file except in compliance with the License.\n", 65 | "# You may obtain a copy of the License at\n", 66 | "#\n", 67 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 68 | "#\n", 69 | "# Unless required by applicable law or agreed to in writing, software\n", 70 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 71 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 72 | "# See the License for the specific language governing permissions and\n", 73 | "# limitations under the License." 74 | ], 75 | "execution_count": null, 76 | "outputs": [] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "metadata": { 81 | "colab": { 82 | "base_uri": "https://localhost:8080/" 83 | }, 84 | "id": "zaCMcjMQifQc", 85 | "outputId": "67697644-f401-4ea6-f1d1-2076d15ee591" 86 | }, 87 | "source": [ 88 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 89 | "\n", 90 | "sentences = [\n", 91 | " 'i love my dog',\n", 92 | " 'I, love my cat',\n", 93 | " 'You love my dog!'\n", 94 | "]\n", 95 | "\n", 96 | "tokenizer = Tokenizer(num_words = 100)\n", 97 | "tokenizer.fit_on_texts(sentences)\n", 98 | "word_index = tokenizer.word_index\n", 99 | "print(word_index)" 100 | ], 101 | "execution_count": 2, 102 | "outputs": [ 103 | { 104 | "output_type": "stream", 105 | "text": [ 106 | "{'love': 1, 'my': 2, 'i': 3, 'dog': 4, 'cat': 5, 'you': 6}\n" 107 | ], 108 | "name": "stdout" 109 | } 110 | ] 111 | } 112 | ] 113 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week1/Course_3_Week_1_Lesson_2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Course 3 - Week 1 - Lesson 2.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "toc_visible": true 10 | }, 11 | "kernelspec": { 12 | "display_name": "Python 3", 13 | "name": "python3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "metadata": { 20 | "id": "zX4Kg8DUTKWO" 21 | }, 22 | "source": [ 23 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 24 | "# you may not use this file except in compliance with the License.\n", 25 | "# You may obtain a copy of the License at\n", 26 | "#\n", 27 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 28 | "#\n", 29 | "# Unless required by applicable law or agreed to in writing, software\n", 30 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 31 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 32 | "# See the License for the specific language governing permissions and\n", 33 | "# limitations under the License." 34 | ], 35 | "execution_count": 1, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": { 41 | "id": "view-in-github" 42 | }, 43 | "source": [ 44 | "\"Open" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "id": "rX8mhOLljYeM" 51 | }, 52 | "source": [ 53 | "##### Copyright 2019 The TensorFlow Authors." 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "metadata": { 59 | "cellView": "form", 60 | "id": "BZSlp3DAjdYf" 61 | }, 62 | "source": [ 63 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 64 | "# you may not use this file except in compliance with the License.\n", 65 | "# You may obtain a copy of the License at\n", 66 | "#\n", 67 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 68 | "#\n", 69 | "# Unless required by applicable law or agreed to in writing, software\n", 70 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 71 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 72 | "# See the License for the specific language governing permissions and\n", 73 | "# limitations under the License." 74 | ], 75 | "execution_count": null, 76 | "outputs": [] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "metadata": { 81 | "colab": { 82 | "base_uri": "https://localhost:8080/" 83 | }, 84 | "id": "ArOPfBwyZtln", 85 | "outputId": "8d644a6a-b167-483f-b9d5-3d81e310c3fe" 86 | }, 87 | "source": [ 88 | "import tensorflow as tf\n", 89 | "from tensorflow import keras\n", 90 | "\n", 91 | "\n", 92 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 93 | "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", 94 | "\n", 95 | "sentences = [\n", 96 | " 'I love my dog',\n", 97 | " 'I love my cat',\n", 98 | " 'You love my dog!',\n", 99 | " 'Do you think my dog is amazing?'\n", 100 | "]\n", 101 | "\n", 102 | "tokenizer = Tokenizer(num_words = 100, oov_token=\"\")\n", 103 | "tokenizer.fit_on_texts(sentences)\n", 104 | "word_index = tokenizer.word_index\n", 105 | "\n", 106 | "sequences = tokenizer.texts_to_sequences(sentences)\n", 107 | "\n", 108 | "padded = pad_sequences(sequences, maxlen=5)\n", 109 | "print(\"\\nWord Index = \" , word_index)\n", 110 | "print(\"\\nSequences = \" , sequences)\n", 111 | "print(\"\\nPadded Sequences:\")\n", 112 | "print(padded)\n", 113 | "\n", 114 | "\n", 115 | "# Try with words that the tokenizer wasn't fit to\n", 116 | "test_data = [\n", 117 | " 'i really love my dog',\n", 118 | " 'my dog loves my manatee'\n", 119 | "]\n", 120 | "\n", 121 | "test_seq = tokenizer.texts_to_sequences(test_data)\n", 122 | "print(\"\\nTest Sequence = \", test_seq)\n", 123 | "\n", 124 | "padded = pad_sequences(test_seq, maxlen=10)\n", 125 | "print(\"\\nPadded Test Sequence: \")\n", 126 | "print(padded)" 127 | ], 128 | "execution_count": 2, 129 | "outputs": [ 130 | { 131 | "output_type": "stream", 132 | "text": [ 133 | "\n", 134 | "Word Index = {'': 1, 'my': 2, 'love': 3, 'dog': 4, 'i': 5, 'you': 6, 'cat': 7, 'do': 8, 'think': 9, 'is': 10, 'amazing': 11}\n", 135 | "\n", 136 | "Sequences = [[5, 3, 2, 4], [5, 3, 2, 7], [6, 3, 2, 4], [8, 6, 9, 2, 4, 10, 11]]\n", 137 | "\n", 138 | "Padded Sequences:\n", 139 | "[[ 0 5 3 2 4]\n", 140 | " [ 0 5 3 2 7]\n", 141 | " [ 0 6 3 2 4]\n", 142 | " [ 9 2 4 10 11]]\n", 143 | "\n", 144 | "Test Sequence = [[5, 1, 3, 2, 4], [2, 4, 1, 2, 1]]\n", 145 | "\n", 146 | "Padded Test Sequence: \n", 147 | "[[0 0 0 0 0 5 1 3 2 4]\n", 148 | " [0 0 0 0 0 2 4 1 2 1]]\n" 149 | ], 150 | "name": "stdout" 151 | } 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "metadata": { 157 | "id": "6adaKRegsmrF" 158 | }, 159 | "source": [ 160 | "" 161 | ], 162 | "execution_count": null, 163 | "outputs": [] 164 | } 165 | ] 166 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week1/Course_3_Week_1_Lesson_3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Course 3 - Week 1 - Lesson 3.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "toc_visible": true 10 | }, 11 | "kernelspec": { 12 | "display_name": "Python 3", 13 | "name": "python3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "metadata": { 20 | "id": "zX4Kg8DUTKWO" 21 | }, 22 | "source": [ 23 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 24 | "# you may not use this file except in compliance with the License.\n", 25 | "# You may obtain a copy of the License at\n", 26 | "#\n", 27 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 28 | "#\n", 29 | "# Unless required by applicable law or agreed to in writing, software\n", 30 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 31 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 32 | "# See the License for the specific language governing permissions and\n", 33 | "# limitations under the License." 34 | ], 35 | "execution_count": null, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": { 41 | "id": "view-in-github" 42 | }, 43 | "source": [ 44 | "\"Open" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "id": "rX8mhOLljYeM" 51 | }, 52 | "source": [ 53 | "##### Copyright 2019 The TensorFlow Authors." 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "metadata": { 59 | "cellView": "form", 60 | "id": "BZSlp3DAjdYf" 61 | }, 62 | "source": [ 63 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 64 | "# you may not use this file except in compliance with the License.\n", 65 | "# You may obtain a copy of the License at\n", 66 | "#\n", 67 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 68 | "#\n", 69 | "# Unless required by applicable law or agreed to in writing, software\n", 70 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 71 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 72 | "# See the License for the specific language governing permissions and\n", 73 | "# limitations under the License." 74 | ], 75 | "execution_count": null, 76 | "outputs": [] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "metadata": { 81 | "id": "OkaBMeNDwMel" 82 | }, 83 | "source": [ 84 | "!wget --no-check-certificate \\\n", 85 | " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sarcasm.json \\\n", 86 | " -O /tmp/sarcasm.json\n", 87 | " \n", 88 | "import json\n", 89 | "\n", 90 | "with open(\"/tmp/sarcasm.json\", 'r') as f:\n", 91 | " datastore = json.load(f)\n", 92 | "\n", 93 | "\n", 94 | "sentences = [] \n", 95 | "labels = []\n", 96 | "urls = []\n", 97 | "for item in datastore:\n", 98 | " sentences.append(item['headline'])\n", 99 | " labels.append(item['is_sarcastic'])\n", 100 | " urls.append(item['article_link'])\n", 101 | "\n", 102 | "\n", 103 | "\n", 104 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 105 | "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", 106 | "tokenizer = Tokenizer(oov_token=\"\")\n", 107 | "tokenizer.fit_on_texts(sentences)\n", 108 | "\n", 109 | "word_index = tokenizer.word_index\n", 110 | "print(len(word_index))\n", 111 | "print(word_index)\n", 112 | "sequences = tokenizer.texts_to_sequences(sentences)\n", 113 | "padded = pad_sequences(sequences, padding='post')\n", 114 | "print(padded[0])\n", 115 | "print(padded.shape)" 116 | ], 117 | "execution_count": null, 118 | "outputs": [] 119 | } 120 | ] 121 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week2/Course_3_Week_2_Lesson_1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 3 - Week 2 - Lesson 1.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "language": "python", 15 | "name": "python3" 16 | }, 17 | "language_info": { 18 | "codemirror_mode": { 19 | "name": "ipython", 20 | "version": 3 21 | }, 22 | "file_extension": ".py", 23 | "mimetype": "text/x-python", 24 | "name": "python", 25 | "nbconvert_exporter": "python", 26 | "pygments_lexer": "ipython3", 27 | "version": "3.6.9" 28 | } 29 | }, 30 | "cells": [ 31 | { 32 | "cell_type": "code", 33 | "metadata": { 34 | "id": "zX4Kg8DUTKWO" 35 | }, 36 | "source": [ 37 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 38 | "# you may not use this file except in compliance with the License.\n", 39 | "# You may obtain a copy of the License at\n", 40 | "#\n", 41 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 42 | "#\n", 43 | "# Unless required by applicable law or agreed to in writing, software\n", 44 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 45 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 46 | "# See the License for the specific language governing permissions and\n", 47 | "# limitations under the License." 48 | ], 49 | "execution_count": null, 50 | "outputs": [] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": { 55 | "id": "view-in-github" 56 | }, 57 | "source": [ 58 | "\"Open" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "metadata": { 64 | "id": "P-AhVYeBWgQ3" 65 | }, 66 | "source": [ 67 | "import tensorflow as tf\n", 68 | "print(tf.__version__)\n", 69 | "\n", 70 | "# !pip install -q tensorflow-datasets" 71 | ], 72 | "execution_count": null, 73 | "outputs": [] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "metadata": { 78 | "id": "_IoM4VFxWpMR" 79 | }, 80 | "source": [ 81 | "import tensorflow_datasets as tfds\n", 82 | "imdb, info = tfds.load(\"imdb_reviews\", with_info=True, as_supervised=True)\n" 83 | ], 84 | "execution_count": null, 85 | "outputs": [] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "metadata": { 90 | "id": "wHQ2Ko0zl7M4" 91 | }, 92 | "source": [ 93 | "import numpy as np\n", 94 | "\n", 95 | "train_data, test_data = imdb['train'], imdb['test']\n", 96 | "\n", 97 | "training_sentences = []\n", 98 | "training_labels = []\n", 99 | "\n", 100 | "testing_sentences = []\n", 101 | "testing_labels = []\n", 102 | "\n", 103 | "# str(s.tonumpy()) is needed in Python3 instead of just s.numpy()\n", 104 | "for s,l in train_data:\n", 105 | " training_sentences.append(s.numpy().decode('utf8'))\n", 106 | " training_labels.append(l.numpy())\n", 107 | " \n", 108 | "for s,l in test_data:\n", 109 | " testing_sentences.append(s.numpy().decode('utf8'))\n", 110 | " testing_labels.append(l.numpy())\n", 111 | " \n", 112 | "training_labels_final = np.array(training_labels)\n", 113 | "testing_labels_final = np.array(testing_labels)\n" 114 | ], 115 | "execution_count": null, 116 | "outputs": [] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "metadata": { 121 | "id": "7n15yyMdmoH1" 122 | }, 123 | "source": [ 124 | "vocab_size = 10000\n", 125 | "embedding_dim = 16\n", 126 | "max_length = 120\n", 127 | "trunc_type='post'\n", 128 | "oov_tok = \"\"\n", 129 | "\n", 130 | "\n", 131 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 132 | "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", 133 | "\n", 134 | "tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\n", 135 | "tokenizer.fit_on_texts(training_sentences)\n", 136 | "word_index = tokenizer.word_index\n", 137 | "sequences = tokenizer.texts_to_sequences(training_sentences)\n", 138 | "padded = pad_sequences(sequences,maxlen=max_length, truncating=trunc_type)\n", 139 | "\n", 140 | "testing_sequences = tokenizer.texts_to_sequences(testing_sentences)\n", 141 | "testing_padded = pad_sequences(testing_sequences,maxlen=max_length)\n" 142 | ], 143 | "execution_count": null, 144 | "outputs": [] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "metadata": { 149 | "id": "9axf0uIXVMhO" 150 | }, 151 | "source": [ 152 | "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", 153 | "\n", 154 | "def decode_review(text):\n", 155 | " return ' '.join([reverse_word_index.get(i, '?') for i in text])\n", 156 | "\n", 157 | "print(decode_review(padded[3]))\n", 158 | "print(training_sentences[3])" 159 | ], 160 | "execution_count": null, 161 | "outputs": [] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "metadata": { 166 | "id": "5NEpdhb8AxID" 167 | }, 168 | "source": [ 169 | "model = tf.keras.Sequential([\n", 170 | " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", 171 | " tf.keras.layers.Flatten(),\n", 172 | " tf.keras.layers.Dense(6, activation='relu'),\n", 173 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 174 | "])\n", 175 | "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", 176 | "model.summary()\n" 177 | ], 178 | "execution_count": null, 179 | "outputs": [] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "metadata": { 184 | "id": "V5LLrXC-uNX6" 185 | }, 186 | "source": [ 187 | "num_epochs = 10\n", 188 | "model.fit(padded, training_labels_final, epochs=num_epochs, validation_data=(testing_padded, testing_labels_final))" 189 | ], 190 | "execution_count": null, 191 | "outputs": [] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "metadata": { 196 | "id": "yAmjJqEyCOF_" 197 | }, 198 | "source": [ 199 | "e = model.layers[0]\n", 200 | "weights = e.get_weights()[0]\n", 201 | "print(weights.shape) # shape: (vocab_size, embedding_dim)" 202 | ], 203 | "execution_count": null, 204 | "outputs": [] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "metadata": { 209 | "id": "jmB0Uxk0ycP6" 210 | }, 211 | "source": [ 212 | "import io\n", 213 | "\n", 214 | "out_v = io.open('vecs.tsv', 'w', encoding='utf-8')\n", 215 | "out_m = io.open('meta.tsv', 'w', encoding='utf-8')\n", 216 | "for word_num in range(1, vocab_size):\n", 217 | " word = reverse_word_index[word_num]\n", 218 | " embeddings = weights[word_num]\n", 219 | " out_m.write(word + \"\\n\")\n", 220 | " out_v.write('\\t'.join([str(x) for x in embeddings]) + \"\\n\")\n", 221 | "out_v.close()\n", 222 | "out_m.close()" 223 | ], 224 | "execution_count": null, 225 | "outputs": [] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "metadata": { 230 | "id": "VDeqpOCVydtq" 231 | }, 232 | "source": [ 233 | "try:\n", 234 | " from google.colab import files\n", 235 | "except ImportError:\n", 236 | " pass\n", 237 | "else:\n", 238 | " files.download('vecs.tsv')\n", 239 | " files.download('meta.tsv')" 240 | ], 241 | "execution_count": null, 242 | "outputs": [] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "metadata": { 247 | "id": "YRxoxc2apscY" 248 | }, 249 | "source": [ 250 | "sentence = \"I really think this is amazing. honest.\"\n", 251 | "sequence = tokenizer.texts_to_sequences([sentence])\n", 252 | "print(sequence)" 253 | ], 254 | "execution_count": null, 255 | "outputs": [] 256 | } 257 | ] 258 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week2/Course_3_Week_2_Lesson_2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 3 - Week 2 - Lesson 2.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "name": "python3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "code", 20 | "metadata": { 21 | "id": "zX4Kg8DUTKWO" 22 | }, 23 | "source": [ 24 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 25 | "# you may not use this file except in compliance with the License.\n", 26 | "# You may obtain a copy of the License at\n", 27 | "#\n", 28 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 29 | "#\n", 30 | "# Unless required by applicable law or agreed to in writing, software\n", 31 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 32 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 33 | "# See the License for the specific language governing permissions and\n", 34 | "# limitations under the License." 35 | ], 36 | "execution_count": null, 37 | "outputs": [] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": { 42 | "id": "view-in-github" 43 | }, 44 | "source": [ 45 | "\"Open" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "metadata": { 51 | "id": "4gs9htvM7n_x" 52 | }, 53 | "source": [ 54 | "# Run this to ensure TensorFlow 2.x is used\n", 55 | "try:\n", 56 | " # %tensorflow_version only exists in Colab.\n", 57 | " %tensorflow_version 2.x\n", 58 | "except Exception:\n", 59 | " pass" 60 | ], 61 | "execution_count": null, 62 | "outputs": [] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "metadata": { 67 | "id": "XYYDvoskkE61" 68 | }, 69 | "source": [ 70 | "import json\n", 71 | "import tensorflow as tf\n", 72 | "\n", 73 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 74 | "from tensorflow.keras.preprocessing.sequence import pad_sequences" 75 | ], 76 | "execution_count": null, 77 | "outputs": [] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "metadata": { 82 | "id": "0eJSTTYnkJQd" 83 | }, 84 | "source": [ 85 | "vocab_size = 10000\n", 86 | "embedding_dim = 16\n", 87 | "max_length = 100\n", 88 | "trunc_type='post'\n", 89 | "padding_type='post'\n", 90 | "oov_tok = \"\"\n", 91 | "training_size = 20000\n" 92 | ], 93 | "execution_count": null, 94 | "outputs": [] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "metadata": { 99 | "id": "BQVuQrZNkPn9" 100 | }, 101 | "source": [ 102 | "!wget --no-check-certificate \\\n", 103 | " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sarcasm.json \\\n", 104 | " -O /tmp/sarcasm.json\n" 105 | ], 106 | "execution_count": null, 107 | "outputs": [] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "metadata": { 112 | "id": "oaLaaqhNkUPd" 113 | }, 114 | "source": [ 115 | "with open(\"/tmp/sarcasm.json\", 'r') as f:\n", 116 | " datastore = json.load(f)\n", 117 | "\n", 118 | "sentences = []\n", 119 | "labels = []\n", 120 | "\n", 121 | "for item in datastore:\n", 122 | " sentences.append(item['headline'])\n", 123 | " labels.append(item['is_sarcastic'])" 124 | ], 125 | "execution_count": null, 126 | "outputs": [] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "metadata": { 131 | "id": "S1sD-7v0kYWk" 132 | }, 133 | "source": [ 134 | "training_sentences = sentences[0:training_size]\n", 135 | "testing_sentences = sentences[training_size:]\n", 136 | "training_labels = labels[0:training_size]\n", 137 | "testing_labels = labels[training_size:]" 138 | ], 139 | "execution_count": null, 140 | "outputs": [] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "metadata": { 145 | "id": "3u8UB0MCkZ5N" 146 | }, 147 | "source": [ 148 | "tokenizer = Tokenizer(num_words=vocab_size, oov_token=oov_tok)\n", 149 | "tokenizer.fit_on_texts(training_sentences)\n", 150 | "\n", 151 | "word_index = tokenizer.word_index\n", 152 | "\n", 153 | "training_sequences = tokenizer.texts_to_sequences(training_sentences)\n", 154 | "training_padded = pad_sequences(training_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n", 155 | "\n", 156 | "testing_sequences = tokenizer.texts_to_sequences(testing_sentences)\n", 157 | "testing_padded = pad_sequences(testing_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)" 158 | ], 159 | "execution_count": null, 160 | "outputs": [] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "metadata": { 165 | "id": "GrAlWBKf99Ya" 166 | }, 167 | "source": [ 168 | "# Need this block to get it to work with TensorFlow 2.x\n", 169 | "import numpy as np\n", 170 | "training_padded = np.array(training_padded)\n", 171 | "training_labels = np.array(training_labels)\n", 172 | "testing_padded = np.array(testing_padded)\n", 173 | "testing_labels = np.array(testing_labels)" 174 | ], 175 | "execution_count": null, 176 | "outputs": [] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "metadata": { 181 | "id": "FufaT4vlkiDE" 182 | }, 183 | "source": [ 184 | "model = tf.keras.Sequential([\n", 185 | " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", 186 | " tf.keras.layers.GlobalAveragePooling1D(),\n", 187 | " tf.keras.layers.Dense(24, activation='relu'),\n", 188 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 189 | "])\n", 190 | "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])" 191 | ], 192 | "execution_count": null, 193 | "outputs": [] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "metadata": { 198 | "id": "XfDt1hmYkiys" 199 | }, 200 | "source": [ 201 | "model.summary()\n" 202 | ], 203 | "execution_count": null, 204 | "outputs": [] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "metadata": { 209 | "id": "2DTKQFf1kkyc" 210 | }, 211 | "source": [ 212 | "num_epochs = 30\n", 213 | "history = model.fit(training_padded, training_labels, epochs=num_epochs, validation_data=(testing_padded, testing_labels), verbose=2)" 214 | ], 215 | "execution_count": null, 216 | "outputs": [] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "metadata": { 221 | "id": "2HYfBKXjkmU8" 222 | }, 223 | "source": [ 224 | "import matplotlib.pyplot as plt\n", 225 | "\n", 226 | "\n", 227 | "def plot_graphs(history, string):\n", 228 | " plt.plot(history.history[string])\n", 229 | " plt.plot(history.history['val_'+string])\n", 230 | " plt.xlabel(\"Epochs\")\n", 231 | " plt.ylabel(string)\n", 232 | " plt.legend([string, 'val_'+string])\n", 233 | " plt.show()\n", 234 | " \n", 235 | "plot_graphs(history, \"accuracy\")\n", 236 | "plot_graphs(history, \"loss\")" 237 | ], 238 | "execution_count": null, 239 | "outputs": [] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "metadata": { 244 | "id": "7SBdAZAenvzL" 245 | }, 246 | "source": [ 247 | "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", 248 | "\n", 249 | "def decode_sentence(text):\n", 250 | " return ' '.join([reverse_word_index.get(i, '?') for i in text])\n", 251 | "\n", 252 | "print(decode_sentence(training_padded[0]))\n", 253 | "print(training_sentences[2])\n", 254 | "print(labels[2])" 255 | ], 256 | "execution_count": null, 257 | "outputs": [] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "metadata": { 262 | "id": "c9MqihtEkzQ9" 263 | }, 264 | "source": [ 265 | "e = model.layers[0]\n", 266 | "weights = e.get_weights()[0]\n", 267 | "print(weights.shape) # shape: (vocab_size, embedding_dim)\n" 268 | ], 269 | "execution_count": null, 270 | "outputs": [] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "metadata": { 275 | "id": "LoBXVffknldU" 276 | }, 277 | "source": [ 278 | "import io\n", 279 | "\n", 280 | "out_v = io.open('vecs.tsv', 'w', encoding='utf-8')\n", 281 | "out_m = io.open('meta.tsv', 'w', encoding='utf-8')\n", 282 | "for word_num in range(1, vocab_size):\n", 283 | " word = reverse_word_index[word_num]\n", 284 | " embeddings = weights[word_num]\n", 285 | " out_m.write(word + \"\\n\")\n", 286 | " out_v.write('\\t'.join([str(x) for x in embeddings]) + \"\\n\")\n", 287 | "out_v.close()\n", 288 | "out_m.close()" 289 | ], 290 | "execution_count": null, 291 | "outputs": [] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "metadata": { 296 | "id": "U4eZ5HtVnnEE" 297 | }, 298 | "source": [ 299 | "try:\n", 300 | " from google.colab import files\n", 301 | "except ImportError:\n", 302 | " pass\n", 303 | "else:\n", 304 | " files.download('vecs.tsv')\n", 305 | " files.download('meta.tsv')" 306 | ], 307 | "execution_count": null, 308 | "outputs": [] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "metadata": { 313 | "id": "cG8-ArY-qDcz" 314 | }, 315 | "source": [ 316 | "sentence = [\"granny starting to fear spiders in the garden might be real\", \"game of thrones season finale showing this sunday night\"]\n", 317 | "sequences = tokenizer.texts_to_sequences(sentence)\n", 318 | "padded = pad_sequences(sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n", 319 | "print(model.predict(padded))" 320 | ], 321 | "execution_count": null, 322 | "outputs": [] 323 | } 324 | ] 325 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week3/Course_3_Week_3_Lesson_1a.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Course 3 - Week 3 - Lesson 1a.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "toc_visible": true 10 | }, 11 | "kernelspec": { 12 | "display_name": "Python 3", 13 | "name": "python3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "metadata": { 20 | "id": "zX4Kg8DUTKWO" 21 | }, 22 | "source": [ 23 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 24 | "# you may not use this file except in compliance with the License.\n", 25 | "# You may obtain a copy of the License at\n", 26 | "#\n", 27 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 28 | "#\n", 29 | "# Unless required by applicable law or agreed to in writing, software\n", 30 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 31 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 32 | "# See the License for the specific language governing permissions and\n", 33 | "# limitations under the License." 34 | ], 35 | "execution_count": null, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": { 41 | "id": "rFiCyWQ-NC5D" 42 | }, 43 | "source": [ 44 | "# Single Layer LSTM" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "metadata": { 50 | "id": "Y20Lud2ZMBhW" 51 | }, 52 | "source": [ 53 | "from __future__ import absolute_import, division, print_function, unicode_literals\n", 54 | "\n", 55 | "\n", 56 | "import tensorflow_datasets as tfds\n", 57 | "import tensorflow as tf\n", 58 | "print(tf.__version__)" 59 | ], 60 | "execution_count": null, 61 | "outputs": [] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "metadata": { 66 | "id": "uAU8g7C0MPZE" 67 | }, 68 | "source": [ 69 | "import tensorflow_datasets as tfds\n", 70 | "import tensorflow as tf\n", 71 | "print(tf.__version__)" 72 | ], 73 | "execution_count": null, 74 | "outputs": [] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "metadata": { 79 | "id": "AW-4Vo4TMUHb" 80 | }, 81 | "source": [ 82 | "# Get the data\n", 83 | "dataset, info = tfds.load('imdb_reviews/subwords8k', with_info=True, as_supervised=True)\n", 84 | "train_dataset, test_dataset = dataset['train'], dataset['test']" 85 | ], 86 | "execution_count": null, 87 | "outputs": [] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "metadata": { 92 | "id": "DVfhKpHsPOxq" 93 | }, 94 | "source": [ 95 | "tokenizer = info.features['text'].encoder" 96 | ], 97 | "execution_count": null, 98 | "outputs": [] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "metadata": { 103 | "id": "ffvRUI0_McDS" 104 | }, 105 | "source": [ 106 | "BUFFER_SIZE = 10000\n", 107 | "BATCH_SIZE = 64\n", 108 | "\n", 109 | "train_dataset = train_dataset.shuffle(BUFFER_SIZE)\n", 110 | "train_dataset = train_dataset.padded_batch(BATCH_SIZE, tf.compat.v1.data.get_output_shapes(train_dataset))\n", 111 | "test_dataset = test_dataset.padded_batch(BATCH_SIZE, tf.compat.v1.data.get_output_shapes(test_dataset))" 112 | ], 113 | "execution_count": null, 114 | "outputs": [] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "metadata": { 119 | "id": "FxQooMEkMgur" 120 | }, 121 | "source": [ 122 | "model = tf.keras.Sequential([\n", 123 | " tf.keras.layers.Embedding(tokenizer.vocab_size, 64),\n", 124 | " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)),\n", 125 | " tf.keras.layers.Dense(64, activation='relu'),\n", 126 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 127 | "])" 128 | ], 129 | "execution_count": null, 130 | "outputs": [] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "metadata": { 135 | "id": "QKI5dfPgMioL" 136 | }, 137 | "source": [ 138 | "model.summary()" 139 | ], 140 | "execution_count": null, 141 | "outputs": [] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "metadata": { 146 | "id": "Uip7QOVzMoMq" 147 | }, 148 | "source": [ 149 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 150 | ], 151 | "execution_count": null, 152 | "outputs": [] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "metadata": { 157 | "id": "7mlgzaRDMtF6" 158 | }, 159 | "source": [ 160 | "NUM_EPOCHS = 10\n", 161 | "history = model.fit(train_dataset, epochs=NUM_EPOCHS, validation_data=test_dataset)" 162 | ], 163 | "execution_count": null, 164 | "outputs": [] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "metadata": { 169 | "id": "Mp1Z7P9pYRSK" 170 | }, 171 | "source": [ 172 | "import matplotlib.pyplot as plt\n", 173 | "\n", 174 | "\n", 175 | "def plot_graphs(history, string):\n", 176 | " plt.plot(history.history[string])\n", 177 | " plt.plot(history.history['val_'+string])\n", 178 | " plt.xlabel(\"Epochs\")\n", 179 | " plt.ylabel(string)\n", 180 | " plt.legend([string, 'val_'+string])\n", 181 | " plt.show()" 182 | ], 183 | "execution_count": null, 184 | "outputs": [] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "metadata": { 189 | "id": "R_sX6ilIM515" 190 | }, 191 | "source": [ 192 | "plot_graphs(history, 'accuracy')" 193 | ], 194 | "execution_count": null, 195 | "outputs": [] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "metadata": { 200 | "id": "RFEXtKtqNARB" 201 | }, 202 | "source": [ 203 | "plot_graphs(history, 'loss')" 204 | ], 205 | "execution_count": null, 206 | "outputs": [] 207 | } 208 | ] 209 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week3/Course_3_Week_3_Lesson_1b.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 3 - Week 3 - Lesson 1b.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "name": "python3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "code", 20 | "metadata": { 21 | "id": "zX4Kg8DUTKWO" 22 | }, 23 | "source": [ 24 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 25 | "# you may not use this file except in compliance with the License.\n", 26 | "# You may obtain a copy of the License at\n", 27 | "#\n", 28 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 29 | "#\n", 30 | "# Unless required by applicable law or agreed to in writing, software\n", 31 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 32 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 33 | "# See the License for the specific language governing permissions and\n", 34 | "# limitations under the License." 35 | ], 36 | "execution_count": null, 37 | "outputs": [] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": { 42 | "id": "rFiCyWQ-NC5D" 43 | }, 44 | "source": [ 45 | "# Multiple Layer LSTM" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "metadata": { 51 | "id": "Y20Lud2ZMBhW" 52 | }, 53 | "source": [ 54 | "from __future__ import absolute_import, division, print_function, unicode_literals\n", 55 | "\n", 56 | "\n", 57 | "import tensorflow_datasets as tfds\n", 58 | "import tensorflow as tf\n", 59 | "print(tf.__version__)" 60 | ], 61 | "execution_count": null, 62 | "outputs": [] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "metadata": { 67 | "id": "uAU8g7C0MPZE" 68 | }, 69 | "source": [ 70 | "import tensorflow_datasets as tfds\n", 71 | "import tensorflow as tf\n", 72 | "print(tf.__version__)" 73 | ], 74 | "execution_count": null, 75 | "outputs": [] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "metadata": { 80 | "id": "AW-4Vo4TMUHb" 81 | }, 82 | "source": [ 83 | "# Get the data\n", 84 | "dataset, info = tfds.load('imdb_reviews/subwords8k', with_info=True, as_supervised=True)\n", 85 | "train_dataset, test_dataset = dataset['train'], dataset['test']\n" 86 | ], 87 | "execution_count": null, 88 | "outputs": [] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "metadata": { 93 | "id": "L11bIR6-PKvs" 94 | }, 95 | "source": [ 96 | "tokenizer = info.features['text'].encoder" 97 | ], 98 | "execution_count": null, 99 | "outputs": [] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "metadata": { 104 | "id": "ffvRUI0_McDS" 105 | }, 106 | "source": [ 107 | "BUFFER_SIZE = 10000\n", 108 | "BATCH_SIZE = 64\n", 109 | "\n", 110 | "train_dataset = train_dataset.shuffle(BUFFER_SIZE)\n", 111 | "train_dataset = train_dataset.padded_batch(BATCH_SIZE, train_dataset.output_shapes)\n", 112 | "test_dataset = test_dataset.padded_batch(BATCH_SIZE, test_dataset.output_shapes)" 113 | ], 114 | "execution_count": null, 115 | "outputs": [] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "metadata": { 120 | "id": "jo1jjO3vn0jo" 121 | }, 122 | "source": [ 123 | "model = tf.keras.Sequential([\n", 124 | " tf.keras.layers.Embedding(tokenizer.vocab_size, 64),\n", 125 | " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)),\n", 126 | " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n", 127 | " tf.keras.layers.Dense(64, activation='relu'),\n", 128 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 129 | "])" 130 | ], 131 | "execution_count": null, 132 | "outputs": [] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "metadata": { 137 | "id": "QKI5dfPgMioL" 138 | }, 139 | "source": [ 140 | "model.summary()" 141 | ], 142 | "execution_count": null, 143 | "outputs": [] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "metadata": { 148 | "id": "Uip7QOVzMoMq" 149 | }, 150 | "source": [ 151 | "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])" 152 | ], 153 | "execution_count": null, 154 | "outputs": [] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "metadata": { 159 | "id": "7mlgzaRDMtF6" 160 | }, 161 | "source": [ 162 | "NUM_EPOCHS = 10\n", 163 | "history = model.fit(train_dataset, epochs=NUM_EPOCHS, validation_data=test_dataset)" 164 | ], 165 | "execution_count": null, 166 | "outputs": [] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "metadata": { 171 | "id": "Mp1Z7P9pYRSK" 172 | }, 173 | "source": [ 174 | "import matplotlib.pyplot as plt\n", 175 | "\n", 176 | "\n", 177 | "def plot_graphs(history, string):\n", 178 | " plt.plot(history.history[string])\n", 179 | " plt.plot(history.history['val_'+string])\n", 180 | " plt.xlabel(\"Epochs\")\n", 181 | " plt.ylabel(string)\n", 182 | " plt.legend([string, 'val_'+string])\n", 183 | " plt.show()" 184 | ], 185 | "execution_count": null, 186 | "outputs": [] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "metadata": { 191 | "id": "R_sX6ilIM515" 192 | }, 193 | "source": [ 194 | "plot_graphs(history, 'accuracy')" 195 | ], 196 | "execution_count": null, 197 | "outputs": [] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "metadata": { 202 | "id": "RFEXtKtqNARB" 203 | }, 204 | "source": [ 205 | "plot_graphs(history, 'loss')" 206 | ], 207 | "execution_count": null, 208 | "outputs": [] 209 | } 210 | ] 211 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week3/Course_3_Week_3_Lesson_2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 3 - Week 3 - Lesson 2.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "language": "python", 15 | "name": "python3" 16 | }, 17 | "language_info": { 18 | "codemirror_mode": { 19 | "name": "ipython", 20 | "version": 3 21 | }, 22 | "file_extension": ".py", 23 | "mimetype": "text/x-python", 24 | "name": "python", 25 | "nbconvert_exporter": "python", 26 | "pygments_lexer": "ipython3", 27 | "version": "3.7.6" 28 | } 29 | }, 30 | "cells": [ 31 | { 32 | "cell_type": "code", 33 | "metadata": { 34 | "id": "zX4Kg8DUTKWO" 35 | }, 36 | "source": [ 37 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 38 | "# you may not use this file except in compliance with the License.\n", 39 | "# You may obtain a copy of the License at\n", 40 | "#\n", 41 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 42 | "#\n", 43 | "# Unless required by applicable law or agreed to in writing, software\n", 44 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 45 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 46 | "# See the License for the specific language governing permissions and\n", 47 | "# limitations under the License." 48 | ], 49 | "execution_count": null, 50 | "outputs": [] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": { 55 | "id": "j4n0vqzE3BDr" 56 | }, 57 | "source": [ 58 | "\"Open" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "metadata": { 64 | "id": "jGwXGIXvFhXW" 65 | }, 66 | "source": [ 67 | "import numpy as np\n", 68 | "\n", 69 | "import json\n", 70 | "import tensorflow as tf\n", 71 | "\n", 72 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 73 | "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", 74 | "\n", 75 | "!wget --no-check-certificate \\\n", 76 | " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sarcasm.json \\\n", 77 | " -O /tmp/sarcasm.json\n", 78 | "\n", 79 | "vocab_size = 1000\n", 80 | "embedding_dim = 16\n", 81 | "max_length = 120\n", 82 | "trunc_type='post'\n", 83 | "padding_type='post'\n", 84 | "oov_tok = \"\"\n", 85 | "training_size = 20000\n", 86 | "\n", 87 | "\n", 88 | "with open(\"/tmp/sarcasm.json\", 'r') as f:\n", 89 | " datastore = json.load(f)\n", 90 | "\n", 91 | "\n", 92 | "sentences = []\n", 93 | "labels = []\n", 94 | "urls = []\n", 95 | "for item in datastore:\n", 96 | " sentences.append(item['headline'])\n", 97 | " labels.append(item['is_sarcastic'])\n", 98 | "\n", 99 | "training_sentences = sentences[0:training_size]\n", 100 | "testing_sentences = sentences[training_size:]\n", 101 | "training_labels = labels[0:training_size]\n", 102 | "testing_labels = labels[training_size:]\n", 103 | "\n", 104 | "tokenizer = Tokenizer(num_words=vocab_size, oov_token=oov_tok)\n", 105 | "tokenizer.fit_on_texts(training_sentences)\n", 106 | "\n", 107 | "word_index = tokenizer.word_index\n", 108 | "\n", 109 | "training_sequences = tokenizer.texts_to_sequences(training_sentences)\n", 110 | "training_padded = pad_sequences(training_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n", 111 | "\n", 112 | "testing_sequences = tokenizer.texts_to_sequences(testing_sentences)\n", 113 | "testing_padded = pad_sequences(testing_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n", 114 | "\n", 115 | "model = tf.keras.Sequential([\n", 116 | " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", 117 | " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n", 118 | " tf.keras.layers.Dense(24, activation='relu'),\n", 119 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 120 | "])\n", 121 | "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", 122 | "model.summary()\n", 123 | "\n", 124 | "num_epochs = 50\n", 125 | "training_padded = np.array(training_padded)\n", 126 | "training_labels = np.array(training_labels)\n", 127 | "testing_padded = np.array(testing_padded)\n", 128 | "testing_labels = np.array(testing_labels)\n", 129 | "history = model.fit(training_padded, training_labels, epochs=num_epochs, validation_data=(testing_padded, testing_labels), verbose=1)\n" 130 | ], 131 | "execution_count": null, 132 | "outputs": [] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "metadata": { 137 | "id": "g9DC6dmLF8DC" 138 | }, 139 | "source": [ 140 | "import matplotlib.pyplot as plt\n", 141 | "\n", 142 | "\n", 143 | "def plot_graphs(history, string):\n", 144 | " plt.plot(history.history[string])\n", 145 | " plt.plot(history.history['val_'+string])\n", 146 | " plt.xlabel(\"Epochs\")\n", 147 | " plt.ylabel(string)\n", 148 | " plt.legend([string, 'val_'+string])\n", 149 | " plt.show()\n", 150 | "\n", 151 | "plot_graphs(history, 'accuracy')\n", 152 | "plot_graphs(history, 'loss')" 153 | ], 154 | "execution_count": null, 155 | "outputs": [] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "metadata": { 160 | "id": "7ZEZIUppGhdi" 161 | }, 162 | "source": [ 163 | "model.save(\"test.h5\")" 164 | ], 165 | "execution_count": null, 166 | "outputs": [] 167 | } 168 | ] 169 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week3/Course_3_Week_3_Lesson_2c.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 3 - Week 3 - Lesson 2c.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "language": "python", 15 | "name": "python3" 16 | }, 17 | "language_info": { 18 | "codemirror_mode": { 19 | "name": "ipython", 20 | "version": 3 21 | }, 22 | "file_extension": ".py", 23 | "mimetype": "text/x-python", 24 | "name": "python", 25 | "nbconvert_exporter": "python", 26 | "pygments_lexer": "ipython3", 27 | "version": "3.7.6" 28 | } 29 | }, 30 | "cells": [ 31 | { 32 | "cell_type": "code", 33 | "metadata": { 34 | "id": "zX4Kg8DUTKWO" 35 | }, 36 | "source": [ 37 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 38 | "# you may not use this file except in compliance with the License.\n", 39 | "# You may obtain a copy of the License at\n", 40 | "#\n", 41 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 42 | "#\n", 43 | "# Unless required by applicable law or agreed to in writing, software\n", 44 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 45 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 46 | "# See the License for the specific language governing permissions and\n", 47 | "# limitations under the License." 48 | ], 49 | "execution_count": null, 50 | "outputs": [] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": { 55 | "id": "D35s04Cv3Lu0" 56 | }, 57 | "source": [ 58 | "\"Open" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "metadata": { 64 | "id": "jGwXGIXvFhXW" 65 | }, 66 | "source": [ 67 | "import numpy as np\n", 68 | "\n", 69 | "import json\n", 70 | "import tensorflow as tf\n", 71 | "\n", 72 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 73 | "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", 74 | "\n", 75 | "!wget --no-check-certificate \\\n", 76 | " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sarcasm.json \\\n", 77 | " -O /tmp/sarcasm.json\n", 78 | "\n", 79 | "vocab_size = 1000\n", 80 | "embedding_dim = 16\n", 81 | "max_length = 120\n", 82 | "trunc_type='post'\n", 83 | "padding_type='post'\n", 84 | "oov_tok = \"\"\n", 85 | "training_size = 20000\n", 86 | "\n", 87 | "\n", 88 | "with open(\"/tmp/sarcasm.json\", 'r') as f:\n", 89 | " datastore = json.load(f)\n", 90 | "\n", 91 | "\n", 92 | "sentences = []\n", 93 | "labels = []\n", 94 | "urls = []\n", 95 | "for item in datastore:\n", 96 | " sentences.append(item['headline'])\n", 97 | " labels.append(item['is_sarcastic'])\n", 98 | "\n", 99 | "training_sentences = sentences[0:training_size]\n", 100 | "testing_sentences = sentences[training_size:]\n", 101 | "training_labels = labels[0:training_size]\n", 102 | "testing_labels = labels[training_size:]\n", 103 | "\n", 104 | "tokenizer = Tokenizer(num_words=vocab_size, oov_token=oov_tok)\n", 105 | "tokenizer.fit_on_texts(training_sentences)\n", 106 | "\n", 107 | "word_index = tokenizer.word_index\n", 108 | "\n", 109 | "training_sequences = tokenizer.texts_to_sequences(training_sentences)\n", 110 | "training_padded = pad_sequences(training_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n", 111 | "\n", 112 | "testing_sequences = tokenizer.texts_to_sequences(testing_sentences)\n", 113 | "testing_padded = pad_sequences(testing_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n", 114 | "\n", 115 | "model = tf.keras.Sequential([\n", 116 | " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", 117 | " tf.keras.layers.Conv1D(128, 5, activation='relu'),\n", 118 | " tf.keras.layers.GlobalMaxPooling1D(),\n", 119 | " tf.keras.layers.Dense(24, activation='relu'),\n", 120 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 121 | "])\n", 122 | "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", 123 | "model.summary()\n", 124 | "\n", 125 | "num_epochs = 50\n", 126 | "\n", 127 | "training_padded = np.array(training_padded)\n", 128 | "training_labels = np.array(training_labels)\n", 129 | "testing_padded = np.array(testing_padded)\n", 130 | "testing_labels = np.array(testing_labels)\n", 131 | "\n", 132 | "history = model.fit(training_padded, training_labels, epochs=num_epochs, validation_data=(testing_padded, testing_labels), verbose=1)\n" 133 | ], 134 | "execution_count": null, 135 | "outputs": [] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "metadata": { 140 | "id": "g9DC6dmLF8DC" 141 | }, 142 | "source": [ 143 | "import matplotlib.pyplot as plt\n", 144 | "\n", 145 | "\n", 146 | "def plot_graphs(history, string):\n", 147 | " plt.plot(history.history[string])\n", 148 | " plt.plot(history.history['val_'+string])\n", 149 | " plt.xlabel(\"Epochs\")\n", 150 | " plt.ylabel(string)\n", 151 | " plt.legend([string, 'val_'+string])\n", 152 | " plt.show()\n", 153 | "\n", 154 | "plot_graphs(history, 'accuracy')\n", 155 | "plot_graphs(history, 'loss')" 156 | ], 157 | "execution_count": null, 158 | "outputs": [] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "metadata": { 163 | "id": "7ZEZIUppGhdi" 164 | }, 165 | "source": [ 166 | "model.save(\"test.h5\")" 167 | ], 168 | "execution_count": null, 169 | "outputs": [] 170 | } 171 | ] 172 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week3/Course_3_Week_3_Lesson_2d.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 3 - Week 3 - Lesson 2d.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "name": "python3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "code", 20 | "metadata": { 21 | "id": "zX4Kg8DUTKWO" 22 | }, 23 | "source": [ 24 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 25 | "# you may not use this file except in compliance with the License.\n", 26 | "# You may obtain a copy of the License at\n", 27 | "#\n", 28 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 29 | "#\n", 30 | "# Unless required by applicable law or agreed to in writing, software\n", 31 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 32 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 33 | "# See the License for the specific language governing permissions and\n", 34 | "# limitations under the License." 35 | ], 36 | "execution_count": null, 37 | "outputs": [] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "metadata": { 42 | "id": "P-AhVYeBWgQ3" 43 | }, 44 | "source": [ 45 | "import tensorflow as tf\n", 46 | "print(tf.__version__)\n", 47 | "\n", 48 | "# !pip install -q tensorflow-datasets" 49 | ], 50 | "execution_count": null, 51 | "outputs": [] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "metadata": { 56 | "id": "_IoM4VFxWpMR" 57 | }, 58 | "source": [ 59 | "import tensorflow_datasets as tfds\n", 60 | "imdb, info = tfds.load(\"imdb_reviews\", with_info=True, as_supervised=True)\n" 61 | ], 62 | "execution_count": null, 63 | "outputs": [] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "metadata": { 68 | "id": "wHQ2Ko0zl7M4" 69 | }, 70 | "source": [ 71 | "import numpy as np\n", 72 | "\n", 73 | "train_data, test_data = imdb['train'], imdb['test']\n", 74 | "\n", 75 | "training_sentences = []\n", 76 | "training_labels = []\n", 77 | "\n", 78 | "testing_sentences = []\n", 79 | "testing_labels = []\n", 80 | "\n", 81 | "# str(s.tonumpy()) is needed in Python3 instead of just s.numpy()\n", 82 | "for s,l in train_data:\n", 83 | " training_sentences.append(str(s.numpy()))\n", 84 | " training_labels.append(l.numpy())\n", 85 | " \n", 86 | "for s,l in test_data:\n", 87 | " testing_sentences.append(str(s.numpy()))\n", 88 | " testing_labels.append(l.numpy())\n", 89 | " \n", 90 | "training_labels_final = np.array(training_labels)\n", 91 | "testing_labels_final = np.array(testing_labels)\n" 92 | ], 93 | "execution_count": null, 94 | "outputs": [] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "metadata": { 99 | "id": "7n15yyMdmoH1" 100 | }, 101 | "source": [ 102 | "vocab_size = 10000\n", 103 | "embedding_dim = 16\n", 104 | "max_length = 120\n", 105 | "trunc_type='post'\n", 106 | "oov_tok = \"\"\n", 107 | "\n", 108 | "\n", 109 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 110 | "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", 111 | "\n", 112 | "tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\n", 113 | "tokenizer.fit_on_texts(training_sentences)\n", 114 | "word_index = tokenizer.word_index\n", 115 | "sequences = tokenizer.texts_to_sequences(training_sentences)\n", 116 | "padded = pad_sequences(sequences,maxlen=max_length, truncating=trunc_type)\n", 117 | "\n", 118 | "testing_sequences = tokenizer.texts_to_sequences(testing_sentences)\n", 119 | "testing_padded = pad_sequences(testing_sequences,maxlen=max_length)\n" 120 | ], 121 | "execution_count": null, 122 | "outputs": [] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "metadata": { 127 | "id": "9axf0uIXVMhO" 128 | }, 129 | "source": [ 130 | "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", 131 | "\n", 132 | "def decode_review(text):\n", 133 | " return ' '.join([reverse_word_index.get(i, '?') for i in text])\n", 134 | "\n", 135 | "print(decode_review(padded[1]))\n", 136 | "print(training_sentences[1])" 137 | ], 138 | "execution_count": null, 139 | "outputs": [] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "metadata": { 144 | "id": "5NEpdhb8AxID" 145 | }, 146 | "source": [ 147 | "model = tf.keras.Sequential([\n", 148 | " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", 149 | " tf.keras.layers.Bidirectional(tf.keras.layers.GRU(32)),\n", 150 | " tf.keras.layers.Dense(6, activation='relu'),\n", 151 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 152 | "])\n", 153 | "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", 154 | "model.summary()\n" 155 | ], 156 | "execution_count": null, 157 | "outputs": [] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "metadata": { 162 | "id": "V5LLrXC-uNX6" 163 | }, 164 | "source": [ 165 | "num_epochs = 50\n", 166 | "history = model.fit(padded, training_labels_final, epochs=num_epochs, validation_data=(testing_padded, testing_labels_final))" 167 | ], 168 | "execution_count": null, 169 | "outputs": [] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "metadata": { 174 | "id": "nHGYuU4jPYaj" 175 | }, 176 | "source": [ 177 | "import matplotlib.pyplot as plt\n", 178 | "\n", 179 | "\n", 180 | "def plot_graphs(history, string):\n", 181 | " plt.plot(history.history[string])\n", 182 | " plt.plot(history.history['val_'+string])\n", 183 | " plt.xlabel(\"Epochs\")\n", 184 | " plt.ylabel(string)\n", 185 | " plt.legend([string, 'val_'+string])\n", 186 | " plt.show()\n", 187 | "\n", 188 | "plot_graphs(history, 'accuracy')\n", 189 | "plot_graphs(history, 'loss')" 190 | ], 191 | "execution_count": null, 192 | "outputs": [] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "metadata": { 197 | "id": "wSualgGPPK0S" 198 | }, 199 | "source": [ 200 | "# Model Definition with LSTM\n", 201 | "model = tf.keras.Sequential([\n", 202 | " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", 203 | " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n", 204 | " tf.keras.layers.Dense(6, activation='relu'),\n", 205 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 206 | "])\n", 207 | "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", 208 | "model.summary()\n" 209 | ], 210 | "execution_count": null, 211 | "outputs": [] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "metadata": { 216 | "id": "K_Jc7cY3Qxke" 217 | }, 218 | "source": [ 219 | "# Model Definition with Conv1D\n", 220 | "model = tf.keras.Sequential([\n", 221 | " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", 222 | " tf.keras.layers.Conv1D(128, 5, activation='relu'),\n", 223 | " tf.keras.layers.GlobalAveragePooling1D(),\n", 224 | " tf.keras.layers.Dense(6, activation='relu'),\n", 225 | " tf.keras.layers.Dense(1, activation='sigmoid')\n", 226 | "])\n", 227 | "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", 228 | "model.summary()\n" 229 | ], 230 | "execution_count": null, 231 | "outputs": [] 232 | } 233 | ] 234 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week4/Course_3_Week_4_Lesson_2_Notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Course 3 - Week 4 - Lesson 2 - Notebook.ipynb", 8 | "provenance": [], 9 | "toc_visible": true 10 | }, 11 | "kernelspec": { 12 | "display_name": "Python 3", 13 | "name": "python3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "metadata": { 20 | "id": "zX4Kg8DUTKWO" 21 | }, 22 | "source": [ 23 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 24 | "# you may not use this file except in compliance with the License.\n", 25 | "# You may obtain a copy of the License at\n", 26 | "#\n", 27 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 28 | "#\n", 29 | "# Unless required by applicable law or agreed to in writing, software\n", 30 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 31 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 32 | "# See the License for the specific language governing permissions and\n", 33 | "# limitations under the License." 34 | ], 35 | "execution_count": null, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": { 41 | "id": "view-in-github" 42 | }, 43 | "source": [ 44 | "\"Open" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "metadata": { 50 | "id": "BOwsuGQQY9OL" 51 | }, 52 | "source": [ 53 | "import tensorflow as tf\n", 54 | "\n", 55 | "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", 56 | "from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional\n", 57 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 58 | "from tensorflow.keras.models import Sequential\n", 59 | "from tensorflow.keras.optimizers import Adam\n", 60 | "import numpy as np " 61 | ], 62 | "execution_count": null, 63 | "outputs": [] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "metadata": { 68 | "id": "pylt5qZYsWPh" 69 | }, 70 | "source": [ 71 | "!wget --no-check-certificate \\\n", 72 | " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/irish-lyrics-eof.txt \\\n", 73 | " -O /tmp/irish-lyrics-eof.txt" 74 | ], 75 | "execution_count": null, 76 | "outputs": [] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "metadata": { 81 | "id": "PRnDnCW-Z7qv" 82 | }, 83 | "source": [ 84 | "tokenizer = Tokenizer()\n", 85 | "\n", 86 | "data = open('/tmp/irish-lyrics-eof.txt').read()\n", 87 | "\n", 88 | "corpus = data.lower().split(\"\\n\")\n", 89 | "\n", 90 | "tokenizer.fit_on_texts(corpus)\n", 91 | "total_words = len(tokenizer.word_index) + 1\n", 92 | "\n", 93 | "print(tokenizer.word_index)\n", 94 | "print(total_words)\n" 95 | ], 96 | "execution_count": null, 97 | "outputs": [] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "metadata": { 102 | "id": "soPGVheskaQP" 103 | }, 104 | "source": [ 105 | "input_sequences = []\n", 106 | "for line in corpus:\n", 107 | "\ttoken_list = tokenizer.texts_to_sequences([line])[0]\n", 108 | "\tfor i in range(1, len(token_list)):\n", 109 | "\t\tn_gram_sequence = token_list[:i+1]\n", 110 | "\t\tinput_sequences.append(n_gram_sequence)\n", 111 | "\n", 112 | "# pad sequences \n", 113 | "max_sequence_len = max([len(x) for x in input_sequences])\n", 114 | "input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))\n", 115 | "\n", 116 | "# create predictors and label\n", 117 | "xs, labels = input_sequences[:,:-1],input_sequences[:,-1]\n", 118 | "\n", 119 | "ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)" 120 | ], 121 | "execution_count": null, 122 | "outputs": [] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "metadata": { 127 | "id": "pJtwVB2NbOAP" 128 | }, 129 | "source": [ 130 | "print(tokenizer.word_index['in'])\n", 131 | "print(tokenizer.word_index['the'])\n", 132 | "print(tokenizer.word_index['town'])\n", 133 | "print(tokenizer.word_index['of'])\n", 134 | "print(tokenizer.word_index['athy'])\n", 135 | "print(tokenizer.word_index['one'])\n", 136 | "print(tokenizer.word_index['jeremy'])\n", 137 | "print(tokenizer.word_index['lanigan'])" 138 | ], 139 | "execution_count": null, 140 | "outputs": [] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "metadata": { 145 | "id": "49Cv68JOakwv" 146 | }, 147 | "source": [ 148 | "print(xs[6])" 149 | ], 150 | "execution_count": null, 151 | "outputs": [] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "metadata": { 156 | "id": "iY-jwvfgbEF8" 157 | }, 158 | "source": [ 159 | "print(ys[6])" 160 | ], 161 | "execution_count": null, 162 | "outputs": [] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "metadata": { 167 | "id": "wtzlUMYadhKt" 168 | }, 169 | "source": [ 170 | "print(xs[5])\n", 171 | "print(ys[5])" 172 | ], 173 | "execution_count": null, 174 | "outputs": [] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "metadata": { 179 | "id": "H4myRpB1c4Gg" 180 | }, 181 | "source": [ 182 | "print(tokenizer.word_index)" 183 | ], 184 | "execution_count": null, 185 | "outputs": [] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "metadata": { 190 | "id": "w9vH8Y59ajYL" 191 | }, 192 | "source": [ 193 | "model = Sequential()\n", 194 | "model.add(Embedding(total_words, 100, input_length=max_sequence_len-1))\n", 195 | "model.add(Bidirectional(LSTM(150)))\n", 196 | "model.add(Dense(total_words, activation='softmax'))\n", 197 | "adam = Adam(lr=0.01)\n", 198 | "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n", 199 | "#earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto')\n", 200 | "history = model.fit(xs, ys, epochs=100, verbose=1)\n", 201 | "#print model.summary()\n", 202 | "print(model)\n" 203 | ], 204 | "execution_count": null, 205 | "outputs": [] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "metadata": { 210 | "id": "3YXGelKThoTT" 211 | }, 212 | "source": [ 213 | "import matplotlib.pyplot as plt\n", 214 | "\n", 215 | "\n", 216 | "def plot_graphs(history, string):\n", 217 | " plt.plot(history.history[string])\n", 218 | " plt.xlabel(\"Epochs\")\n", 219 | " plt.ylabel(string)\n", 220 | " plt.show()" 221 | ], 222 | "execution_count": null, 223 | "outputs": [] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "metadata": { 228 | "id": "poeprYK8h-c7" 229 | }, 230 | "source": [ 231 | "plot_graphs(history, 'accuracy')\n" 232 | ], 233 | "execution_count": null, 234 | "outputs": [] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "metadata": { 239 | "id": "6Vc6PHgxa6Hm" 240 | }, 241 | "source": [ 242 | "seed_text = \"I've got a bad feeling about this\"\n", 243 | "next_words = 100\n", 244 | " \n", 245 | "for _ in range(next_words):\n", 246 | "\ttoken_list = tokenizer.texts_to_sequences([seed_text])[0]\n", 247 | "\ttoken_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n", 248 | "\tpredicted = model.predict_classes(token_list, verbose=0)\n", 249 | "\toutput_word = \"\"\n", 250 | "\tfor word, index in tokenizer.word_index.items():\n", 251 | "\t\tif index == predicted:\n", 252 | "\t\t\toutput_word = word\n", 253 | "\t\t\tbreak\n", 254 | "\tseed_text += \" \" + output_word\n", 255 | "print(seed_text)" 256 | ], 257 | "execution_count": null, 258 | "outputs": [] 259 | } 260 | ] 261 | } -------------------------------------------------------------------------------- /Natural Language Processing in TensorFlow/week4/NLP_Week4_Exercise_Shakespeare_Answer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "NLP_Week4_Exercise_Shakespeare_Answer.ipynb", 8 | "provenance": [], 9 | "toc_visible": true 10 | }, 11 | "kernelspec": { 12 | "display_name": "Python 3", 13 | "name": "python3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "metadata": { 20 | "id": "zX4Kg8DUTKWO" 21 | }, 22 | "source": [ 23 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 24 | "# you may not use this file except in compliance with the License.\n", 25 | "# You may obtain a copy of the License at\n", 26 | "#\n", 27 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 28 | "#\n", 29 | "# Unless required by applicable law or agreed to in writing, software\n", 30 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 31 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 32 | "# See the License for the specific language governing permissions and\n", 33 | "# limitations under the License." 34 | ], 35 | "execution_count": null, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "metadata": { 41 | "id": "BOwsuGQQY9OL" 42 | }, 43 | "source": [ 44 | "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", 45 | "from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout, Bidirectional\n", 46 | "from tensorflow.keras.preprocessing.text import Tokenizer\n", 47 | "from tensorflow.keras.models import Sequential\n", 48 | "from tensorflow.keras.optimizers import Adam\n", 49 | "from tensorflow.keras import regularizers\n", 50 | "import tensorflow.keras.utils as ku \n", 51 | "import numpy as np " 52 | ], 53 | "execution_count": null, 54 | "outputs": [] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "metadata": { 59 | "id": "PRnDnCW-Z7qv" 60 | }, 61 | "source": [ 62 | "tokenizer = Tokenizer()\n", 63 | "!wget --no-check-certificate \\\n", 64 | " https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sonnets.txt \\\n", 65 | " -O /tmp/sonnets.txt\n", 66 | "data = open('/tmp/sonnets.txt').read()\n", 67 | "\n", 68 | "corpus = data.lower().split(\"\\n\")\n", 69 | "\n", 70 | "\n", 71 | "tokenizer.fit_on_texts(corpus)\n", 72 | "total_words = len(tokenizer.word_index) + 1\n", 73 | "\n", 74 | "# create input sequences using list of tokens\n", 75 | "input_sequences = []\n", 76 | "for line in corpus:\n", 77 | "\ttoken_list = tokenizer.texts_to_sequences([line])[0]\n", 78 | "\tfor i in range(1, len(token_list)):\n", 79 | "\t\tn_gram_sequence = token_list[:i+1]\n", 80 | "\t\tinput_sequences.append(n_gram_sequence)\n", 81 | "\n", 82 | "\n", 83 | "# pad sequences \n", 84 | "max_sequence_len = max([len(x) for x in input_sequences])\n", 85 | "input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))\n", 86 | "\n", 87 | "# create predictors and label\n", 88 | "predictors, label = input_sequences[:,:-1],input_sequences[:,-1]\n", 89 | "\n", 90 | "label = ku.to_categorical(label, num_classes=total_words)" 91 | ], 92 | "execution_count": null, 93 | "outputs": [] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "metadata": { 98 | "id": "w9vH8Y59ajYL" 99 | }, 100 | "source": [ 101 | "model = Sequential()\n", 102 | "model.add(Embedding(total_words, 100, input_length=max_sequence_len-1))\n", 103 | "model.add(Bidirectional(LSTM(150, return_sequences = True)))\n", 104 | "model.add(Dropout(0.2))\n", 105 | "model.add(LSTM(100))\n", 106 | "model.add(Dense(total_words/2, activation='relu', kernel_regularizer=regularizers.l2(0.01)))\n", 107 | "model.add(Dense(total_words, activation='softmax'))\n", 108 | "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n", 109 | "print(model.summary())\n" 110 | ], 111 | "execution_count": null, 112 | "outputs": [] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "metadata": { 117 | "id": "AIg2f1HBxqof" 118 | }, 119 | "source": [ 120 | " history = model.fit(predictors, label, epochs=100, verbose=1)" 121 | ], 122 | "execution_count": null, 123 | "outputs": [] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "metadata": { 128 | "id": "1fXTEO3GJ282" 129 | }, 130 | "source": [ 131 | "import matplotlib.pyplot as plt\n", 132 | "acc = history.history['accuracy']\n", 133 | "loss = history.history['loss']\n", 134 | "\n", 135 | "epochs = range(len(acc))\n", 136 | "\n", 137 | "plt.plot(epochs, acc, 'b', label='Training accuracy')\n", 138 | "plt.title('Training accuracy')\n", 139 | "\n", 140 | "plt.figure()\n", 141 | "\n", 142 | "plt.plot(epochs, loss, 'b', label='Training Loss')\n", 143 | "plt.title('Training loss')\n", 144 | "plt.legend()\n", 145 | "\n", 146 | "plt.show()" 147 | ], 148 | "execution_count": null, 149 | "outputs": [] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "metadata": { 154 | "id": "6Vc6PHgxa6Hm" 155 | }, 156 | "source": [ 157 | "seed_text = \"Help me Obi Wan Kenobi, you're my only hope\"\n", 158 | "next_words = 100\n", 159 | " \n", 160 | "for _ in range(next_words):\n", 161 | "\ttoken_list = tokenizer.texts_to_sequences([seed_text])[0]\n", 162 | "\ttoken_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n", 163 | "\tpredicted = model.predict_classes(token_list, verbose=0)\n", 164 | "\toutput_word = \"\"\n", 165 | "\tfor word, index in tokenizer.word_index.items():\n", 166 | "\t\tif index == predicted:\n", 167 | "\t\t\toutput_word = word\n", 168 | "\t\t\tbreak\n", 169 | "\tseed_text += \" \" + output_word\n", 170 | "print(seed_text)" 171 | ], 172 | "execution_count": null, 173 | "outputs": [] 174 | } 175 | ] 176 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow-Certification 2 | Contains all the notebooks made while preparing for TensorFlow Certification Exam 3 | -------------------------------------------------------------------------------- /Screen Shot 2021-07-21 at 11.33.06 PM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chanakya1310/TensorFlow-Certification/6639a3b84e5d3d13e1a6a8fb4d6ea6f5d0e91960/Screen Shot 2021-07-21 at 11.33.06 PM.png -------------------------------------------------------------------------------- /Sequences, Time Series, and Prediction/week1/Week_1_Exercise_Answer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "Week 1 Exercise Answer.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "name": "python3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "code", 20 | "metadata": { 21 | "id": "zX4Kg8DUTKWO" 22 | }, 23 | "source": [ 24 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 25 | "# you may not use this file except in compliance with the License.\n", 26 | "# You may obtain a copy of the License at\n", 27 | "#\n", 28 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 29 | "#\n", 30 | "# Unless required by applicable law or agreed to in writing, software\n", 31 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 32 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 33 | "# See the License for the specific language governing permissions and\n", 34 | "# limitations under the License." 35 | ], 36 | "execution_count": null, 37 | "outputs": [] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "metadata": { 42 | "id": "t9HrvPfrSlzS" 43 | }, 44 | "source": [ 45 | "import tensorflow as tf\n", 46 | "print(tf.__version__)\n" 47 | ], 48 | "execution_count": null, 49 | "outputs": [] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "metadata": { 54 | "id": "gqWabzlJ63nL" 55 | }, 56 | "source": [ 57 | "import numpy as np\n", 58 | "import matplotlib.pyplot as plt\n", 59 | "import tensorflow as tf\n", 60 | "from tensorflow import keras\n", 61 | "\n", 62 | "def plot_series(time, series, format=\"-\", start=0, end=None):\n", 63 | " plt.plot(time[start:end], series[start:end], format)\n", 64 | " plt.xlabel(\"Time\")\n", 65 | " plt.ylabel(\"Value\")\n", 66 | " plt.grid(True)\n", 67 | "\n", 68 | "def trend(time, slope=0):\n", 69 | " return slope * time\n", 70 | "\n", 71 | "def seasonal_pattern(season_time):\n", 72 | " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", 73 | " return np.where(season_time < 0.1,\n", 74 | " np.cos(season_time * 7 * np.pi),\n", 75 | " 1 / np.exp(5 * season_time))\n", 76 | "\n", 77 | "def seasonality(time, period, amplitude=1, phase=0):\n", 78 | " \"\"\"Repeats the same pattern at each period\"\"\"\n", 79 | " season_time = ((time + phase) % period) / period\n", 80 | " return amplitude * seasonal_pattern(season_time)\n", 81 | "\n", 82 | "def noise(time, noise_level=1, seed=None):\n", 83 | " rnd = np.random.RandomState(seed)\n", 84 | " return rnd.randn(len(time)) * noise_level\n", 85 | "\n", 86 | "time = np.arange(4 * 365 + 1, dtype=\"float32\")\n", 87 | "baseline = 10\n", 88 | "series = trend(time, 0.1) \n", 89 | "baseline = 10\n", 90 | "amplitude = 40\n", 91 | "slope = 0.01\n", 92 | "noise_level = 2\n", 93 | "\n", 94 | "# Create the series\n", 95 | "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", 96 | "# Update with noise\n", 97 | "series += noise(time, noise_level, seed=42)\n", 98 | "\n", 99 | "plt.figure(figsize=(10, 6))\n", 100 | "plot_series(time, series)\n", 101 | "plt.show()" 102 | ], 103 | "execution_count": null, 104 | "outputs": [] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "metadata": { 109 | "id": "UfdyqJJ1VZVu" 110 | }, 111 | "source": [ 112 | "Now that we have the time series, let's split it so we can start forecasting" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "metadata": { 118 | "id": "_w0eKap5uFNP" 119 | }, 120 | "source": [ 121 | "split_time = 1100\n", 122 | "time_train = time[:split_time]\n", 123 | "x_train = series[:split_time]\n", 124 | "time_valid = time[split_time:]\n", 125 | "x_valid = series[split_time:]\n", 126 | "plt.figure(figsize=(10, 6))\n", 127 | "plot_series(time_train, x_train)\n", 128 | "plt.show()\n", 129 | "\n", 130 | "plt.figure(figsize=(10, 6))\n", 131 | "plot_series(time_valid, x_valid)\n", 132 | "plt.show()" 133 | ], 134 | "execution_count": null, 135 | "outputs": [] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": { 140 | "id": "bjD8ncEZbjEW" 141 | }, 142 | "source": [ 143 | "# Naive Forecast" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "metadata": { 149 | "id": "Pj_-uCeYxcAb" 150 | }, 151 | "source": [ 152 | "naive_forecast = series[split_time - 1:-1]" 153 | ], 154 | "execution_count": null, 155 | "outputs": [] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "metadata": { 160 | "id": "JtxwHj9Ig0jT" 161 | }, 162 | "source": [ 163 | "plt.figure(figsize=(10, 6))\n", 164 | "plot_series(time_valid, x_valid)\n", 165 | "plot_series(time_valid, naive_forecast)" 166 | ], 167 | "execution_count": null, 168 | "outputs": [] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": { 173 | "id": "fw1SP5WeuixH" 174 | }, 175 | "source": [ 176 | "Let's zoom in on the start of the validation period:" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "metadata": { 182 | "id": "D0MKg7FNug9V" 183 | }, 184 | "source": [ 185 | "plt.figure(figsize=(10, 6))\n", 186 | "plot_series(time_valid, x_valid, start=0, end=150)\n", 187 | "plot_series(time_valid, naive_forecast, start=1, end=151)" 188 | ], 189 | "execution_count": null, 190 | "outputs": [] 191 | }, 192 | { 193 | "cell_type": "markdown", 194 | "metadata": { 195 | "id": "35gIlQLfu0TT" 196 | }, 197 | "source": [ 198 | "You can see that the naive forecast lags 1 step behind the time series." 199 | ] 200 | }, 201 | { 202 | "cell_type": "markdown", 203 | "metadata": { 204 | "id": "Uh_7244Gsxfx" 205 | }, 206 | "source": [ 207 | "Now let's compute the mean squared error and the mean absolute error between the forecasts and the predictions in the validation period:" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "metadata": { 213 | "id": "byNnC7IbsnMZ" 214 | }, 215 | "source": [ 216 | "print(keras.metrics.mean_squared_error(x_valid, naive_forecast).numpy())\n", 217 | "print(keras.metrics.mean_absolute_error(x_valid, naive_forecast).numpy())" 218 | ], 219 | "execution_count": null, 220 | "outputs": [] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "metadata": { 225 | "id": "WGPBC9QttI1u" 226 | }, 227 | "source": [ 228 | "That's our baseline, now let's try a moving average:" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "metadata": { 234 | "id": "YGz5UsUdf2tV" 235 | }, 236 | "source": [ 237 | "def moving_average_forecast(series, window_size):\n", 238 | " \"\"\"Forecasts the mean of the last few values.\n", 239 | " If window_size=1, then this is equivalent to naive forecast\"\"\"\n", 240 | " forecast = []\n", 241 | " for time in range(len(series) - window_size):\n", 242 | " forecast.append(series[time:time + window_size].mean())\n", 243 | " return np.array(forecast)" 244 | ], 245 | "execution_count": null, 246 | "outputs": [] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "metadata": { 251 | "id": "HHFhGXQji7_r" 252 | }, 253 | "source": [ 254 | "moving_avg = moving_average_forecast(series, 30)[split_time - 30:]\n", 255 | "\n", 256 | "plt.figure(figsize=(10, 6))\n", 257 | "plot_series(time_valid, x_valid)\n", 258 | "plot_series(time_valid, moving_avg)" 259 | ], 260 | "execution_count": null, 261 | "outputs": [] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "metadata": { 266 | "id": "wG7pTAd7z0e8" 267 | }, 268 | "source": [ 269 | "print(keras.metrics.mean_squared_error(x_valid, moving_avg).numpy())\n", 270 | "print(keras.metrics.mean_absolute_error(x_valid, moving_avg).numpy())" 271 | ], 272 | "execution_count": null, 273 | "outputs": [] 274 | }, 275 | { 276 | "cell_type": "markdown", 277 | "metadata": { 278 | "id": "JMYPnJqwz8nS" 279 | }, 280 | "source": [ 281 | "That's worse than naive forecast! The moving average does not anticipate trend or seasonality, so let's try to remove them by using differencing. Since the seasonality period is 365 days, we will subtract the value at time *t* – 365 from the value at time *t*." 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "metadata": { 287 | "id": "5pqySF7-rJR4" 288 | }, 289 | "source": [ 290 | "diff_series = (series[365:] - series[:-365])\n", 291 | "diff_time = time[365:]\n", 292 | "\n", 293 | "plt.figure(figsize=(10, 6))\n", 294 | "plot_series(diff_time, diff_series)\n", 295 | "plt.show()" 296 | ], 297 | "execution_count": null, 298 | "outputs": [] 299 | }, 300 | { 301 | "cell_type": "markdown", 302 | "metadata": { 303 | "id": "xPlPlS7DskWg" 304 | }, 305 | "source": [ 306 | "Great, the trend and seasonality seem to be gone, so now we can use the moving average:" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "metadata": { 312 | "id": "QmZpz7arsjbb" 313 | }, 314 | "source": [ 315 | "diff_moving_avg = moving_average_forecast(diff_series, 50)[split_time - 365 - 50:]\n", 316 | "\n", 317 | "plt.figure(figsize=(10, 6))\n", 318 | "plot_series(time_valid, diff_series[split_time - 365:])\n", 319 | "plot_series(time_valid, diff_moving_avg)\n", 320 | "plt.show()" 321 | ], 322 | "execution_count": null, 323 | "outputs": [] 324 | }, 325 | { 326 | "cell_type": "markdown", 327 | "metadata": { 328 | "id": "Gno9S2lyecnc" 329 | }, 330 | "source": [ 331 | "Now let's bring back the trend and seasonality by adding the past values from t – 365:" 332 | ] 333 | }, 334 | { 335 | "cell_type": "code", 336 | "metadata": { 337 | "id": "Dv6RWFq7TFGB" 338 | }, 339 | "source": [ 340 | "diff_moving_avg_plus_past = series[split_time - 365:-365] + diff_moving_avg\n", 341 | "\n", 342 | "plt.figure(figsize=(10, 6))\n", 343 | "plot_series(time_valid, x_valid)\n", 344 | "plot_series(time_valid, diff_moving_avg_plus_past)\n", 345 | "plt.show()" 346 | ], 347 | "execution_count": null, 348 | "outputs": [] 349 | }, 350 | { 351 | "cell_type": "code", 352 | "metadata": { 353 | "id": "59jmBrwcTFCx" 354 | }, 355 | "source": [ 356 | "print(keras.metrics.mean_squared_error(x_valid, diff_moving_avg_plus_past).numpy())\n", 357 | "print(keras.metrics.mean_absolute_error(x_valid, diff_moving_avg_plus_past).numpy())" 358 | ], 359 | "execution_count": null, 360 | "outputs": [] 361 | }, 362 | { 363 | "cell_type": "markdown", 364 | "metadata": { 365 | "id": "vx9Et1Hkeusl" 366 | }, 367 | "source": [ 368 | "Better than naive forecast, good. However the forecasts look a bit too random, because we're just adding past values, which were noisy. Let's use a moving averaging on past values to remove some of the noise:" 369 | ] 370 | }, 371 | { 372 | "cell_type": "code", 373 | "metadata": { 374 | "id": "K81dtROoTE_r" 375 | }, 376 | "source": [ 377 | "diff_moving_avg_plus_smooth_past = moving_average_forecast(series[split_time - 370:-360], 10) + diff_moving_avg\n", 378 | "\n", 379 | "plt.figure(figsize=(10, 6))\n", 380 | "plot_series(time_valid, x_valid)\n", 381 | "plot_series(time_valid, diff_moving_avg_plus_smooth_past)\n", 382 | "plt.show()" 383 | ], 384 | "execution_count": null, 385 | "outputs": [] 386 | }, 387 | { 388 | "cell_type": "code", 389 | "metadata": { 390 | "id": "iN2MsBxWTE3m" 391 | }, 392 | "source": [ 393 | "print(keras.metrics.mean_squared_error(x_valid, diff_moving_avg_plus_smooth_past).numpy())\n", 394 | "print(keras.metrics.mean_absolute_error(x_valid, diff_moving_avg_plus_smooth_past).numpy())" 395 | ], 396 | "execution_count": null, 397 | "outputs": [] 398 | } 399 | ] 400 | } -------------------------------------------------------------------------------- /Sequences, Time Series, and Prediction/week2/S+P_Week_2_Exercise_Answer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "S+P_Week_2_Exercise_Answer.ipynb", 8 | "provenance": [], 9 | "toc_visible": true 10 | }, 11 | "kernelspec": { 12 | "display_name": "Python 3", 13 | "name": "python3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "metadata": { 20 | "id": "zX4Kg8DUTKWO" 21 | }, 22 | "source": [ 23 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 24 | "# you may not use this file except in compliance with the License.\n", 25 | "# You may obtain a copy of the License at\n", 26 | "#\n", 27 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 28 | "#\n", 29 | "# Unless required by applicable law or agreed to in writing, software\n", 30 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 31 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 32 | "# See the License for the specific language governing permissions and\n", 33 | "# limitations under the License." 34 | ], 35 | "execution_count": null, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "metadata": { 41 | "cellView": "both", 42 | "id": "D1J15Vh_1Jih" 43 | }, 44 | "source": [ 45 | "!pip install tf-nightly-2.0-preview\n" 46 | ], 47 | "execution_count": null, 48 | "outputs": [] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "metadata": { 53 | "id": "BOjujz601HcS" 54 | }, 55 | "source": [ 56 | "import tensorflow as tf\n", 57 | "import numpy as np\n", 58 | "import matplotlib.pyplot as plt\n", 59 | "print(tf.__version__)" 60 | ], 61 | "execution_count": null, 62 | "outputs": [] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "metadata": { 67 | "id": "Zswl7jRtGzkk" 68 | }, 69 | "source": [ 70 | "def plot_series(time, series, format=\"-\", start=0, end=None):\n", 71 | " plt.plot(time[start:end], series[start:end], format)\n", 72 | " plt.xlabel(\"Time\")\n", 73 | " plt.ylabel(\"Value\")\n", 74 | " plt.grid(False)\n", 75 | "\n", 76 | "def trend(time, slope=0):\n", 77 | " return slope * time\n", 78 | "\n", 79 | "def seasonal_pattern(season_time):\n", 80 | " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", 81 | " return np.where(season_time < 0.1,\n", 82 | " np.cos(season_time * 6 * np.pi),\n", 83 | " 2 / np.exp(9 * season_time))\n", 84 | "\n", 85 | "def seasonality(time, period, amplitude=1, phase=0):\n", 86 | " \"\"\"Repeats the same pattern at each period\"\"\"\n", 87 | " season_time = ((time + phase) % period) / period\n", 88 | " return amplitude * seasonal_pattern(season_time)\n", 89 | "\n", 90 | "def noise(time, noise_level=1, seed=None):\n", 91 | " rnd = np.random.RandomState(seed)\n", 92 | " return rnd.randn(len(time)) * noise_level\n", 93 | "\n", 94 | "time = np.arange(10 * 365 + 1, dtype=\"float32\")\n", 95 | "baseline = 10\n", 96 | "series = trend(time, 0.1) \n", 97 | "baseline = 10\n", 98 | "amplitude = 40\n", 99 | "slope = 0.005\n", 100 | "noise_level = 3\n", 101 | "\n", 102 | "# Create the series\n", 103 | "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", 104 | "# Update with noise\n", 105 | "series += noise(time, noise_level, seed=51)\n", 106 | "\n", 107 | "split_time = 3000\n", 108 | "time_train = time[:split_time]\n", 109 | "x_train = series[:split_time]\n", 110 | "time_valid = time[split_time:]\n", 111 | "x_valid = series[split_time:]\n", 112 | "\n", 113 | "window_size = 20\n", 114 | "batch_size = 32\n", 115 | "shuffle_buffer_size = 1000\n", 116 | "\n", 117 | "plot_series(time, series)" 118 | ], 119 | "execution_count": null, 120 | "outputs": [] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "metadata": { 125 | "id": "4sTTIOCbyShY" 126 | }, 127 | "source": [ 128 | "def windowed_dataset(series, window_size, batch_size, shuffle_buffer):\n", 129 | " dataset = tf.data.Dataset.from_tensor_slices(series)\n", 130 | " dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)\n", 131 | " dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))\n", 132 | " dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1]))\n", 133 | " dataset = dataset.batch(batch_size).prefetch(1)\n", 134 | " return dataset" 135 | ], 136 | "execution_count": null, 137 | "outputs": [] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "metadata": { 142 | "id": "TW-vT7eLYAdb" 143 | }, 144 | "source": [ 145 | "dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)\n", 146 | "\n", 147 | "\n", 148 | "model = tf.keras.models.Sequential([\n", 149 | " tf.keras.layers.Dense(100, input_shape=[window_size], activation=\"relu\"), \n", 150 | " tf.keras.layers.Dense(10, activation=\"relu\"), \n", 151 | " tf.keras.layers.Dense(1)\n", 152 | "])\n", 153 | "\n", 154 | "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(lr=1e-6, momentum=0.9))\n", 155 | "model.fit(dataset,epochs=100,verbose=0)\n" 156 | ], 157 | "execution_count": null, 158 | "outputs": [] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "metadata": { 163 | "id": "efhco2rYyIFF" 164 | }, 165 | "source": [ 166 | "forecast = []\n", 167 | "for time in range(len(series) - window_size):\n", 168 | " forecast.append(model.predict(series[time:time + window_size][np.newaxis]))\n", 169 | "\n", 170 | "forecast = forecast[split_time-window_size:]\n", 171 | "results = np.array(forecast)[:, 0, 0]\n", 172 | "\n", 173 | "\n", 174 | "plt.figure(figsize=(10, 6))\n", 175 | "\n", 176 | "plot_series(time_valid, x_valid)\n", 177 | "plot_series(time_valid, results)" 178 | ], 179 | "execution_count": null, 180 | "outputs": [] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "metadata": { 185 | "id": "-kT6j186YO6K" 186 | }, 187 | "source": [ 188 | "tf.keras.metrics.mean_absolute_error(x_valid, results).numpy()" 189 | ], 190 | "execution_count": null, 191 | "outputs": [] 192 | } 193 | ] 194 | } -------------------------------------------------------------------------------- /Sequences, Time Series, and Prediction/week2/S+P_Week_2_Exercise_Question.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "S+P_Week_2_Exercise_Question.ipynb", 7 | "provenance": [], 8 | "toc_visible": true 9 | }, 10 | "kernelspec": { 11 | "display_name": "Python 3", 12 | "name": "python3" 13 | } 14 | }, 15 | "cells": [ 16 | { 17 | "cell_type": "code", 18 | "metadata": { 19 | "id": "zX4Kg8DUTKWO" 20 | }, 21 | "source": [ 22 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 23 | "# you may not use this file except in compliance with the License.\n", 24 | "# You may obtain a copy of the License at\n", 25 | "#\n", 26 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 27 | "#\n", 28 | "# Unless required by applicable law or agreed to in writing, software\n", 29 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 30 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 31 | "# See the License for the specific language governing permissions and\n", 32 | "# limitations under the License." 33 | ], 34 | "execution_count": null, 35 | "outputs": [] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "metadata": { 40 | "cellView": "both", 41 | "id": "D1J15Vh_1Jih" 42 | }, 43 | "source": [ 44 | "!pip install tf-nightly-2.0-preview\n" 45 | ], 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "metadata": { 52 | "id": "BOjujz601HcS" 53 | }, 54 | "source": [ 55 | "import tensorflow as tf\n", 56 | "import numpy as np\n", 57 | "import matplotlib.pyplot as plt\n", 58 | "print(tf.__version__)" 59 | ], 60 | "execution_count": null, 61 | "outputs": [] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "metadata": { 66 | "id": "Zswl7jRtGzkk" 67 | }, 68 | "source": [ 69 | "def plot_series(time, series, format=\"-\", start=0, end=None):\n", 70 | " plt.plot(time[start:end], series[start:end], format)\n", 71 | " plt.xlabel(\"Time\")\n", 72 | " plt.ylabel(\"Value\")\n", 73 | " plt.grid(False)\n", 74 | "\n", 75 | "def trend(time, slope=0):\n", 76 | " return slope * time\n", 77 | "\n", 78 | "def seasonal_pattern(season_time):\n", 79 | " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", 80 | " return np.where(season_time < 0.1,\n", 81 | " np.cos(season_time * # YOUR CODE HERE # * np.pi),\n", 82 | " #YOUR CODE HERE# / np.exp(#YOUR CODE HERE# * season_time))\n", 83 | "\n", 84 | "def seasonality(time, period, amplitude=1, phase=0):\n", 85 | " \"\"\"Repeats the same pattern at each period\"\"\"\n", 86 | " season_time = ((time + phase) % period) / period\n", 87 | " return amplitude * seasonal_pattern(season_time)\n", 88 | "\n", 89 | "def noise(time, noise_level=1, seed=None):\n", 90 | " rnd = np.random.RandomState(seed)\n", 91 | " return rnd.randn(len(time)) * noise_level\n", 92 | "\n", 93 | "time = np.arange(10 * 365 + 1, dtype=\"float32\")\n", 94 | "baseline = # YOUR CODE HERE #\n", 95 | "series = trend(time, # YOUR CODE HERE#) \n", 96 | "baseline = 10\n", 97 | "amplitude = 40\n", 98 | "slope = # YOUR CODE HERE#\n", 99 | "noise_level = # YOUR CODE HERE#\n", 100 | "\n", 101 | "# Create the series\n", 102 | "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", 103 | "# Update with noise\n", 104 | "series += noise(time, noise_level, seed=51)\n", 105 | "\n", 106 | "split_time = 3000\n", 107 | "time_train = time[:split_time]\n", 108 | "x_train = series[:split_time]\n", 109 | "time_valid = time[split_time:]\n", 110 | "x_valid = series[split_time:]\n", 111 | "\n", 112 | "window_size = 20\n", 113 | "batch_size = 32\n", 114 | "shuffle_buffer_size = 1000\n", 115 | "\n", 116 | "plot_series(time, series)" 117 | ], 118 | "execution_count": null, 119 | "outputs": [] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": { 124 | "id": "GfUTNqti_lNC" 125 | }, 126 | "source": [ 127 | "Desired output -- a chart that looks like this:\n", 128 | "\n", 129 | "![Chart showing upward trend and seasonailty](http://www.laurencemoroney.com/wp-content/uploads/2019/07/plot1.png)" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "metadata": { 135 | "id": "4sTTIOCbyShY" 136 | }, 137 | "source": [ 138 | "def windowed_dataset(series, window_size, batch_size, shuffle_buffer):\n", 139 | " dataset = tf.data.Dataset.from_tensor_slices(series)\n", 140 | " dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)\n", 141 | " dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))\n", 142 | " dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1]))\n", 143 | " dataset = dataset.batch(batch_size).prefetch(1)\n", 144 | " return dataset" 145 | ], 146 | "execution_count": null, 147 | "outputs": [] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "metadata": { 152 | "id": "TW-vT7eLYAdb" 153 | }, 154 | "source": [ 155 | "dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)\n", 156 | "\n", 157 | "\n", 158 | "model = tf.keras.models.Sequential([\n", 159 | " tf.keras.layers.Dense(# YOUR CODE HERE #),\n", 160 | " tf.keras.layers.Dense(# YOUR CODE HERE #, activation=\"relu\"), \n", 161 | " tf.keras.layers.Dense(1)\n", 162 | "])\n", 163 | "\n", 164 | "model.compile(loss=# YOUR CODE HERE #, optimizer=# YOUR CODE HERE#))\n", 165 | "model.fit(dataset,epochs=100,verbose=0)\n" 166 | ], 167 | "execution_count": null, 168 | "outputs": [] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "metadata": { 173 | "id": "efhco2rYyIFF" 174 | }, 175 | "source": [ 176 | "forecast = []\n", 177 | "for time in range(len(series) - window_size):\n", 178 | " forecast.append(model.predict(series[time:time + window_size][np.newaxis]))\n", 179 | "\n", 180 | "forecast = forecast[split_time-window_size:]\n", 181 | "results = np.array(forecast)[:, 0, 0]\n", 182 | "\n", 183 | "\n", 184 | "plt.figure(figsize=(10, 6))\n", 185 | "\n", 186 | "plot_series(time_valid, x_valid)\n", 187 | "plot_series(time_valid, results)" 188 | ], 189 | "execution_count": null, 190 | "outputs": [] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "metadata": { 195 | "id": "-kT6j186YO6K" 196 | }, 197 | "source": [ 198 | "tf.keras.metrics.mean_absolute_error(x_valid, results).numpy()\n", 199 | "# EXPECTED OUTPUT\n", 200 | "# A Value less than 3" 201 | ], 202 | "execution_count": null, 203 | "outputs": [] 204 | } 205 | ] 206 | } -------------------------------------------------------------------------------- /Sequences, Time Series, and Prediction/week2/S+P_Week_2_Lesson_1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "S+P Week 2 Lesson 1.ipynb", 8 | "provenance": [], 9 | "toc_visible": true 10 | }, 11 | "kernelspec": { 12 | "display_name": "Python 3", 13 | "name": "python3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "metadata": { 20 | "id": "zX4Kg8DUTKWO" 21 | }, 22 | "source": [ 23 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 24 | "# you may not use this file except in compliance with the License.\n", 25 | "# You may obtain a copy of the License at\n", 26 | "#\n", 27 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 28 | "#\n", 29 | "# Unless required by applicable law or agreed to in writing, software\n", 30 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 31 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 32 | "# See the License for the specific language governing permissions and\n", 33 | "# limitations under the License." 34 | ], 35 | "execution_count": null, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "metadata": { 41 | "id": "s6eq-RBcQ_Zr" 42 | }, 43 | "source": [ 44 | "try:\n", 45 | " # %tensorflow_version only exists in Colab.\n", 46 | " %tensorflow_version 2.x\n", 47 | "except Exception:\n", 48 | " pass" 49 | ], 50 | "execution_count": null, 51 | "outputs": [] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "metadata": { 56 | "id": "BOjujz601HcS" 57 | }, 58 | "source": [ 59 | "import tensorflow as tf\n", 60 | "import numpy as np\n", 61 | "import matplotlib.pyplot as plt\n", 62 | "print(tf.__version__)" 63 | ], 64 | "execution_count": null, 65 | "outputs": [] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "metadata": { 70 | "id": "asEdslR_05O_" 71 | }, 72 | "source": [ 73 | "dataset = tf.data.Dataset.range(10)\n", 74 | "for val in dataset:\n", 75 | " print(val.numpy())" 76 | ], 77 | "execution_count": null, 78 | "outputs": [] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "metadata": { 83 | "id": "Lrv_ghSt1lgQ" 84 | }, 85 | "source": [ 86 | "dataset = tf.data.Dataset.range(10)\n", 87 | "dataset = dataset.window(5, shift=1)\n", 88 | "for window_dataset in dataset:\n", 89 | " for val in window_dataset:\n", 90 | " print(val.numpy(), end=\" \")\n", 91 | " print()" 92 | ], 93 | "execution_count": null, 94 | "outputs": [] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "metadata": { 99 | "id": "QLEq6MG-2DN2" 100 | }, 101 | "source": [ 102 | "dataset = tf.data.Dataset.range(10)\n", 103 | "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", 104 | "for window_dataset in dataset:\n", 105 | " for val in window_dataset:\n", 106 | " print(val.numpy(), end=\" \")\n", 107 | " print()" 108 | ], 109 | "execution_count": null, 110 | "outputs": [] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "metadata": { 115 | "id": "PJ9CAHlJ2ODe" 116 | }, 117 | "source": [ 118 | "dataset = tf.data.Dataset.range(10)\n", 119 | "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", 120 | "dataset = dataset.flat_map(lambda window: window.batch(5))\n", 121 | "for window in dataset:\n", 122 | " print(window.numpy())\n" 123 | ], 124 | "execution_count": null, 125 | "outputs": [] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "metadata": { 130 | "id": "DryEZ2Mz2nNV" 131 | }, 132 | "source": [ 133 | "dataset = tf.data.Dataset.range(10)\n", 134 | "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", 135 | "dataset = dataset.flat_map(lambda window: window.batch(5))\n", 136 | "dataset = dataset.map(lambda window: (window[:-1], window[-1:]))\n", 137 | "for x,y in dataset:\n", 138 | " print(x.numpy(), y.numpy())" 139 | ], 140 | "execution_count": null, 141 | "outputs": [] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "metadata": { 146 | "id": "1tl-0BOKkEtk" 147 | }, 148 | "source": [ 149 | "dataset = tf.data.Dataset.range(10)\n", 150 | "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", 151 | "dataset = dataset.flat_map(lambda window: window.batch(5))\n", 152 | "dataset = dataset.map(lambda window: (window[:-1], window[-1:]))\n", 153 | "dataset = dataset.shuffle(buffer_size=10)\n", 154 | "for x,y in dataset:\n", 155 | " print(x.numpy(), y.numpy())\n" 156 | ], 157 | "execution_count": null, 158 | "outputs": [] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "metadata": { 163 | "id": "Wa0PNwxMGapy" 164 | }, 165 | "source": [ 166 | "dataset = tf.data.Dataset.range(10)\n", 167 | "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", 168 | "dataset = dataset.flat_map(lambda window: window.batch(5))\n", 169 | "dataset = dataset.map(lambda window: (window[:-1], window[-1:]))\n", 170 | "dataset = dataset.shuffle(buffer_size=10)\n", 171 | "dataset = dataset.batch(2).prefetch(1)\n", 172 | "for x,y in dataset:\n", 173 | " print(\"x = \", x.numpy())\n", 174 | " print(\"y = \", y.numpy())\n" 175 | ], 176 | "execution_count": null, 177 | "outputs": [] 178 | } 179 | ] 180 | } -------------------------------------------------------------------------------- /Sequences, Time Series, and Prediction/week3/S+P_Week_3_Exercise_Answer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "S+P Week 3 Exercise Answer.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "name": "python3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "code", 20 | "metadata": { 21 | "id": "zX4Kg8DUTKWO" 22 | }, 23 | "source": [ 24 | "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", 25 | "# you may not use this file except in compliance with the License.\n", 26 | "# You may obtain a copy of the License at\n", 27 | "#\n", 28 | "# https://www.apache.org/licenses/LICENSE-2.0\n", 29 | "#\n", 30 | "# Unless required by applicable law or agreed to in writing, software\n", 31 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 32 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 33 | "# See the License for the specific language governing permissions and\n", 34 | "# limitations under the License." 35 | ], 36 | "execution_count": null, 37 | "outputs": [] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "metadata": { 42 | "id": "D1J15Vh_1Jih" 43 | }, 44 | "source": [ 45 | "!pip install tf-nightly-2.0-preview\n" 46 | ], 47 | "execution_count": null, 48 | "outputs": [] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "metadata": { 53 | "id": "BOjujz601HcS" 54 | }, 55 | "source": [ 56 | "import tensorflow as tf\n", 57 | "import numpy as np\n", 58 | "import matplotlib.pyplot as plt\n", 59 | "print(tf.__version__)" 60 | ], 61 | "execution_count": null, 62 | "outputs": [] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "metadata": { 67 | "id": "Zswl7jRtGzkk" 68 | }, 69 | "source": [ 70 | "def plot_series(time, series, format=\"-\", start=0, end=None):\n", 71 | " plt.plot(time[start:end], series[start:end], format)\n", 72 | " plt.xlabel(\"Time\")\n", 73 | " plt.ylabel(\"Value\")\n", 74 | " plt.grid(False)\n", 75 | "\n", 76 | "def trend(time, slope=0):\n", 77 | " return slope * time\n", 78 | "\n", 79 | "def seasonal_pattern(season_time):\n", 80 | " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", 81 | " return np.where(season_time < 0.1,\n", 82 | " np.cos(season_time * 6 * np.pi),\n", 83 | " 2 / np.exp(9 * season_time))\n", 84 | "\n", 85 | "def seasonality(time, period, amplitude=1, phase=0):\n", 86 | " \"\"\"Repeats the same pattern at each period\"\"\"\n", 87 | " season_time = ((time + phase) % period) / period\n", 88 | " return amplitude * seasonal_pattern(season_time)\n", 89 | "\n", 90 | "def noise(time, noise_level=1, seed=None):\n", 91 | " rnd = np.random.RandomState(seed)\n", 92 | " return rnd.randn(len(time)) * noise_level\n", 93 | "\n", 94 | "time = np.arange(10 * 365 + 1, dtype=\"float32\")\n", 95 | "baseline = 10\n", 96 | "series = trend(time, 0.1) \n", 97 | "baseline = 10\n", 98 | "amplitude = 40\n", 99 | "slope = 0.005\n", 100 | "noise_level = 3\n", 101 | "\n", 102 | "# Create the series\n", 103 | "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", 104 | "# Update with noise\n", 105 | "series += noise(time, noise_level, seed=51)\n", 106 | "\n", 107 | "split_time = 3000\n", 108 | "time_train = time[:split_time]\n", 109 | "x_train = series[:split_time]\n", 110 | "time_valid = time[split_time:]\n", 111 | "x_valid = series[split_time:]\n", 112 | "\n", 113 | "window_size = 20\n", 114 | "batch_size = 32\n", 115 | "shuffle_buffer_size = 1000\n", 116 | "\n", 117 | "plot_series(time, series)" 118 | ], 119 | "execution_count": null, 120 | "outputs": [] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "metadata": { 125 | "id": "4sTTIOCbyShY" 126 | }, 127 | "source": [ 128 | "def windowed_dataset(series, window_size, batch_size, shuffle_buffer):\n", 129 | " dataset = tf.data.Dataset.from_tensor_slices(series)\n", 130 | " dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)\n", 131 | " dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))\n", 132 | " dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1]))\n", 133 | " dataset = dataset.batch(batch_size).prefetch(1)\n", 134 | " return dataset" 135 | ], 136 | "execution_count": null, 137 | "outputs": [] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "metadata": { 142 | "id": "A1Hl39rklkLm" 143 | }, 144 | "source": [ 145 | "tf.keras.backend.clear_session()\n", 146 | "tf.random.set_seed(51)\n", 147 | "np.random.seed(51)\n", 148 | "\n", 149 | "tf.keras.backend.clear_session()\n", 150 | "dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)\n", 151 | "\n", 152 | "model = tf.keras.models.Sequential([\n", 153 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=-1),\n", 154 | " input_shape=[None]),\n", 155 | " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, return_sequences=True)),\n", 156 | " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n", 157 | " tf.keras.layers.Dense(1),\n", 158 | " tf.keras.layers.Lambda(lambda x: x * 10.0)\n", 159 | "])\n", 160 | "\n", 161 | "lr_schedule = tf.keras.callbacks.LearningRateScheduler(\n", 162 | " lambda epoch: 1e-8 * 10**(epoch / 20))\n", 163 | "optimizer = tf.keras.optimizers.SGD(lr=1e-8, momentum=0.9)\n", 164 | "model.compile(loss=tf.keras.losses.Huber(),\n", 165 | " optimizer=optimizer,\n", 166 | " metrics=[\"mae\"])\n", 167 | "history = model.fit(dataset, epochs=100, callbacks=[lr_schedule])" 168 | ], 169 | "execution_count": null, 170 | "outputs": [] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "metadata": { 175 | "id": "AkBsrsXMzoWR" 176 | }, 177 | "source": [ 178 | "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n", 179 | "plt.axis([1e-8, 1e-4, 0, 30])" 180 | ], 181 | "execution_count": null, 182 | "outputs": [] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "metadata": { 187 | "id": "4uh-97bpLZCA" 188 | }, 189 | "source": [ 190 | "tf.keras.backend.clear_session()\n", 191 | "tf.random.set_seed(51)\n", 192 | "np.random.seed(51)\n", 193 | "\n", 194 | "tf.keras.backend.clear_session()\n", 195 | "dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)\n", 196 | "\n", 197 | "model = tf.keras.models.Sequential([\n", 198 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=-1),\n", 199 | " input_shape=[None]),\n", 200 | " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, return_sequences=True)),\n", 201 | " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n", 202 | " tf.keras.layers.Dense(1),\n", 203 | " tf.keras.layers.Lambda(lambda x: x * 100.0)\n", 204 | "])\n", 205 | "\n", 206 | "\n", 207 | "model.compile(loss=\"mse\", optimizer=tf.keras.optimizers.SGD(lr=1e-5, momentum=0.9),metrics=[\"mae\"])\n", 208 | "history = model.fit(dataset,epochs=500,verbose=1)" 209 | ], 210 | "execution_count": null, 211 | "outputs": [] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "metadata": { 216 | "id": "icGDaND7z0ne" 217 | }, 218 | "source": [ 219 | "forecast = []\n", 220 | "results = []\n", 221 | "for time in range(len(series) - window_size):\n", 222 | " forecast.append(model.predict(series[time:time + window_size][np.newaxis]))\n", 223 | "\n", 224 | "forecast = forecast[split_time-window_size:]\n", 225 | "results = np.array(forecast)[:, 0, 0]\n", 226 | "\n", 227 | "\n", 228 | "plt.figure(figsize=(10, 6))\n", 229 | "\n", 230 | "plot_series(time_valid, x_valid)\n", 231 | "plot_series(time_valid, results)" 232 | ], 233 | "execution_count": null, 234 | "outputs": [] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "metadata": { 239 | "id": "KfPeqI7rz4LD" 240 | }, 241 | "source": [ 242 | "tf.keras.metrics.mean_absolute_error(x_valid, results).numpy()" 243 | ], 244 | "execution_count": null, 245 | "outputs": [] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "metadata": { 250 | "id": "JUsdZB_tzDLe" 251 | }, 252 | "source": [ 253 | "import matplotlib.image as mpimg\n", 254 | "import matplotlib.pyplot as plt\n", 255 | "\n", 256 | "#-----------------------------------------------------------\n", 257 | "# Retrieve a list of list results on training and test data\n", 258 | "# sets for each training epoch\n", 259 | "#-----------------------------------------------------------\n", 260 | "mae=history.history['mae']\n", 261 | "loss=history.history['loss']\n", 262 | "\n", 263 | "epochs=range(len(loss)) # Get number of epochs\n", 264 | "\n", 265 | "#------------------------------------------------\n", 266 | "# Plot MAE and Loss\n", 267 | "#------------------------------------------------\n", 268 | "plt.plot(epochs, mae, 'r')\n", 269 | "plt.plot(epochs, loss, 'b')\n", 270 | "plt.title('MAE and Loss')\n", 271 | "plt.xlabel(\"Epochs\")\n", 272 | "plt.ylabel(\"Accuracy\")\n", 273 | "plt.legend([\"MAE\", \"Loss\"])\n", 274 | "\n", 275 | "plt.figure()\n", 276 | "\n", 277 | "epochs_zoom = epochs[200:]\n", 278 | "mae_zoom = mae[200:]\n", 279 | "loss_zoom = loss[200:]\n", 280 | "\n", 281 | "#------------------------------------------------\n", 282 | "# Plot Zoomed MAE and Loss\n", 283 | "#------------------------------------------------\n", 284 | "plt.plot(epochs_zoom, mae_zoom, 'r')\n", 285 | "plt.plot(epochs_zoom, loss_zoom, 'b')\n", 286 | "plt.title('MAE and Loss')\n", 287 | "plt.xlabel(\"Epochs\")\n", 288 | "plt.ylabel(\"Accuracy\")\n", 289 | "plt.legend([\"MAE\", \"Loss\"])\n", 290 | "\n", 291 | "plt.figure()" 292 | ], 293 | "execution_count": null, 294 | "outputs": [] 295 | } 296 | ] 297 | } --------------------------------------------------------------------------------