├── Session 4 - Modifications of NN ├── MNIST_batch_norm_notebook.ipynb └── nn_keras.ipynb ├── Assignment Keras └── Assignment+Using+Keras.ipynb └── Session 3 - Backpropagation in NN └── nn_from_scratch.ipynb /Session 4 - Modifications of NN/MNIST_batch_norm_notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "id": "g4ghKGG1f8fS" 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import pandas as pd\n", 13 | "import matplotlib.pyplot as plt\n", 14 | "import tensorflow as tf\n", 15 | "from tensorflow import keras\n", 16 | "from sklearn.model_selection import train_test_split\n", 17 | "%matplotlib inline" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "metadata": { 24 | "colab": { 25 | "base_uri": "https://localhost:8080/", 26 | "height": 204 27 | }, 28 | "id": "KYeZzPYwQ_14", 29 | "outputId": "bec00135-36eb-4086-960d-18de16546972" 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "data = pd.read_csv('../train.csv')\n", 34 | "X_test = pd.read_csv(\"../test.csv\")" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "data.head(5)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "metadata": { 50 | "colab": { 51 | "base_uri": "https://localhost:8080/", 52 | "height": 89, 53 | "resources": { 54 | "http://localhost:8080/nbextensions/google.colab/files.js": { 55 | "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=", 56 | "headers": [ 57 | [ 58 | "content-type", 59 | "application/javascript" 60 | ] 61 | ], 62 | "ok": true, 63 | "status": 200, 64 | "status_text": "" 65 | } 66 | } 67 | }, 68 | "id": "ZzgfjB-RJ_fn", 69 | "outputId": "2adeb1a2-92b3-4c06-c135-92e8c78f3382" 70 | }, 71 | "outputs": [], 72 | "source": [ 73 | "data_y = data['label'].values\n", 74 | "data_x = data.drop('label',axis=1)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": { 81 | "id": "082Yj5G-J-qU" 82 | }, 83 | "outputs": [], 84 | "source": [ 85 | "data_x.head(2)" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": { 92 | "colab": { 93 | "base_uri": "https://localhost:8080/", 94 | "height": 170 95 | }, 96 | "id": "tfk0BpyKSNFH", 97 | "outputId": "53da59a7-e512-4011-b668-eb6ff5a6ed08" 98 | }, 99 | "outputs": [], 100 | "source": [ 101 | "img = data_x.iloc[3].values.reshape((28,28))\n", 102 | "plt.imshow(img,cmap='gray')" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": { 109 | "colab": { 110 | "base_uri": "https://localhost:8080/", 111 | "height": 34 112 | }, 113 | "id": "HBU1DRAhcxPT", 114 | "outputId": "9fd0dd9e-818b-46f4-cdb0-bd87025c66fb" 115 | }, 116 | "outputs": [], 117 | "source": [ 118 | "set(data_y)" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": { 125 | "colab": { 126 | "base_uri": "https://localhost:8080/", 127 | "height": 34 128 | }, 129 | "id": "gMc_whwFf8fa", 130 | "outputId": "c814502d-7574-4523-a1ad-8a026f60f10b" 131 | }, 132 | "outputs": [], 133 | "source": [ 134 | "# If batch normalization is used once, we show the accuracies of each example of different locations of batch normalization layer\n", 135 | "model = keras.Sequential([ #loss: 0.0152 - accuracy: 0.9946 - val_loss: 0.1050 - val_accuracy: 0.9758\n", 136 | " keras.layers.Flatten(input_shape=(28, 28)),\n", 137 | " #keras.layers.BatchNormalization() #loss: 0.0191 - accuracy: 0.9941 - val_loss: 0.1239 - val_accuracy: 0.9762\n", 138 | " keras.layers.Dense(128, activation=tf.nn.relu, input_dim=16), \n", 139 | " #keras.layers.BatchNormalization() #loss: 0.0186 - accuracy: 0.9940 - val_loss: 0.1034 - val_accuracy: 0.9783\n", 140 | " keras.layers.Dense(128, activation=tf.nn.relu),\n", 141 | " #keras.layers.BatchNormalization() #loss: 0.0179 - accuracy: 0.9939 - val_loss: 0.1348 - val_accuracy: 0.9747\n", 142 | " keras.layers.Dense(128, activation=tf.nn.relu), \n", 143 | " #keras.layers.BatchNormalization() #loss: 0.0134 - accuracy: 0.9953 - val_loss: 0.1254 - val_accuracy: 0.9740\n", 144 | " keras.layers.Dense(10, activation=tf.nn.softmax)])\n", 145 | "\n", 146 | "'''\n", 147 | "Observation - \n", 148 | "1) Adding one batch normalization layer anywhere has no significant increase in accuracy.\n", 149 | "2) A model such as given below gives loss: 0.0173 - accuracy: 0.9940 - val_loss: 0.0872 - val_accuracy: 0.9790\n", 150 | "Very similar to a model with no batch normalization. Shows that the outputs of each layers are regularized wtih respect to their amplitude.\n", 151 | "model = keras.Sequential([\n", 152 | " keras.layers.Flatten(input_shape=(28, 28)),\n", 153 | " keras.layers.Dense(128, activation=tf.nn.relu, input_dim=16),\n", 154 | " keras.layers.BatchNormalization(),\n", 155 | " keras.layers.Dense(128, activation=tf.nn.relu),\n", 156 | " keras.layers.BatchNormalization(),\n", 157 | " keras.layers.Dense(128, activation=tf.nn.relu),\n", 158 | " keras.layers.BatchNormalization(),\n", 159 | " keras.layers.Dense(10, activation=tf.nn.softmax)])\n", 160 | " \n", 161 | "Conclusion - Batch normalization, multiple or singular, has no significant change in training and validation accuracy.\n", 162 | "Showing that the model is stable, regularized and consistent.\n", 163 | "'''" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": { 170 | "colab": { 171 | "base_uri": "https://localhost:8080/", 172 | "height": 34 173 | }, 174 | "id": "l0DFq3w5ffwY", 175 | "outputId": "171c2436-1801-4fda-c3c4-cb382cdf7ec9" 176 | }, 177 | "outputs": [], 178 | "source": [ 179 | "model.compile(optimizer='adam',\n", 180 | " loss='sparse_categorical_crossentropy',\n", 181 | " metrics=['accuracy'])\n", 182 | "model.summary()" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "metadata": { 189 | "id": "Wam0bgHHgAtt" 190 | }, 191 | "outputs": [], 192 | "source": [ 193 | "X_train, X_val, y_train, y_val = train_test_split(data_x, data_y, test_size = 0.1, random_state=42)" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": null, 199 | "metadata": { 200 | "id": "CCIyvvHvm0Ik" 201 | }, 202 | "outputs": [], 203 | "source": [ 204 | "model.fit(X_train, y_train, batch_size=64, epochs=5, validation_data=(X_val, y_val))" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": { 211 | "colab": { 212 | "base_uri": "https://localhost:8080/", 213 | "height": 34 214 | }, 215 | "id": "FKCrk4x_nUKL", 216 | "outputId": "5cb95ec6-8a52-410d-c021-b679a1c7bbb9" 217 | }, 218 | "outputs": [], 219 | "source": [ 220 | "model.predict(X_test).argmax(axis=1)" 221 | ] 222 | }, 223 | { 224 | "cell_type": "code", 225 | "execution_count": null, 226 | "metadata": { 227 | "colab": { 228 | "base_uri": "https://localhost:8080/", 229 | "height": 34 230 | }, 231 | "id": "RyexoL4wuzYW", 232 | "outputId": "be6b2a8d-caba-4c55-d5d9-734e3be8a7c5" 233 | }, 234 | "outputs": [], 235 | "source": [ 236 | "img = X_test.iloc[1].values.reshape((28,28))\n", 237 | "plt.imshow(img,cmap='gray')" 238 | ] 239 | } 240 | ], 241 | "metadata": { 242 | "colab": { 243 | "collapsed_sections": [], 244 | "name": "MNIST digit csv DL.ipynb", 245 | "provenance": [] 246 | }, 247 | "kernelspec": { 248 | "display_name": "Python 3", 249 | "language": "python", 250 | "name": "python3" 251 | }, 252 | "language_info": { 253 | "codemirror_mode": { 254 | "name": "ipython", 255 | "version": 3 256 | }, 257 | "file_extension": ".py", 258 | "mimetype": "text/x-python", 259 | "name": "python", 260 | "nbconvert_exporter": "python", 261 | "pygments_lexer": "ipython3", 262 | "version": "3.9.4" 263 | } 264 | }, 265 | "nbformat": 4, 266 | "nbformat_minor": 1 267 | } 268 | -------------------------------------------------------------------------------- /Session 4 - Modifications of NN/nn_keras.ipynb: -------------------------------------------------------------------------------- 1 | {"cells":[{"cell_type":"code","execution_count":null,"id":"bbcde453-fbc3-4b80-a663-01212599fa9c","metadata":{"id":"bbcde453-fbc3-4b80-a663-01212599fa9c"},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","from sklearn.preprocessing import StandardScaler\n","from sklearn.metrics import accuracy_score"]},{"cell_type":"markdown","id":"173476da-ac46-435c-81bb-858825c1908d","metadata":{"id":"173476da-ac46-435c-81bb-858825c1908d"},"source":["## Part 1: Import the Housing data and do feature transformations"]},{"cell_type":"code","execution_count":null,"id":"999b20bb-ff06-47ce-9824-3431ade3d628","metadata":{"id":"999b20bb-ff06-47ce-9824-3431ade3d628","outputId":"c9961c6e-e367-44c1-8635-b7c6a62c4cb7"},"outputs":[{"data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
bedroomssqft_livingprice
031340313000
1536502384000
231930342000
332000420000
441940550000
\n","
"],"text/plain":[" bedrooms sqft_living price\n","0 3 1340 313000\n","1 5 3650 2384000\n","2 3 1930 342000\n","3 3 2000 420000\n","4 4 1940 550000"]},"execution_count":2,"metadata":{},"output_type":"execute_result"}],"source":["df= pd.read_csv('house_price_full.csv')\n","df.head()"]},{"cell_type":"code","execution_count":null,"id":"9ac1d055-4622-4fc9-a2a3-d463bcf8befd","metadata":{"id":"9ac1d055-4622-4fc9-a2a3-d463bcf8befd"},"outputs":[],"source":["X = df.copy()\n","# Remove target\n","Y = X.pop('price')\n","\n","# perform a scaler transform of the input data\n","scaler = StandardScaler()\n","X = scaler.fit_transform(X)\n","\n","# perform log transformation of target variable (For Sandeep: Is this needed?)\n","Y = np.log(Y)"]},{"cell_type":"code","execution_count":null,"id":"82b4c4a9-00c6-42d5-aede-b01c0faeb61a","metadata":{"id":"82b4c4a9-00c6-42d5-aede-b01c0faeb61a","outputId":"14e3cdc8-62a0-4dce-c657-a6937a09b761"},"outputs":[{"data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
01
0-0.433198-0.753258
11.6757351.457330
2-0.433198-0.188649
3-0.433198-0.121661
40.621269-0.179079
.........
4940.6212690.873582
4951.6757352.299459
496-0.433198-0.724549
497-0.433198-0.179079
498-0.433198-1.040347
\n","

499 rows × 2 columns

\n","
"],"text/plain":[" 0 1\n","0 -0.433198 -0.753258\n","1 1.675735 1.457330\n","2 -0.433198 -0.188649\n","3 -0.433198 -0.121661\n","4 0.621269 -0.179079\n",".. ... ...\n","494 0.621269 0.873582\n","495 1.675735 2.299459\n","496 -0.433198 -0.724549\n","497 -0.433198 -0.179079\n","498 -0.433198 -1.040347\n","\n","[499 rows x 2 columns]"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["df_scaled = pd.DataFrame(X)\n","df_scaled"]},{"cell_type":"code","execution_count":null,"id":"8034fbca-2081-4de3-9fd3-8e5d379b2cbf","metadata":{"id":"8034fbca-2081-4de3-9fd3-8e5d379b2cbf","outputId":"22931110-1276-44a7-e878-8d189355c9cc"},"outputs":[{"data":{"text/plain":["0 12.653958\n","1 14.684290\n","2 12.742566\n","3 12.948010\n","4 13.217674\n"," ... \n","494 13.380102\n","495 13.764217\n","496 12.128111\n","497 12.721886\n","498 12.254863\n","Name: price, Length: 499, dtype: float64"]},"execution_count":5,"metadata":{},"output_type":"execute_result"}],"source":["Y"]},{"cell_type":"markdown","id":"754adc57-4f6c-4b96-a348-77bd82a4b304","metadata":{"id":"754adc57-4f6c-4b96-a348-77bd82a4b304"},"source":["## Part 2: Create Model Using `keras`\n","\n","![](multiple_neurons.png)"]},{"cell_type":"code","execution_count":null,"id":"9e7e3de3-c446-4fbe-be67-08c1431450d5","metadata":{"id":"9e7e3de3-c446-4fbe-be67-08c1431450d5"},"outputs":[],"source":["from tensorflow import keras"]},{"cell_type":"code","execution_count":null,"id":"7bf2231d-a13c-4e82-aae9-597b3caae6c0","metadata":{"id":"7bf2231d-a13c-4e82-aae9-597b3caae6c0"},"outputs":[],"source":["model = keras.Sequential(\n"," [\n"," keras.layers.Dense(\n"," 2, activation=\"sigmoid\", input_shape=(X.shape[-1],)\n"," ),\n"," keras.layers.Dense(1, activation=\"linear\")\n"," ]\n",")\n","model.summary()"]},{"cell_type":"markdown","id":"304a97ac-488e-44cf-b86b-b6f44dc81d72","metadata":{"id":"304a97ac-488e-44cf-b86b-b6f44dc81d72"},"source":["```python\n","def random_init_params():\n"," w1 = tf.Variable(tf.random.uniform((2, 2)))\n"," b1 = tf.Variable(tf.random.uniform((1, 2)))\n"," w2 = tf.Variable(tf.random.uniform((2, 1)))\n"," b2 = tf.Variable(tf.random.uniform((1, 1)))\n"," return w1,b1,w2,b2\n","\n","\n","def forward_prop(x, w1, b1, w2, b2):\n"," z1 = tf.matmul(x,w1) + b1\n"," h1 = tf.math.sigmoid(z1)\n"," z2 = tf.matmul(h1,w2) + b2\n"," h2 = z2\n"," return h2\n","```"]},{"cell_type":"code","execution_count":null,"id":"867ac40d-01c2-4f93-8977-1a3a1b37b320","metadata":{"id":"867ac40d-01c2-4f93-8977-1a3a1b37b320"},"outputs":[],"source":["model.compile(\n"," optimizer=keras.optimizers.SGD(), loss=\"mean_squared_error\"\n",")"]},{"cell_type":"markdown","id":"78a51db7-4b84-4338-9745-ece25b503892","metadata":{"id":"78a51db7-4b84-4338-9745-ece25b503892"},"source":["```python\n","def train(x, y, w1, b1, w2, b2):\n"," y_true = y\n"," with tf.GradientTape() as g:\n"," y_pred = forward_prop(x, w1, b1, w2, b2)\n","\n"," # loss\n"," loss = 0.5*(y_true - y_pred)** 2\n"," \n"," #Gradient calculation \n"," print(\"**************************************************\")\n"," print(\"GRADIENTS\")\n"," print(\"**************************************************\")\n"," gw1, gb1, gw2, gb2 = g.gradient(loss, [w1, b1, w2, b2])\n"," print(\" the gradient for 1st layer weights are:\\n\",gw1.numpy())\n"," print(\"--------------------------------------------------\")\n"," print(\" the gradient for 2nd layer weights are:\\n\",gw2.numpy())\n"," print(\"--------------------------------------------------\")\n"," print(\" the gradient for 1st layer bias are:\\n\",gb1.numpy())\n"," print(\"--------------------------------------------------\")\n"," print(\" the gradient for 2nd layer bias are:\\n\",gb2.numpy())\n"," print(\"--------------------------------------------------\")\n","\n"," # Gradient descent:\n"," lr=0.2\n"," w1.assign_sub(lr*gw1)\n"," b1.assign_sub(lr*gb1) \n"," w2.assign_sub(lr*gw2)\n"," b2.assign_sub(lr*gb2)\n"," print(\"**************************************************\")\n"," print(\"NEW UPDATES\")\n"," print(\"**************************************************\")\n"," print(\" the updated 1st layer weights are:\\n\",w1.numpy())\n"," print(\"--------------------------------------------------\")\n"," print(\" the updated 2nd layer weights are:\\n\",w2.numpy())\n"," print(\"--------------------------------------------------\")\n"," print(\" the updated 1st layer bias are:\\n\",b1.numpy())\n"," print(\"--------------------------------------------------\")\n"," print(\" the updated 2nd layer bias are:\\n\",b2.numpy())\n","\n","\n"," return w1, b1, w2, b2,loss\n","\n","```"]},{"cell_type":"code","execution_count":null,"id":"72ab0367-85da-46b9-bce9-4da2c72dfaa7","metadata":{"id":"72ab0367-85da-46b9-bce9-4da2c72dfaa7","outputId":"d50e1675-e868-44fd-b708-a0459249cadd"},"outputs":[{"name":"stderr","output_type":"stream","text":["2021-12-28 16:39:52.206212: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n","2021-12-28 16:39:52.302809: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled.\n"]},{"name":"stdout","output_type":"stream","text":["Epoch 1/10\n","16/16 [==============================] - 1s 6ms/step - loss: 130.8849\n","Epoch 2/10\n","16/16 [==============================] - 0s 3ms/step - loss: 48.3066\n","Epoch 3/10\n","16/16 [==============================] - 0s 3ms/step - loss: 15.0036\n","Epoch 4/10\n","16/16 [==============================] - 0s 3ms/step - loss: 4.2597\n","Epoch 5/10\n","16/16 [==============================] - 0s 3ms/step - loss: 1.6313\n","Epoch 6/10\n","16/16 [==============================] - 0s 3ms/step - loss: 0.9751\n","Epoch 7/10\n","16/16 [==============================] - 0s 3ms/step - loss: 0.7346\n","Epoch 8/10\n","16/16 [==============================] - 0s 3ms/step - loss: 0.5919\n","Epoch 9/10\n","16/16 [==============================] - 0s 3ms/step - loss: 0.4938\n","Epoch 10/10\n","16/16 [==============================] - 0s 3ms/step - loss: 0.4205\n"]},{"data":{"text/plain":[""]},"execution_count":9,"metadata":{},"output_type":"execute_result"}],"source":["model.fit(X,Y.values,epochs=10,batch_size=32)"]},{"cell_type":"code","execution_count":null,"id":"ac6189a0-f5a7-48e3-9bbd-03f219c6ab2c","metadata":{"id":"ac6189a0-f5a7-48e3-9bbd-03f219c6ab2c","outputId":"fccf1023-1b62-450d-9d92-0870bf5f9d9a"},"outputs":[{"data":{"text/plain":["array([13.128477 , 12.628965 , 13.178449 , 13.180457 , 13.151085 ,\n"," 12.900153 , 13.024193 , 13.047293 , 13.174511 , 13.168509 ,\n"," 13.166689 , 13.005024 , 13.178679 , 13.0899315, 12.836474 ,\n"," 13.108126 , 13.154926 , 12.962559 , 13.132423 , 13.10496 ,\n"," 13.180997 , 13.168523 , 13.169441 , 13.043495 , 13.167471 ,\n"," 13.18181 , 13.177605 , 13.125883 , 12.860254 , 13.163628 ,\n"," 13.168126 , 13.017628 , 13.159059 , 13.025801 , 12.830246 ,\n"," 13.08225 , 12.876249 , 13.175127 , 13.179413 , 13.176378 ,\n"," 13.155876 , 13.168619 , 12.891278 , 13.170714 , 12.990326 ,\n"," 13.101732 , 13.179699 , 13.178449 , 13.118698 , 12.91412 ,\n"," 13.047293 , 13.131868 , 13.154086 , 13.172497 , 13.004393 ,\n"," 13.152577 , 13.127048 , 12.626102 , 13.125765 , 13.08344 ,\n"," 12.91412 , 13.156879 , 13.043495 , 13.168009 , 13.005024 ,\n"," 12.90601 , 13.162054 , 13.075506 , 13.194309 , 13.181587 ,\n"," 13.079155 , 13.13918 , 13.11602 , 13.177332 , 13.127129 ,\n"," 13.005175 , 13.129332 , 13.043495 , 13.134769 , 13.171211 ,\n"," 13.147072 , 12.813833 , 13.190196 , 13.172497 , 13.107994 ,\n"," 13.156761 , 12.9723835, 13.145622 , 13.037858 , 12.987804 ,\n"," 12.76259 , 13.181906 , 12.640232 , 13.17798 , 13.153959 ,\n"," 12.880949 , 13.164448 , 12.7875805, 13.137455 , 12.526696 ,\n"," 12.946011 , 13.1806755, 13.127129 , 13.103783 , 13.072294 ,\n"," 13.007153 , 13.139817 , 12.628907 , 12.888296 , 13.16354 ,\n"," 13.16787 , 13.112074 , 13.082844 , 12.928171 , 13.145354 ,\n"," 13.148248 , 13.1811695, 12.879278 , 12.759796 , 13.163666 ,\n"," 13.1487255, 13.142221 , 11.773012 , 13.176501 , 13.176501 ,\n"," 13.137455 , 13.168619 , 12.9723835, 13.178093 , 13.007153 ,\n"," 13.08225 , 13.152975 , 13.035875 , 13.168279 , 13.108615 ,\n"," 13.113403 , 13.172497 , 13.145354 , 13.17192 , 13.093452 ,\n"," 13.178787 , 12.977577 , 12.891238 , 12.605877 , 13.155022 ,\n"," 13.159508 , 13.137209 , 13.11418 , 12.396421 , 13.075692 ,\n"," 13.091919 , 13.168457 , 12.961836 , 13.129808 , 13.131553 ,\n"," 13.146723 , 13.180457 , 12.942875 , 12.851604 , 13.0414295,\n"," 13.155876 , 13.135811 , 13.165733 , 13.160373 , 12.898933 ,\n"," 13.154694 , 13.168509 , 12.90601 , 13.082844 , 13.117235 ,\n"," 12.485647 , 13.114719 , 13.027815 , 13.162054 , 13.1612215,\n"," 13.175127 , 13.146221 , 13.137455 , 13.168598 , 13.1729355,\n"," 13.075506 , 12.995331 , 13.109232 , 12.998156 , 13.091919 ,\n"," 12.964886 , 13.1407795, 13.175127 , 13.157893 , 12.800024 ,\n"," 13.138671 , 13.148874 , 13.181967 , 12.867771 , 13.160373 ,\n"," 13.17192 , 13.146221 , 13.118576 , 12.980893 , 13.124384 ,\n"," 12.76082 , 12.79447 , 12.80132 , 12.836474 , 13.124014 ,\n"," 13.08909 , 12.941199 , 13.052895 , 13.112753 , 13.041578 ,\n"," 12.896376 , 13.093382 , 13.078556 , 13.177332 , 12.914703 ,\n"," 13.165212 , 13.138671 , 12.391222 , 13.179969 , 12.980154 ,\n"," 13.181587 , 13.032988 , 13.145622 , 12.841802 , 13.155965 ,\n"," 12.846737 , 13.118698 , 12.396421 , 12.978636 , 13.103354 ,\n"," 13.0339985, 13.071745 , 13.094971 , 12.594386 , 13.115758 ,\n"," 13.151976 , 13.157724 , 13.093382 , 13.174636 , 13.169339 ,\n"," 13.156879 , 11.996012 , 13.097965 , 13.163887 , 13.181628 ,\n"," 13.170086 , 13.164448 , 13.067701 , 12.885763 , 13.1584835,\n"," 13.136221 , 13.17606 , 11.89765 , 12.9456215, 13.153123 ,\n"," 13.075506 , 13.155876 , 13.06727 , 13.178093 , 13.17798 ,\n"," 12.777402 , 13.08243 , 13.08225 , 13.1729355, 12.937342 ,\n"," 12.882296 , 12.958685 , 13.170714 , 13.024193 , 13.179413 ,\n"," 12.915373 , 13.1906 , 12.493273 , 13.112074 , 13.166273 ,\n"," 13.058382 , 13.154694 , 13.181692 , 13.181061 , 13.180376 ,\n"," 13.158625 , 12.8640175, 13.085339 , 13.160373 , 12.461103 ,\n"," 13.1253605, 13.178787 , 13.137455 , 13.081032 , 12.882296 ,\n"," 12.908919 , 12.2551365, 13.128197 , 13.157724 , 13.147807 ,\n"," 13.128477 , 13.175602 , 13.168009 , 12.9456215, 13.133705 ,\n"," 13.1612215, 12.812229 , 13.132641 , 13.170086 , 13.1628685,\n"," 13.174127 , 13.124702 , 13.168617 , 13.117235 , 12.45493 ,\n"," 12.755163 , 12.974987 , 13.057295 , 12.960966 , 13.158267 ,\n"," 12.882296 , 13.065666 , 13.18181 , 12.76817 , 13.163479 ,\n"," 13.163666 , 13.075827 , 12.80132 , 13.115758 , 13.14447 ,\n"," 13.056566 , 12.928946 , 13.179413 , 12.851604 , 13.026506 ,\n"," 13.124712 , 13.133713 , 13.159616 , 13.166334 , 13.165212 ,\n"," 13.178787 , 13.110728 , 13.1407795, 12.888659 , 13.119831 ,\n"," 13.134972 , 12.951078 , 13.146723 , 13.087236 , 12.954094 ,\n"," 13.180997 , 13.178093 , 13.176925 , 13.119831 , 13.005175 ,\n"," 12.923286 , 12.919119 , 13.146221 , 12.940114 , 13.157893 ,\n"," 13.179969 , 12.474701 , 12.995331 , 13.11602 , 13.174511 ,\n"," 13.130596 , 13.11602 , 13.148874 , 13.064037 , 13.106551 ,\n"," 13.124384 , 12.865469 , 13.169441 , 13.125765 , 13.090372 ,\n"," 13.114264 , 13.117235 , 12.668648 , 12.777711 , 13.151976 ,\n"," 13.067701 , 13.143372 , 13.134972 , 13.160382 , 13.156809 ,\n"," 13.153299 , 13.1807785, 13.137455 , 12.964493 , 12.894249 ,\n"," 13.179313 , 12.940114 , 12.928946 , 12.660626 , 13.1612215,\n"," 12.780514 , 13.031808 , 13.162613 , 13.168373 , 13.175127 ,\n"," 12.923286 , 13.1718025, 12.797249 , 13.06727 , 13.134972 ,\n"," 13.118576 , 13.088812 , 12.879278 , 13.166273 , 13.119887 ,\n"," 12.851604 , 13.134972 , 13.14164 , 13.12005 , 13.181874 ,\n"," 13.084639 , 13.147807 , 13.1487255, 13.137455 , 12.805561 ,\n"," 13.127048 , 13.178093 , 13.101732 , 12.231955 , 13.146221 ,\n"," 13.1301155, 12.677629 , 13.021738 , 12.974987 , 12.964886 ,\n"," 13.138671 , 13.094971 , 13.15184 , 13.1628685, 13.136221 ,\n"," 13.056566 , 13.146221 , 13.133713 , 13.130074 , 13.169441 ,\n"," 12.981035 , 13.040086 , 13.151085 , 13.137455 , 13.1311245,\n"," 13.111227 , 13.023775 , 12.788899 , 13.109368 , 13.020288 ,\n"," 13.168604 , 12.9740925, 13.0792055, 13.15184 , 13.171326 ,\n"," 13.180457 , 13.165212 , 12.228783 , 13.122987 , 13.099442 ,\n"," 12.974987 , 13.112074 , 13.043495 , 13.021738 , 12.940114 ,\n"," 13.128477 , 12.854502 , 11.922558 , 13.0732565, 12.795837 ,\n"," 13.181967 , 13.080799 , 13.151085 , 13.129808 , 13.130449 ,\n"," 13.132423 , 12.334373 , 13.023775 , 13.1806755, 12.916623 ,\n"," 12.995331 , 13.167402 , 12.674873 , 13.153959 , 13.166615 ,\n"," 12.52608 , 12.992836 , 12.628965 , 13.181852 , 13.0774975,\n"," 12.908919 , 12.992022 , 13.134769 , 12.875654 , 12.9447975,\n"," 13.181587 , 13.163479 , 13.130596 , 13.071745 , 12.978636 ,\n"," 12.3860445, 13.132423 , 13.178787 , 13.081032 ], dtype=float32)"]},"execution_count":17,"metadata":{},"output_type":"execute_result"}],"source":["model.predict(X)[:,0]"]},{"cell_type":"code","execution_count":null,"id":"e70387c2-c04d-4ff1-8bc1-a13a064daf6a","metadata":{"id":"e70387c2-c04d-4ff1-8bc1-a13a064daf6a"},"outputs":[],"source":[""]}],"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.9"},"colab":{"name":"nn_keras.ipynb","provenance":[]}},"nbformat":4,"nbformat_minor":5} -------------------------------------------------------------------------------- /Assignment Keras/Assignment+Using+Keras.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "In this assignment, you'll implement a L-layer deep model on MNIST dataset using Keras. The MNIST dataset contains tens of thousands of scanned images of handwritten digits, together with their correct classifications. MNIST's name comes from the fact that it is a modified subset of two data sets collected by NIST, the United States' National Institute of Standards and Technology.
\n", 8 | "
\n", 9 | "
\n", 10 | "To use Keras, you'll need to install Keras and Tensorflow.\n", 11 | "
\n", 12 | "Please run the following commands if you don't have Keras and TensorFlow already installed.\n", 13 | "
\n", 14 | "1. ! pip install TensorFlow\n", 15 | "
\n", 16 | "2. ! pip install keras\n", 17 | "
\n", 18 | "3. ! pip install msgpack\n" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": {}, 25 | "outputs": [ 26 | { 27 | "name": "stderr", 28 | "output_type": "stream", 29 | "text": [ 30 | "C:\\Users\\Kaustabh Singh\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", 31 | " from ._conv import register_converters as _register_converters\n", 32 | "Using TensorFlow backend.\n" 33 | ] 34 | } 35 | ], 36 | "source": [ 37 | "import numpy as np\n", 38 | "import pickle\n", 39 | "import gzip\n", 40 | "import matplotlib.pyplot as plt\n", 41 | "import pandas as pd\n", 42 | "import numpy as np\n", 43 | "import matplotlib.pyplot as plt\n", 44 | "import h5py\n", 45 | "import sklearn\n", 46 | "import sklearn.datasets\n", 47 | "import scipy\n", 48 | "\n", 49 | "from PIL import Image\n", 50 | "from scipy import ndimage\n", 51 | "\n", 52 | "\n", 53 | "from keras.models import Sequential\n", 54 | "from keras.layers import Dense, Dropout, BatchNormalization, Activation\n", 55 | "from keras import regularizers\n", 56 | "\n", 57 | "np.random.seed(7)\n", 58 | "\n", 59 | "\n", 60 | "%matplotlib inline" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": {}, 66 | "source": [ 67 | "The MNIST dataset we use here is 'mnist.pkl.gz' which is divided into training, validation and test data. The following function load_data() unpacks the file and extracts the training, validation and test data." 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 2, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "def load_data():\n", 77 | " f = gzip.open('mnist.pkl.gz', 'rb')\n", 78 | " f.seek(0)\n", 79 | " training_data, validation_data, test_data = pickle.load(f, encoding='latin1')\n", 80 | " f.close()\n", 81 | " return (training_data, validation_data, test_data)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "Let's see how the data looks:" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 3, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "training_data, validation_data, test_data = load_data()" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 4, 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "data": { 107 | "text/plain": [ 108 | "(array([[0., 0., 0., ..., 0., 0., 0.],\n", 109 | " [0., 0., 0., ..., 0., 0., 0.],\n", 110 | " [0., 0., 0., ..., 0., 0., 0.],\n", 111 | " ...,\n", 112 | " [0., 0., 0., ..., 0., 0., 0.],\n", 113 | " [0., 0., 0., ..., 0., 0., 0.],\n", 114 | " [0., 0., 0., ..., 0., 0., 0.]], dtype=float32),\n", 115 | " array([5, 0, 4, ..., 8, 4, 8], dtype=int64))" 116 | ] 117 | }, 118 | "execution_count": 4, 119 | "metadata": {}, 120 | "output_type": "execute_result" 121 | } 122 | ], 123 | "source": [ 124 | "training_data" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 5, 130 | "metadata": {}, 131 | "outputs": [ 132 | { 133 | "name": "stdout", 134 | "output_type": "stream", 135 | "text": [ 136 | "The feature dataset is:[[0. 0. 0. ... 0. 0. 0.]\n", 137 | " [0. 0. 0. ... 0. 0. 0.]\n", 138 | " [0. 0. 0. ... 0. 0. 0.]\n", 139 | " ...\n", 140 | " [0. 0. 0. ... 0. 0. 0.]\n", 141 | " [0. 0. 0. ... 0. 0. 0.]\n", 142 | " [0. 0. 0. ... 0. 0. 0.]]\n", 143 | "The target dataset is:[5 0 4 ... 8 4 8]\n", 144 | "The number of examples in the training dataset is:50000\n", 145 | "The number of points in a single input is:784\n" 146 | ] 147 | } 148 | ], 149 | "source": [ 150 | "print(\"The feature dataset is:\" + str(training_data[0]))\n", 151 | "print(\"The target dataset is:\" + str(training_data[1]))\n", 152 | "print(\"The number of examples in the training dataset is:\" + str(len(training_data[0])))\n", 153 | "print(\"The number of points in a single input is:\" + str(len(training_data[0][1])))" 154 | ] 155 | }, 156 | { 157 | "cell_type": "markdown", 158 | "metadata": {}, 159 | "source": [ 160 | "Now, as discussed earlier in the lectures, the target variable is converted to a one hot matrix. We use the function one_hot to convert the target dataset to one hot encoding." 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 6, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "def one_hot(j):\n", 170 | " # input is the target dataset of shape (1, m) where m is the number of data points\n", 171 | " # returns a 2 dimensional array of shape (10, m) where each target value is converted to a one hot encoding\n", 172 | " # Look at the next block of code for a better understanding of one hot encoding\n", 173 | " n = j.shape[0]\n", 174 | " new_array = np.zeros((10, n))\n", 175 | " index = 0\n", 176 | " for res in j:\n", 177 | " new_array[res][index] = 1.0\n", 178 | " index = index + 1\n", 179 | " return new_array" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 7, 185 | "metadata": {}, 186 | "outputs": [ 187 | { 188 | "data": { 189 | "text/plain": [ 190 | "array([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", 191 | " [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n", 192 | " [0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],\n", 193 | " [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n", 194 | " [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],\n", 195 | " [0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],\n", 196 | " [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.],\n", 197 | " [0., 0., 0., 0., 0., 0., 0., 1., 0., 0.],\n", 198 | " [0., 0., 0., 0., 0., 0., 0., 0., 1., 0.],\n", 199 | " [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]])" 200 | ] 201 | }, 202 | "execution_count": 7, 203 | "metadata": {}, 204 | "output_type": "execute_result" 205 | } 206 | ], 207 | "source": [ 208 | "data = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n", 209 | "one_hot(data)" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": 8, 215 | "metadata": {}, 216 | "outputs": [], 217 | "source": [ 218 | "def data_wrapper():\n", 219 | " tr_d, va_d, te_d = load_data()\n", 220 | " \n", 221 | " training_inputs = np.array(tr_d[0][:]).T\n", 222 | " training_results = np.array(tr_d[1][:])\n", 223 | " train_set_y = one_hot(training_results)\n", 224 | " \n", 225 | " validation_inputs = np.array(va_d[0][:]).T\n", 226 | " validation_results = np.array(va_d[1][:])\n", 227 | " validation_set_y = one_hot(validation_results)\n", 228 | " \n", 229 | " test_inputs = np.array(te_d[0][:]).T\n", 230 | " test_results = np.array(te_d[1][:])\n", 231 | " test_set_y = one_hot(test_results)\n", 232 | " \n", 233 | " return (training_inputs, train_set_y, validation_inputs, validation_set_y)" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": 9, 239 | "metadata": {}, 240 | "outputs": [], 241 | "source": [ 242 | "train_set_x, train_set_y, test_set_x, test_set_y = data_wrapper()" 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "For implementing in Keras, the input training and input target dataset are supposed to have shape (m, n) where m is the number of training samples and n is the number of parts in a single input.\n", 250 | "
Hence, let create the desired dataset shapes by taking transpose." 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 10, 256 | "metadata": {}, 257 | "outputs": [], 258 | "source": [ 259 | "train_set_x = train_set_x.T\n", 260 | "train_set_y = train_set_y.T\n", 261 | "test_set_x = test_set_x.T\n", 262 | "test_set_y = test_set_y.T" 263 | ] 264 | }, 265 | { 266 | "cell_type": "markdown", 267 | "metadata": {}, 268 | "source": [ 269 | "Now, let's see if the datasets are in the desired shape:" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 11, 275 | "metadata": {}, 276 | "outputs": [ 277 | { 278 | "name": "stdout", 279 | "output_type": "stream", 280 | "text": [ 281 | "train_set_x shape: (50000, 784)\n", 282 | "train_set_y shape: (50000, 10)\n", 283 | "test_set_x shape: (10000, 784)\n", 284 | "test_set_y shape: (10000, 10)\n" 285 | ] 286 | } 287 | ], 288 | "source": [ 289 | "print (\"train_set_x shape: \" + str(train_set_x.shape))\n", 290 | "print (\"train_set_y shape: \" + str(train_set_y.shape))\n", 291 | "print (\"test_set_x shape: \" + str(test_set_x.shape))\n", 292 | "print (\"test_set_y shape: \" + str(test_set_y.shape))" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "Now let us visualise the dataset. Feel free to change the index to see if the training data has been correctly tagged." 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": 12, 305 | "metadata": {}, 306 | "outputs": [ 307 | { 308 | "data": { 309 | "text/plain": [ 310 | "" 311 | ] 312 | }, 313 | "execution_count": 12, 314 | "metadata": {}, 315 | "output_type": "execute_result" 316 | }, 317 | { 318 | "data": { 319 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEJJJREFUeJzt3X2QVfV9x/H3x5WYBrABKQ+uGBJER9saUxiaKUxKTJNSx45mRA01lY5Y0jaMzRi16uhI01ohk0Ts1MmU1AfABMSIyhinieNoTOxIXRgJRBqDDCphZQVU0Oog+O0f92xnXe89d/c+nbv7+7xmdu7d8z0PX+7w2XPOPefenyICM0vPMUU3YGbFcPjNEuXwmyXK4TdLlMNvliiH3yxRDn8iJD0h6fJGLyvpekn/UV93VgSHf4iRtEvSnxTdR6+I+JeIGPQfFUljJT0g6S1JL0r6i2b0Z5UdW3QDlqzbgcPABOAs4EeStkTEL4ttKx3e8w8TksZIeljSq5Jey56f1G+2qZL+W9Ibkh6SNLbP8p+W9F+SXpe0RdKcAW53iaR7sucflnSPpP3Zep6RNKHMMiOBC4AbI+LNiPg5sAH4y1r//TZ4Dv/wcQxwF/Ax4GTgbeDf+s1zKXAZcCJwBPhXAEmdwI+AfwbGAlcB90v6nUH2sAD4bWAycALwN1kf/Z0KHI2I5/tM2wL87iC3Z3Vw+IeJiNgfEfdHxP9GxCHgZuCP+822OiK2RcRbwI3ARZI6gC8Dj0TEIxHxXkQ8CnQB5wyyjXcphf6UiDgaEZsi4mCZ+UYBb/Sb9gYwepDbszo4/MOEpI9I+vfszbODwJPAR7Nw93q5z/MXgRHAOEpHCxdmh+qvS3odmA1MGmQbq4EfA2sl7ZH0TUkjysz3JnB8v2nHA4cGuT2rg8M/fHwdOA34w4g4HvhMNl195pnc5/nJlPbU+yj9UVgdER/t8zMyIpYOpoGIeDci/jEizgD+CDiX0qlGf88Dx0qa1mfaJwG/2ddCDv/QNCJ7c63351hKh8xvA69nb+TdVGa5L0s6Q9JHgG8AP4yIo8A9wJ9L+lNJHdk655R5wzCXpM9K+v3saOMgpT8uR/vPl512rAe+IWmkpFnAeZSOHKxFHP6h6RFKQe/9WQIsB36L0p78aeA/yyy3GrgbeAX4MHAFQES8TCl81wOvUjoSuJrB//+YCPyQUvC3Az+l9IelnL/L+u0B1gB/68t8rSV/mYdZmrznN0uUw2+WKIffLFEOv1miWvrBHkl+d9GsySJC1eeqc88vaa6kX0naIenaetZlZq1V86W+7EaO54HPA7uBZ4D5EfFczjLe85s1WSv2/DOBHRGxMyIOA2sp3ShiZkNAPeHv5P0fFNmdTXsfSYskdUnqqmNbZtZg9bzhV+7Q4gOH9RGxAlgBPuw3ayf17Pl38/5PiZ0E7KmvHTNrlXrC/wwwTdLHJX0I+BKlr2IysyGg5sP+iDgiaTGlL2/oAO70p7LMho6WfqrP5/xmzdeSm3zMbOhy+M0S5fCbJcrhN0uUw2+WKIffLFEOv1miHH6zRDn8Zoly+M0S5fCbJcrhN0uUw2+WKIffLFEOv1miHH6zRDn8Zoly+M0S5fCbJcrhN0uUw2+WKIffLFEOv1miHH6zRDn8Zoly+M0S5fCbJcrhN0uUw2+WqJqH6LY0nHLKKbn1K664Ire+ePHiijUpfzDZI0eO5NYvv/zy3PqaNWsq1g4fPpy7bArqCr+kXcAh4ChwJCJmNKIpM2u+Ruz5PxsR+xqwHjNrIZ/zmyWq3vAH8BNJmyQtKjeDpEWSuiR11bktM2ugeg/7Z0XEHknjgUcl/U9EPNl3hohYAawAkBR1bs/MGqSuPX9E7Mkee4AHgJmNaMrMmq/m8EsaKWl073PgC8C2RjVmZs2liNqOxCV9gtLeHkqnDz+IiJurLOPD/hbr6OjIrV966aW59WXLluXWx40bN+ieevX09OTWx48fX/O6AaZNm1ax9sILL9S17nYWEfk3UGRqPuePiJ3AJ2td3syK5Ut9Zoly+M0S5fCbJcrhN0uUw2+WqJov9dW0MV/qa4r58+dXrE2fPj132SuvvLKubT/44IO59dtvv71irdrltrVr1+bWZ87Mv6fsiSeeqFg7++yzc5cdygZ6qc97frNEOfxmiXL4zRLl8JslyuE3S5TDb5Yoh98sUb7OPwTkff01wG233VaxVu3rsffv359bnzt3bm598+bNufV6/n+NGjUqt37w4MGatz1r1qzcZZ9++uncejvzdX4zy+XwmyXK4TdLlMNvliiH3yxRDr9Zohx+s0R5iO42UO16drXr/HnX8t96663cZc8999zc+qZNm3LrzVRtGO3t27fn1k8//fRGtjPseM9vliiH3yxRDr9Zohx+s0Q5/GaJcvjNEuXwmyXK1/nbwOjRo3Prp556as3rXr58eW5948aNNa+72apd59+6dWtu3df581Xd80u6U1KPpG19po2V9KikX2ePY5rbppk12kAO++8G+n+dy7XAYxExDXgs+93MhpCq4Y+IJ4ED/SafB6zMnq8Ezm9wX2bWZLWe80+IiG6AiOiWNL7SjJIWAYtq3I6ZNUnT3/CLiBXACvAXeJq1k1ov9e2VNAkge+xpXEtm1gq1hn8DsCB7vgB4qDHtmFmrVD3sl7QGmAOMk7QbuAlYCqyTtBB4CbiwmU0OdyeccEJdy+d9Zv+uu+6qa902fFUNf0TMr1D6XIN7MbMW8u29Zoly+M0S5fCbJcrhN0uUw2+WKH+ktw3MmzevruXXrVtXsbZz58661m3Dl/f8Zoly+M0S5fCbJcrhN0uUw2+WKIffLFEOv1mifJ2/Bap9ZHfhwoV1rb+rq6uu5dvVcccdl1ufNWtWizoZnrznN0uUw2+WKIffLFEOv1miHH6zRDn8Zoly+M0S5ev8LXDaaafl1js7O+ta/4ED/YdSHB46Ojpy69Vet3feeadi7e23366pp+HEe36zRDn8Zoly+M0S5fCbJcrhN0uUw2+WKIffLFG+zj8MbNiwoegW2tKOHTsq1rZs2dLCTtpT1T2/pDsl9Uja1mfaEkm/kfRs9nNOc9s0s0YbyGH/3cDcMtNvjYizsp9HGtuWmTVb1fBHxJPA8Lx/1Cxh9bzht1jSL7LTgjGVZpK0SFKXpOH5RXNmQ1St4f8uMBU4C+gGvl1pxohYEREzImJGjdsysyaoKfwRsTcijkbEe8D3gJmNbcvMmq2m8Eua1OfXLwLbKs1rZu2p6nV+SWuAOcA4SbuBm4A5ks4CAtgFfKWJPVqiFixYUNfyy5Yta1Anw1PV8EfE/DKT72hCL2bWQr691yxRDr9Zohx+s0Q5/GaJcvjNEqWIaN3GpNZtrI2MGDEit/7cc8/l1qdOnZpbHzlyZMVaO39F9cSJE3Prmzdvrmv5E088sWLtlVdeyV12KIsIDWQ+7/nNEuXwmyXK4TdLlMNvliiH3yxRDr9Zohx+s0T5q7tb4N13382tHz16tEWdtJfZs2fn1qtdx6/2urXyHpahyHt+s0Q5/GaJcvjNEuXwmyXK4TdLlMNvliiH3yxRvs4/DHR2dlas5Q1T3Qrjx4+vWLvhhhtyl612HX/hwoW59b179+bWU+c9v1miHH6zRDn8Zoly+M0S5fCbJcrhN0uUw2+WqIEM0T0ZWAVMBN4DVkTEbZLGAvcCUygN031RRLzWvFaHr3vvvTe3fuONN+bW582bV7G2dOnSmnoaqI6Ojtz6NddcU7F25pln5i7b3d2dW1+1alVu3fINZM9/BPh6RJwOfBr4qqQzgGuBxyJiGvBY9ruZDRFVwx8R3RGxOXt+CNgOdALnASuz2VYC5zerSTNrvEGd80uaAnwK2AhMiIhuKP2BACrfx2lmbWfA9/ZLGgXcD3wtIg5KAxoODEmLgEW1tWdmzTKgPb+kEZSC//2IWJ9N3itpUlafBPSUWzYiVkTEjIiY0YiGzawxqoZfpV38HcD2iPhOn9IGYEH2fAHwUOPbM7NmqTpEt6TZwM+ArZQu9QFcT+m8fx1wMvAScGFEHKiyLn+XchkXXHBBbv2+++7Lre/atatibfr06bnLvvZafVdnL7nkktz66tWrK9YOHMj978LcuXNz611dXbn1VA10iO6q5/wR8XOg0so+N5imzKx9+A4/s0Q5/GaJcvjNEuXwmyXK4TdLlMNvlih/dXcbePzxx3Pr+/fvz61PmTKlYu3qq6/OXfbWW2/NrV922WW59byP7FazfPny3Lqv4zeX9/xmiXL4zRLl8JslyuE3S5TDb5Yoh98sUQ6/WaKqfp6/oRvz5/lrMmNG/pcgPfXUUxVrI0aMyF123759ufWxY8fm1o85Jn//sX79+oq1iy++OHfZakN0W3kD/Ty/9/xmiXL4zRLl8JslyuE3S5TDb5Yoh98sUQ6/WaJ8nX8YuOqqqyrWrrvuutxlx4wZU9e2b7nlltx63vcFVLvHwGrj6/xmlsvhN0uUw2+WKIffLFEOv1miHH6zRDn8Zomqep1f0mRgFTAReA9YERG3SVoC/DXwajbr9RHxSJV1+Tq/WZMN9Dr/QMI/CZgUEZsljQY2AecDFwFvRsS3BtqUw2/WfAMNf9UReyKiG+jOnh+StB3orK89MyvaoM75JU0BPgVszCYtlvQLSXdKKnufqKRFkrokeewlszYy4Hv7JY0CfgrcHBHrJU0A9gEB/BOlU4Pcgd182G/WfA075weQNAJ4GPhxRHynTH0K8HBE/F6V9Tj8Zk3WsA/2SBJwB7C9b/CzNwJ7fRHYNtgmzaw4A3m3fzbwM2ArpUt9ANcD84GzKB327wK+kr05mLcu7/nNmqyhh/2N4vCbNZ8/z29muRx+s0Q5/GaJcvjNEuXwmyXK4TdLlMNvliiH3yxRDr9Zohx+s0Q5/GaJcvjNEuXwmyXK4TdLVNUv8GywfcCLfX4fl01rR+3aW7v2Be6tVo3s7WMDnbGln+f/wMalroiYUVgDOdq1t3btC9xbrYrqzYf9Zoly+M0SVXT4VxS8/Tzt2lu79gXurVaF9FboOb+ZFafoPb+ZFcThN0tUIeGXNFfSryTtkHRtET1UImmXpK2Sni16fMFsDMQeSdv6TBsr6VFJv84ey46RWFBvSyT9JnvtnpV0TkG9TZb0uKTtkn4p6e+z6YW+djl9FfK6tfycX1IH8DzweWA38AwwPyKea2kjFUjaBcyIiMJvCJH0GeBNYFXvUGiSvgkciIil2R/OMRHxD23S2xIGOWx7k3qrNKz8X1Hga9fI4e4boYg9/0xgR0TsjIjDwFrgvAL6aHsR8SRwoN/k84CV2fOVlP7ztFyF3tpCRHRHxObs+SGgd1j5Ql+7nL4KUUT4O4GX+/y+mwJfgDIC+ImkTZIWFd1MGRN6h0XLHscX3E9/VYdtb6V+w8q3zWtXy3D3jVZE+MsNJdRO1xtnRcQfAH8GfDU7vLWB+S4wldIYjt3At4tsJhtW/n7gaxFxsMhe+irTVyGvWxHh3w1M7vP7ScCeAvooKyL2ZI89wAOUTlPayd7eEZKzx56C+/l/EbE3Io5GxHvA9yjwtcuGlb8f+H5ErM8mF/7aleurqNetiPA/A0yT9HFJHwK+BGwooI8PkDQyeyMGSSOBL9B+Q49vABZkzxcADxXYy/u0y7DtlYaVp+DXrt2Guy/kDr/sUsZyoAO4MyJubnkTZUj6BKW9PZQ+7vyDInuTtAaYQ+kjn3uBm4AHgXXAycBLwIUR0fI33ir0NodBDtvepN4qDSu/kQJfu0YOd9+Qfnx7r1mafIefWaIcfrNEOfxmiXL4zRLl8JslyuE3S5TDb5ao/wPMtvAoDfUVWwAAAABJRU5ErkJggg==\n", 320 | "text/plain": [ 321 | "
" 322 | ] 323 | }, 324 | "metadata": {}, 325 | "output_type": "display_data" 326 | } 327 | ], 328 | "source": [ 329 | "index = 1000\n", 330 | "k = train_set_x[index,:]\n", 331 | "k = k.reshape((28, 28))\n", 332 | "plt.title('Label is {label}'.format(label= training_data[1][index]))\n", 333 | "plt.imshow(k, cmap='gray')" 334 | ] 335 | }, 336 | { 337 | "cell_type": "markdown", 338 | "metadata": {}, 339 | "source": [ 340 | "Keras is a framework. So, to implement a neural network model in Keras, we first create an instance of Sequential().
\n", 341 | "The Sequential model is a linear stack of layers. We then keep adding Dense layers that are fully connected layers as we desire.

\n", 342 | "We have included Dropout using nn_model.add(Dropout(0.3))

\n", 343 | "We can also include regularization using the command
nn_model.add(Dense(21, activation='relu', kernel_regularizer=regularizers.l2(0.01)))
instead of
nn_model.add(Dense(21, activation='relu')) " 344 | ] 345 | }, 346 | { 347 | "cell_type": "code", 348 | "execution_count": 13, 349 | "metadata": {}, 350 | "outputs": [], 351 | "source": [ 352 | "# create model\n", 353 | "nn_model = Sequential()\n", 354 | "nn_model.add(Dense(35, input_dim=784, activation='relu'))\n", 355 | "nn_model.add(Dropout(0.3))\n", 356 | "nn_model.add(Dense(21, activation = 'relu'))\n", 357 | "nn_model.add(Dense(10, activation='softmax'))" 358 | ] 359 | }, 360 | { 361 | "cell_type": "markdown", 362 | "metadata": {}, 363 | "source": [ 364 | "Before we run the model on the training datasets, we compile the model in which we define various things like the loss function, the optimizer and the evaluation metric." 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": 14, 370 | "metadata": {}, 371 | "outputs": [], 372 | "source": [ 373 | "nn_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])" 374 | ] 375 | }, 376 | { 377 | "cell_type": "markdown", 378 | "metadata": {}, 379 | "source": [ 380 | "Now, to fit the model on the training input and training target dataset, we run the following command using a minibatch of size 10 and 10 epochs." 381 | ] 382 | }, 383 | { 384 | "cell_type": "code", 385 | "execution_count": 15, 386 | "metadata": {}, 387 | "outputs": [ 388 | { 389 | "name": "stdout", 390 | "output_type": "stream", 391 | "text": [ 392 | "Epoch 1/10\n", 393 | "50000/50000 [==============================] - 10s 204us/step - loss: 0.5016 - acc: 0.8458\n", 394 | "Epoch 2/10\n", 395 | "50000/50000 [==============================] - 10s 203us/step - loss: 0.3222 - acc: 0.9018\n", 396 | "Epoch 3/10\n", 397 | "50000/50000 [==============================] - 10s 195us/step - loss: 0.2861 - acc: 0.9119\n", 398 | "Epoch 4/10\n", 399 | "50000/50000 [==============================] - 10s 196us/step - loss: 0.2657 - acc: 0.9172\n", 400 | "Epoch 5/10\n", 401 | "50000/50000 [==============================] - 10s 203us/step - loss: 0.2498 - acc: 0.9227\n", 402 | "Epoch 6/10\n", 403 | "50000/50000 [==============================] - 10s 195us/step - loss: 0.2467 - acc: 0.9225\n", 404 | "Epoch 7/10\n", 405 | "50000/50000 [==============================] - 10s 195us/step - loss: 0.2307 - acc: 0.9281\n", 406 | "Epoch 8/10\n", 407 | "50000/50000 [==============================] - 10s 191us/step - loss: 0.2283 - acc: 0.9286\n", 408 | "Epoch 9/10\n", 409 | "50000/50000 [==============================] - 9s 182us/step - loss: 0.2217 - acc: 0.9301\n", 410 | "Epoch 10/10\n", 411 | "50000/50000 [==============================] - 10s 202us/step - loss: 0.2195 - acc: 0.9305\n" 412 | ] 413 | }, 414 | { 415 | "data": { 416 | "text/plain": [ 417 | "" 418 | ] 419 | }, 420 | "execution_count": 15, 421 | "metadata": {}, 422 | "output_type": "execute_result" 423 | } 424 | ], 425 | "source": [ 426 | "nn_model.fit(train_set_x, train_set_y, epochs=10, batch_size=10)" 427 | ] 428 | }, 429 | { 430 | "cell_type": "code", 431 | "execution_count": 16, 432 | "metadata": {}, 433 | "outputs": [ 434 | { 435 | "name": "stdout", 436 | "output_type": "stream", 437 | "text": [ 438 | "50000/50000 [==============================] - 2s 30us/step\n", 439 | "\n", 440 | "acc: 96.89%\n" 441 | ] 442 | } 443 | ], 444 | "source": [ 445 | "scores_train = nn_model.evaluate(train_set_x, train_set_y)\n", 446 | "print(\"\\n%s: %.2f%%\" % (nn_model.metrics_names[1], scores_train[1]*100))" 447 | ] 448 | }, 449 | { 450 | "cell_type": "markdown", 451 | "metadata": {}, 452 | "source": [ 453 | "We can see that the model has ~ 97% accuracy on the training dataset." 454 | ] 455 | }, 456 | { 457 | "cell_type": "markdown", 458 | "metadata": {}, 459 | "source": [ 460 | "Now, let's make predictions on the test dataset." 461 | ] 462 | }, 463 | { 464 | "cell_type": "code", 465 | "execution_count": 17, 466 | "metadata": {}, 467 | "outputs": [ 468 | { 469 | "data": { 470 | "text/plain": [ 471 | "array([3, 8, 6, ..., 5, 6, 8], dtype=int64)" 472 | ] 473 | }, 474 | "execution_count": 17, 475 | "metadata": {}, 476 | "output_type": "execute_result" 477 | } 478 | ], 479 | "source": [ 480 | "predictions = nn_model.predict(test_set_x)\n", 481 | "predictions = np.argmax(predictions, axis = 1)\n", 482 | "predictions" 483 | ] 484 | }, 485 | { 486 | "cell_type": "code", 487 | "execution_count": 18, 488 | "metadata": {}, 489 | "outputs": [ 490 | { 491 | "name": "stdout", 492 | "output_type": "stream", 493 | "text": [ 494 | "10000/10000 [==============================] - 0s 31us/step\n", 495 | "\n", 496 | "acc: 96.46%\n" 497 | ] 498 | } 499 | ], 500 | "source": [ 501 | "scores_test = nn_model.evaluate(test_set_x, test_set_y)\n", 502 | "print(\"\\n%s: %.2f%%\" % (nn_model.metrics_names[1], scores_test[1]*100))" 503 | ] 504 | }, 505 | { 506 | "cell_type": "markdown", 507 | "metadata": {}, 508 | "source": [ 509 | "We can see that the model has ~96% accuracy on the training dataset." 510 | ] 511 | }, 512 | { 513 | "cell_type": "markdown", 514 | "metadata": {}, 515 | "source": [ 516 | "Try and look at the different test cases and check which all have gone wrong. Feel free to change the index numbers." 517 | ] 518 | }, 519 | { 520 | "cell_type": "code", 521 | "execution_count": 19, 522 | "metadata": {}, 523 | "outputs": [ 524 | { 525 | "data": { 526 | "text/plain": [ 527 | "" 528 | ] 529 | }, 530 | "execution_count": 19, 531 | "metadata": {}, 532 | "output_type": "execute_result" 533 | }, 534 | { 535 | "data": { 536 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAESFJREFUeJzt3X2wVPV9x/H3B9RJRKgPVKSKwVCbSnXElhKmOj5MqiU2PmRGnTBqwERJZqKNM7FTpDrSVoPGYk2noxMUBdRgnOADQ22M4xRp7Wi9OkZJkKiMKA+CqMglOpWHb//Yc5Plunv23t2zexZ+n9fMnbt7vufhexc+e87u2bM/RQRmlp4hZTdgZuVw+M0S5fCbJcrhN0uUw2+WKIffLFEO/15K0nJJlxe9rKRZku5ucr1zJF3dzLKD3M5tkr7d7u3s6xz+kkl6U9Jflt1Hn4j4fkQM+klF0u8DXwd+lN0fKykkba/6uX4Q6wtJv6latvoJ6Vbg7yUdMNg+7Xf2K7sB22dMBx6PiI/7TT84InY2uc4TI+L1/hMjYqOkV4FzgZ82ue7kec/fpSQdImmZpHclfZDdPqrfbOMk/a+kDyU9JunQquUnS/ofSVsl/ULS6QPc7mxJ92e3PyPpfknvZet5XtKoOot+GXi6mb+1ScuBv+7g9vY5Dn/3GgLcC3wOOBr4GPi3fvN8HfgG8AfATuBfASQdCfw7cCNwKHANsCQ7NB+MacDvAWOAw4BvZ33UcgKwusb0tZLWSbpX0shBbn+FpHckPSxpbL/aKuDEQa7Pqjj8XSoi3ouIJRHxUUT0AjcBp/Wb7b6IWBkRvwGuBy6SNBS4hMoh+OMRsTsingR6gLMH2cYOKqH/w4jYFREvRMS2OvMeDPRW3d8C/DmVJ68/A4YDDwxi26cBY4E/BjYAyyRVv0ztzbZpTXL4u5SkAyX9SNJaSduAFcDBWbj7vF11ey2wPzCSSuAuzA7Vt0raCpwCjB5kG/cBTwAPStog6QeS9q8z7wdUAg5ARGyPiJ6I2BkRm4ArgbMkjRjIhiNiRUR8EhFbge8CxwDHVc0yHNg6yL/Hqjj83et7wBeAL0bECODUbLqq5hlTdftoKnvqLVSeFO6LiIOrfoZFxM2DaSAidkTEP0TEeOAvgK9QealRy8vAH+Wtrkb/g2qn37LHAb9ocl2Gw98t9s/eXOv72Y/Knu1jYGv2Rt4NNZa7RNJ4SQcC/wj8NCJ2AfcD50j6K0lDs3WeXuMNw1ySzpB0Qna0sY3Kk8uuOrM/TtXLEklflPQFSUMkHUbl/YjlEfFhVp8taXmd7f6JpAlZ7wcBc4H1VF7n9zkN+I/B/D22J4e/OzxOJeh9P7OB24HPUtmTPwv8rMZy9wELgHeAzwB/AxARbwPnAbOAd6kcCfwtg//3PoLKqbRtVIL3NJUnlloWAWdL+mx2//NZz73ASuD/gKlV848BnqmzrlHAT7LtrqHy2v8rEbEDQNJoYDzw6CD/Hqsif5mHFUXS94HNEXH7AOZ9CfhSRLzXxHbmAm9ExB1NtGkZh98sUT7sN0uUw2+WKIffLFEdvbBHkt9gMGuziBjQZyla2vNLmiJptaTXJc1sZV1m1llNv9ufffDj18CZwDrgeWBqRPwqZxnv+c3arBN7/knA6xGxJiI+AR6k8sESM9sLtBL+I9nzwpJ12bQ9SJohqUdSTwvbMrOCtfKGX61Di08d1kfEPGAe+LDfrJu0sudfx55XlR1F5bprM9sLtBL+54FjJR2TfZHi14ClxbRlZu3W9GF/ROyUdCWVL3sYCtwTEb8srDMza6uOXtjj1/xm7deRD/mY2d7L4TdLlMNvliiH3yxRDr9Zohx+s0Q5/GaJcvjNEuXwmyXK4TdLlMNvliiH3yxRDr9Zohx+s0Q5/GaJcvjNEuXwmyXK4TdLlMNvliiH3yxRDr9Zohx+s0Q5/GaJcvjNEuXwmyXK4TdLlMNvliiH3yxRDr9ZopoeotusbNOnT8+tjx49um7tkksuyV12/PjxzbT0W7Nmzcqtz5kzp6X1F6Gl8Et6E+gFdgE7I2JiEU2ZWfsVsec/IyK2FLAeM+sgv+Y3S1Sr4Q/g55JekDSj1gySZkjqkdTT4rbMrECtHvafHBEbJB0OPCnp1YhYUT1DRMwD5gFIiha3Z2YFaWnPHxEbst+bgUeASUU0ZWbt13T4JQ2TNLzvNnAWsLKoxsysvVo57B8FPCKpbz0/joifFdKV7TUmTsw/uzt58uS6tZNOOil32QsuuCC3fuCBB+bWhwxp/sC2t7c3t75gwYLcek9P97/F1XT4I2INcGKBvZhZB/lUn1miHH6zRDn8Zoly+M0S5fCbJUoRnfvQnT/h13mNToc1Op125pln5tbPPffc3PpBBx2UW8/z1ltv5daXL1+eW1+3bl3d2p133pm77K5du3LrmzZtyq2XKSI0kPm85zdLlMNvliiH3yxRDr9Zohx+s0Q5/GaJcvjNEuWv7t4LnHLKKbn1vK+Zvuaaa3KXHTduXFM99Xnttddy6/Pnz69bW7p0ae6y27dvz62vX78+t275vOc3S5TDb5Yoh98sUQ6/WaIcfrNEOfxmiXL4zRLl6/m7wKRJ+WOdNLr2fMKECUW2s4c77rgjt37LLbfk1vOuqbf28PX8ZpbL4TdLlMNvliiH3yxRDr9Zohx+s0Q5/GaJ8nn+Dmh0Pf6yZcty68OHDy+ynT189NFHufXjjz8+t7527doi27ECFHaeX9I9kjZLWlk17VBJT0p6Lft9SCvNmlnnDeSwfwEwpd+0mcBTEXEs8FR238z2Ig3DHxErgPf7TT4PWJjdXgicX3BfZtZmzX6H36iI2AgQERslHV5vRkkzgBlNbsfM2qTtX+AZEfOAeZDuG35m3ajZU32bJI0GyH5vLq4lM+uEZsO/FJiW3Z4GPFZMO2bWKQ0P+yUtBk4HRkpaB9wA3Aw8JOmbwFvAhe1scm+3Zs2a3Pr06dNz67Nnz86tn3DCCYPs6HcWLFiQW/d5/H1Xw/BHxNQ6pS8V3IuZdZA/3muWKIffLFEOv1miHH6zRDn8ZonyEN0dsGHDhtz6ypUrc+sjRowosp09NOrN9l3e85slyuE3S5TDb5Yoh98sUQ6/WaIcfrNEOfxmifJXd+8FHnss/+sSzjjjjLq1YcOG5S67Y8eO3Ppdd92VW2/k2WefrVtbvHhx7rK7d+9uadup8hDdZpbL4TdLlMNvliiH3yxRDr9Zohx+s0Q5/GaJ8nn+fcBhhx1Wt3bFFVfkLnvOOefk1idPnpxbl/JPKef9/7r88stzl7333ntz61abz/ObWS6H3yxRDr9Zohx+s0Q5/GaJcvjNEuXwmyXK5/kt15QpU3LrM2fOzK2feuqpTW97/PjxufVXX3216XXvywo7zy/pHkmbJa2smjZb0npJL2U/Z7fSrJl13kAO+xcAtZ7+/yUiJmQ/jxfblpm1W8PwR8QK4P0O9GJmHdTKG35XSno5e1lwSL2ZJM2Q1COpp4VtmVnBmg3/ncA4YAKwEZhbb8aImBcREyNiYpPbMrM2aCr8EbEpInZFxG7gLmBSsW2ZWbs1FX5Jo6vufhXIH2PazLpOw/P8khYDpwMjgU3ADdn9CUAAbwLfioiNDTfm8/z7nAMOOCC3vmXLlrq1RmMKNDrPv3r16tx6qgZ6nn+/Aaxoao3J8wfdkZl1FX+81yxRDr9Zohx+s0Q5/GaJcvjNEuVLeq2trr322rq1G2+8MXfZRYsW5dYvu+yypnra1/mru80sl8NvliiH3yxRDr9Zohx+s0Q5/GaJcvjNEtXwqj6zVvT29ja97IgRIwrsxPrznt8sUQ6/WaIcfrNEOfxmiXL4zRLl8JslyuE3S5TDb5Yoh98sUQ6/WaIcfrNEOfxmiXL4zRLl8JslyuE3S1TD6/kljQEWAUcAu4F5EfFDSYcCPwHGUhmm+6KI+KB9rVpq7r777rJb2KcNZM+/E/heRBwHTAa+I2k8MBN4KiKOBZ7K7pvZXqJh+CNiY0S8mN3uBVYBRwLnAQuz2RYC57erSTMr3qBe80saC5wEPAeMioiNUHmCAA4vujkza58Bf4efpIOAJcDVEbFNGtBwYEiaAcxorj0za5cB7fkl7U8l+A9ExMPZ5E2SRmf10cDmWstGxLyImBgRE4to2MyK0TD8quzi5wOrIuK2qtJSYFp2exrwWPHtmVm7DOSw/2TgUuAVSS9l02YBNwMPSfom8BZwYXta7IzDD89/y2Lz5poHNskbOXJkbv3SSy9tet1r1qxpellrrGH4I+K/gXov8L9UbDtm1in+hJ9Zohx+s0Q5/GaJcvjNEuXwmyXK4TdLlIfozixZsiS3nnfOec6cObnLvvHGG7n1HTt25NZbMXTo0Nz6EUcckVs/66yzcutXXXVVbv3EE0+sW3vmmWdyl33nnXdy69Ya7/nNEuXwmyXK4TdLlMNvliiH3yxRDr9Zohx+s0T5PH/m0Ucfza1ff/31dWsXX3xx7rKLFy/OrW/dujW33ophw4bl1qdNm5Zbb2TIkPz9xxNPPFG3duutt+Yu++GHHzbVkw2M9/xmiXL4zRLl8JslyuE3S5TDb5Yoh98sUQ6/WaIUEZ3bmNS5jXXQ/Pnzc+vTp0/vTCMluO6663Lrc+fOrVv75JNPim7HgIgY0Fh63vObJcrhN0uUw2+WKIffLFEOv1miHH6zRDn8ZolqeJ5f0hhgEXAEsBuYFxE/lDQbuAJ4N5t1VkQ83mBd++R5frNuMtDz/AMJ/2hgdES8KGk48AJwPnARsD0i/nmgTTn8Zu030PA3/CafiNgIbMxu90paBRzZWntmVrZBveaXNBY4CXgum3SlpJcl3SPpkDrLzJDUI6mnpU7NrFAD/my/pIOAp4GbIuJhSaOALUAA/0TlpcE3GqzDh/1mbVbYa34ASfsDy4AnIuK2GvWxwLKIOL7Behx+szYr7MIeSQLmA6uqg5+9Edjnq8DKwTZpZuUZyLv9pwD/BbxC5VQfwCxgKjCBymH/m8C3sjcH89blPb9ZmxV62F8Uh9+s/Xw9v5nlcvjNEuXwmyXK4TdLlMNvliiH3yxRDr9Zohx+s0Q5/GaJcvjNEuXwmyXK4TdLlMNvliiH3yxRDb/As2BbgLVV90dm07pRt/bWrX2Be2tWkb19bqAzdvR6/k9tXOqJiImlNZCjW3vr1r7AvTWrrN582G+WKIffLFFlh39eydvP0629dWtf4N6aVUpvpb7mN7PylL3nN7OSOPxmiSol/JKmSFot6XVJM8vooR5Jb0p6RdJLZY8vmI2BuFnSyqpph0p6UtJr2e+aYySW1NtsSeuzx+4lSWeX1NsYSf8paZWkX0r6bja91Mcup69SHreOv+aXNBT4NXAmsA54HpgaEb/qaCN1SHoTmBgRpX8gRNKpwHZgUd9QaJJ+ALwfETdnT5yHRMTfdUlvsxnksO1t6q3esPLTKfGxK3K4+yKUseefBLweEWsi4hPgQeC8EvroehGxAni/3+TzgIXZ7YVU/vN0XJ3eukJEbIyIF7PbvUDfsPKlPnY5fZWijPAfCbxddX8dJT4ANQTwc0kvSJpRdjM1jOobFi37fXjJ/fTXcNj2Tuo3rHzXPHbNDHdftDLCX2sooW4633hyRPwp8GXgO9nhrQ3MncA4KmM4bgTmltlMNqz8EuDqiNhWZi/VavRVyuNWRvjXAWOq7h8FbCihj5oiYkP2ezPwCJWXKd1kU98IydnvzSX381sRsSkidkXEbuAuSnzssmHllwAPRMTD2eTSH7tafZX1uJUR/ueBYyUdI+kA4GvA0hL6+BRJw7I3YpA0DDiL7ht6fCkwLbs9DXisxF720C3DttcbVp6SH7tuG+6+lE/4ZacybgeGAvdExE0db6IGSZ+nsreHyuXOPy6zN0mLgdOpXPK5CbgBeBR4CDgaeAu4MCI6/sZbnd5OZ5DDtrept3rDyj9HiY9dkcPdF9KPP95rliZ/ws8sUQ6/WaIcfrNEOfxmiXL4zRLl8JslyuE3S9T/A5RYLU0jvWKuAAAAAElFTkSuQmCC\n", 537 | "text/plain": [ 538 | "
" 539 | ] 540 | }, 541 | "metadata": {}, 542 | "output_type": "display_data" 543 | } 544 | ], 545 | "source": [ 546 | "index = 9997\n", 547 | "k = test_set_x[index, :]\n", 548 | "k = k.reshape((28, 28))\n", 549 | "plt.title('Label is {label}'.format(label=(predictions[index], np.argmax(test_set_y, axis = 1)[index])))\n", 550 | "plt.imshow(k, cmap='gray')" 551 | ] 552 | } 553 | ], 554 | "metadata": { 555 | "kernelspec": { 556 | "display_name": "Python 3", 557 | "language": "python", 558 | "name": "python3" 559 | }, 560 | "language_info": { 561 | "codemirror_mode": { 562 | "name": "ipython", 563 | "version": 3 564 | }, 565 | "file_extension": ".py", 566 | "mimetype": "text/x-python", 567 | "name": "python", 568 | "nbconvert_exporter": "python", 569 | "pygments_lexer": "ipython3", 570 | "version": "3.6.5" 571 | } 572 | }, 573 | "nbformat": 4, 574 | "nbformat_minor": 2 575 | } 576 | -------------------------------------------------------------------------------- /Session 3 - Backpropagation in NN/nn_from_scratch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "911dae06-d967-45e5-afde-ff00761e6f52", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import tensorflow as tf\n", 11 | "import pandas as pd\n", 12 | "import numpy as np\n", 13 | "from sklearn.preprocessing import StandardScaler\n", 14 | "from sklearn.metrics import accuracy_score" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "id": "09133cf8-90f1-4695-a669-fb05f8501f4d", 20 | "metadata": {}, 21 | "source": [ 22 | "### Part 1: Import the Housing data and do feature transformations" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 2, 28 | "id": "f7efcf68-b2ca-4cb8-8c11-490ca1a745d5", 29 | "metadata": {}, 30 | "outputs": [ 31 | { 32 | "data": { 33 | "text/html": [ 34 | "
\n", 35 | "\n", 48 | "\n", 49 | " \n", 50 | " \n", 51 | " \n", 52 | " \n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | "
bedroomssqft_livingprice
031340313000
1536502384000
231930342000
332000420000
441940550000
\n", 90 | "
" 91 | ], 92 | "text/plain": [ 93 | " bedrooms sqft_living price\n", 94 | "0 3 1340 313000\n", 95 | "1 5 3650 2384000\n", 96 | "2 3 1930 342000\n", 97 | "3 3 2000 420000\n", 98 | "4 4 1940 550000" 99 | ] 100 | }, 101 | "execution_count": 2, 102 | "metadata": {}, 103 | "output_type": "execute_result" 104 | } 105 | ], 106 | "source": [ 107 | "df= pd.read_csv('house_price_full.csv')\n", 108 | "df.head()" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 3, 114 | "id": "85fd63b2-defd-4ba3-b042-9afc73d18a99", 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "X = df.copy()\n", 119 | "# Remove target\n", 120 | "Y = X.pop('price')\n", 121 | "\n", 122 | "# perform a scaler transform of the input data\n", 123 | "scaler = StandardScaler()\n", 124 | "X = scaler.fit_transform(X)\n", 125 | "\n", 126 | "# perform log transformation of target variable (For Sandeep: Is this needed?)\n", 127 | "Y = np.log(Y)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 4, 133 | "id": "95196b58-d932-456a-aa16-f9763822259f", 134 | "metadata": {}, 135 | "outputs": [ 136 | { 137 | "data": { 138 | "text/html": [ 139 | "
\n", 140 | "\n", 153 | "\n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | "
01
0-0.433198-0.753258
11.6757351.457330
2-0.433198-0.188649
3-0.433198-0.121661
40.621269-0.179079
.........
4940.6212690.873582
4951.6757352.299459
496-0.433198-0.724549
497-0.433198-0.179079
498-0.433198-1.040347
\n", 219 | "

499 rows × 2 columns

\n", 220 | "
" 221 | ], 222 | "text/plain": [ 223 | " 0 1\n", 224 | "0 -0.433198 -0.753258\n", 225 | "1 1.675735 1.457330\n", 226 | "2 -0.433198 -0.188649\n", 227 | "3 -0.433198 -0.121661\n", 228 | "4 0.621269 -0.179079\n", 229 | ".. ... ...\n", 230 | "494 0.621269 0.873582\n", 231 | "495 1.675735 2.299459\n", 232 | "496 -0.433198 -0.724549\n", 233 | "497 -0.433198 -0.179079\n", 234 | "498 -0.433198 -1.040347\n", 235 | "\n", 236 | "[499 rows x 2 columns]" 237 | ] 238 | }, 239 | "execution_count": 4, 240 | "metadata": {}, 241 | "output_type": "execute_result" 242 | } 243 | ], 244 | "source": [ 245 | "df_scaled = pd.DataFrame(X)\n", 246 | "df_scaled" 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": 5, 252 | "id": "4059a87f-e14b-462c-8fc2-0d8171cea2cc", 253 | "metadata": {}, 254 | "outputs": [ 255 | { 256 | "data": { 257 | "text/plain": [ 258 | "0 12.653958\n", 259 | "1 14.684290\n", 260 | "2 12.742566\n", 261 | "3 12.948010\n", 262 | "4 13.217674\n", 263 | " ... \n", 264 | "494 13.380102\n", 265 | "495 13.764217\n", 266 | "496 12.128111\n", 267 | "497 12.721886\n", 268 | "498 12.254863\n", 269 | "Name: price, Length: 499, dtype: float64" 270 | ] 271 | }, 272 | "execution_count": 5, 273 | "metadata": {}, 274 | "output_type": "execute_result" 275 | } 276 | ], 277 | "source": [ 278 | "Y" 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": 6, 284 | "id": "7c6c4da9-aa2b-46a2-acaa-cb900ac2b024", 285 | "metadata": {}, 286 | "outputs": [], 287 | "source": [ 288 | "#Taking 1 sample: x0,x1\n", 289 | "x1, x2 = df_scaled.iloc[0]" 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "execution_count": 7, 295 | "id": "bd37b5c7-9e4c-4919-ba3a-3718132eddf2", 296 | "metadata": {}, 297 | "outputs": [ 298 | { 299 | "name": "stdout", 300 | "output_type": "stream", 301 | "text": [ 302 | "Metal device set to: Apple M1\n" 303 | ] 304 | }, 305 | { 306 | "name": "stderr", 307 | "output_type": "stream", 308 | "text": [ 309 | "2021-12-28 16:15:59.169774: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n", 310 | "2021-12-28 16:15:59.170350: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: )\n" 311 | ] 312 | }, 313 | { 314 | "data": { 315 | "text/plain": [ 316 | "" 317 | ] 318 | }, 319 | "execution_count": 7, 320 | "metadata": {}, 321 | "output_type": "execute_result" 322 | } 323 | ], 324 | "source": [ 325 | "x = tf.constant([[x1,x2]], dtype=tf.float32)\n", 326 | "x" 327 | ] 328 | }, 329 | { 330 | "cell_type": "markdown", 331 | "id": "c919a12a-a45a-4cdb-b359-f75127de1f0b", 332 | "metadata": {}, 333 | "source": [ 334 | "### Part2: Forward Propagation with a single Neuron\n", 335 | "\n", 336 | "The simplest way to describe a neural network is that we have some inputs , which get combined into an auxilliary variable . The auxilliary variable is passed through the activation function and the result is the output.\n", 337 | "\n", 338 | "Here is another image showing each step.\n", 339 | "![](neuron.png)\n", 340 | "Notice that the inputs are linearly combined according to some weights and a bias . This transformation is also sometimes called an affine transformation. The perceptron transforms the weighted inputs according to the rule of the activation function. For a single perceptron, the output is just the output from the perceptron. The linear transformation and activation of the neuron occurs within a single layer of the network (shown in the dotted box).\n", 341 | "\n", 342 | "Let's see what the single-layer, single neuron network give us. We have a couple of choices to make:\n", 343 | "\n", 344 | "We must choose some weights and some biases\n", 345 | "We must choose an activation function\n", 346 | "For now, we will manually specify the weights and biases.\n", 347 | "\n", 348 | "We choose a sigmoid activation function" 349 | ] 350 | }, 351 | { 352 | "cell_type": "code", 353 | "execution_count": 8, 354 | "id": "d22f12ae-177d-43ba-8afa-62e62afe6bb0", 355 | "metadata": {}, 356 | "outputs": [], 357 | "source": [ 358 | "#weights\n", 359 | "w1 = tf.Variable([0.2], dtype=tf.float32)\n", 360 | "w2 = tf.Variable([0.15], dtype=tf.float32)\n", 361 | "#bias\n", 362 | "b = tf.Variable([0.1], dtype=tf.float32)" 363 | ] 364 | }, 365 | { 366 | "cell_type": "code", 367 | "execution_count": 9, 368 | "id": "f2e7b9c3-4f70-46dc-a6bc-b75a83803e67", 369 | "metadata": {}, 370 | "outputs": [ 371 | { 372 | "name": "stdout", 373 | "output_type": "stream", 374 | "text": [ 375 | "The output from the first neuron is tf.Tensor([0.4751135], shape=(1,), dtype=float32)\n" 376 | ] 377 | } 378 | ], 379 | "source": [ 380 | "#Cumulative input\n", 381 | "z = b + w1*x1 +w2*x2\n", 382 | "h = tf.math.sigmoid(z)\n", 383 | "print(\"The output from the first neuron is\",h)" 384 | ] 385 | }, 386 | { 387 | "cell_type": "markdown", 388 | "id": "a34f3985-fe07-439f-a6ef-4b6ffbba1f1c", 389 | "metadata": {}, 390 | "source": [ 391 | "### Part3: Forward Propagation with multiple neurons\n", 392 | "\n", 393 | "![](multiple_neurons.png)" 394 | ] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "execution_count": 10, 399 | "id": "a708032a-8898-4c75-8a6b-19708e5db798", 400 | "metadata": {}, 401 | "outputs": [], 402 | "source": [ 403 | "## layer1 weights\n", 404 | "# neuron1\n", 405 | "b1 = tf.Variable([0.1])\n", 406 | "w11 = tf.Variable([0.2])\n", 407 | "w12 = tf.Variable([0.15])\n", 408 | "#neuron2\n", 409 | "b2 = tf.Variable([0.25])\n", 410 | "w21 = tf.Variable([0.5])\n", 411 | "w22 = tf.Variable([0.6])\n" 412 | ] 413 | }, 414 | { 415 | "cell_type": "code", 416 | "execution_count": 11, 417 | "id": "df953e00-859b-4559-94c9-9ac7b128fa5c", 418 | "metadata": {}, 419 | "outputs": [ 420 | { 421 | "name": "stdout", 422 | "output_type": "stream", 423 | "text": [ 424 | "The output from the first neuron is tf.Tensor([0.4751135], shape=(1,), dtype=float32)\n" 425 | ] 426 | } 427 | ], 428 | "source": [ 429 | "## forward pass\n", 430 | "# neuron 1\n", 431 | "z1 = b1+w11*x1+w12*x2\n", 432 | "h1 = tf.math.sigmoid(z1)\n", 433 | "print(\"The output from the first neuron is\",h1)" 434 | ] 435 | }, 436 | { 437 | "cell_type": "code", 438 | "execution_count": 12, 439 | "id": "383503a1-fdcf-4f53-a0ee-03077dd6f5ca", 440 | "metadata": {}, 441 | "outputs": [ 442 | { 443 | "name": "stdout", 444 | "output_type": "stream", 445 | "text": [ 446 | "The output from the second neuron is tf.Tensor([0.39686298], shape=(1,), dtype=float32)\n" 447 | ] 448 | } 449 | ], 450 | "source": [ 451 | "## forward pass\n", 452 | "# neuron 2\n", 453 | "z2 = b2+w21*x1+w22*x2\n", 454 | "h2 = tf.math.sigmoid(z2)\n", 455 | "print(\"The output from the second neuron is\",h2)" 456 | ] 457 | }, 458 | { 459 | "cell_type": "code", 460 | "execution_count": 13, 461 | "id": "3156f647-fd4f-4f6a-aa6c-158dfd7f1e8b", 462 | "metadata": {}, 463 | "outputs": [], 464 | "source": [ 465 | "## layer2 weights\n", 466 | "b1 = tf.Variable([0.4])\n", 467 | "w11 = tf.Variable([0.3])\n", 468 | "w12 = tf.Variable([0.2])" 469 | ] 470 | }, 471 | { 472 | "cell_type": "code", 473 | "execution_count": 14, 474 | "id": "81be3f6c-a1bd-42d3-9576-c325f90804b4", 475 | "metadata": {}, 476 | "outputs": [ 477 | { 478 | "name": "stdout", 479 | "output_type": "stream", 480 | "text": [ 481 | "The output from the first neuron is tf.Tensor([0.62190664], shape=(1,), dtype=float32)\n" 482 | ] 483 | } 484 | ], 485 | "source": [ 486 | "## forward pass\n", 487 | "# second layer\n", 488 | "z1 = b1+w11*h1+w12*h2\n", 489 | "h1 = z1\n", 490 | "print(\"The output from the first neuron is\",h1)" 491 | ] 492 | }, 493 | { 494 | "cell_type": "code", 495 | "execution_count": 15, 496 | "id": "2668ce90-6f54-4f40-9bbe-ea6f0db4bff4", 497 | "metadata": {}, 498 | "outputs": [], 499 | "source": [ 500 | "y_true = Y[0]\n", 501 | "y_pred = h1.numpy()" 502 | ] 503 | }, 504 | { 505 | "cell_type": "code", 506 | "execution_count": 16, 507 | "id": "320dc2df-3489-4cb8-aa27-3162426357ac", 508 | "metadata": {}, 509 | "outputs": [ 510 | { 511 | "name": "stdout", 512 | "output_type": "stream", 513 | "text": [ 514 | "The MSE error is [72.38514]\n" 515 | ] 516 | } 517 | ], 518 | "source": [ 519 | "#loss\n", 520 | "L = 0.5*(y_true - y_pred)**2\n", 521 | "print(\"The MSE error is\",L)" 522 | ] 523 | }, 524 | { 525 | "cell_type": "markdown", 526 | "id": "8442d282-5547-46e2-8a9d-d5769bb99dab", 527 | "metadata": {}, 528 | "source": [ 529 | "## Part 4: Forward pass matrix multiplication\n", 530 | "![](multiple_neurons.png)\n", 531 | "\n", 532 | "![](Matrix.gif)\n", 533 | "\n", 534 | "This network can be described as follows:\n", 535 | "\n", 536 | "- Input vector = $X = (x1,x2)$\n", 537 | "- Weight Matrix (hidden layer) = $$W^1 = \\begin{bmatrix}\n", 538 | "w^1_{11}&&w^1_{12}\\\\\n", 539 | "w^1_{21}&&w^1_{22}\\\\\n", 540 | "\\end{bmatrix}\n", 541 | "$$\n", 542 | "*note the subscripts are being mapped to weights in the figure\n", 543 | "\n", 544 | "- Bias/offset Matrix (hidden layer) = $$\n", 545 | "B^1_0 = \\begin{bmatrix}\n", 546 | "b^1_{1}\\\\\n", 547 | "b^1_{2}\\\\\n", 548 | "\\end{bmatrix}\n", 549 | "$$\n", 550 | "\n", 551 | "Now the forward pass for the hidden layer can be described as \n", 552 | "\n", 553 | "$$W^1 \\times X^T + B^1_0= Z^1 = \\begin{bmatrix}\n", 554 | "z^1_{1}\\\\\n", 555 | "z^1_{2}\\\\\n", 556 | "\\end{bmatrix}\n", 557 | "$$\n", 558 | "\n", 559 | "Applying the activation function $f$ over the matrix $Z$ will complete the forward pass.\n", 560 | "\n", 561 | "$$f(W^1 \\times X^T + B^1_0)= f(Z^1) = f(\\begin{bmatrix}\n", 562 | "z^1_{1}\\\\\n", 563 | "z^1_{2}\\\\\n", 564 | "\\end{bmatrix}) = \n", 565 | "\\begin{bmatrix}\n", 566 | "f(z^1_{1})\\\\\n", 567 | "f(z^1_{2})\\\\\n", 568 | "\\end{bmatrix}\n", 569 | "=\n", 570 | "\\begin{bmatrix}\n", 571 | "h^1_1\\\\\n", 572 | "h^1_2\\\\\n", 573 | "\\end{bmatrix}\n", 574 | "= H^1\n", 575 | "$$\n", 576 | "\n", 577 | "For the output layer:\n", 578 | "\n", 579 | "- The weight matrix is $$W^2 = \\begin{bmatrix}\n", 580 | "w^2_{11}&&w^2_{12}\\\\\n", 581 | "\\end{bmatrix}\n", 582 | "$$\n", 583 | "\n", 584 | "- The bias/offset matrix is $$B^2_0 = \\begin{bmatrix}\n", 585 | "b^2_{1}\\\\\n", 586 | "\\end{bmatrix}\n", 587 | "$$\n", 588 | "\n", 589 | "Now the forward pass can be written as:\n", 590 | "\n", 591 | "\n", 592 | "$$ B_0^2+W^2 \\times H^1$$\n" 593 | ] 594 | }, 595 | { 596 | "cell_type": "code", 597 | "execution_count": 17, 598 | "id": "21fab65a-f968-421e-80fa-86cb280685c6", 599 | "metadata": {}, 600 | "outputs": [], 601 | "source": [ 602 | "## layer 1 weights\n", 603 | "W1 = tf.Variable([[0.2, 0.15],\n", 604 | " [0.5, 0.6]], dtype=tf.float32)\n", 605 | "## layer 1 bias\n", 606 | "B1 = tf.Variable([[0.1],\n", 607 | " [0.25]], dtype=tf.float32)" 608 | ] 609 | }, 610 | { 611 | "cell_type": "code", 612 | "execution_count": 18, 613 | "id": "0f3cc3e1-574a-47dc-84c4-d66bf18b35bd", 614 | "metadata": {}, 615 | "outputs": [], 616 | "source": [ 617 | "## layer 2 weights\n", 618 | "W2 = tf.Variable([[0.3, 0.2]], dtype=tf.float32)\n", 619 | "#bias\n", 620 | "B2 = tf.Variable([0.4], dtype=tf.float32)" 621 | ] 622 | }, 623 | { 624 | "cell_type": "code", 625 | "execution_count": 19, 626 | "id": "095a917a-2901-4dd5-9ca2-80c0d997a9ec", 627 | "metadata": {}, 628 | "outputs": [], 629 | "source": [ 630 | "## data\n", 631 | "X = tf.constant([[x1,x2]], dtype=tf.float32)" 632 | ] 633 | }, 634 | { 635 | "cell_type": "code", 636 | "execution_count": 20, 637 | "id": "00f72506-52a9-43d5-9a06-5ebb270891bf", 638 | "metadata": {}, 639 | "outputs": [ 640 | { 641 | "name": "stdout", 642 | "output_type": "stream", 643 | "text": [ 644 | "tf.Tensor(\n", 645 | "[[0.4751135 ]\n", 646 | " [0.39686298]], shape=(2, 1), dtype=float32)\n" 647 | ] 648 | } 649 | ], 650 | "source": [ 651 | "## forward pass layer 1\n", 652 | "Z1 = tf.matmul(W1, tf.transpose(X)) + B1\n", 653 | "H1 = tf.math.sigmoid(Z1)\n", 654 | "print(H1)" 655 | ] 656 | }, 657 | { 658 | "cell_type": "code", 659 | "execution_count": 21, 660 | "id": "028e2e88-6417-47a1-9cee-ddb4f2b874f8", 661 | "metadata": {}, 662 | "outputs": [], 663 | "source": [ 664 | "## forward pass layer 2\n", 665 | "Z2 = tf.matmul(W2,H1)+B2" 666 | ] 667 | }, 668 | { 669 | "cell_type": "code", 670 | "execution_count": 22, 671 | "id": "2e00bbaa-8b6f-4e6a-b43d-fc0a851e7739", 672 | "metadata": {}, 673 | "outputs": [ 674 | { 675 | "data": { 676 | "text/plain": [ 677 | "" 678 | ] 679 | }, 680 | "execution_count": 22, 681 | "metadata": {}, 682 | "output_type": "execute_result" 683 | } 684 | ], 685 | "source": [ 686 | "Z2" 687 | ] 688 | }, 689 | { 690 | "cell_type": "code", 691 | "execution_count": 23, 692 | "id": "5ffd39a5-e9eb-47b5-9ac7-925c16acd3ec", 693 | "metadata": {}, 694 | "outputs": [ 695 | { 696 | "name": "stdout", 697 | "output_type": "stream", 698 | "text": [ 699 | "[[72.38514]]\n" 700 | ] 701 | } 702 | ], 703 | "source": [ 704 | "y_pred = Z2.numpy()\n", 705 | "loss = 0.5*(y_true-y_pred)**2\n", 706 | "print(loss)" 707 | ] 708 | }, 709 | { 710 | "cell_type": "markdown", 711 | "id": "8d35c31e-38e1-4c4f-aae9-6862e38063a8", 712 | "metadata": {}, 713 | "source": [ 714 | "## Part5: Random Weight Initialization\n", 715 | "\n", 716 | "![](multiple_neurons.png)" 717 | ] 718 | }, 719 | { 720 | "cell_type": "code", 721 | "execution_count": 24, 722 | "id": "d4f13145-94d3-40c3-a010-b5085660bfad", 723 | "metadata": {}, 724 | "outputs": [], 725 | "source": [ 726 | "def random_init_params():\n", 727 | " w1 = tf.Variable(tf.random.uniform((2, 2)))\n", 728 | " b1 = tf.Variable(tf.random.uniform((1, 2)))\n", 729 | " w2 = tf.Variable(tf.random.uniform((2, 1)))\n", 730 | " b2 = tf.Variable(tf.random.uniform((1, 1)))\n", 731 | " return w1,b1,w2,b2" 732 | ] 733 | }, 734 | { 735 | "cell_type": "code", 736 | "execution_count": 25, 737 | "id": "d62f7eab-49b0-4594-b5b4-37e6088c9a3b", 738 | "metadata": {}, 739 | "outputs": [], 740 | "source": [ 741 | "x = tf.constant([[x1,x2]], dtype=tf.float32)\n", 742 | "y = Y[0]\n", 743 | "w1,b1,w2,b2 = random_init_params()" 744 | ] 745 | }, 746 | { 747 | "cell_type": "code", 748 | "execution_count": 26, 749 | "id": "27c5870a-6e1f-41e7-a90b-5fa693b81df7", 750 | "metadata": {}, 751 | "outputs": [ 752 | { 753 | "name": "stdout", 754 | "output_type": "stream", 755 | "text": [ 756 | " the initial 1st layer weights are:\n", 757 | " [[0.23209727 0.01658845]\n", 758 | " [0.83517206 0.99959767]]\n", 759 | "--------------------------------------------------\n", 760 | " the initial 2nd layer weights are:\n", 761 | " [[0.23209727]\n", 762 | " [0.01658845]]\n", 763 | "--------------------------------------------------\n", 764 | " the initial 1st layer bias are:\n", 765 | " [[0.23209727 0.01658845]]\n", 766 | "--------------------------------------------------\n", 767 | " the initial 2nd layer bias are:\n", 768 | " [[0.23209727]]\n" 769 | ] 770 | } 771 | ], 772 | "source": [ 773 | "print(\" the initial 1st layer weights are:\\n\",w1.numpy())\n", 774 | "print(\"--------------------------------------------------\")\n", 775 | "print(\" the initial 2nd layer weights are:\\n\",w2.numpy())\n", 776 | "print(\"--------------------------------------------------\")\n", 777 | "print(\" the initial 1st layer bias are:\\n\",b1.numpy())\n", 778 | "print(\"--------------------------------------------------\")\n", 779 | "print(\" the initial 2nd layer bias are:\\n\",b2.numpy())" 780 | ] 781 | }, 782 | { 783 | "cell_type": "code", 784 | "execution_count": 27, 785 | "id": "ab9abe7a-f3d9-49bb-be50-bb38a51a1700", 786 | "metadata": {}, 787 | "outputs": [], 788 | "source": [ 789 | "def forward_prop(x, w1, b1, w2, b2):\n", 790 | " z1 = tf.matmul(x,w1) + b1\n", 791 | " h1 = tf.math.sigmoid(z1)\n", 792 | " z2 = tf.matmul(h1,w2) + b2\n", 793 | " h2 = z2\n", 794 | " return h2" 795 | ] 796 | }, 797 | { 798 | "cell_type": "code", 799 | "execution_count": 28, 800 | "id": "f29e7996-476d-429d-9af4-59205bf21db1", 801 | "metadata": {}, 802 | "outputs": [ 803 | { 804 | "name": "stdout", 805 | "output_type": "stream", 806 | "text": [ 807 | "The MSE error is tf.Tensor([[75.99908]], shape=(1, 1), dtype=float32)\n" 808 | ] 809 | } 810 | ], 811 | "source": [ 812 | "y_pred = forward_prop(x, w1, b1, w2, b2)\n", 813 | "#loss\n", 814 | "L = 0.5*(y - y_pred)**2\n", 815 | "print(\"The MSE error is\",L)" 816 | ] 817 | }, 818 | { 819 | "cell_type": "markdown", 820 | "id": "844f9c21-ff86-4767-8d91-1d6a399789c1", 821 | "metadata": {}, 822 | "source": [ 823 | "## Part6: Backpropagation\n", 824 | "\n", 825 | "Find the value of x that minimises $y = x^2+4x$\n", 826 | "\n", 827 | "Gradient descent update equation\n", 828 | "\n", 829 | "$x_{new} := x_{old}-\\eta\\frac{dy}{dx}$" 830 | ] 831 | }, 832 | { 833 | "cell_type": "code", 834 | "execution_count": 29, 835 | "id": "423c86c8-bf03-495e-8972-e69247822584", 836 | "metadata": {}, 837 | "outputs": [], 838 | "source": [ 839 | "x = tf.Variable(0.0) ## add gradient tape\n", 840 | "lr = eta = 0.1" 841 | ] 842 | }, 843 | { 844 | "cell_type": "code", 845 | "execution_count": 30, 846 | "id": "98fe645c-fb0e-4257-84e0-9a47b3a81b4d", 847 | "metadata": {}, 848 | "outputs": [], 849 | "source": [ 850 | "with tf.GradientTape() as tape:\n", 851 | " y = x**2+4*x\n", 852 | "grad = tape.gradient(y,x) ## dy/dx" 853 | ] 854 | }, 855 | { 856 | "cell_type": "code", 857 | "execution_count": 31, 858 | "id": "529b973d-6a0d-424b-9b19-3c44b3e05672", 859 | "metadata": {}, 860 | "outputs": [ 861 | { 862 | "data": { 863 | "text/plain": [ 864 | "4.0" 865 | ] 866 | }, 867 | "execution_count": 31, 868 | "metadata": {}, 869 | "output_type": "execute_result" 870 | } 871 | ], 872 | "source": [ 873 | "grad.numpy() #dy/dx = 2x+4, x=0 => dy/dx = 4" 874 | ] 875 | }, 876 | { 877 | "cell_type": "code", 878 | "execution_count": 32, 879 | "id": "7f9b9642-3290-41a6-a4ce-eeb20b412594", 880 | "metadata": {}, 881 | "outputs": [ 882 | { 883 | "data": { 884 | "text/plain": [ 885 | "" 886 | ] 887 | }, 888 | "execution_count": 32, 889 | "metadata": {}, 890 | "output_type": "execute_result" 891 | } 892 | ], 893 | "source": [ 894 | "x.assign_sub(lr*grad) ## x_new = x_old -lr*dy/dx" 895 | ] 896 | }, 897 | { 898 | "cell_type": "code", 899 | "execution_count": 33, 900 | "id": "76ffc91d-4154-40d4-8c8e-f46a76c3847f", 901 | "metadata": {}, 902 | "outputs": [ 903 | { 904 | "data": { 905 | "text/plain": [ 906 | "-0.4" 907 | ] 908 | }, 909 | "execution_count": 33, 910 | "metadata": {}, 911 | "output_type": "execute_result" 912 | } 913 | ], 914 | "source": [ 915 | "x.numpy()" 916 | ] 917 | }, 918 | { 919 | "cell_type": "code", 920 | "execution_count": 34, 921 | "id": "ca154ab8-599c-4ca6-8254-ad4f7fe8a472", 922 | "metadata": {}, 923 | "outputs": [ 924 | { 925 | "name": "stdout", 926 | "output_type": "stream", 927 | "text": [ 928 | "-0.4\n", 929 | "-0.72\n", 930 | "-0.9760001\n", 931 | "-1.1808001\n", 932 | "-1.34464\n", 933 | "-1.4757121\n", 934 | "-1.5805696\n", 935 | "-1.6644558\n", 936 | "-1.7315646\n", 937 | "-1.7852517\n" 938 | ] 939 | } 940 | ], 941 | "source": [ 942 | "## full loop\n", 943 | "x = tf.Variable(0.0) ## add gradient tape\n", 944 | "lr = eta = 0.1\n", 945 | "for i in range(10):\n", 946 | " with tf.GradientTape() as tape:\n", 947 | " y = x**2+4*x\n", 948 | " grad = tape.gradient(y,x)\n", 949 | " x.assign_sub(lr*grad)\n", 950 | " print(x.numpy())" 951 | ] 952 | }, 953 | { 954 | "cell_type": "markdown", 955 | "id": "b25dc73f-f2e2-47e9-be8a-495ca52658e3", 956 | "metadata": {}, 957 | "source": [ 958 | "\n", 959 | "![](gradients.png)" 960 | ] 961 | }, 962 | { 963 | "cell_type": "code", 964 | "execution_count": 35, 965 | "id": "6797e506-2b65-4a8d-8f5c-24defd7411a8", 966 | "metadata": {}, 967 | "outputs": [], 968 | "source": [ 969 | "x = tf.constant([[x1,x2]], dtype=tf.float32)\n", 970 | "y = Y[0]\n", 971 | "\n", 972 | "def random_init_params():\n", 973 | " w1 = tf.Variable(tf.random.uniform((2, 2)))\n", 974 | " b1 = tf.Variable(tf.random.uniform((1, 2)))\n", 975 | " w2 = tf.Variable(tf.random.uniform((2, 1)))\n", 976 | " b2 = tf.Variable(tf.random.uniform((1, 1)))\n", 977 | " return w1,b1,w2,b2\n", 978 | "\n", 979 | "def forward_prop(x, w1, b1, w2, b2):\n", 980 | " z1 = tf.matmul(x,w1) + b1\n", 981 | " h1 = tf.math.sigmoid(z1)\n", 982 | " z2 = tf.matmul(h1,w2) + b2\n", 983 | " h2 = z2\n", 984 | " return h2" 985 | ] 986 | }, 987 | { 988 | "cell_type": "code", 989 | "execution_count": 36, 990 | "id": "b0dd8c3e-473b-4a06-be20-b4230d0fc027", 991 | "metadata": {}, 992 | "outputs": [], 993 | "source": [ 994 | "w1,b1,w2,b2 = random_init_params()" 995 | ] 996 | }, 997 | { 998 | "cell_type": "code", 999 | "execution_count": 37, 1000 | "id": "66195d4a-d966-4ecd-977f-678177f8b6fe", 1001 | "metadata": {}, 1002 | "outputs": [], 1003 | "source": [ 1004 | "with tf.GradientTape() as tape:\n", 1005 | " y_pred = forward_prop(x,w1,b1,w2,b2)\n", 1006 | " loss = 0.5*(y-y_pred)**2" 1007 | ] 1008 | }, 1009 | { 1010 | "cell_type": "code", 1011 | "execution_count": 38, 1012 | "id": "4a83bfbc-f1ba-4ca9-bfb4-cb8b5aa6fd26", 1013 | "metadata": {}, 1014 | "outputs": [], 1015 | "source": [ 1016 | "gw1, gb1, gw2, gb2 = tape.gradient(loss, [w1, b1, w2, b2])" 1017 | ] 1018 | }, 1019 | { 1020 | "cell_type": "code", 1021 | "execution_count": 39, 1022 | "id": "c1e63266-99eb-4650-9034-0ce23e0beb12", 1023 | "metadata": {}, 1024 | "outputs": [ 1025 | { 1026 | "data": { 1027 | "text/plain": [ 1028 | "" 1031 | ] 1032 | }, 1033 | "execution_count": 39, 1034 | "metadata": {}, 1035 | "output_type": "execute_result" 1036 | } 1037 | ], 1038 | "source": [ 1039 | "gw1" 1040 | ] 1041 | }, 1042 | { 1043 | "cell_type": "code", 1044 | "execution_count": 40, 1045 | "id": "b202f6b6-99e4-457e-8c9e-3abb11840bbc", 1046 | "metadata": {}, 1047 | "outputs": [ 1048 | { 1049 | "data": { 1050 | "text/plain": [ 1051 | "" 1052 | ] 1053 | }, 1054 | "execution_count": 40, 1055 | "metadata": {}, 1056 | "output_type": "execute_result" 1057 | } 1058 | ], 1059 | "source": [ 1060 | "gb1" 1061 | ] 1062 | }, 1063 | { 1064 | "cell_type": "code", 1065 | "execution_count": 41, 1066 | "id": "95600402-76fa-44b5-9d05-9e71c4ebbb8c", 1067 | "metadata": {}, 1068 | "outputs": [ 1069 | { 1070 | "data": { 1071 | "text/plain": [ 1072 | "" 1075 | ] 1076 | }, 1077 | "execution_count": 41, 1078 | "metadata": {}, 1079 | "output_type": "execute_result" 1080 | } 1081 | ], 1082 | "source": [ 1083 | "gw2" 1084 | ] 1085 | }, 1086 | { 1087 | "cell_type": "code", 1088 | "execution_count": 42, 1089 | "id": "880ad697-3224-4527-ad47-84eb6ae09815", 1090 | "metadata": {}, 1091 | "outputs": [ 1092 | { 1093 | "data": { 1094 | "text/plain": [ 1095 | "" 1096 | ] 1097 | }, 1098 | "execution_count": 42, 1099 | "metadata": {}, 1100 | "output_type": "execute_result" 1101 | } 1102 | ], 1103 | "source": [ 1104 | "gb2" 1105 | ] 1106 | }, 1107 | { 1108 | "cell_type": "code", 1109 | "execution_count": 43, 1110 | "id": "1d34744a-69df-45c8-a158-00c128f42a59", 1111 | "metadata": {}, 1112 | "outputs": [ 1113 | { 1114 | "name": "stdout", 1115 | "output_type": "stream", 1116 | "text": [ 1117 | "Value of w1 before gradient update is \n", 1120 | "Value of w1 after gradient update is \n" 1123 | ] 1124 | } 1125 | ], 1126 | "source": [ 1127 | "lr=0.01\n", 1128 | "print(f\"Value of w1 before gradient update is {w1}\")\n", 1129 | "w1.assign_sub(lr*gw1)\n", 1130 | "print(f\"Value of w1 after gradient update is {w1}\")" 1131 | ] 1132 | }, 1133 | { 1134 | "cell_type": "code", 1135 | "execution_count": 44, 1136 | "id": "fe8bb25d-b7aa-4bd7-9765-28731e5944d3", 1137 | "metadata": {}, 1138 | "outputs": [ 1139 | { 1140 | "name": "stdout", 1141 | "output_type": "stream", 1142 | "text": [ 1143 | "Value of b1 before gradient update is \n", 1144 | "Value of w1 after gradient update is \n" 1145 | ] 1146 | } 1147 | ], 1148 | "source": [ 1149 | "lr=0.01\n", 1150 | "print(f\"Value of b1 before gradient update is {b1}\")\n", 1151 | "b1.assign_sub(lr*gb1)\n", 1152 | "print(f\"Value of w1 after gradient update is {b1}\")" 1153 | ] 1154 | }, 1155 | { 1156 | "cell_type": "code", 1157 | "execution_count": 45, 1158 | "id": "809b9b9e-acea-48dd-8386-c40eb3b20a8e", 1159 | "metadata": {}, 1160 | "outputs": [], 1161 | "source": [ 1162 | "def train(x, y, w1, b1, w2, b2):\n", 1163 | " y_true = y\n", 1164 | " with tf.GradientTape() as g:\n", 1165 | " y_pred = forward_prop(x, w1, b1, w2, b2)\n", 1166 | "\n", 1167 | " # loss\n", 1168 | " loss = 0.5*(y_true - y_pred)** 2\n", 1169 | " \n", 1170 | " #Gradient calculation \n", 1171 | " print(\"**************************************************\")\n", 1172 | " print(\"GRADIENTS\")\n", 1173 | " print(\"**************************************************\")\n", 1174 | " gw1, gb1, gw2, gb2 = g.gradient(loss, [w1, b1, w2, b2])\n", 1175 | " print(\" the gradient for 1st layer weights are:\\n\",gw1.numpy())\n", 1176 | " print(\"--------------------------------------------------\")\n", 1177 | " print(\" the gradient for 2nd layer weights are:\\n\",gw2.numpy())\n", 1178 | " print(\"--------------------------------------------------\")\n", 1179 | " print(\" the gradient for 1st layer bias are:\\n\",gb1.numpy())\n", 1180 | " print(\"--------------------------------------------------\")\n", 1181 | " print(\" the gradient for 2nd layer bias are:\\n\",gb2.numpy())\n", 1182 | " print(\"--------------------------------------------------\")\n", 1183 | "\n", 1184 | " # Gradient descent:\n", 1185 | " lr=0.2\n", 1186 | " w1.assign_sub(lr*gw1)\n", 1187 | " b1.assign_sub(lr*gb1) \n", 1188 | " w2.assign_sub(lr*gw2)\n", 1189 | " b2.assign_sub(lr*gb2)\n", 1190 | " print(\"**************************************************\")\n", 1191 | " print(\"NEW UPDATES\")\n", 1192 | " print(\"**************************************************\")\n", 1193 | " print(\" the updated 1st layer weights are:\\n\",w1.numpy())\n", 1194 | " print(\"--------------------------------------------------\")\n", 1195 | " print(\" the updated 2nd layer weights are:\\n\",w2.numpy())\n", 1196 | " print(\"--------------------------------------------------\")\n", 1197 | " print(\" the updated 1st layer bias are:\\n\",b1.numpy())\n", 1198 | " print(\"--------------------------------------------------\")\n", 1199 | " print(\" the updated 2nd layer bias are:\\n\",b2.numpy())\n", 1200 | "\n", 1201 | "\n", 1202 | " return w1, b1, w2, b2,loss" 1203 | ] 1204 | }, 1205 | { 1206 | "cell_type": "code", 1207 | "execution_count": 46, 1208 | "id": "8c176262-1ea8-447b-b714-c30041457cd7", 1209 | "metadata": {}, 1210 | "outputs": [ 1211 | { 1212 | "name": "stdout", 1213 | "output_type": "stream", 1214 | "text": [ 1215 | "**************************************************\n", 1216 | "GRADIENTS\n", 1217 | "**************************************************\n", 1218 | " the gradient for 1st layer weights are:\n", 1219 | " [[0.2914811 0.01934897]\n", 1220 | " [0.50683635 0.03364459]]\n", 1221 | "--------------------------------------------------\n", 1222 | " the gradient for 2nd layer weights are:\n", 1223 | " [[-4.661717 ]\n", 1224 | " [-3.9726682]]\n", 1225 | "--------------------------------------------------\n", 1226 | " the gradient for 1st layer bias are:\n", 1227 | " [[-0.6728594 -0.04466546]]\n", 1228 | "--------------------------------------------------\n", 1229 | " the gradient for 2nd layer bias are:\n", 1230 | " [[-12.328753]]\n", 1231 | "--------------------------------------------------\n", 1232 | "**************************************************\n", 1233 | "NEW UPDATES\n", 1234 | "**************************************************\n", 1235 | " the updated 1st layer weights are:\n", 1236 | " [[0.17380105 0.01271866]\n", 1237 | " [0.73380476 0.9928687 ]]\n", 1238 | "--------------------------------------------------\n", 1239 | " the updated 2nd layer weights are:\n", 1240 | " [[1.1644406]\n", 1241 | " [0.8111221]]\n", 1242 | "--------------------------------------------------\n", 1243 | " the updated 1st layer bias are:\n", 1244 | " [[0.36666915 0.02552154]]\n", 1245 | "--------------------------------------------------\n", 1246 | " the updated 2nd layer bias are:\n", 1247 | " [[2.6978478]]\n" 1248 | ] 1249 | } 1250 | ], 1251 | "source": [ 1252 | "w1,b1,w2,b2 = random_init_params()\n", 1253 | "w1, b1, w2, b2,loss = train(x, y, w1, b1, w2, b2)" 1254 | ] 1255 | }, 1256 | { 1257 | "cell_type": "code", 1258 | "execution_count": null, 1259 | "id": "37c8d3c6-de91-4a19-926a-6bd7b85720b0", 1260 | "metadata": {}, 1261 | "outputs": [], 1262 | "source": [] 1263 | } 1264 | ], 1265 | "metadata": { 1266 | "kernelspec": { 1267 | "display_name": "Python 3", 1268 | "language": "python", 1269 | "name": "python3" 1270 | }, 1271 | "language_info": { 1272 | "codemirror_mode": { 1273 | "name": "ipython", 1274 | "version": 3 1275 | }, 1276 | "file_extension": ".py", 1277 | "mimetype": "text/x-python", 1278 | "name": "python", 1279 | "nbconvert_exporter": "python", 1280 | "pygments_lexer": "ipython3", 1281 | "version": "3.9.9" 1282 | } 1283 | }, 1284 | "nbformat": 4, 1285 | "nbformat_minor": 5 1286 | } 1287 | --------------------------------------------------------------------------------