├── .ipynb_checkpoints ├── Convolution layer to train MNIST-checkpoint.ipynb ├── Draw_boxes-checkpoint.ipynb └── Pretrained model Pytorch-checkpoint.ipynb ├── Linear regression using GSD.ipynb ├── Linear regression using Normal equations .ipynb ├── Locally Weighted Regression .ipynb ├── Logistic regression.ipynb ├── MNIST training Pytorch.ipynb ├── Multi-input Network Pytorch.ipynb ├── Newton Method.ipynb ├── Pretrained model Pytorch.ipynb ├── README.md ├── README.md~ ├── Transfere Learning.ipynb ├── dog.jpeg ├── imagenet_class_index.json └── softmax.ipynb /.ipynb_checkpoints/Convolution layer to train MNIST-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Import libraries " 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import torch\n", 17 | "import torch.nn as nn\n", 18 | "from torch.autograd import Variable\n", 19 | "import torchvision\n", 20 | "import torchvision.transforms as T\n", 21 | "import torch.optim as optim\n", 22 | "from torch.utils.data import DataLoader\n", 23 | "from torch.utils.data import sampler\n", 24 | "import torchvision.datasets as dset\n", 25 | "import matplotlib.pyplot as plt\n", 26 | "import numpy as np" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "### Sampler function" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "A function that returns a sample/patch of specific size " 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "class ChunkSampler(sampler.Sampler):\n", 50 | " \"\"\"Samples elements sequentially from some offset. \n", 51 | " Arguments:\n", 52 | " num_samples: # of desired datapoints\n", 53 | " start: offset where we should start selecting from\n", 54 | " \"\"\"\n", 55 | " def __init__(self, num_samples, start=0):\n", 56 | " self.num_samples = num_samples\n", 57 | " self.start = start\n", 58 | "\n", 59 | " def __iter__(self):\n", 60 | " return iter(range(self.start, self.start + self.num_samples))\n", 61 | "\n", 62 | " def __len__(self):\n", 63 | " return self.num_samples" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "### Sample data from training and validation " 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "Download the data set if it doesn't exist and use the sampler to divide it using the batch size " 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 3, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "NUM_TRAIN = 50000\n", 87 | "NUM_VAL = 5000\n", 88 | "batch_size = 128\n", 89 | "\n", 90 | "#download mnist training dataset and sample it \n", 91 | "mnist_train = dset.MNIST('./datasets/MNIST_data', train=True, download=True,\n", 92 | " transform=T.ToTensor())\n", 93 | "loader_train = DataLoader(mnist_train, batch_size=batch_size,\n", 94 | " sampler=ChunkSampler(NUM_TRAIN, 0))\n", 95 | "\n", 96 | "#download mnist validation dataset and sample it \n", 97 | "mnist_val = dset.MNIST('./datasets/MNIST_data', train=True, download=True,\n", 98 | " transform=T.ToTensor())\n", 99 | "loader_val = DataLoader(mnist_val, batch_size=batch_size,\n", 100 | " sampler=ChunkSampler(NUM_VAL, NUM_TRAIN))" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "### Flatten the data for fully connected layers" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": {}, 113 | "source": [ 114 | "In order to forward the current patch of size $(N,C,H,W)$ we have to reshape it to size $(N,C \\times H \\times W)$ " 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 4, 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [ 123 | "class Flatten(nn.Module):\n", 124 | " def forward(self,x):\n", 125 | " N,_,_,_ = x.size()\n", 126 | " return x.view(N,-1)" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": {}, 132 | "source": [ 133 | "### Model design " 134 | ] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "metadata": {}, 139 | "source": [ 140 | "The function which a signle convolution layer + relu + norm + 2FC layers " 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 5, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "model=nn.Sequential(nn.Conv2d(in_channels=1,out_channels=32,kernel_size=7,stride=1),\n", 150 | " nn.ReLU(inplace=True),\n", 151 | " nn.BatchNorm2d(num_features=32),\n", 152 | " nn.MaxPool2d(kernel_size=2,stride=2),\n", 153 | " Flatten(),\n", 154 | " nn.Linear(3872,1024), \n", 155 | " nn.ReLU(inplace=True),\n", 156 | " nn.Linear(1024,10),\n", 157 | " ).cuda()" 158 | ] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "metadata": {}, 163 | "source": [ 164 | "### Optimizer " 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 6, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "optimizer = optim.Adam(model.parameters())" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "metadata": {}, 179 | "source": [ 180 | "### Trianer " 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 7, 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "def train(model,solver,epochs = 10):\n", 190 | " \n", 191 | " #loss function \n", 192 | " loss = nn.CrossEntropyLoss()\n", 193 | " \n", 194 | " #variable to save the loss of each epoch \n", 195 | " training_loss = []\n", 196 | " l = 0 \n", 197 | " \n", 198 | " #iterate over the epochs \n", 199 | " for i in range(0,epochs):\n", 200 | " \n", 201 | " #iterate over the training set \n", 202 | " for x,target in loader_train:\n", 203 | " #zero out the gradient first \n", 204 | " solver.zero_grad()\n", 205 | " \n", 206 | " #convert the input set to a cuda variable to run on gpu \n", 207 | " data = Variable(x).cuda()\n", 208 | " trgt = Variable(target).cuda()\n", 209 | " \n", 210 | " #forward pass\n", 211 | " y = model(data)\n", 212 | " \n", 213 | " #error calculation \n", 214 | " outpt= loss(y,trgt)\n", 215 | " \n", 216 | " #calculate the gradient \n", 217 | " outpt.backward()\n", 218 | " \n", 219 | " #update the parameters \n", 220 | " solver.step()\n", 221 | "\n", 222 | " # the current loss \n", 223 | " l = outpt.cpu().data.numpy()[0]\n", 224 | " training_loss.append(l)\n", 225 | " \n", 226 | " #plot the loss of each epoch run \n", 227 | " plt.xlabel('Epoch')\n", 228 | " plt.ylabel('Loss')\n", 229 | " epoch = list(range(1, 1+len(training_loss)))\n", 230 | " t_line, = plt.plot(epoch, training_loss, 'co-', label='Training Loss')\n", 231 | " plt.legend(handles=[t_line])\n", 232 | " plt.show()\n", 233 | " \n", 234 | " return model " 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": 8, 240 | "metadata": {}, 241 | "outputs": [ 242 | { 243 | "data": { 244 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt8XHWd//HXJ7dJm0lvIbS5tE1r\nK5oKjTW2iwiy4qUoS1XKUgRkXdaqC96Q3a26P378+pBd9PF7KF7KKggIWLlY1p9dqLC7IOsitVCQ\nWynd1go0l4Y0Lb0kTZpJPr8/5qRMptMmTXMyt/fz8ZhHzpz5TuaTSTLvOed75nzM3RERETmWgnQX\nICIimU9hISIiQ1JYiIjIkBQWIiIyJIWFiIgMSWEhIiJDUliIiMiQFBYiIjIkhYWIiAypKN0FjJaT\nTjrJ6+rq0l2GiEhWefrpp3e5e+VQ43ImLOrq6ti4cWO6yxARySpm9upwxmk3lIiIDElhISIiQ1JY\niIjIkHJmzkJEMkdvby9NTU10d3enuxQJlJaWUltbS3Fx8Yjur7AQkVHX1NREeXk5dXV1mFm6y8l7\n7k5HRwdNTU3MmjVrRN8j73dDrW5ro279egoee4y69etZ3daW7pJEsl53dzcVFRUKigxhZlRUVJzQ\nll5eb1msbmtj+ZYtdPX3A/BqTw/Lt2wB4JKpU9NZmkjWU1BklhP9fYS6ZWFmi81si5ltM7MVKW6P\nmNm9we0bzKwuWH+JmT2bcOk3s4bRru8b27cfDooBXf39fGP79tF+KBGRrBZaWJhZIbAKOBeoBy42\ns/qkYVcAe9x9DvBd4FsA7r7a3RvcvQG4DPiTuz872jW+1tNzXOtFJDt0dHTQ0NBAQ0MD06ZNo6am\n5vD1Q4cODet7fPrTn2ZLsKfhaFatWsXq1atHo2Te+9738uyzo/4yN2rC3A21ENjm7tsBzOweYAnw\nUsKYJcB1wfIa4IdmZu7uCWMuBu4Jo8AZkQivpgiGGZFIGA8nIkexuq2Nb2zfzms9PcyIRLh+9uwT\n2hVcUVFx+IX3uuuuIxqNcs011wwa4+64OwUFqd8z33777UM+zpVXXjniGrNNmLuhaoAdCdebgnUp\nx7h7DNgLVCSNuQi4O4wCr589m/FJfyjjCwq4fvbsMB5ORFIYmDt8tacH5825wzAONtm2bRv19fVc\ncsklzJs3j9bWVpYvX05jYyPz5s1j5cqVh8cOvNOPxWJMmjSJFStWMH/+fE4//XRef/11AP7xH/+R\nG2+88fD4FStWsHDhQk455RSeeOIJADo7O7nggguor69n6dKlNDY2DnsL4uDBg1x++eWceuqpLFiw\ngN/+9rcAvPDCC7z73e+moaGB0047je3bt7N//37OPfdc5s+fzzve8Q7WrFkzmk9dZk9wm9kioMvd\nXzzK7cuB5QAzZsw47u8/8M7ls1u20Nnfz4xIhH86wXc0IjLYl7du5dkDB456++/37aNn0M6E+Nzh\nFS+/zC0tLSnv0xCNcuPcuSOq5+WXX+bOO++ksbERgBtuuIEpU6YQi8X48z//c5YuXUp9/eA95nv3\n7uV973sfN9xwA1dffTW33XYbK1YcMQ2Lu/Pkk0+ydu1aVq5cyUMPPcQPfvADpk2bxv33389zzz3H\nggULhl3r97//fSKRCC+88AKbNm3iIx/5CFu3buWmm27immuu4aKLLqKnpwd351e/+hV1dXX8+te/\nPlzzaApzy6IZmJ5wvTZYl3KMmRUBE4GOhNuXcYytCne/2d0b3b2xsnLIkyamdMnUqfxzsCWxfsEC\nBYXIGEsOiqHWn6i3vOUth4MC4O6772bBggUsWLCAzZs389JLLx1xn3HjxnHuuecC8K53vYtXXnkl\n5ff+xCc+ccSYxx9/nGXLlgEwf/585s2bN+xaH3/8cS699FIA5s2bR3V1Ndu2beM973kP3/zmN/n2\nt7/Njh07KC0t5bTTTuOhhx5ixYoV/O53v2PixInDfpzhCHPL4ilgrpnNIh4Ky4BPJo1ZC1wOrAeW\nAo8OzFeYWQHwl8CZIdYIxN+lADx74ADVmq8QGVVDbQHUrV+fcu5wZiTCY+9856jXU1ZWdnh569at\nfO973+PJJ59k0qRJXHrppSk/i1BSUnJ4ubCwkFgslvJ7R4LXj2ONGQ2XXXYZp59+Og8++CCLFy/m\ntttu46yzzmLjxo2sW7eOFStWcO655/L1r3991B4ztC2LYA7iKuBhYDNwn7tvMrOVZnZ+MOxWoMLM\ntgFXA4nbdWcBOwYmyMN0WkJYiMjYSufc4b59+ygvL2fChAm0trby8MMPj/pjnHHGGdx3331AfK4h\n1ZbL0Zx55pmHj7bavHkzra2tzJkzh+3btzNnzhy+9KUvcd555/H888/T3NxMNBrlsssu46tf/SrP\nPPPMqP4coc5ZuPs6YF3SumsTlruBC49y38eAPwuzvgETi4qYXVqqsBBJg4Fdv6N5NNRwLViwgPr6\net72trcxc+ZMzjjjjFF/jC984Qt86lOfor6+/vDlaLuIPvzhDx8+d9OZZ57Jbbfdxmc/+1lOPfVU\niouLufPOOykpKeHnP/85d999N8XFxVRXV3PdddfxxBNPsGLFCgoKCigpKeFHP/rRqP4c5iHtFxxr\njY2NfiLNjy548UVe6OzkfxYtGsWqRPLT5s2befvb357uMjJCLBYjFotRWlrK1q1b+dCHPsTWrVsp\nKhr744tS/V7M7Gl3bzzKXQ7L6KOhxlJDNMovd+1ifyxGeRp+iSKSmw4cOMA555xDLBbD3fnxj3+c\nlqA4UdlXcUgaolEceKGzk/eM8lEEIpK/Jk2axNNPP53uMk5Y3p91dsDAEVHPad5CZFTkyi7uXHGi\nvw+FRaA2EmFKUZEmuUVGQWlpKR0dHQqMDDHQz6K0tHTE30O7oQJmRkM0qrAQGQW1tbU0NTXR3t6e\n7lIkMNApb6QUFgkaolFuamkh1t9P0VFOLiYiQysuLh5xRzbJTHpFTNAQjdLd38/WgwfTXYqISEZR\nWCRo0Ce5RURSUlgkeNv48ZSYKSxERJIoLBIUFxTwjrIyhYWISBKFRZKGaJQ/HDigQ/5ERBIoLJI0\nRKO09/ayc5h9ekVE8oHCIokmuUVEjqSwSKLeFiIiR1JYJFFvCxGRIyksUtBpP0REBlNYpNAQjbL1\n4EEOhNhDV0QkmygsUkjsbSEiIiGHhZktNrMtZrbNzFakuD1iZvcGt28ws7qE204zs/VmtsnMXjCz\nkZ9b9zjN1yS3iMggoYWFmRUCq4BzgXrgYjOrTxp2BbDH3ecA3wW+Fdy3CPgZ8Dl3nwecDfSGVWuy\n6ZEIk9XbQkTksDC3LBYC29x9u7sfAu4BliSNWQLcESyvAc4xMwM+BDzv7s8BuHuHu/eFWOsg6m0h\nIjJYmGFRA+xIuN4UrEs5xt1jwF6gAngr4Gb2sJk9Y2Z/H2KdKTVEozzf2Umsv3+sH1pEJONk6gR3\nEfBe4JLg68fN7JzkQWa23Mw2mtnG0e7Ipd4WIiJvCjMsmoHpCddrg3UpxwTzFBOBDuJbIb91913u\n3gWsAxYkP4C73+zuje7eWFlZOarF67QfIiJvCjMsngLmmtksMysBlgFrk8asBS4PlpcCj3r8dK8P\nA6ea2fggRN4HvBRirUdQbwsRkTeF1oPb3WNmdhXxF/5C4DZ332RmK4GN7r4WuBW4y8y2AbuJBwru\nvsfMvkM8cBxY5+4PhlVrKiUFBcxTbwsRESDEsABw93XEdyElrrs2YbkbuPAo9/0Z8cNn06YhGuWB\njg7cnfhBWiIi+SlTJ7gzgnpbiIjEKSyOQZPcIiJxCotj0Gk/RETiFBbHMLGoiFmlpTynsBCRPKew\nGIJO+yEiorAYUkM0yv8cPEhn35idmkpEJOMoLIZwuLeFti5EJI8pLIagI6JERBQWQ1JvCxERhcWQ\n1NtCRERhMSwDvS363NNdiohIWigshqEhGuVgfz9bu7rSXYqISFooLIZBk9wiku8UFsOg3hYiku8U\nFsOg3hYiku8UFsOkI6JEJJ8pLIapIRqlrbeXnT096S5FRGTMKSyGSacrF5F8prAYpvllZYDCQkTy\nU6hhYWaLzWyLmW0zsxUpbo+Y2b3B7RvMrC5YX2dmB83s2eDyozDrHI5JxcXUlZYqLEQkLxWF9Y3N\nrBBYBXwQaAKeMrO17v5SwrArgD3uPsfMlgHfAi4KbvujuzeEVd9IaJJbRPJVmFsWC4Ft7r7d3Q8B\n9wBLksYsAe4IltcA55iZhVjTCVFvCxHJV2GGRQ2wI+F6U7Au5Rh3jwF7gYrgtllm9gcz+y8zOzPE\nOodNvS1EJF9l6gR3KzDD3d8JXA383MwmJA8ys+VmttHMNra3t4delE77ISL5KsywaAamJ1yvDdal\nHGNmRcBEoMPde9y9A8Ddnwb+CLw1+QHc/WZ3b3T3xsrKyhB+hMFmRCJMUm8LEclDYYbFU8BcM5tl\nZiXAMmBt0pi1wOXB8lLgUXd3M6sMJsgxs9nAXGB7iLUOi3pbiEi+Ci0sgjmIq4CHgc3Afe6+ycxW\nmtn5wbBbgQoz20Z8d9PA4bVnAc+b2bPEJ74/5+67w6r1eKi3hYjko9AOnQVw93XAuqR11yYsdwMX\nprjf/cD9YdY2Uom9Ld4WfFBPRCTXZeoEd8bSJLeI5COFxXF6+/jxFKu3hYjkGYXFcVJvCxHJRwqL\nEWiIRnmuszPdZYiIjBmFxQg0RKPsPHRIvS1EJG8oLEZgYJJbWxciki8UFiOg3hYikm8UFiOg3hYi\nkm8UFiOk036ISD5RWIxQQzTKlq4u9bYQkbygsBihgd4WL2qSW0TygMJihHTaDxHJJwqLEVJvCxHJ\nJwqLEVJvCxHJJwqLEzC/rIznDxxQbwsRyXkKixPQEI3S1d/PtoMH012KiEioFBYnQJPcIpIvFBYn\noL6sTL0tRCQvKCxOQElBAfXjxyssRCTnhRoWZrbYzLaY2TYzW5Hi9oiZ3RvcvsHM6pJun2FmB8zs\nmjDrPBE6IkpE8kFoYWFmhcAq4FygHrjYzOqThl0B7HH3OcB3gW8l3f4d4Ndh1Tga1NtCRPJBmFsW\nC4Ft7r7d3Q8B9wBLksYsAe4IltcA55iZAZjZx4A/AZtCrPGEqbeFiOSDMMOiBtiRcL0pWJdyjLvH\ngL1AhZlFgX8A/k+I9Y2K+ToiSkTyQKZOcF8HfNfdj/kKbGbLzWyjmW1sb28fm8qSTC4uZmYkorAQ\nkZxWFOL3bgamJ1yvDdalGtNkZkXARKADWAQsNbNvA5OAfjPrdvcfJt7Z3W8GbgZobGxM28eoNckt\nIrkuzLB4CphrZrOIh8Iy4JNJY9YClwPrgaXAo+7uwJkDA8zsOuBAclBkkoZolLUdHXT29VFWWJju\nckRERt2wdkOZ2VvMLBIsn21mXzSzSce6TzAHcRXwMLAZuM/dN5nZSjM7Pxh2K/E5im3A1cARh9dm\nA/W2EJFcN9wti/uBRjObQ3y3z6+AnwMfOdad3H0dsC5p3bUJy93AhUN8j+uGWWPaJJ72Y9GECWmu\nRkRk9A13grs/2FL4OPADd/87oCq8srLLzNJSJhYWat5CRHLWcMOi18wuJj6/8ECwrjickrKPeluI\nSK4bblh8GjgduN7d/xRMWt8VXlnZpyEaVW8LEclZw5qzcPeXgC8CmNlkoNzdk0/NkdcSe1ucMn58\nussRERlVwz0a6jEzm2BmU4BngFvM7DvhlpZdDp/2Q7uiRCQHDXc31ER33wd8ArjT3RcBHwivrOyj\n3hYiksuGGxZFZlYF/CVvTnBLAvW2EJFcNtywWEn8w3V/dPenzGw2sDW8srKTjogSkVw1rLBw91+4\n+2nu/vng+nZ3vyDc0rJPQzRK66FDtB06lO5SRERG1XAnuGvN7Jdm9npwud/MasMuLttokltEctVw\nd0PdTvykf9XB5d+CdZJAvS1EJFcNNywq3f12d48Fl58ClSHWlZXU20JEctVww6LDzC41s8Lgcinx\nvhOSZL4muUUkBw03LP6a+GGzO4FW4r0n/iqkmrJaQzTKlq4uuvr60l2KiMioGe7RUK+6+/nuXunu\nJ7v7xwAdDZVCQzRKP+ptISK55UR6cF89alXkkAZNcotIDjqRsLBRqyKH1JWWMkG9LUQkx5xIWOhc\n3Cmot4WI5KJjnqLczPaTOhQMGBdKRTmgIRrl1tZW+twpNG2AiUj2O+aWhbuXu/uEFJdydx+yF4aZ\nLTazLWa2zcxWpLg9Ymb3BrdvMLO6YP1CM3s2uDxnZh8f6Q+YDg3RKJ39/fzx4MF0lyIiMipOZDfU\nMZlZIbAKOBeoBy42s/qkYVcAe9x9DvBdYKCh0otAo7s3AIuBH5vZsBo1ZQJNcotIrgktLICFwLbg\npIOHgHuAJUljlgB3BMtrgHPMzNy9y91jwfpSsmx+pL6sjCL1thCRHBJmWNQAOxKuNwXrUo4JwmEv\nUAFgZovMbBPwAvC5hPA4zMyWm9lGM9vY3t4ewo8wMhH1thCRHBNmWJwQd9/g7vOAdwNfM7PSFGNu\ndvdGd2+srMysU1XpiCgRySVhhkUzMD3hem2wLuWYYE5iIknnnHL3zcAB4B2hVRoC9bYQkVwSZlg8\nBcw1s1lmVgIsI36a80RrgcuD5aXAo+7uwX2KAMxsJvA24JUQax116m0hIrkktLAI5hiuIt6OdTNw\nn7tvMrOVZnZ+MOxWoMLMthE/fcjA4bXvBZ4zs2eBXwJ/6+67wqo1DOptISK5JNTDUd19HbAuad21\nCcvdwIUp7ncXcFeYtYVtSnExM9TbQkRyRMZOcOcCTXKLSK5QWIRIvS1EJFcoLEKk3hYikisUFiHS\naT9EJFcoLEI00NtCh8+KSLZTWIRIvS1EJFcoLELWEI3y3IED9HtWnQtRRGQQhUXI1NtCRHKBwiJk\nmuQWkVygsAiZeluISC5QWIQsUlDA29XbQkSynMJiDOiIKBHJdgqLMdAQjdJy6BCvq7eFiGQphcUY\nUG8LEcl2CosxoN4WIpLtFBZjoKK4mOnqbSEiWUxhMUY0yS0i2UxhMUYaolFe7urioHpbiEgWUliM\nEfW2EJFsFmpYmNliM9tiZtvMbEWK2yNmdm9w+wYzqwvWf9DMnjazF4Kv7w+zzrGg036ISDYLLSzM\nrBBYBZwL1AMXm1l90rArgD3uPgf4LvCtYP0u4C/c/VTgcuCusOocKwO9LRQWIpKNwtyyWAhsc/ft\n7n4IuAdYkjRmCXBHsLwGOMfMzN3/4O4twfpNwDgzi4RYa+gKzJivSW4RyVJhhkUNsCPhelOwLuUY\nd48Be4GKpDEXAM+4e0/yA5jZcjPbaGYb29vbR63wsKi3hYhkq4ye4DazecR3TX021e3ufrO7N7p7\nY2Vl5dgWNwLqbSEi2SrMsGgGpidcrw3WpRxjZkXARKAjuF4L/BL4lLv/McQ6x4wmuUUkW4UZFk8B\nc81slpmVAMuAtUlj1hKfwAZYCjzq7m5mk4AHgRXu/rsQaxxT9ePHq7eFiGSl0MIimIO4CngY2Azc\n5+6bzGylmZ0fDLsVqDCzbcDVwMDhtVcBc4BrzezZ4HJyWLWOldLCQvW2EJGsVBTmN3f3dcC6pHXX\nJix3AxemuN83gW+GWVu6NESjPLJnT7rLEBE5Lhk9wZ2L1NtCRLKRwmKMqbeFiGQjhcUYU28LEclG\nCosxpt4WIpKNFBZpMPBJbhGRbKGwSAP1thCRbKOwSIOGaJQ+YJN6W4hIllBYpIFO+yEi2UZhkQZ1\npaWUq7eFiGQRhUUaqLeFiGQbhUWaNESjPNfZqd4WIpIVFBZp0hCNcqCvj+3qbSEiWUBhkSaa5BaR\nbKKwSJN548dTiMJCRLKDwiJNSgsLeXtZmcJCRLKCwiKNGnRElIhkCYVFGjVEozQfOkS7eluISIZT\nWKSReluISLYINSzMbLGZbTGzbWa2IsXtETO7N7h9g5nVBesrzOw3ZnbAzH4YZo3pNL+sDNAkt4hk\nvtDCwswKgVXAuUA9cLGZ1ScNuwLY4+5zgO8C3wrWdwP/C7gmrPoywUklJdSqt4WIZIEwtywWAtvc\nfbu7HwLuAZYkjVkC3BEsrwHOMTNz9053f5x4aOQ0TXKLSDYIMyxqgB0J15uCdSnHuHsM2AtUhFhT\nxlFvCxHJBlk9wW1my81so5ltbG9vT3c5I6LeFiKSDcIMi2ZgesL12mBdyjFmVgRMBDqG+wDufrO7\nN7p7Y2Vl5QmWmx467YeIZIMww+IpYK6ZzTKzEmAZsDZpzFrg8mB5KfCoe36dhnWWeluISBYoCusb\nu3vMzK4CHgYKgdvcfZOZrQQ2uvta4FbgLjPbBuwmHigAmNkrwASgxMw+BnzI3V8Kq950UW8LEckG\noYUFgLuvA9Ylrbs2YbkbuPAo960Ls7ZM0hCN8tOdO+l3p8As3eWIiBwhqye4c4V6W4hIplNYZABN\ncotIplNYZAD1thCRTKewyAD379pFgRnXv/YadevXs7qtLd0liYgMorBIs9VtbSzfsoXe4IjhV3t6\nWL5liwJDRDKKwiLNvrF9O139/YPWdfX38+WtW/nTwYPk2cdORI5pdVsbdevXU/DYY9oKH2OhHjor\nQ3utpyfl+l2xGLM3bKCyuJiF5eUsmjCBRRMmsLC8nEnFxWNc5dha3dbGN7Zv57WeHmZEIlw/ezaX\nTJ2a7rIkzQa2wgfeXA1shQN5+/cxlv8rCos0mxGJ8GqKwJhWUsL/njmTDfv3s2HfPh7cvfvwbaeM\nG8fCIDwWlZdzWjRKSUFubCTqBUGO5utH2Qr/xvbtefm3Mdb/K5YruzkaGxt948aN6S7juCX/wgHG\nFxRw8ymnDPqF743FeGrfPp4MwmPDvn209fYCEDFjQbD1MbAVMqu0FMuyD/j1uzPz97+nKUV4zoxE\neOX009NQlaRTZ18f/7lnDw92dHBLa2vKMQYcOussinLkDdNw1T7xBM0pWjIf7/+KmT3t7o1DjdOW\nRZoNBMJQm5ITi4r4wJQpfGDKFADcndd6etiwbx9P7tvHhv37+XFLCzcGoZO8++rd5eVMTuPuq+6+\nPloOHaK5pyd+CZabBq739NBy6NDhif5kR9tdJ7nnlYMHeaCjgwd37+Y3e/bQ486EwkLGFxQcsWUB\n4MDsDRv4Qk0Nf1NVlda/87C5O4/v3cuq5uaUQQHh/a9oyyKH9Pb382Jn56Ctj81dXQz8ht86btzh\n8Ei1+2ok+z/dnT2x2KAQSAyAgXW7gq2gRGUFBdREItREItQGX3/c0sKeWOyIsQZ8aupUPlNdzXsm\nTMi6rSY5ulh/P+v37YsHREcHm7q6gPjf63kVFXy0ooL3TpzIL9rbU26Ff666mj8cOMBv3niDsoIC\n/mraNL5YW8tbx49P14806jr7+ljd1saq5mae7+xkUlERfe7sT9EHJ6wtC4VFjtsbi7Fx//741kew\nBbIzeEcysPtqYXk5ve7cvnMnB5P+Ef951iwWTpgQD4DELYOE6wdTvNs7ubh4UAjUlJQcEQwTCguP\neNFPtVuu1Iz3TJjAUwcOsL+vj/rx4/lMVRWXTZtGRQ6/i8xlu3t7eWj3bh7o6OCh3bvZE4tRZMb7\nJk48HBBzU7zYH+sNzbP79/O95mZ+3tbGIXfOq6jgy7W1vH/SpKx9c7G1q4ubWlq4vbWVvX19zC8r\n46qaGj45dSq/3LVrWLuwh6KwkJTcnR0Du6+CLZCN+/enfMFPpcRsyBCoKik5oQn3o70gHIjFuLe9\nnVtaWtiwfz8lZlxQWclnqqo4O4tfEPKBu/NSVxcPdHTwQEcHT+zdSz/x3aUfrajgvIoKPjh5MhOK\nTnzP+M6eHn7U0sJNLS209/ZyalkZX66t5ZMnn0xpYeGJ/zAh63NnXUcHq5qbeXjPHorMuLCykitr\nao7Yqh6No6EUFjJssf5+Sn77W472l/DgqaceDoaK4uKMeFF+4cABbmlt5a62Nt6IxZg7bhx/U1XF\n5dOmMbWkJN3lCfF5qsfeeONwQAwc9ffOaJTzgoBoLC8P7UzL3X193P3669zY1MTznZ1UFhfz+epq\nPl9dzbRIJJTHPBEdvb3c2trKv7S08Ep3NzUlJXy2uprPVFWFWq/CQo5L3fr1KQ/hzfSjkA729bGm\nvZ1bWlv57717KTJjSUUFy6ur+cDkyTrl+xhr7ulhXRAO/7lnD139/YwvKOADkydzXkUFH6mooGaM\nX6jdnd+88QY3NjXxQEcHRWZcfPLJfKW2loby8jGtJZWN+/axqqWFu9va6HHn7EmTuLK6miUnnUTx\nGBzhpbCQ4zLcQ3gz2cudnfyktZWf7txJRyxGXWkpV0ybxqerqsb8BSrXHG13R787G/fvP7z18Ifg\nZJh1paXxuYcpUzh70qSM2f2ztauL7zc3c3trK539/bxv4kS+Mn0651VUUDiGbyy6+/r4RXs7P2xu\n5sn9+ykrKOBT06bxt9XVvCM4C/VYUVjIccuVT0739Pfz/3bt4paWFh554w0KgI9WVLC8qorFU6Zk\n3fH46f69pHojUWLGovJythw8yOu9vRQAZyRMTtePH58RuyuP5o3eXm7duZPvNzXxWk8Pbykt5Yu1\ntXx62jTKR2He5Ghe7e7mRy0t/KS1lV29vZwybhxX1tTwqWnTmBji4x6LwkIE+OPBg/yktZXbW1tp\n6+2lpqSEv66q4oqqKmaWlqa7vCGlPDqsoICVM2fy4YoKYu70use/9vcPvp78dYS33/f663SmOACi\nAFh28sl8tKKCxVOmMCULj0yLBW8sbmxq4nf79jGhsJArqqr4Qk0Ns8aNG5XHcHce2bOHHzY3828d\nHQCcf9JJXFldzTmTJ6c9VBUWIgl6+/t5IPgU8EPBqVM+NHkyy6ur+YuKijHZN3w0fe609vTwak8P\nr3R382rC5ZE33iA2Bv+jBUCxGUVmFBcUxL8G13cc5UNeBvSffXbotY2VJ/ft43tNTdzX3k6/Ox87\n6SS+UlvLGRMnjugFfW8sxh07d3JTczNbDh7kpOJiPlNVxeeqq5mRQW9UMiIszGwx8D2gEPiJu9+Q\ndHsEuBN4F9ABXOTurwS3fQ24AugDvujuDx/rsRQWMlyvdndzW2srt+3cSVNPD1OLi/mradP4m6oq\n5oTwQa5D/f3s6OkZFAKvdHeqR1OAAAAH0ElEQVTzarBuR0/PEYFwUnExMyMRnj5KQywDfjFv3psv\n8Mlfk17wU90+sFxkdswDAbL14IeRauru5qaWFn7c0sLuWIx3RaN8Zfp0LqysHNYh4S8eOMCqlhbu\n2rmTzv5+FpWXc2VNDRdWVmbM3E2itIeFmRUC/wN8EGgCngIudveXEsb8LXCau3/OzJYBH3f3i8ys\nHrgbWAhUA/8JvNXdj/y4YkBhIcerz52Hdu/m5pYWHuzooA94/6RJfKaqio9XVrKmvX1YcwWdfX2D\ngmAgBAa2EloPHRp0WLIB1SUlzCwtffMSiQy6Xha8qGTCC3UuHPwwEl19fdzV1saNTU283NVFdUkJ\nV9bUsLyqiof37Bn0t7Gyro5xhYWsam7mv/buJWLGxVOncmV1NY0TJqT7RzmmTAiL04Hr3P3DwfWv\nAbj7PyeMeTgYs97MioCdQCWwInFs4rijPZ7CQk5ES08Pt+/cyU9aW3mlu5syM3pg0Dv+SPDhqMqS\nkkHBkHwqkyIzpkci1B0lCKZHIsP+0GKmvFCne5I9nfrd+ffdu7mxqSn+IblgfeJJaYz4OarqSkv5\nfHU1fz1tGidlyed9MuFEgjXAjoTrTcCio41x95iZ7QUqgvW/T7pvTXilSr6rjkT4xsyZfG3GDB7Z\ns4ePvfgisaRJ3R53fvb664wrKDgcAo3l5UdsIVRFIqN2GOZwTzQZtkumTs2bcEhWYMbiigoWV1Tw\nUmcni55+mgNJfxtO/NPo2xYtGtNDcMdSVp911syWA8sBZsyYkeZqJBcUmPHBKVOOevoTAzrPPHNM\nj2DJ5xfqTFNfVpbyyDCAXb29ORsUEG5b1WZgesL12mBdyjHBbqiJxCe6h3Nf3P1md29098bKyspR\nLF3y3YyjfIhvRiSS9kMdJb2O9beRy8IMi6eAuWY2y8xKgGXA2qQxa4HLg+WlwKMen0RZCywzs4iZ\nzQLmAk+GWKvIINfPns34pHmF8QUFXD97dpoqkkyRr38boe2GCuYgrgIeJn7o7G3uvsnMVgIb3X0t\ncCtwl5ltA3YTDxSCcfcBLxGfR7ryWEdCiYy2TJkrkMyTr38b+lCeiEgeG+7RUNl1khwREUkLhYWI\niAxJYSEiIkNSWIiIyJAUFiIiMqScORrKzNqBV9Ndxwk6CdiV7iIyiJ6PwfR8vEnPxWAn8nzMdPch\nP9WcM2GRC8xs43AOYcsXej4G0/PxJj0Xg43F86HdUCIiMiSFhYiIDElhkVluTncBGUbPx2B6Pt6k\n52Kw0J8PzVmIiMiQtGUhIiJDUlhkADObbma/MbOXzGyTmX0p3TWlm5kVmtkfzOyBdNeSbmY2yczW\nmNnLZrY5aFmct8zsK8H/yYtmdreZlaa7prFkZreZ2etm9mLCuilm9h9mtjX4Onm0H1dhkRliwFfd\nvR74M+BKM6tPc03p9iVgc7qLyBDfAx5y97cB88nj58XMaoAvAo3u/g7i7Q+WpbeqMfdTYHHSuhXA\nI+4+F3gkuD6qFBYZwN1b3f2ZYHk/8ReDvO05bma1wEeBn6S7lnQzs4nAWcR7v+Duh9z9jfRWlXZF\nwLigu+Z4oCXN9Ywpd/8t8f4/iZYAdwTLdwAfG+3HVVhkGDOrA94JbEhvJWl1I/D3QOpmx/llFtAO\n3B7slvuJmZWlu6h0cfdm4P8CrwGtwF53//f0VpURprp7a7C8Exj1TkwKiwxiZlHgfuDL7r4v3fWk\ng5mdB7zu7k+nu5YMUQQsAP7F3d8JdBLCLoZsEeyLX0I8RKuBMjO7NL1VZZagNfWoH+aqsMgQZlZM\nPChWu/u/prueNDoDON/MXgHuAd5vZj9Lb0lp1QQ0ufvAluYa4uGRrz4A/Mnd2929F/hX4D1prikT\ntJlZFUDw9fXRfgCFRQYwMyO+T3qzu38n3fWkk7t/zd1r3b2O+MTlo+6et+8c3X0nsMPMTglWnUO8\nN32+eg34MzMbH/zfnEMeT/gnWAtcHixfDvxqtB9AYZEZzgAuI/4u+tng8pF0FyUZ4wvAajN7HmgA\n/inN9aRNsIW1BngGeIH4a1hefZrbzO4G1gOnmFmTmV0B3AB80My2Et/6umHUH1ef4BYRkaFoy0JE\nRIaksBARkSEpLEREZEgKCxERGZLCQkREhqSwEDkOZtaXcHjzs2Y2ap+mNrO6xDOJimSSonQXIJJl\nDrp7Q7qLEBlr2rIQGQVm9oqZfdvMXjCzJ81sTrC+zsweNbPnzewRM5sRrJ9qZr80s+eCy8ApKwrN\n7JagX8O/m9m4tP1QIgkUFiLHZ1zSbqiLEm7b6+6nAj8kfuZcgB8Ad7j7acBq4PvB+u8D/+Xu84mf\n62lTsH4usMrd5wFvABeE/POIDIs+wS1yHMzsgLtHU6x/BXi/u28PTgq5090rzGwXUOXuvcH6Vnc/\nyczagVp370n4HnXAfwQNbDCzfwCK3f2b4f9kIsemLQuR0eNHWT4ePQnLfWheUTKEwkJk9FyU8HV9\nsPwEb7b9vAT472D5EeDzcLjf+MSxKlJkJPSuReT4jDOzZxOuP+TuA4fPTg7ODNsDXBys+wLxLnd/\nR7zj3aeD9V8Cbg7OGNpHPDhaEclQmrMQGQXBnEWju+9Kdy0iYdBuKBERGZK2LEREZEjashARkSEp\nLEREZEgKCxERGZLCQkREhqSwEBGRISksRERkSP8fs1xjAfHI1XkAAAAASUVORK5CYII=\n", 245 | "text/plain": [ 246 | "" 247 | ] 248 | }, 249 | "metadata": {}, 250 | "output_type": "display_data" 251 | } 252 | ], 253 | "source": [ 254 | "trained_model = train(model,optimizer)" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": {}, 260 | "source": [ 261 | "### Validation " 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 9, 267 | "metadata": {}, 268 | "outputs": [], 269 | "source": [ 270 | "from sklearn.metrics import accuracy_score\n", 271 | "\n", 272 | "def valid(model):\n", 273 | " cnt = 0\n", 274 | " \n", 275 | " #save true labels and predicted \n", 276 | " y_tru = []\n", 277 | " y_prd = [] \n", 278 | " \n", 279 | " for x,target in loader_val:\n", 280 | " \n", 281 | " #same as training \n", 282 | " data = Variable(x).cuda()\n", 283 | " trgt = Variable(target).cuda()\n", 284 | " y = model(data)\n", 285 | " \n", 286 | " #convert the data to numpy types \n", 287 | " y_batch_prd = y.cpu().data.numpy()\n", 288 | " y_batch_prd = np.argmax(y_batch_prd,axis=1)\n", 289 | " y_batch_tru = trgt.cpu().data.numpy() \n", 290 | " \n", 291 | " #conactenate calcuated and true labels \n", 292 | " y_prd = np.concatenate((y_prd,y_batch_prd))\n", 293 | " y_tru = np.concatenate((y_tru,y_batch_tru))\n", 294 | " \n", 295 | " print('Accuracy on validation' , accuracy_score(y_tru,y_prd)*100 ,'%')\n", 296 | " " 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": 10, 302 | "metadata": {}, 303 | "outputs": [ 304 | { 305 | "name": "stdout", 306 | "output_type": "stream", 307 | "text": [ 308 | "Accuracy on validation 98.68 %\n" 309 | ] 310 | } 311 | ], 312 | "source": [ 313 | "valid(trained_model)" 314 | ] 315 | } 316 | ], 317 | "metadata": { 318 | "kernelspec": { 319 | "display_name": "Python 3", 320 | "language": "python", 321 | "name": "python3" 322 | }, 323 | "language_info": { 324 | "codemirror_mode": { 325 | "name": "ipython", 326 | "version": 3 327 | }, 328 | "file_extension": ".py", 329 | "mimetype": "text/x-python", 330 | "name": "python", 331 | "nbconvert_exporter": "python", 332 | "pygments_lexer": "ipython3", 333 | "version": "3.5.2" 334 | } 335 | }, 336 | "nbformat": 4, 337 | "nbformat_minor": 2 338 | } 339 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/Draw_boxes-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import cv2\n", 10 | "import numpy as np\n", 11 | "\n", 12 | "drawing = False # true if mouse is pressed\n", 13 | "ix,iy = -1,-1\n", 14 | "\n", 15 | "# mouse callback function\n", 16 | "def draw_rectangle(event,x,y,flags,param):\n", 17 | " global ix,iy,drawing,img_res,img_cpy,f_x,f_y\n", 18 | "\n", 19 | " if event == cv2.EVENT_LBUTTONDOWN:\n", 20 | " drawing = True\n", 21 | " ix,iy = x,y\n", 22 | "\n", 23 | " elif event == cv2.EVENT_MOUSEMOVE:\n", 24 | " if drawing == True:\n", 25 | " img_cpy = img_res.copy()\n", 26 | " cv2.rectangle(img_cpy,(ix,iy),(x,y),(255,255,0),1)\n", 27 | "\n", 28 | "\n", 29 | " elif event == cv2.EVENT_LBUTTONUP:\n", 30 | " drawing = False\n", 31 | " cv2.rectangle(img_cpy,(ix,iy),(x,y),(255,255,0),1)\n", 32 | " print(str(int(ix / f_x)) +' '+ str(int(iy / f_y)) +' '+ str(int((x-ix) / f_x))+ ' '+str(int((y-iy) / f_y)))\n" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 3, 38 | "metadata": {}, 39 | "outputs": [ 40 | { 41 | "name": "stdout", 42 | "output_type": "stream", 43 | "text": [ 44 | "77 22 274 186\n" 45 | ] 46 | } 47 | ], 48 | "source": [ 49 | "img = cv2.imread('img.jpg')\n", 50 | "\n", 51 | "#check if the shape is too large \n", 52 | "shape = img.shape\n", 53 | "\n", 54 | "f_x = 1.0\n", 55 | "f_y = 1.0 \n", 56 | "\n", 57 | "img_res = cv2.resize(img.copy(), (0,0), fx=f_x, fy=f_y) \n", 58 | "img_cpy = img_res.copy()\n", 59 | "\n", 60 | "cv2.namedWindow('image')\n", 61 | "cv2.setMouseCallback('image',draw_rectangle)\n", 62 | "\n", 63 | "while(1):\n", 64 | " cv2.imshow('image',img_cpy)\n", 65 | " k = cv2.waitKey(1) & 0xFF\n", 66 | " if k == 27:\n", 67 | " break\n", 68 | "\n", 69 | "cv2.destroyAllWindows()\n" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [] 78 | } 79 | ], 80 | "metadata": { 81 | "kernelspec": { 82 | "display_name": "Python 3", 83 | "language": "python", 84 | "name": "python3" 85 | }, 86 | "language_info": { 87 | "codemirror_mode": { 88 | "name": "ipython", 89 | "version": 3 90 | }, 91 | "file_extension": ".py", 92 | "mimetype": "text/x-python", 93 | "name": "python", 94 | "nbconvert_exporter": "python", 95 | "pygments_lexer": "ipython3", 96 | "version": "3.5.2" 97 | } 98 | }, 99 | "nbformat": 4, 100 | "nbformat_minor": 2 101 | } 102 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/Pretrained model Pytorch-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 2 6 | } 7 | -------------------------------------------------------------------------------- /Linear regression using GSD.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Import libraries " 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "from sklearn import datasets\n", 17 | "import matplotlib.pyplot as plt\n", 18 | "import numpy as np\n", 19 | "from numpy.linalg import inv\n", 20 | "from numpy.linalg import norm" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": {}, 26 | "source": [ 27 | "The big disadvantage of using the normal equations is the complexity of finding the inverse of $X^T X$ which can be arbitrary huge depending on the data on the training set. A pretty preferrrable approach is using gradient descent which tracks the decrease on the loss function and moves towards it. " 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "### Generated random data\n", 35 | "\n", 36 | "Use the diabetes data offered by skicit learn " 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 2, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "# Load the diabetes dataset\n", 46 | "diabetes = datasets.load_diabetes()\n", 47 | "\n", 48 | "# Use only one feature\n", 49 | "diabetes_X = diabetes.data[:, np.newaxis, 2]\n", 50 | "\n", 51 | "# Use 50 data points\n", 52 | "X = diabetes_X[0:50]\n", 53 | "\n", 54 | "# The true value of y\n", 55 | "y = diabetes.target[0:50]\n" 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "### Plot the data \n" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 3, 68 | "metadata": {}, 69 | "outputs": [ 70 | { 71 | "data": { 72 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF0hJREFUeJzt3W2MHVd9x/HvzyYENqSKnSzG8cNuIhlVDi8MuXUjUSGV\nAgl549AXyNWSWmrUBSlKQwVSHfyiqZAlQKSIqiWSo0Y18ZLUKqBYKFDZUaQoEk1Yozw54MY0tmPL\nsRcMDZElN7H/fTGz2evNvXvnPs7T7yON7twzM3vPzLX/98w5/5lRRGBmZtW1LO8KmJnZcDnQm5lV\nnAO9mVnFOdCbmVWcA72ZWcU50JuZVZwDvZlZxTnQm5lVnAO9mVnFvSvvCgBcc801MTk5mXc1zMxK\n5eDBg7+OiPFO63UM9JLeAzwJXJ6u/x8R8feS7gX+GphLV/1KRDyWbnMPcAdwAfibiPjPpT5jcnKS\n2dnZTlUxM7Mmko5lWS9Li/488PGIeEPSZcBTkn6cLvtWRHxz0QdvBLYCNwDXAgckfTAiLmSvvpmZ\nDUrHPvpIvJG+vSydlroT2hbgkYg4HxGvAEeAzX3X1MzMepJpMFbScknPAmeA/RHxdLroLknPS3pQ\n0oq0bA3watPmJ9KyxX9zWtKspNm5ubnFi83MbEAyBfqIuBARm4C1wGZJHwLuB64HNgGngPu6+eCI\n2BURjYhojI93HEswM7MedZVeGRG/A54AbomI0+kPwEXgARa6Z04C65o2W5uWmZlZDjoGeknjkq5K\n598LfBL4paTVTat9Bngxnd8HbJV0uaTrgA3AM4OttplZwczMwOQkLFuWvM7M5F2jt2XJulkN7Ja0\nnOSHYW9E/EjSQ5I2kQzMHgU+DxARhyTtBV4C3gLudMaNmVXazAxMT8O5c8n7Y8eS9wBTU/nVK6Ui\nPEqw0WiE8+jNrLQmJ5PgvtjEBBw9OrSPlXQwIhqd1vMtEMzM+nX8eHflI+ZAb2bWr/XruysfMQd6\nM7N+7dwJY2OXlo2NJeUF4EBvZtavqSnYtSvpk5eS1127CjEQCwW5e6WZWelNTRUmsC/mFr2ZWcU5\n0JuZVZwDvZlZxTnQm5lVnAO9mVnFOdCbmVWcA72ZWcU50JuZVZwDvZlZxTnQm5lVnAO9mVnFOdCb\nmVWcA72ZWcU50JvlpcAPk7Zq8W2KzfJQ8IdJW7W4RW+Whx07FoL8vHPnknKzAXOgN8tDwR8mbdXi\nQG+Wh4I/TNqqxYHeLA8Ff5i0VUvHQC/pPZKekfScpEOS/iEtXylpv6SX09cVTdvcI+mIpMOSbh7m\nDpiVUsEfJm3VoohYegVJwBUR8Yaky4CngLuBPwfORsTXJG0HVkTE30naCDwMbAauBQ4AH4yIC+0+\no9FoxOzs7GD2yMysJiQdjIhGp/U6tugj8Ub69rJ0CmALsDst3w3cls5vAR6JiPMR8QpwhCTom5lZ\nDjL10UtaLulZ4AywPyKeBlZFxKl0ldeAVen8GuDVps1PpGVmZpaDTIE+Ii5ExCZgLbBZ0ocWLQ+S\nVn5mkqYlzUqanZub62ZTMzPrQldZNxHxO+AJ4BbgtKTVAOnrmXS1k8C6ps3WpmWL/9auiGhERGN8\nfLyXupuZWQZZsm7GJV2Vzr8X+CTwS2AfsC1dbRvwaDq/D9gq6XJJ1wEbgGcGXXEzM8smy71uVgO7\nJS0n+WHYGxE/kvRTYK+kO4BjwGcBIuKQpL3AS8BbwJ1LZdyYmdlwdUyvHAWnV5qZdW9g6ZVmZlZu\nDvRmZhXnQG9mVnEO9GZmFedAb8XhR+uZDYUfJWjF4EfrmQ2NW/RWDH60ntnQONBbMfjRemZD40Bv\nxTDsR+u5/99qzIHeimGYj9ab7/8/dgwiFvr/HeytJhzorRiG+Wg99/9bzfleN1Z9y5YlLfnFJLh4\ncfT1MRsQ3+vGbN6w+//NCs6B3qpvmP3/ZiXgQG/VN8z+f7MS8JWxVg9TUw7sVltu0Ztl4Tx8KzG3\n6M068X14rOTcojfrxHn4VnIO9Gad+D48VnIO9Gad1DEP32MSleJAb9ZJ3fLwfW+gynGgN+ukbnn4\nHpOoHN/rxswu5XsDlYbvdWNmvWk39rBypfvtS6pjoJe0TtITkl6SdEjS3Wn5vZJOSno2nW5t2uYe\nSUckHZZ08zB3wMwGrNWYxLvfDa+/7n77ksrSon8L+FJEbARuAu6UtDFd9q2I2JROjwGky7YCNwC3\nAN+RtHwIdTezYWg1JnHllfDmm5eu53770ugY6CPiVET8PJ3/PfALYM0Sm2wBHomI8xHxCnAE2DyI\nyprZiExNwdGjSZ/80aNw9mzr9XwtQSl01UcvaRL4MPB0WnSXpOclPShpRVq2Bni1abMTLP3DYGZF\nV8drCSokc6CX9D7g+8AXI+J14H7gemATcAq4r5sPljQtaVbS7NzcXDebmlmver0Qqm7XElRMpkAv\n6TKSID8TET8AiIjTEXEhIi4CD7DQPXMSWNe0+dq07BIRsSsiGhHRGB8f72cfzCyLfi6Eqtu1BBXT\nMY9ekoDdwNmI+GJT+eqIOJXO/y3wxxGxVdINwPdIAv+1wOPAhoi40O4znEdvNgKTk0lwX2xiIumH\nt9LJmkef5TbFHwVuB16Q9Gxa9hXgLyRtAgI4CnweICIOSdoLvESSsXPnUkHezEbEN2errY6BPiKe\nAtRi0WNLbLMTcOedWZGsX9+6Re8B1crzlbFmdeEB1dpyoDerCw+o1pYfJWhWJ35Iei25RW9mVnEO\n9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRmZhXnQG9mVnEO9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRm\nZhXnQG9mVnEO9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRmZhXnQG9mVnEO9DY6MzMwOQnLliWvMzN5\n18isFvxwcBuNmRmYnoZz55L3x44l78EPqzYbMrfobTTuvnshyM87dw527MinPmY10jHQS1on6QlJ\nL0k6JOnutHylpP2SXk5fVzRtc4+kI5IOS7p5mDtgJTAzA7/5Tetlx4+Pti5mNZSlRf8W8KWI2Ajc\nBNwpaSOwHXg8IjYAj6fvSZdtBW4AbgG+I2n5MCpvJbFUq339+tHVw6ymOgb6iDgVET9P538P/AJY\nA2wBdqer7QZuS+e3AI9ExPmIeAU4AmwedMWtRJZqte/cObp6mNVUV330kiaBDwNPA6si4lS66DVg\nVTq/Bni1abMTaZnVVbtW+9VXeyDWbAQyB3pJ7wO+D3wxIl5vXhYRAUQ3HyxpWtKspNm5ubluNrWy\n2bkTxsYuLRsbg29/O5/6mNVMpkAv6TKSID8TET9Ii09LWp0uXw2cSctPAuuaNl+bll0iInZFRCMi\nGuPj473W38pgagp27YKJCZCS11273Jo3G5EsWTcC/hX4RUT8Y9OifcC2dH4b8GhT+VZJl0u6DtgA\nPDO4KlspTU3B0aNw8WLy6iBvNjJZWvQfBW4HPi7p2XS6Ffga8ElJLwOfSN8TEYeAvcBLwE+AOyPi\nwlBqb+XiK2PNctHxytiIeApQm8V/1mabnYDTKWyBr4w1y42vjLXR2LHDV8aa5cSB3kajXS69r4w1\nGzoHehuNdrn0na6Mdb++Wd8c6G002uXSL3Vl7Hy//rFjELHQr1/WYO8fLcuJA72NRi+59FXq16/a\nj5aVipKLWvPVaDRidnY272pY0SxblgTFxaQkH79MJieT4L7YxERyXYFZDyQdjIhGp/XcordLFal7\nodd+/SLyYLTlyIHeFhSte6GXfv2iqtKPlpWOA70tKFqfeJXukVOlHy0rHffR24Iq9YkX0cxM8qN5\n/HjSkt+5s5w/WlYYWfvo/XBwW7B+fesBQ3cvDMbUlAO75cJdN7ag3+6FIg3kmtnbHOhtQT994kUb\nyDWzt7mP3gbDeeJmI+c8ehutVkF+qXIzGxkHehuM5cu7K7fWPM5hQ+CsGxuMC20eItau3N7JD2ex\nIXGL3gZjYqK7cnunol2wZpXhQG+D4Ss/++f74diQONDbYFTpdgV58f1wbEgc6G1wpqaSVMqLF5NX\nB/nu+KzIhsSB3qwofFZkQ+KsG7Mi8f1wbAjcojczqzgHejOziusY6CU9KOmMpBebyu6VdFLSs+l0\na9OyeyQdkXRY0s3DqriZ9cFX4NZKlj76fwP+GfjuovJvRcQ3mwskbQS2AjcA1wIHJH0wInx5pFlR\n+Arc2unYoo+IJ4GzGf/eFuCRiDgfEa8AR4DNfdTPzAbNV+DWTj999HdJej7t2lmRlq0BXm1a50Ra\nZmZF4Stwa6fXQH8/cD2wCTgF3NftH5A0LWlW0uzc3FyP1TCzrvkK3NrpKdBHxOmIuBARF4EHWOie\nOQmsa1p1bVrW6m/siohGRDTGx8d7qYZZdzwAmfAVuLXTU6CXtLrp7WeA+YycfcBWSZdLug7YADzT\nXxULwAGi/Fo96vD225MrUOv2nfoK3PqJiCUn4GGS7pk3Sfrc7wAeAl4AnicJ7qub1t8B/Ao4DHy6\n09+PCG688cYorD17IsbGIpLwkExjY0m5lcfExKXf4eKpiN/pnj1JvaXktWj1s9wBs5EhxvqZsZ34\nWajVsGxZEtKXUqTvdHEKJCTdK255WxM/M7YXrbpoipah4G6k3mQZaCxS1olTIG2AHOjnterDnZ6G\nlStbr59HhkK7OjrYd9ZqAHKxImWdFK2BYaXmQD+vXQsKipOh0E8rr+5nAs0DkJAMQjYrWtaJUyBt\ngBzo57VrKZ09W5wMhV5beT4TSMw/GCUCHnqoGN9pO06BtEHKMmI77KkQWTftsjImJvKu2YJe6ziM\nfXNGyPD5GFsHZMy6cYt+XhlaUL3WcdD9vT5DGA0/mtEGxIF+XhkuIum1joPu73VGyGjUfVzFBsZ5\n9HUw6JzsdjnpUtL6tP45j94ycB69LRj02YozQobPZ002QA70dTHI/t4yjGeUnfPobYAc6K17ZRjP\nKDufNdkAOdBbb5wRMlw+a7IBcqA3KyKfNdkAOdBbZ07zy4fPmmxA3pV3BazgFqf5zV8cBQ48ZiXh\nFr0tzWl+ZqXnQG9Lc5qfWek50NvSnOZnVnoO9LY0p/mZlZ4DvS3NaX7V5EyqWnHWjXU2NeXAXiXO\npKodt+jN6saZVLXjQG9WN86kqh0HerO6cSZV7TjQmwfm6saZVLXTMdBLelDSGUkvNpWtlLRf0svp\n64qmZfdIOiLpsKSbh1XxnjigvVPez3/1dzJ6zqSqn05PDwc+BnwEeLGp7BvA9nR+O/D1dH4j8Bxw\nOXAd8CtgeafPuPHGG4f/uPQ9eyLGxiKScJZMY2NJeZ1NTFx6TOaniYnhf7a/E7O+ALPRIb5GROcW\nfUQ8CZxdVLwF2J3O7wZuayp/JCLOR8QrwBFgc0+/QINWl0yDblvIeQzMzdfxc5+rx3dilrNe++hX\nRcSpdP41YFU6vwZ4tWm9E2nZO0ialjQraXZubq7HanShDpkGvXTDjHpgrrmO7VTpOzErgL4HY9PT\nh+hhu10R0YiIxvj4eL/V6KwOmQa9nLWMemCuVR0Xq9J3YlYAvQb605JWA6SvZ9Lyk8C6pvXWpmX5\nq0OmQS9nLaMemOvUWq/ad2JWAL0G+n3AtnR+G/BoU/lWSZdLug7YADzTXxUHpA6ZBr2etWR9ktEg\nMmSWqksVvxOzIug0Wgs8DJwC3iTpc78DuBp4HHgZOACsbFp/B0m2zWHg01lGhEeSdVMHw8xiGdTf\nLkqmzZ49SWaRlLw608dKiIxZNx1XGMXkQD9Awwpgg0zDzDvIFuXHxqxPWQO9knXz1Wg0YnZ2Nu9q\nFM/MTDJ4efx40uWxc2d+3RrLliUhcTEp6fIpk8nJ1lk/ExNJ15VZSUg6GBGNTuv5FghFlfcVq4tV\nKWupDqm2Zk0c6IuqaBd4VSlrqUo/Wt3w7SZqy4G+qIrW6qxS1lKVfrSyKtoZoo2U++iLyv3Iw1Wk\n8Y9R8L+nSnIffdnVsdU5SlmvHaiKop0h2kg50BdVlbpKLH91HZcwwIG+2OrW6rTh8RlirTnQF1Uv\nGRLOqrB2fIZYax6MLaL5DInm9MqxsaX/Y/ayjZmVWtbBWAf6IuolQ8JZFWa146ybMuslQ8JZFWbW\nhgN9EfWSIeGsiv54fMMqzIE+T+2CSy8ZElXKqhh10PVVo1Z1WW5xOeyplrcp7nSr3F5u5Zv37X8H\nIY9bCA/yFsxmI4RvU5yzTpfYe/C0tTyOS5VuwWy14sHYPGXpCvDgaWvt9v/YseF143h8wyrOgX4Y\nstxi2MGltaX2f1h951Ua3zBrwYF+GLK01uscXJYabG11XJoN4578vmrUqi5LR/6wp8oMxs4PhrYa\n2Gs1uFeFwdNuZRls7XQcpdyqb1YkeDB2xFrdgqCZb0eQ6Gaw1QPWZkvyYOyoteqXn+eugAXdDELX\nuXvLbIAc6Ps139/cquUJSZ+vbzG8oJtBaPedmw2EA30/mtMo2ylaFk3el/p320r3PfnN+uZA34+l\numugeN0MRbjU3610s5HrazBW0lHg98AF4K2IaEhaCfw7MAkcBT4bEb9d6u+UdjC23RWVkASwoj1w\n2oObZpUyysHYP42ITU0fth14PCI2AI+n76upXbfMfOAsUpCHwV2Nm3f3j5l1ZRhdN1uA3en8buC2\nIXxGMZQtK2QQV+MWofvHzLrSb6AP4ICkg5Km07JVEXEqnX8NWNVqQ0nTkmYlzc7NzfVZjZyUrb95\nED9MWW7vYGaF0m8f/ZqIOCnp/cB+4C5gX0Rc1bTObyNixVJ/p7R99GXU6a6anfhOj2aFkbWP/l39\nfEhEnExfz0j6IbAZOC1pdUSckrQaONPPZ9iATU31d8axfn3rAd2ipZGa2dt67rqRdIWkK+fngU8B\nLwL7gG3patuAR/utpBVI2cYlzKyvFv0q4IeS5v/O9yLiJ5J+BuyVdAdwDPhs/9W0wpg/G+in+8fM\nRso3NTMzKynf1MzMzAAHejOzynOgNzOrOAd6M7OKc6A3M6u4QmTdSJojScVc7Brg1yOuTlH5WCzw\nsVjgY7GgjsdiIiLGO61UiEDfjqTZLKlDdeBjscDHYoGPxQIfi/bcdWNmVnEO9GZmFVf0QL8r7woU\niI/FAh+LBT4WC3ws2ih0H72ZmfWv6C16MzPrU+6BXtJKSfslvZy+tnxIiaQHJZ2R9GIv25dBF8fi\nFkmHJR2RtL2p/F5JJyU9m063jq72/Wu3X03LJemf0uXPS/pI1m3Lps9jcVTSC+m/gdLfLTDDsfhD\nST+VdF7Sl7vZtjYiItcJ+AawPZ3fDny9zXofAz4CvNjL9mWYsuwLsBz4FXA98G7gOWBjuuxe4Mt5\n70eP+952v5rWuRX4MSDgJuDprNuWaernWKTLjgLX5L0fIzwW7wf+CNjZ/O+/av8u+plyb9GT8WHi\nEfEkcLbX7Usiy75sBo5ExP9ExP8Bj6TblV2W/doCfDcS/wVclT7FrGrHpJ9jUTUdj0VEnImInwFv\ndrttXRQh0Gd6mPgQty+SLPuyBni16f2JtGzeXemp/IMl68bqtF9LrZNl2zLp51gABHBA0kFJ00Or\n5Wj0891W7d9Fz/p6ZmxWkg4AH2ixaEfzm4gIST2nAfW7/SgM+VjcD3yV5D/6V4H7gL/qpZ5Wan8S\nESclvR/YL+mX6Rmx1dRIAn1EfKLdMkn9Pky8VA8jH8CxOAmsa3q/Ni0jIk43/a0HgB8NptYj0Xa/\nMqxzWYZty6SfY0FEzL+ekfRDki6Msgb6LMdiGNtWShG6bvp9mHiVHkaeZV9+BmyQdJ2kdwNb0+1Y\n1Ef7GZKHtZdF2/1qsg/4yzTj5Cbgf9OurizblknPx0LSFZKuBJB0BfApyvXvYLF+vtuq/bvoXd6j\nwcDVwOPAy8ABYGVafi3wWNN6DwOnSAZcTgB3LLV9GacujsWtwH+TZBTsaCp/CHgBeJ7kH/TqvPep\ny/1/x34BXwC+kM4L+Jd0+QtAo9MxKevU67EgyTB5Lp0O1eRYfCCNCa8Dv0vn/6CK/y56nXxlrJlZ\nxRWh68bMzIbIgd7MrOIc6M3MKs6B3sys4hzozcwqzoHezKziHOjNzCrOgd7MrOL+HwLGhCgHTK97\nAAAAAElFTkSuQmCC\n", 73 | "text/plain": [ 74 | "" 75 | ] 76 | }, 77 | "metadata": {}, 78 | "output_type": "display_data" 79 | } 80 | ], 81 | "source": [ 82 | "plt.scatter(X, y, color='red')\n", 83 | "plt.show()" 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": {}, 89 | "source": [ 90 | "The length of training and testing " 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 4, 96 | "metadata": {}, 97 | "outputs": [ 98 | { 99 | "name": "stdout", 100 | "output_type": "stream", 101 | "text": [ 102 | "The length of the data set 50\n" 103 | ] 104 | } 105 | ], 106 | "source": [ 107 | "print (\"The length of the data set \" , len(X))" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": {}, 113 | "source": [ 114 | "### Insert columns of 1's\n", 115 | "In order to rewrite $X\\theta+b$ to $X\\theta$ hence we don't need to worry about the bias term " 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 5, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "X=np.insert(X, 0, values=1.0, axis=1)" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "metadata": {}, 130 | "source": [ 131 | "### Initialize the parameters " 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 6, 137 | "metadata": {}, 138 | "outputs": [ 139 | { 140 | "name": "stdout", 141 | "output_type": "stream", 142 | "text": [ 143 | "(50, 2)\n", 144 | "(50,)\n", 145 | "(2,)\n" 146 | ] 147 | } 148 | ], 149 | "source": [ 150 | "M = X.shape[0]\n", 151 | "N = X.shape[1]\n", 152 | "\n", 153 | "theta = np.random.randn(N)\n", 154 | "print(X.shape)\n", 155 | "print(y.shape)\n", 156 | "print(theta.shape)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "### Loss function " 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "$$L(\\theta) = \\frac{1}{2} \\sum_{i=1}^m ||y^{(i)}-x^{(i)}\\theta^T||^2$$\n", 171 | "\n", 172 | "In a vectorized notation we have \n", 173 | "\n", 174 | "$$L(\\theta) = \\frac{1}{2} \\left(y-X\\theta \\right)^T \\left(y-X\\theta \\right)$$" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 7, 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [ 183 | "def ssd(X,y,theta):\n", 184 | " return norm(y-X.dot(theta))/2" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 8, 190 | "metadata": {}, 191 | "outputs": [ 192 | { 193 | "name": "stdout", 194 | "output_type": "stream", 195 | "text": [ 196 | "567.698929301\n" 197 | ] 198 | } 199 | ], 200 | "source": [ 201 | "print(ssd(X,y,theta))" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "metadata": {}, 207 | "source": [ 208 | "### Gradient of the hyper-parameters " 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "Note the loss function can be written \n", 216 | "\n", 217 | "$$L(\\theta) = \\frac{1}{2} \\left(y^Ty-\\theta^TX^Ty-yX\\theta+\\theta^T X^TX \\theta \\right)$$\n", 218 | "\n", 219 | "By taking the gradient with respect to the parameters \n", 220 | "\n", 221 | "$$\\frac{\\partial L}{\\partial \\theta} = \\frac{1}{2} \\left(-2X^Ty +2X^TX \\theta \\right) $$\n", 222 | "\n", 223 | "Which simplifies to the following \n", 224 | "\n", 225 | "$$\\frac{\\partial L}{\\partial \\theta}= X^T \\left(X \\theta -y \\right)$$\n", 226 | "\n" 227 | ] 228 | }, 229 | { 230 | "cell_type": "markdown", 231 | "metadata": {}, 232 | "source": [ 233 | "### Training" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": 76, 239 | "metadata": { 240 | "scrolled": true 241 | }, 242 | "outputs": [], 243 | "source": [ 244 | "def train(X,theta,y,itr = 1000 , lrate = 1e-2):\n", 245 | " for j in range(0, itr):\n", 246 | " f = X.dot(theta)\n", 247 | " gradient = (X.T).dot(f - y)\n", 248 | " theta = theta - gradient * lrate\n", 249 | " \n", 250 | " if j % 100 == 0:\n", 251 | " print (\"The current loss is \" , ssd(X,y,theta))\n", 252 | " return theta" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 77, 258 | "metadata": {}, 259 | "outputs": [ 260 | { 261 | "name": "stdout", 262 | "output_type": "stream", 263 | "text": [ 264 | "The current loss is 362.683362338\n", 265 | "The current loss is 252.933456772\n", 266 | "The current loss is 246.402064324\n", 267 | "The current loss is 240.948561338\n", 268 | "The current loss is 236.411339741\n", 269 | "The current loss is 232.648405716\n", 270 | "The current loss is 229.536273318\n", 271 | "The current loss is 226.9685568\n", 272 | "The current loss is 224.854371602\n", 273 | "The current loss is 223.11664994\n" 274 | ] 275 | } 276 | ], 277 | "source": [ 278 | "thetahat = train(X,theta,y)" 279 | ] 280 | }, 281 | { 282 | "cell_type": "markdown", 283 | "metadata": {}, 284 | "source": [ 285 | "### Prediction " 286 | ] 287 | }, 288 | { 289 | "cell_type": "markdown", 290 | "metadata": {}, 291 | "source": [ 292 | "Note that \n", 293 | "\n", 294 | "$$\\hat{y} = X \\hat{\\theta}$$" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 78, 300 | "metadata": {}, 301 | "outputs": [], 302 | "source": [ 303 | "yhat = X.dot(thetahat)" 304 | ] 305 | }, 306 | { 307 | "cell_type": "markdown", 308 | "metadata": {}, 309 | "source": [ 310 | "### Plot the regression solution " 311 | ] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "execution_count": 79, 316 | "metadata": {}, 317 | "outputs": [ 318 | { 319 | "data": { 320 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG5FJREFUeJzt3X2MHPd93/H3hyeKyUkqLB7PDM2HOxlgEFD5g7GvjIsU\nBhrbkaJ/aBeFQePsErEQRpSgyEECVMz9URUBgySoayhtJYNCmDLixYrQxDBhKAkkwoAjIJF8CmQ9\n2azo6kiRoMWjZFdmicrk8ds/Zla7XO7D7PPM7OcFDHZ3dmZ3Zm7vu7/9/p4UEZiZWXmtGfUBmJnZ\nYDnQm5mVnAO9mVnJOdCbmZWcA72ZWck50JuZlZwDvZlZyTnQm5mVnAO9mVnJ3TDqAwDYsGFDzM7O\njvowzMwK5YUXXrgQEdPttmsb6CX9DPBtYF26/f+MiP8o6SHgN4GVdNPfj4in0n0OAHcDq8BvR8Tf\nt3qP2dlZlpaW2h2KmZnVkHQqy3ZZSvTvAb8aERclrQWelfS36XNfiYj/XPfGO4A9wO3Ah4BnJP18\nRKxmP3wzM+uXtjn6SFxMH65Nl1Yjoe0GnoiI9yLiDeAksKvnIzUzs65kqoyVNCHpReA88HREPJc+\ndb+klyQdlnRrum4z8GbN7mfSdfWvuU/SkqSllZWV+qfNzKxPMgX6iFiNiJ3AFmCXpF8EHgU+DOwE\nzgFf7uSNI+JQRMxFxNz0dNu6BDMz61JHzSsj4sfAt4A7I+Kt9AvgKvAY1fTMWWBrzW5b0nVmZjYC\nbQO9pGlJH0jv/yzwKeD7kjbVbPYZ4JX0/jFgj6R1km4DtgPP9/ewzcxyZnERZmdhzZrkdnFx1Ef0\nviytbjYBRyRNkHwxPBkR35T0uKSdJBWzy8BvAUTEq5KeBF4DrgD3ucWNmZXa4iLs2weXLiWPT51K\nHgPMz4/uuFLKw1SCc3Nz4Xb0ZlZYs7NJcK83MwPLywN7W0kvRMRcu+08BIKZWa9On+5s/ZA50JuZ\n9Wrbts7WD5kDvZlZrw4ehMnJa9dNTibrc8CB3sysV/PzcOhQkpOXkttDh3JREQs5Gb3SzKzw5udz\nE9jruURvZlZyDvRmZiXnQG9mVnIO9GZmJedAb2ZWcg70ZmYl50BvZlZyDvRmZiXnQG9mVnIO9GZm\nJedAb2ZWcg70ZmYl50BvZlZyDvRmo5LjyaStXDxMsdko5HwyaSsXl+jNRmFhoRrkKy5dStab9ZkD\nvdko5HwyaSsXB3qzUcj5ZNJWLg70ZqOQ88mkrVzaBnpJPyPpeUnflfSqpP+Url8v6WlJr6e3t9bs\nc0DSSUknJN0xyBMwK6ScTyZt5aKIaL2BJOCmiLgoaS3wLPAA8G+BdyLijyQ9CNwaEf9B0g7ga8Au\n4EPAM8DPR8Rqs/eYm5uLpaWl/pyRmdmYkPRCRMy1265tiT4SF9OHa9MlgN3AkXT9EeDT6f3dwBMR\n8V5EvAGcJAn6ZmY2Aply9JImJL0InAeejojngI0RcS7d5IfAxvT+ZuDNmt3PpOvMzGwEMgX6iFiN\niJ3AFmCXpF+sez5ISvmZSdonaUnS0srKSie7mplZBzpqdRMRPwa+BdwJvCVpE0B6ez7d7CywtWa3\nLem6+tc6FBFzETE3PT3dzbGbmVkGWVrdTEv6QHr/Z4FPAd8HjgF70832At9I7x8D9khaJ+k2YDvw\nfL8P3MzMssky1s0m4IikCZIvhicj4puS/hF4UtLdwCngswAR8aqkJ4HXgCvAfa1a3JiZ2WC1bV45\nDG5eaWbWub41rzQzs2JzoDczKzkHejOzknOgNzMrOQd6yw9PrWc2EJ5K0PLBU+uZDYxL9JYPnlrP\nbGAc6C0fPLWe2cA40Fs+DHpqPef/bYw50Fs+DHJqvUr+/9QpiKjm/x3sbUw40Fs+DHJqPef/bcx5\nrBsrvzVrkpJ8PQmuXh3+8Zj1ice6MasYdP7fLOcc6K38Bpn/NysAB3orv0Hm/80KwD1jbTzMzzuw\n29hyid4sC7fDtwJzid6sHY/DYwXnEr1ZO26HbwXnQG/WjsfhsYJzoDdrZxzb4btOolQc6M3aGbd2\n+B4bqHQc6M3aGbd2+K6TKB2PdWNm1/LYQIXhsW7MrDvN6h7Wr3fevqDaBnpJWyV9S9Jrkl6V9EC6\n/iFJZyW9mC531exzQNJJSSck3THIEzCzPmtUJ3HjjfDuu87bF1SWEv0V4HcjYgfwMeA+STvS574S\nETvT5SmA9Lk9wO3AncAjkiYGcOxmNgiN6iRuuQUuX752O+ftC6NtoI+IcxHxz+n9nwDfAza32GU3\n8EREvBcRbwAngV39OFgzG5L5eVheTnLyy8vwzjuNt3NfgkLoKEcvaRb4JeC5dNX9kl6SdFjSrem6\nzcCbNbudofUXg5nl3Tj2JSiRzIFe0s3AXwNfioh3gUeBDwM7gXPAlzt5Y0n7JC1JWlpZWelkVzPr\nVrcdocatL0HJZAr0ktaSBPnFiPgbgIh4KyJWI+Iq8BjV9MxZYGvN7lvSddeIiEMRMRcRc9PT072c\ng5ll0UtHqHHrS1AybdvRSxJwBHgnIr5Us35TRJxL7/8O8MsRsUfS7cBfkgT+DwHHge0RsdrsPdyO\n3mwIZmeT4F5vZibJw1vhZG1Hn2WY4l8BvgC8LOnFdN3vA5+TtBMIYBn4LYCIeFXSk8BrJC127msV\n5M1sSDw429hqG+gj4llADZ56qsU+BwEn78zyZNu2xiV6V6iWnnvGmo0LV6iOLQd6s3HhCtWx5akE\nzcaJJ0kfSy7Rm5mVnAO9mVnJOdCbmZWcA72ZWck50JuZlZwDvZlZyTnQm5mVnAO9mVnJOdCbmZWc\nA72ZWck50JuZ9UG3k3cNg8e6MTPrUWXyrkuXkseVybsgH0MLuURvZtajhYVqkK+4dClZnwcO9GZm\nPcr75F0O9GZmPWo2SVdeJu9yoDcz61HeJ+9yoDcz61HeJ+9yqxszsz7I8+RdLtGbmZWcA70NT557\nlJiVmFM3Nhx571FiVmIu0dtwPPBAvnuUmJVY20Avaaukb0l6TdKrkh5I16+X9LSk19PbW2v2OSDp\npKQTku4Y5AlYASwuwttvN34uLz1KzEosS4n+CvC7EbED+Bhwn6QdwIPA8YjYDhxPH5M+twe4HbgT\neETSxCAO3gqiVak9Lz1KzEqsbaCPiHMR8c/p/Z8A3wM2A7uBI+lmR4BPp/d3A09ExHsR8QZwEtjV\n7wO3AmlVas9LjxKzEusoRy9pFvgl4DlgY0ScS5/6IbAxvb8ZeLNmtzPpOhtXzUrtU1OuiDUbgsyB\nXtLNwF8DX4qId2ufi4gAopM3lrRP0pKkpZWVlU52taJp1j/84YdHczxmYyZToJe0liTIL0bE36Sr\n35K0KX1+E3A+XX8W2Fqz+5Z03TUi4lBEzEXE3PT0dLfHb0WQ9/7hZiWXpdWNgD8DvhcR/6XmqWPA\n3vT+XuAbNev3SFon6TZgO/B8/w7ZCml+HpaX4erV5NZB3mxospTofwX4AvCrkl5Ml7uAPwI+Jel1\n4JPpYyLiVeBJ4DXg74D7ImJ1IEdvxeKesWYj0bZnbEQ8C6jJ059oss9BwM0prMo9Y81Gxj1jbTjy\nPteateUfZMXlsW5sOPI+15q15B9kxeYSvQ1Ht3OtuRg5EvWX3UMVFZsDvQ1HN3OtVYqRp05BRLUY\nWdRgX5AvrUaX3UMVFZsDvQ1HN23py5TXz9mX1r33wg03JH+KG25IHlc0uuzNeKiiYlDSqXW05ubm\nYmlpadSHYXmzZk0SFOtJSXv8IpmdTYJ7vZmZpF/BECwuJimYZqXz/fvhkUeaX/Z6k5Pu9zZqkl6I\niLl227lEb9fKU3qh27x+Ho24MnpxEb74xeZBHpKgDa2HJnLn5mJyoLeqnKUXusrr59WIv7QWFuCn\nP229zWrarbHV0ETu3FxMDvRWlbeceJnGyBnxl1aWHw4T6awRZbrslnCO3qrKlBPPo8XF5Evz9Omk\nJH/w4NCiZ7MqglqVHL0Vh3P01rky5cTzaIQDux08CDfe2Pg5yUG+7BzorarX9EKeKnJLoFUTyE7N\nz8Phw0mFasXUFBw9mnzvOMiXXESMfPnoRz8alhNHj0bMzERIye3Ro9n3m5yMSJI/yTI5mX1/i4jq\n5a+9jLXL/v2jPkLLE2ApMsRY5+itP3LQTrxo6lP2d90FR4607qw0MQFXrgzvGC3fsuboPaiZ9Uez\nmr52NYBjqtEgYV/9avuOSque2cG64By99UelbV7W9WOotgpj797rS+5Zflz7clo3XKK3/mhW1HQR\nFLi+BN/tZakMDWzWCZforT9mZjpbP2ayDhQm6vsrBBBMTLgJpHXPgd76o0zDFQxAlp6pk/xf7uER\nZlhGXGWGZY4yT2iCK1cc5K17Tt1Yf1Q6/4yo52febdvWuF56YiJpx75tGxy8eID5t/8rcH/dzv5V\nZL1xid76Z4Q9Pwet175gzX7wHDlSc7ke/mX/KrKBcKA3a6Mfg3pmGijMo4nZgLjDlBmtxxtzXzDL\nK3eYMmvh3nuTwvLqapKOgeoAnZUSOyTBfsRzhpj1zKkbGwuLi7BhQ5IRkeDRR6tt2a9evX4U5tph\n+D2opxVd20Av6bCk85JeqVn3kKSzkl5Ml7tqnjsg6aSkE5LuGNSBm2WVZRq9Riol9lK2HPVIo2Ml\nS4n+fwB3Nlj/lYjYmS5PAUjaAewBbk/3eUSSO23bSGWZRq+RSom9dHWkeZsy0gaubaCPiG8D72R8\nvd3AExHxXkS8AZwEdvVwfGY96yaXXl9iL1XL0bxNGWkD10uO/n5JL6WpnVvTdZuBN2u2OZOuMxuZ\nrLn0m24qSYm9Hdcuj51uA/2jwIeBncA54MudvoCkfZKWJC2trKx0eRhm7bWaRg94fxyZixdLUmJv\nx7XLY6erQB8Rb0XEakRcBR6jmp45C2yt2XRLuq7RaxyKiLmImJuenu7mMKykausJb7klue1lOr33\np9G7+f9RGSRsSm9zdP+zRDB+48iUsnbZWukq0EvaVPPwM0ClRc4xYI+kdZJuA7YDz/d2iDngFgpD\nUWkC+fnPV+sJL16sjtO+upo0i+wq2LPIhatTBGsI1nAhNjD/1Y8n3yDj9jctXe2ytdVurkHgayTp\nmcskOfe7gceBl4GXSIL7pprtF4AfACeAX88yn2Gu54z1XKgDVTtHqtR8rtTaZWKiizdqNRFrXv+m\n3c7fa2MDzxnbJ+7/3je1vVF71fHHds2a9jvl6W9aP1MJJOkVl7ytRtYhENwztlajFE3eWigUNI30\nyU9e2xu1F11Np5elojFPrU7cBNL6yIG+olknkvXrG28/ihYKBe3osrgIx4/37/W6mk6vUQVkvTy1\nOslbAcMKzYG+olkJCvLTQqGXUt4Qfgnce2/SMqa+hUwvhdB165LXA3qbTq+2AhKqL1qRt1YnbgJp\n/ZQlkT/oJReVsc1qAqX8VIq1OsZWBlChXH9JPvGJxoe2f3/2StbaCtmBX+a8/E2bcSMAy4CMlbEj\nD/KRl0DfrFXGzMyoj6yq22Ps07nVtpCBq3UvV/843m8h067BS+3hOI7VyPuXkY1c1kDv1E1FETqR\ndHuMPeZ769u3J+pSH9c9Tqyutk6PV9IxEWPQI7VTpRpgx0bJgb6iCJ1Iuj3GDvK99an8e+9NKj87\nHeK3YmKi8WEfPZoE97HrldqJgrawshzKUuwf9JKL1E2Ztcn3tuq01El+vVH6Zv/+0Z56YTlHbxng\n1I29r8UvgdoWm3B9n6L6x82Iq3zixn94v417Ty1kzO3ora88Z+yYWGSeBeY5FTBxBlY/DzMLyVgy\n9fGkU+Iq96x5jEcO3ww5ynQVmtvRWx+5RD8G6kvtld6pp05ly73XNzkHWKOrwFVmWObxqS/xyF/c\nnK/6jKJzO3rrIwf6EqmfAHvDhmRdoyxAVpOTcM8911ekrl5dQ8QalmOW+Qt/6iDfb0VoBWaF4dRN\nSVQmwK6dG/Xtt+E3fgMuX+7staQkNz8zk8QVx/ARqFz0hYUkXbNtm/8Y1jWX6Eui2QTYly+3HwRs\nauraEvvjj9e1a3czv9FwO3rrEwf6gmkWc1vV0a2uNu+wNDkJDz/cIp4UdCA1M6tyoM+pRvn2Suel\nRjG3VR1dpTVlZTyvSgk/U38rN/MzKzxPPJJDi4ud5dYrufT6HD3A2rXw53/ew6/+ZhN2SMlPADMb\nGU88UgDN0jALC51VoJ4+XTMB9lR1/dRUj0Ee3MzPrATc6mZE6meKq6RhoPM+MZWYOz8/gPq6gwcb\nT2nnZn5mheES/Yi0Sn23KiwPfb6MIgz2Zp1zS6qx4kA/IO3+j1r1cD94MMmt17vxxus7Lw0l5rqZ\nX7m4JdXYcWVsH1V6oZ46Ve10VDE5eW1Qnp2tHdu9amYmiaWLi/DAA9UhCqamkmaQjrHWs3YfPisM\nV8YOSaXkLsEXvtB8FMj6FonterjPz8OFC9Uxai9ccJC3PvGAaWPHgb4H7Yb4rVf7f+TUt42MW1KN\nHQf6DFo1g+xksLD6/6PcpL5dMTdePGDa2Gkb6CUdlnRe0is169ZLelrS6+ntrTXPHZB0UtIJSXcM\n6sC70kVAa1Vv1ckv3dz+H426Ys5fMsPnn5Pjp90UVMDHgY8Ar9Ss+xPgwfT+g8Afp/d3AN8F1gG3\nAT8AJtq9x1CmEswwNVtlSj0pua2dYq9+mZlp/lz9NHyV18qlVic4aJ4uz6wnZJxKMNOcrsBsXaA/\nAWxK728CTqT3DwAHarb7e+BftXv9oQT6NgGtWcxpFcQb7TPy4N7o26qVZpPCSoM/xmYXdxhfMmYl\nkDXQd5uj3xgR59L7PwQ2pvc3A2/WbHcmXXcdSfskLUlaWllZ6fIwOnD6NIt8jlneYA2rzPIGi3zu\n/fxLsw5MzYb43bat8S/g64b4HaZu0jDDrpirr8FuxK0/zPory7cB15fof1z3/I/S2/8GfL5m/Z8B\n/67d6w+jRH906v6Y5OK1JXYuxtGp+yOiecG2Uck+t9mFbtIww06ftMt3uURvlhkDLtG/JWkTQHp7\nPl1/Fthas92WdN3ILfCHXOKma9Zd4iYW+EOgeQG2dojf3NdbddM+etgVc+1K67mttTYrrm4D/TFg\nb3p/L/CNmvV7JK2TdBuwHXi+t0Psj9Pv3NxyfasWZ7lpBtlOt2mYrCfYjxYyWQbOz+0FNiuodkV+\n4GvAOeAySc79bmAKOA68DjwDrK/ZfoGktc0J4Nez/KwYRuomS1aj03rM3BlkGqZfr52XljaF/2Ob\nZU/dZMrRD3oZSo4+J/Fl4AYVwPrZDHPUQXZsPgxWdlkD/VgNalYZdOz06SSDUEnL5FaeDrhMM015\nUC8riayDmo1VoC+U+plJ4PohMIepTMGxTF9aNtY8emXR5W1S7jKNjzKug3p5uImx5UCfV3kbSrZM\n46OU6Usrq1GPaWQj5dRNXpUpVZJHear/GAZ/nkrJqZuiG8dS5zAVpnNEn+TtF6INlQN9XpUpVWKj\nN671EgY40OfbuJU6bXD8C3GsOdDnVTctJNyqwprxL8Sx5srYPOqmDX3e2t2b2cC5w1SRddNCwq0q\nzMaOW90UWTctJNyqwsyacKDPo25aSLhVRW9cv2El5kA/Ss2CSzctJMrUqmLYQde9Rq3ssgxxOehl\nKJOD5027oXK7Gcp31MP/9sMohhDu5xDMZkOEhykesXZd7F152tgorotHs7SCcmXsKGVJBbjytLFm\n53/q1ODSOK7fsJJzoB+ELEMMO7g01ur8B5U7L1P9hlkDDvSDkKW0Ps7BpVVla6PrUmsQY/K716iV\nXZZE/qCX0lTGVipDG1XsNarcK0PlaaeyVLa2u47SyA7fLE9wZeyQNRqCoJaHI0h0UtnqCmuzllwZ\nO2yN8vIVTgVUdVIJPc7pLbM+cqDvVSXf3KjkCUnO10MMV3VSCe3cuVlfOND3orYZZTN5a0Uz6q7+\nnZbSPSa/Wc8c6HvRKl0D+Usz5KGrv0vpZkPXU2WspGXgJ8AqcCUi5iStB/4KmAWWgc9GxI9avU5h\nK2Ob9aiEJIDlbcJpV26alcowK2P/TUTsrHmzB4HjEbEdOJ4+LqdmaZlK4MxTkIf+9cYddfrHzDoy\niNTNbuBIev8I8OkBvEc+FK1VSD964+Yh/WNmHek10AfwjKQXJO1L122MiHPp/R8CGxvtKGmfpCVJ\nSysrKz0exogULd/cjy+mLMM7mFmu9Jqj3xwRZyV9EHgauB84FhEfqNnmRxFxa6vXKWyOvojajarZ\njkd6NMuNrDn6G3p5k4g4m96el/R1YBfwlqRNEXFO0ibgfC/vYX02P9/bL45t2xpX6OatGamZva/r\n1I2kmyTdUrkP/BrwCnAM2Jtuthf4Rq8HaTlStHoJM+upRL8R+Lqkyuv8ZUT8naTvAE9Kuhs4BXy2\n98O03Kj8Gugl/WNmQ+VBzczMCsqDmpmZGeBAb2ZWeg70ZmYl50BvZlZyDvRmZiWXi1Y3klZImmLW\n2wBcGPLh5JWvRZWvRZWvRdU4XouZiJhut1EuAn0zkpayNB0aB74WVb4WVb4WVb4WzTl1Y2ZWcg70\nZmYll/dAf2jUB5AjvhZVvhZVvhZVvhZN5DpHb2Zmvct7id7MzHo08kAvab2kpyW9nt42nKRE0mFJ\n5yW90s3+RdDBtbhT0glJJyU9WLP+IUlnJb2YLncN7+h71+y8ap6XpD9Nn39J0key7ls0PV6LZUkv\np5+Bwo8WmOFa/IKkf5T0nqTf62TfsRERI12APwEeTO8/CPxxk+0+DnwEeKWb/YuwZDkXYAL4AfBh\n4Ebgu8CO9LmHgN8b9Xl0ee5Nz6tmm7uAvwUEfAx4Luu+RVp6uRbpc8vAhlGfxxCvxQeBfwkcrP38\nl+1z0csy8hI9GScTj4hvA+90u39BZDmXXcDJiPjfEfFT4Il0v6LLcl67gb+IxD8BH0hnMSvbNenl\nWpRN22sREecj4jvA5U73HRd5CPSZJhMf4P55kuVcNgNv1jw+k66ruD/9KX+4YGmsdufVapss+xZJ\nL9cCIIBnJL0gad/AjnI4evnblu1z0bWe5ozNStIzwM81eGqh9kFEhKSumwH1uv8wDPhaPAr8Ack/\n+h8AXwa+2M1xWqH964g4K+mDwNOSvp/+IrYxNZRAHxGfbPacpF4nEy/UZOR9uBZnga01j7ek64iI\nt2pe6zHgm/056qFoel4ZtlmbYd8i6eVaEBGV2/OSvk6SwihqoM9yLQaxb6nkIXXT62TiZZqMPMu5\nfAfYLuk2STcCe9L9qMvRfoZksvaiaHpeNY4B/z5tcfIx4P+kqa4s+xZJ19dC0k2SbgGQdBPwaxTr\nc1Cvl79t2T4X3Rt1bTAwBRwHXgeeAdan6z8EPFWz3deAcyQVLmeAu1vtX8Slg2txF/C/SFoULNSs\nfxx4GXiJ5AO9adTn1OH5X3dewD3APel9Af89ff5lYK7dNSnq0u21IGlh8t10eXVMrsXPpTHhXeDH\n6f1/UcbPRbeLe8aamZVcHlI3ZmY2QA70ZmYl50BvZlZyDvRmZiXnQG9mVnIO9GZmJedAb2ZWcg70\nZmYl9/8BcJ+zLfVPEWcAAAAASUVORK5CYII=\n", 321 | "text/plain": [ 322 | "" 323 | ] 324 | }, 325 | "metadata": {}, 326 | "output_type": "display_data" 327 | } 328 | ], 329 | "source": [ 330 | "plt.scatter(X[:, 1], y, color='red')\n", 331 | "plt.scatter(X[:, 1], yhat, color='blue')\n", 332 | "\n", 333 | "plt.show()" 334 | ] 335 | }, 336 | { 337 | "cell_type": "markdown", 338 | "metadata": {}, 339 | "source": [ 340 | "### Regularization " 341 | ] 342 | }, 343 | { 344 | "cell_type": "markdown", 345 | "metadata": {}, 346 | "source": [ 347 | "$$L(\\theta) = \\frac{1}{2} \\left(y-X\\theta \\right)^T \\left(y-X\\theta \\right) + \\lambda ||\\theta||^2$$" 348 | ] 349 | }, 350 | { 351 | "cell_type": "code", 352 | "execution_count": 80, 353 | "metadata": {}, 354 | "outputs": [], 355 | "source": [ 356 | "def reg_ssd(X,y,theta , lbda):\n", 357 | " return norm(y-X.dot(theta))/2+lbda*norm(theta)" 358 | ] 359 | }, 360 | { 361 | "cell_type": "markdown", 362 | "metadata": {}, 363 | "source": [ 364 | "### Gradient with regularization " 365 | ] 366 | }, 367 | { 368 | "cell_type": "markdown", 369 | "metadata": {}, 370 | "source": [ 371 | "$$\\frac{\\partial L}{\\partial \\theta}= X^T \\left(X \\theta -y \\right)+ 2 \\lambda \\theta$$\n" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": 81, 377 | "metadata": {}, 378 | "outputs": [], 379 | "source": [ 380 | "def reg_train(X,theta,y,lbda = 1e-2, itr = 1000 , lrate = 1e-2):\n", 381 | " for j in range(0, itr):\n", 382 | " f = X.dot(theta)\n", 383 | " gradient = (X.T).dot(f - y)\n", 384 | " theta = theta*(1-2*lbda) - gradient * lrate\n", 385 | " \n", 386 | " if j % 100 == 0:\n", 387 | " print (\"The current loss is \" , reg_ssd(X,y,theta , lbda))\n", 388 | " return theta" 389 | ] 390 | }, 391 | { 392 | "cell_type": "code", 393 | "execution_count": 82, 394 | "metadata": {}, 395 | "outputs": [ 396 | { 397 | "name": "stdout", 398 | "output_type": "stream", 399 | "text": [ 400 | "The current loss is 363.372076092\n", 401 | "The current loss is 259.429646109\n", 402 | "The current loss is 259.039807226\n", 403 | "The current loss is 258.993424224\n", 404 | "The current loss is 258.987885554\n", 405 | "The current loss is 258.987223887\n", 406 | "The current loss is 258.987144838\n", 407 | "The current loss is 258.987135394\n", 408 | "The current loss is 258.987134266\n", 409 | "The current loss is 258.987134131\n" 410 | ] 411 | } 412 | ], 413 | "source": [ 414 | "thetahat = reg_train(X,theta,y)" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": 83, 420 | "metadata": {}, 421 | "outputs": [], 422 | "source": [ 423 | "yhat = X.dot(thetahat)" 424 | ] 425 | }, 426 | { 427 | "cell_type": "code", 428 | "execution_count": 84, 429 | "metadata": {}, 430 | "outputs": [ 431 | { 432 | "data": { 433 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGchJREFUeJzt3W+MHPd93/H3504kE9IuLJ7ODMU/dzJAo6DygLG3rIEU\nARrHkaIntPPAoHtxVVQoE0JQ5CIBKoUPqiIgkARxjRRpXNCoYMZ3sSo0MUwYjgNJMGAbSCUfA1n/\nbFV0+E8ELV6sOJLKQKF43z6YWd1yubsz+3/+fF7AYHdnZ3Z/M3v32d/+fr+ZUURgZmbVNTPtApiZ\n2Xg56M3MKs5Bb2ZWcQ56M7OKc9CbmVWcg97MrOIc9GZmFeegNzOrOAe9mVnF3TLtAgDcdtttsbi4\nOO1imJmVyunTp/8uIuazlssMekk/BXwL2JIu/78j4j9LegT4D8BauujvRMTX03UeBu4DrgO/GRF/\n1es9FhcXWV1dzSqKmZm1kHQ+z3J5avRvA78YEW9J2gR8R9Jfps99LiL+sO2N9wOHgTuB24EnJX0w\nIq7nL76ZmY1KZht9JN5KH25Kp15nQjsEPBYRb0fEWeAMcHDokpqZ2UBydcZKmpX0LHAFeCIink6f\nekDSc5IelXRrOm8XcLFl9VfTee2veUTSqqTVtbW19qfNzGxEcgV9RFyPiAPAbuCgpJ8FPg98ADgA\nXAY+288bR8SJiGhERGN+PrMvwczMBtTX8MqI+AnwTeDuiHgt/QJYB77ARvPMJWBPy2q703lmZjYF\nmUEvaV7S+9L7Pw18DPiBpJ0ti30CeCG9fwo4LGmLpDuAfcAzoy22mVnBrKzA4iLMzCS3KyvTLtG7\n8oy62QmclDRL8sXweER8TdKXJB0g6Zg9B/w6QES8KOlx4CXgHeB+j7gxs0pbWYEjR+Dq1eTx+fPJ\nY4ClpemVK6UiXEqw0WiEx9GbWWktLibh3m5hAc6dG9vbSjodEY2s5XwKBDOzYV240N/8CXPQm5kN\na+/e/uZPmIPezGxYx4/D1q03ztu6NZlfAA56M7NhLS3BiRNJm7yU3J44UYiOWCjI2SvNzEpvaakw\nwd7ONXozs4pz0JuZVZyD3sys4hz0ZmYV56A3M6s4B72ZWcU56M3MKs5Bb2ZWcQ56M7OKc9CbmVWc\ng97MrOIc9GZmFeegNzOrOAe92bQU+GLSVi0+TbHZNBT8YtJWLa7Rm03DsWMbId909Woy32zEHPRm\n01Dwi0lbtTjozaah4BeTtmpx0JtNQ8EvJm3Vkhn0kn5K0jOSvifpRUn/JZ2/XdITkl5Jb29tWedh\nSWckvSzprnFugFkpFfxi0lYtiojeC0gCtkXEW5I2Ad8BHgR+FXg9In5P0kPArRHxnyTtB74MHARu\nB54EPhgR17u9R6PRiNXV1dFskZlZTUg6HRGNrOUya/SReCt9uCmdAjgEnEznnwQ+nt4/BDwWEW9H\nxFngDEnom5nZFORqo5c0K+lZ4ArwREQ8DeyIiMvpIj8CdqT3dwEXW1Z/NZ1nZmZTkCvoI+J6RBwA\ndgMHJf1s2/NBUsvPTdIRSauSVtfW1vpZ1czM+tDXqJuI+AnwTeBu4DVJOwHS2yvpYpeAPS2r7U7n\ntb/WiYhoRERjfn5+kLKbmVkOeUbdzEt6X3r/p4GPAT8ATgH3povdC3w1vX8KOCxpi6Q7gH3AM6Mu\nuJmZ5ZPnXDc7gZOSZkm+GB6PiK9J+mvgcUn3AeeBTwJExIuSHgdeAt4B7u814sbMzMYrc3jlJHh4\npZlZ/0Y2vNLMzMrNQW9mVnEOejOzinPQm5lVnIPeisOX1jMbC19K0IrBl9YzGxvX6K0YfGk9s7Fx\n0Fsx+NJ6ZmPjoLdiGPel9dz+bzXmoLdiGOel9Zrt/+fPQ8RG+7/D3mrCQW/FMM5L67n932rO57qx\n6puZSWry7SRYX598ecxGxOe6MWsad/u/WcE56K36xtn+b1YCDnqrvnG2/5uVgI+MtXpYWnKwW225\nRm+Wh8fhW4m5Rm+WxefhsZJzjd4si8fhW8k56M2y+Dw8VnIOerMsdRyH7z6JSnHQm2Wp2zh8nxuo\nchz0ZlnqNg7ffRKV43PdmNmNfG6g0vC5bsxsMN36HrZvd7t9SWUGvaQ9kr4p6SVJL0p6MJ3/iKRL\nkp5Np3ta1nlY0hlJL0u6a5wbYGYj1qlPYvNmeOMNt9uXVJ4a/TvAb0XEfuAjwP2S9qfPfS4iDqTT\n1wHS5w4DdwJ3A38iaXYMZTezcejUJ/He98K1azcu53b70sgM+oi4HBF/k95/E/g+sKvHKoeAxyLi\n7Yg4C5wBDo6isGY2IUtLcO5c0iZ/7hy8/nrn5XwsQSn01UYvaRH4OeDpdNYDkp6T9KikW9N5u4CL\nLau9Su8vBjMrujoeS1AhuYNe0nuAPwc+ExFvAJ8HPgAcAC4Dn+3njSUdkbQqaXVtba2fVc1sUIMe\nCFW3YwkqJlfQS9pEEvIrEfEXABHxWkRcj4h14AtsNM9cAva0rL47nXeDiDgREY2IaMzPzw+zDWaW\nxzAHQtXtWIKKyRxHL0nASeD1iPhMy/ydEXE5vf8fgX8ZEYcl3Qn8GUnw3w48BeyLiOvd3sPj6M0m\nYHExCfd2CwtJO7yVTt5x9HlOU/zzwKeB5yU9m877HeBTkg4AAZwDfh0gIl6U9DjwEsmInft7hbyZ\nTYhPzlZbmUEfEd8B1OGpr/dY5zjgxjuzItm7t3ON3h2qlecjY83qwh2qteWgN6sLd6jWli8laFYn\nvkh6LblGb2ZWcQ56M7OKc9CbmVWcg97MrOIc9GZmFeegNzOrOAe9mVnFOejNzCrOQW9mVnEOejOz\ninPQm5lVnIPezKziHPRmZhXnoDczqzgHvZlZxTnozcwqzkFvZlZxDnozs4pz0JuZVZyD3iZnZQUW\nF2FmJrldWZl2icxqwRcHt8lYWYEjR+Dq1eTx+fPJY/DFqs3GzDV6m4wHH9wI+aarV+HYsemUx6xG\nMoNe0h5J35T0kqQXJT2Yzt8u6QlJr6S3t7as87CkM5JelnTXODfASmBlBX78487PXbgw2bKY1VCe\nGv07wG9FxH7gI8D9kvYDDwFPRcQ+4Kn0Melzh4E7gbuBP5E0O47CW0n0qrXv3Tu5cpjVVGbQR8Tl\niPib9P6bwPeBXcAh4GS62Eng4+n9Q8BjEfF2RJwFzgAHR11wK5FetfbjxydXDrOa6quNXtIi8HPA\n08COiLicPvUjYEd6fxdwsWW1V9N5Vlfdau1zc+6INZuA3EEv6T3AnwOfiYg3Wp+LiACinzeWdETS\nqqTVtbW1fla1sjl+HLZuvXHe1q3wR380nfKY1UyuoJe0iSTkVyLiL9LZr0namT6/E7iSzr8E7GlZ\nfXc67wYRcSIiGhHRmJ+fH7T8VgZLS3DiBCwsgJTcnjjh2rzZhOQZdSPgfwLfj4j/2vLUKeDe9P69\nwFdb5h+WtEXSHcA+4JnRFdlKaWkJzp2D9fXk1iFvNjF5avQ/D3wa+EVJz6bTPcDvAR+T9ArwS+lj\nIuJF4HHgJeAbwP0RcX0spbdy8ZGxZlOReWRsRHwHUJenP9plneOAh1PYBh8ZazY1PjLWJuPYMR8Z\nazYlDnqbjG5j6X1krNnYOehtMrqNpc86Mtbt+mZDc9DbZHQbS9/ryNhmu/758xCx0a5f1rD3l5ZN\niYPeJmOQsfRVatev2peWlYqSg1qnq9FoxOrq6rSLYUUzM5OEYjspGY9fJouLSbi3W1hIjiswG4Ck\n0xHRyFrONXq7UZGaFwZt1y8id0bbFDnobUPRmhcGadcvqip9aVnpOOhtQ9HaxKt0jpwqfWlZ6biN\n3jZUqU28iFZWki/NCxeSmvzx4+X80rLCyNtG74uD24a9ezt3GLp5YTSWlhzsNhVuurENwzYvFKkj\n18ze5aC3DcO0iRetI9fM3uU2ehsNjxM3mziPo7fJ6hTyveab2cQ46G00Zmf7m2+duZ/DxsCjbmw0\nrne5iFi3+XYzX5zFxsQ1ehuNhYX+5tvNinbAmlWGg95Gw0d+Ds/nw7ExcdDbaFTpdAXT4vPh2Jg4\n6G10lpaSoZTr68mtQ74//lVkY+KgNysK/yqyMfGoG7Mi8flwbAxcozczqzgHvZlZxWUGvaRHJV2R\n9ELLvEckXZL0bDrd0/Lcw5LOSHpZ0l3jKriZDcFH4NZKnjb6LwJ/DPxp2/zPRcQfts6QtB84DNwJ\n3A48KemDEeHDI82Kwkfg1k5mjT4ivgW8nvP1DgGPRcTbEXEWOAMcHKJ8ZjZqPgK3doZpo39A0nNp\n086t6bxdwMWWZV5N55lZUfgI3NoZNOg/D3wAOABcBj7b7wtIOiJpVdLq2tragMUws775CNzaGSjo\nI+K1iLgeEevAF9honrkE7GlZdHc6r9NrnIiIRkQ05ufnBymGWX/cAZnwEbi1M1DQS9rZ8vATQHNE\nzingsKQtku4A9gHPDFfEAnBAlF+nSx1++tPJEah1+0x9BO5YFDomIqLnBHyZpHnmGkmb+33Al4Dn\ngedIwn1ny/LHgB8CLwO/kvX6EcGHP/zhKKzl5YitWyOSeEimrVuT+VYeCws3fobtUxE/0+XlpNxS\ncjvB8nV66ykWp/CmFRPAauTI2MwFJjEVOui7BcTCwrRLZv2Qegf9BD7T9qA8erRHmLIe2/iHmOGd\ngPWY5VocnfkfQyXH8nLE3NzNmz07m5Sldbn20Nq8OWLTpuJ/N07LtGLCQT+ITlWWbgEhFaeMFdO6\nie95z8ZH0B5Ifb3m3AMxx5WA9YD12Mz/uyFEP8o3YoGzXXfrsLu9U3i2T5s2JYHafZn1OLrti4Nt\n/3LWa2/s26wfP67v3GxaMeGg71e3316dqkDT+gsf4vdhM6iagdnchPaf5HNzydSpltm67szMRhHm\n5pJljh7deL5TLXHbtpt348xMMr/53p2W6RZI/ey2zbe8k/G661136yh+lvcTnr2mWa71t/F9vP/s\nbLJsnh8/067vFI1r9GUJ+m6f1Nxcz//ybiHZKTDb9aoltv/MnptLaqXLfCqpeXI9Fjgby3zq3b+m\njuukQb1187WOm9fpJ3nrlF3LjJuCvz2Ul5c3vgBGEnazo/los6bmP+ko/on7Cc/e03p/G9/n+/e7\nv1yjT7iNPsc0qaDv+fNb6hyiUiwf/XYszF5M5s9ejOWj33739bJ+jnf70Hv9YSwvdw7fW/in2MQ/\n3rgOb8Uy/6brT/NNmyLmtv1jrjKOY5qdHV1ttj2Q8ho0ZJu11VH8LB9ZjX7men8b38f7N79A3UY/\nmGm0qjro22R94y7PPRBbeeumED267Ytd1+v3n7e19tOrltj3685ezFhnva/XG/U0utps3BBIeRWh\nRj+yNvpB+yj6aKNvLu9RN8XnoG+T9c+6MPdm51CZud51vX4DrLUG2KuW2Pfrsp6xzvSCfhw1+oHa\n6HM0P7VOo26jb75O7lE3Svormk1iw3REt75/nlE3Vh4O+jZZP78HCe3C1Ogz1pmb+fFNv1aa08ja\n6NMRLJ1CeZg2+i1bRjTqpi3kNm++MUQ/+tHetdWp1GZdhbYMDvo2mTX6Ls93C6jm/91E2+hv6d5O\n2quNfvnot2N507+LBc4GXI9ZrgWsx8LcmwOOullvCfb1mONKLPOpOMofDz3qJqsDuzZ8oJ7l4KBv\nk9lG3+X5o0ez15vYqJuMdtJu62S+Wb98ENn4eR9bDg76DrKyrtvz/gXdxrXN8SvagXpWSHmDXsmy\n09VoNGJ1dXXaxbB+rKwkF6q4cCE5ve3x4z4p1igtLiYnXmu3sADnzk26NFZQkk5HRCNrOV8c3Aaz\ntJQEzvp6cuuQHy2fSthGyEFvVkQ+lbCNkIPeshX6RNsV5l9NNiK3TLsAVnDNC3Y0LyZ9/nzyGBw8\nZiXhGr31duzYRsg3Xb2azDezUnDQW28XLvQ338wKx0Fvve3d2998MyscB7315mF+ZqXnoLfePMyv\nmjySqlY86sayLS052KvEI6lqxzV6s7rxSKracdCb1Y1HUtWOg96sbjySqnYc9OaOubrxSKrayQx6\nSY9KuiLphZZ52yU9IemV9PbWlucelnRG0suS7hpXwQfiQLtZs2Pu/PnkjOfNjrlJ7Rt/JpPnkVT1\nk3XCeuAXgA8BL7TM+wPgofT+Q8Dvp/f3A98DtgB3AD8EZrPeYyIXHvHFMjqb5pWM/JmYDYWcFx7J\nrNFHxLeA19tmHwJOpvdPAh9vmf9YRLwdEWeBM8DBgb6BRq0uIw36rSFPo2OuWcZf+7V6fCZmUzZo\nG/2OiLic3v8RsCO9vwu42LLcq+m8m0g6ImlV0ura2tqAxehDHUYaDNIMM+mOudYydlOlz8SsAIbu\njE1/PvR9PcKIOBERjYhozM/PD1uMbHUYaTDIr5ZJd8x1KmO7Kn0mZgUwaNC/JmknQHp7JZ1/CdjT\nstzudN701WGkwSC/WibdMZdVW6/aZ2JWAIMG/Sng3vT+vcBXW+YflrRF0h3APuCZ4Yo4InUYaTDo\nr5a8VzIaxQiZXmWp4mdiVgRZvbXAl4HLwDWSNvf7gDngKeAV4Elge8vyx0hG27wM/EqeHuGJjLqp\ng3GOYhnVaxdlpM3ycjKySEpuPdLHSoico24yF5jE5KAfoXEF2CiHYU47ZIvyZWM2pLxBr2TZ6Wo0\nGrG6ujrtYhTPykrSeXnhQtLkcfz49Jo1ZmaSSGwnJU0+ZbK42HnUz8JC0nRlVhKSTkdEI2s5nwKh\nqKZ9xGq7Ko1aqsNQW7MWDvqiKtoBXlUatVSlL61++HQTteWgL6qi1TqrNGqpSl9aeRXtF6JNlNvo\ni8rtyONVpP6PSfDfUyW5jb7s6ljrnKS8xw5URdF+IdpEOeiLqkpNJTZ9de2XMMBBX2x1q3Xa+PgX\nYq056ItqkBESHlVh3fgXYq25M7aImiMkWodXbt3a+x9zkHXMrNTydsY66ItokBESHlVhVjsedVNm\ng4yQ8KgKM+vCQV9Eg4yQ8KiK4bh/wyrMQT9N3cJlkBESVRpVMenQ9VGjVnV5TnE57qmWpynOOlXu\nIKfynfbpf0dhGqcQHuUpmM0mCJ+meMqyDrF352ln09gvVToFs9WKO2OnKU9TgDtPO+u2/efPj68Z\nx/0bVnEO+nHIc4phh0tnvbZ/XG3nVerfMOvAQT8OeWrrdQ6XXp2tnfZLq3Gck99HjVrV5WnIH/dU\nmc7YZmdop469Tp17Veg87Veeztas/ShNrfhmRYI7Yyes0ykIWvl0BIl+OlvdYW3WkztjJ61Tu3yT\nmwI29NMJXefmLbMRctAPq9ne3KnmCUmbr08xvKGfTmi3nZuNhIN+GK3DKLsp2iiaaR/q328t3efk\nNxuag34YvZproHjNDEU41N+1dLOJG6ozVtI54E3gOvBORDQkbQf+F7AInAM+GRF/3+t1StsZ2+2I\nSkgCrGgXnHbnplmlTLIz9l9HxIGWN3sIeCoi9gFPpY+rqVuzTDM4ixTyMLqjcafd/GNmfRlH080h\n4GR6/yTw8TG8RzGUbVTIKI7GLULzj5n1ZdigD+BJSaclHUnn7YiIy+n9HwE7Oq0o6YikVUmra2tr\nQxZjSsrW3jyKL6Y8p3cws0IZto1+V0RckvR+4AngAeBURLyvZZm/j4hbe71OadvoyyjrrJpZfKZH\ns8LI20Z/yzBvEhGX0tsrkr4CHARek7QzIi5L2glcGeY9bMSWlob7xbF3b+cO3aINIzWzdw3cdCNp\nm6T3Nu8Dvwy8AJwC7k0Xuxf46rCFtAIpW7+EmQ1Vo98BfEVS83X+LCK+Iem7wOOS7gPOA58cvphW\nGM1fA8M0/5jZRPmkZmZmJeWTmpmZGeCgNzOrPAe9mVnFOejNzCrOQW9mVnGFGHUjaY1kKGa724C/\nm3Bxisr7YoP3xQbviw113BcLETGftVAhgr4bSat5hg7VgffFBu+LDd4XG7wvunPTjZlZxTnozcwq\nruhBf2LaBSgQ74sN3hcbvC82eF90Ueg2ejMzG17Ra/RmZjakqQe9pO2SnpD0Snrb8SIlkh6VdEXS\nC4OsXwZ97Iu7Jb0s6Yykh1rmPyLpkqRn0+meyZV+eN22q+V5Sfpv6fPPSfpQ3nXLZsh9cU7S8+nf\nQOnPFphjX/xzSX8t6W1Jv93PurUREVOdgD8AHkrvPwT8fpflfgH4EPDCIOuXYcqzLcAs8EPgA8Bm\n4HvA/vS5R4DfnvZ2DLjtXberZZl7gL8EBHwEeDrvumWahtkX6XPngNumvR0T3BfvB/4FcLz1779q\nfxfDTFOv0ZPzYuIR8S3g9UHXL4k823IQOBMRfxsR/wQ8lq5Xdnm26xDwp5H4P8D70quYVW2fDLMv\nqiZzX0TElYj4LnCt33XroghBn+ti4mNcv0jybMsu4GLL41fTeU0PpD/lHy1ZM1bWdvVaJs+6ZTLM\nvgAI4ElJpyUdGVspJ2OYz7ZqfxcDG+qasXlJehL4mQ5PHWt9EBEhaeBhQMOuPwlj3hefB36X5B/9\nd4HPAv9+kHJaqf2riLgk6f3AE5J+kP4itpqaSNBHxC91e07SsBcTL9XFyEewLy4Be1oe707nERGv\ntbzWF4CvjabUE9F1u3IssynHumUyzL4gIpq3VyR9haQJo6xBn2dfjGPdSilC082wFxOv0sXI82zL\nd4F9ku6QtBk4nK5HWxvtJ0gu1l4WXberxSng36YjTj4C/EPa1JVn3TIZeF9I2ibpvQCStgG/TLn+\nDtoN89lW7e9icNPuDQbmgKeAV4Ange3p/NuBr7cs92XgMkmHy6vAfb3WL+PUx764B/i/JCMKjrXM\n/xLwPPAcyR/0zmlvU5/bf9N2Ab8B/EZ6X8B/T59/Hmhk7ZOyToPuC5IRJt9Lpxdrsi9+Js2EN4Cf\npPf/WRX/LgadfGSsmVnFFaHpxszMxshBb2ZWcQ56M7OKc9CbmVWcg97MrOIc9GZmFeegNzOrOAe9\nmVnF/X/e7GXVqK/AtQAAAABJRU5ErkJggg==\n", 434 | "text/plain": [ 435 | "" 436 | ] 437 | }, 438 | "metadata": {}, 439 | "output_type": "display_data" 440 | } 441 | ], 442 | "source": [ 443 | "plt.scatter(X[:, 1], y, color='red')\n", 444 | "plt.scatter(X[:, 1], yhat, color='blue')\n", 445 | "\n", 446 | "plt.show()" 447 | ] 448 | }, 449 | { 450 | "cell_type": "markdown", 451 | "metadata": {}, 452 | "source": [ 453 | "Note the line becomes less senstive to outliers " 454 | ] 455 | }, 456 | { 457 | "cell_type": "code", 458 | "execution_count": null, 459 | "metadata": {}, 460 | "outputs": [], 461 | "source": [] 462 | } 463 | ], 464 | "metadata": { 465 | "kernelspec": { 466 | "display_name": "Python 3", 467 | "language": "python", 468 | "name": "python3" 469 | }, 470 | "language_info": { 471 | "codemirror_mode": { 472 | "name": "ipython", 473 | "version": 3 474 | }, 475 | "file_extension": ".py", 476 | "mimetype": "text/x-python", 477 | "name": "python", 478 | "nbconvert_exporter": "python", 479 | "pygments_lexer": "ipython3", 480 | "version": "3.5.4" 481 | } 482 | }, 483 | "nbformat": 4, 484 | "nbformat_minor": 2 485 | } 486 | -------------------------------------------------------------------------------- /Linear regression using Normal equations .ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Import libraries " 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 62, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "from sklearn import datasets\n", 17 | "import matplotlib.pyplot as plt\n", 18 | "import numpy as np\n", 19 | "from numpy.linalg import inv" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": {}, 25 | "source": [ 26 | "### Generated random data\n", 27 | "\n", 28 | "Use the diabetes data offered by skicit learn " 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 64, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "# Load the diabetes dataset\n", 38 | "diabetes = datasets.load_diabetes()\n", 39 | "\n", 40 | "# Use only one feature\n", 41 | "diabetes_X = diabetes.data[:, np.newaxis, 2]\n", 42 | "\n", 43 | "# Use 50 data points\n", 44 | "X = diabetes_X[0:50]\n", 45 | "\n", 46 | "# The true value of y\n", 47 | "y = diabetes.target[0:50]\n" 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "### Plot the data \n" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 65, 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "data": { 64 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF0hJREFUeJzt3W2MHVd9x/HvzyYENqSKnSzG8cNuIhlVDi8MuXUjUSGV\nAgl549AXyNWSWmrUBSlKQwVSHfyiqZAlQKSIqiWSo0Y18ZLUKqBYKFDZUaQoEk1Yozw54MY0tmPL\nsRcMDZElN7H/fTGz2evNvXvnPs7T7yON7twzM3vPzLX/98w5/5lRRGBmZtW1LO8KmJnZcDnQm5lV\nnAO9mVnFOdCbmVWcA72ZWcU50JuZVZwDvZlZxTnQm5lVnAO9mVnFvSvvCgBcc801MTk5mXc1zMxK\n5eDBg7+OiPFO63UM9JLeAzwJXJ6u/x8R8feS7gX+GphLV/1KRDyWbnMPcAdwAfibiPjPpT5jcnKS\n2dnZTlUxM7Mmko5lWS9Li/488PGIeEPSZcBTkn6cLvtWRHxz0QdvBLYCNwDXAgckfTAiLmSvvpmZ\nDUrHPvpIvJG+vSydlroT2hbgkYg4HxGvAEeAzX3X1MzMepJpMFbScknPAmeA/RHxdLroLknPS3pQ\n0oq0bA3watPmJ9KyxX9zWtKspNm5ubnFi83MbEAyBfqIuBARm4C1wGZJHwLuB64HNgGngPu6+eCI\n2BURjYhojI93HEswM7MedZVeGRG/A54AbomI0+kPwEXgARa6Z04C65o2W5uWmZlZDjoGeknjkq5K\n598LfBL4paTVTat9Bngxnd8HbJV0uaTrgA3AM4OttplZwczMwOQkLFuWvM7M5F2jt2XJulkN7Ja0\nnOSHYW9E/EjSQ5I2kQzMHgU+DxARhyTtBV4C3gLudMaNmVXazAxMT8O5c8n7Y8eS9wBTU/nVK6Ui\nPEqw0WiE8+jNrLQmJ5PgvtjEBBw9OrSPlXQwIhqd1vMtEMzM+nX8eHflI+ZAb2bWr/XruysfMQd6\nM7N+7dwJY2OXlo2NJeUF4EBvZtavqSnYtSvpk5eS1127CjEQCwW5e6WZWelNTRUmsC/mFr2ZWcU5\n0JuZVZwDvZlZxTnQm5lVnAO9mVnFOdCbmVWcA72ZWcU50JuZVZwDvZlZxTnQm5lVnAO9mVnFOdCb\nmVWcA72ZWcU50JvlpcAPk7Zq8W2KzfJQ8IdJW7W4RW+Whx07FoL8vHPnknKzAXOgN8tDwR8mbdXi\nQG+Wh4I/TNqqxYHeLA8Ff5i0VUvHQC/pPZKekfScpEOS/iEtXylpv6SX09cVTdvcI+mIpMOSbh7m\nDpiVUsEfJm3VoohYegVJwBUR8Yaky4CngLuBPwfORsTXJG0HVkTE30naCDwMbAauBQ4AH4yIC+0+\no9FoxOzs7GD2yMysJiQdjIhGp/U6tugj8Ub69rJ0CmALsDst3w3cls5vAR6JiPMR8QpwhCTom5lZ\nDjL10UtaLulZ4AywPyKeBlZFxKl0ldeAVen8GuDVps1PpGVmZpaDTIE+Ii5ExCZgLbBZ0ocWLQ+S\nVn5mkqYlzUqanZub62ZTMzPrQldZNxHxO+AJ4BbgtKTVAOnrmXS1k8C6ps3WpmWL/9auiGhERGN8\nfLyXupuZWQZZsm7GJV2Vzr8X+CTwS2AfsC1dbRvwaDq/D9gq6XJJ1wEbgGcGXXEzM8smy71uVgO7\nJS0n+WHYGxE/kvRTYK+kO4BjwGcBIuKQpL3AS8BbwJ1LZdyYmdlwdUyvHAWnV5qZdW9g6ZVmZlZu\nDvRmZhXnQG9mVnEO9GZmFedAb8XhR+uZDYUfJWjF4EfrmQ2NW/RWDH60ntnQONBbMfjRemZD40Bv\nxTDsR+u5/99qzIHeimGYj9ab7/8/dgwiFvr/HeytJhzorRiG+Wg99/9bzfleN1Z9y5YlLfnFJLh4\ncfT1MRsQ3+vGbN6w+//NCs6B3qpvmP3/ZiXgQG/VN8z+f7MS8JWxVg9TUw7sVltu0Ztl4Tx8KzG3\n6M068X14rOTcojfrxHn4VnIO9Gad+D48VnIO9Gad1DEP32MSleJAb9ZJ3fLwfW+gynGgN+ukbnn4\nHpOoHN/rxswu5XsDlYbvdWNmvWk39rBypfvtS6pjoJe0TtITkl6SdEjS3Wn5vZJOSno2nW5t2uYe\nSUckHZZ08zB3wMwGrNWYxLvfDa+/7n77ksrSon8L+FJEbARuAu6UtDFd9q2I2JROjwGky7YCNwC3\nAN+RtHwIdTezYWg1JnHllfDmm5eu53770ugY6CPiVET8PJ3/PfALYM0Sm2wBHomI8xHxCnAE2DyI\nyprZiExNwdGjSZ/80aNw9mzr9XwtQSl01UcvaRL4MPB0WnSXpOclPShpRVq2Bni1abMTLP3DYGZF\nV8drCSokc6CX9D7g+8AXI+J14H7gemATcAq4r5sPljQtaVbS7NzcXDebmlmver0Qqm7XElRMpkAv\n6TKSID8TET8AiIjTEXEhIi4CD7DQPXMSWNe0+dq07BIRsSsiGhHRGB8f72cfzCyLfi6Eqtu1BBXT\nMY9ekoDdwNmI+GJT+eqIOJXO/y3wxxGxVdINwPdIAv+1wOPAhoi40O4znEdvNgKTk0lwX2xiIumH\nt9LJmkef5TbFHwVuB16Q9Gxa9hXgLyRtAgI4CnweICIOSdoLvESSsXPnUkHezEbEN2errY6BPiKe\nAtRi0WNLbLMTcOedWZGsX9+6Re8B1crzlbFmdeEB1dpyoDerCw+o1pYfJWhWJ35Iei25RW9mVnEO\n9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRmZhXnQG9mVnEO9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRm\nZhXnQG9mVnEO9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRmZhXnQG9mVnEO9DY6MzMwOQnLliWvMzN5\n18isFvxwcBuNmRmYnoZz55L3x44l78EPqzYbMrfobTTuvnshyM87dw527MinPmY10jHQS1on6QlJ\nL0k6JOnutHylpP2SXk5fVzRtc4+kI5IOS7p5mDtgJTAzA7/5Tetlx4+Pti5mNZSlRf8W8KWI2Ajc\nBNwpaSOwHXg8IjYAj6fvSZdtBW4AbgG+I2n5MCpvJbFUq339+tHVw6ymOgb6iDgVET9P538P/AJY\nA2wBdqer7QZuS+e3AI9ExPmIeAU4AmwedMWtRJZqte/cObp6mNVUV330kiaBDwNPA6si4lS66DVg\nVTq/Bni1abMTaZnVVbtW+9VXeyDWbAQyB3pJ7wO+D3wxIl5vXhYRAUQ3HyxpWtKspNm5ubluNrWy\n2bkTxsYuLRsbg29/O5/6mNVMpkAv6TKSID8TET9Ii09LWp0uXw2cSctPAuuaNl+bll0iInZFRCMi\nGuPj473W38pgagp27YKJCZCS11273Jo3G5EsWTcC/hX4RUT8Y9OifcC2dH4b8GhT+VZJl0u6DtgA\nPDO4KlspTU3B0aNw8WLy6iBvNjJZWvQfBW4HPi7p2XS6Ffga8ElJLwOfSN8TEYeAvcBLwE+AOyPi\nwlBqb+XiK2PNctHxytiIeApQm8V/1mabnYDTKWyBr4w1y42vjLXR2LHDV8aa5cSB3kajXS69r4w1\nGzoHehuNdrn0na6Mdb++Wd8c6G002uXSL3Vl7Hy//rFjELHQr1/WYO8fLcuJA72NRi+59FXq16/a\nj5aVipKLWvPVaDRidnY272pY0SxblgTFxaQkH79MJieT4L7YxERyXYFZDyQdjIhGp/XcordLFal7\nodd+/SLyYLTlyIHeFhSte6GXfv2iqtKPlpWOA70tKFqfeJXukVOlHy0rHffR24Iq9YkX0cxM8qN5\n/HjSkt+5s5w/WlYYWfvo/XBwW7B+fesBQ3cvDMbUlAO75cJdN7ag3+6FIg3kmtnbHOhtQT994kUb\nyDWzt7mP3gbDeeJmI+c8ehutVkF+qXIzGxkHehuM5cu7K7fWPM5hQ+CsGxuMC20eItau3N7JD2ex\nIXGL3gZjYqK7cnunol2wZpXhQG+D4Ss/++f74diQONDbYFTpdgV58f1wbEgc6G1wpqaSVMqLF5NX\nB/nu+KzIhsSB3qwofFZkQ+KsG7Mi8f1wbAjcojczqzgHejOziusY6CU9KOmMpBebyu6VdFLSs+l0\na9OyeyQdkXRY0s3DqriZ9cFX4NZKlj76fwP+GfjuovJvRcQ3mwskbQS2AjcA1wIHJH0wInx5pFlR\n+Arc2unYoo+IJ4GzGf/eFuCRiDgfEa8AR4DNfdTPzAbNV+DWTj999HdJej7t2lmRlq0BXm1a50Ra\nZmZF4Stwa6fXQH8/cD2wCTgF3NftH5A0LWlW0uzc3FyP1TCzrvkK3NrpKdBHxOmIuBARF4EHWOie\nOQmsa1p1bVrW6m/siohGRDTGx8d7qYZZdzwAmfAVuLXTU6CXtLrp7WeA+YycfcBWSZdLug7YADzT\nXxULwAGi/Fo96vD225MrUOv2nfoK3PqJiCUn4GGS7pk3Sfrc7wAeAl4AnicJ7qub1t8B/Ao4DHy6\n09+PCG688cYorD17IsbGIpLwkExjY0m5lcfExKXf4eKpiN/pnj1JvaXktWj1s9wBs5EhxvqZsZ34\nWajVsGxZEtKXUqTvdHEKJCTdK255WxM/M7YXrbpoipah4G6k3mQZaCxS1olTIG2AHOjnterDnZ6G\nlStbr59HhkK7OjrYd9ZqAHKxImWdFK2BYaXmQD+vXQsKipOh0E8rr+5nAs0DkJAMQjYrWtaJUyBt\ngBzo57VrKZ09W5wMhV5beT4TSMw/GCUCHnqoGN9pO06BtEHKMmI77KkQWTftsjImJvKu2YJe6ziM\nfXNGyPD5GFsHZMy6cYt+XhlaUL3WcdD9vT5DGA0/mtEGxIF+XhkuIum1joPu73VGyGjUfVzFBsZ5\n9HUw6JzsdjnpUtL6tP45j94ycB69LRj02YozQobPZ002QA70dTHI/t4yjGeUnfPobYAc6K17ZRjP\nKDufNdkAOdBbb5wRMlw+a7IBcqA3KyKfNdkAOdBbZ07zy4fPmmxA3pV3BazgFqf5zV8cBQ48ZiXh\nFr0tzWl+ZqXnQG9Lc5qfWek50NvSnOZnVnoO9LY0p/mZlZ4DvS3NaX7V5EyqWnHWjXU2NeXAXiXO\npKodt+jN6saZVLXjQG9WN86kqh0HerO6cSZV7TjQmwfm6saZVLXTMdBLelDSGUkvNpWtlLRf0svp\n64qmZfdIOiLpsKSbh1XxnjigvVPez3/1dzJ6zqSqn05PDwc+BnwEeLGp7BvA9nR+O/D1dH4j8Bxw\nOXAd8CtgeafPuPHGG4f/uPQ9eyLGxiKScJZMY2NJeZ1NTFx6TOaniYnhf7a/E7O+ALPRIb5GROcW\nfUQ8CZxdVLwF2J3O7wZuayp/JCLOR8QrwBFgc0+/QINWl0yDblvIeQzMzdfxc5+rx3dilrNe++hX\nRcSpdP41YFU6vwZ4tWm9E2nZO0ialjQraXZubq7HanShDpkGvXTDjHpgrrmO7VTpOzErgL4HY9PT\nh+hhu10R0YiIxvj4eL/V6KwOmQa9nLWMemCuVR0Xq9J3YlYAvQb605JWA6SvZ9Lyk8C6pvXWpmX5\nq0OmQS9nLaMemOvUWq/ad2JWAL0G+n3AtnR+G/BoU/lWSZdLug7YADzTXxUHpA6ZBr2etWR9ktEg\nMmSWqksVvxOzIug0Wgs8DJwC3iTpc78DuBp4HHgZOACsbFp/B0m2zWHg01lGhEeSdVMHw8xiGdTf\nLkqmzZ49SWaRlLw608dKiIxZNx1XGMXkQD9Awwpgg0zDzDvIFuXHxqxPWQO9knXz1Wg0YnZ2Nu9q\nFM/MTDJ4efx40uWxc2d+3RrLliUhcTEp6fIpk8nJ1lk/ExNJ15VZSUg6GBGNTuv5FghFlfcVq4tV\nKWupDqm2Zk0c6IuqaBd4VSlrqUo/Wt3w7SZqy4G+qIrW6qxS1lKVfrSyKtoZoo2U++iLyv3Iw1Wk\n8Y9R8L+nSnIffdnVsdU5SlmvHaiKop0h2kg50BdVlbpKLH91HZcwwIG+2OrW6rTh8RlirTnQF1Uv\nGRLOqrB2fIZYax6MLaL5DInm9MqxsaX/Y/ayjZmVWtbBWAf6IuolQ8JZFWa146ybMuslQ8JZFWbW\nhgN9EfWSIeGsiv54fMMqzIE+T+2CSy8ZElXKqhh10PVVo1Z1WW5xOeyplrcp7nSr3F5u5Zv37X8H\nIY9bCA/yFsxmI4RvU5yzTpfYe/C0tTyOS5VuwWy14sHYPGXpCvDgaWvt9v/YseF143h8wyrOgX4Y\nstxi2MGltaX2f1h951Ua3zBrwYF+GLK01uscXJYabG11XJoN4578vmrUqi5LR/6wp8oMxs4PhrYa\n2Gs1uFeFwdNuZRls7XQcpdyqb1YkeDB2xFrdgqCZb0eQ6Gaw1QPWZkvyYOyoteqXn+eugAXdDELX\nuXvLbIAc6Ps139/cquUJSZ+vbzG8oJtBaPedmw2EA30/mtMo2ylaFk3el/p320r3PfnN+uZA34+l\numugeN0MRbjU3610s5HrazBW0lHg98AF4K2IaEhaCfw7MAkcBT4bEb9d6u+UdjC23RWVkASwoj1w\n2oObZpUyysHYP42ITU0fth14PCI2AI+n76upXbfMfOAsUpCHwV2Nm3f3j5l1ZRhdN1uA3en8buC2\nIXxGMZQtK2QQV+MWofvHzLrSb6AP4ICkg5Km07JVEXEqnX8NWNVqQ0nTkmYlzc7NzfVZjZyUrb95\nED9MWW7vYGaF0m8f/ZqIOCnp/cB+4C5gX0Rc1bTObyNixVJ/p7R99GXU6a6anfhOj2aFkbWP/l39\nfEhEnExfz0j6IbAZOC1pdUSckrQaONPPZ9iATU31d8axfn3rAd2ipZGa2dt67rqRdIWkK+fngU8B\nLwL7gG3patuAR/utpBVI2cYlzKyvFv0q4IeS5v/O9yLiJ5J+BuyVdAdwDPhs/9W0wpg/G+in+8fM\nRso3NTMzKynf1MzMzAAHejOzynOgNzOrOAd6M7OKc6A3M6u4QmTdSJojScVc7Brg1yOuTlH5WCzw\nsVjgY7GgjsdiIiLGO61UiEDfjqTZLKlDdeBjscDHYoGPxQIfi/bcdWNmVnEO9GZmFVf0QL8r7woU\niI/FAh+LBT4WC3ws2ih0H72ZmfWv6C16MzPrU+6BXtJKSfslvZy+tnxIiaQHJZ2R9GIv25dBF8fi\nFkmHJR2RtL2p/F5JJyU9m063jq72/Wu3X03LJemf0uXPS/pI1m3Lps9jcVTSC+m/gdLfLTDDsfhD\nST+VdF7Sl7vZtjYiItcJ+AawPZ3fDny9zXofAz4CvNjL9mWYsuwLsBz4FXA98G7gOWBjuuxe4Mt5\n70eP+952v5rWuRX4MSDgJuDprNuWaernWKTLjgLX5L0fIzwW7wf+CNjZ/O+/av8u+plyb9GT8WHi\nEfEkcLbX7Usiy75sBo5ExP9ExP8Bj6TblV2W/doCfDcS/wVclT7FrGrHpJ9jUTUdj0VEnImInwFv\ndrttXRQh0Gd6mPgQty+SLPuyBni16f2JtGzeXemp/IMl68bqtF9LrZNl2zLp51gABHBA0kFJ00Or\n5Wj0891W7d9Fz/p6ZmxWkg4AH2ixaEfzm4gIST2nAfW7/SgM+VjcD3yV5D/6V4H7gL/qpZ5Wan8S\nESclvR/YL+mX6Rmx1dRIAn1EfKLdMkn9Pky8VA8jH8CxOAmsa3q/Ni0jIk43/a0HgB8NptYj0Xa/\nMqxzWYZty6SfY0FEzL+ekfRDki6Msgb6LMdiGNtWShG6bvp9mHiVHkaeZV9+BmyQdJ2kdwNb0+1Y\n1Ef7GZKHtZdF2/1qsg/4yzTj5Cbgf9OurizblknPx0LSFZKuBJB0BfApyvXvYLF+vtuq/bvoXd6j\nwcDVwOPAy8ABYGVafi3wWNN6DwOnSAZcTgB3LLV9GacujsWtwH+TZBTsaCp/CHgBeJ7kH/TqvPep\ny/1/x34BXwC+kM4L+Jd0+QtAo9MxKevU67EgyTB5Lp0O1eRYfCCNCa8Dv0vn/6CK/y56nXxlrJlZ\nxRWh68bMzIbIgd7MrOIc6M3MKs6B3sys4hzozcwqzoHezKziHOjNzCrOgd7MrOL+HwLGhCgHTK97\nAAAAAElFTkSuQmCC\n", 65 | "text/plain": [ 66 | "" 67 | ] 68 | }, 69 | "metadata": {}, 70 | "output_type": "display_data" 71 | } 72 | ], 73 | "source": [ 74 | "plt.scatter(X, y, color='red')\n", 75 | "plt.show()" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": {}, 81 | "source": [ 82 | "The length of training and testing " 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 66, 88 | "metadata": {}, 89 | "outputs": [ 90 | { 91 | "name": "stdout", 92 | "output_type": "stream", 93 | "text": [ 94 | "The length of the data set 50\n" 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "print (\"The length of the data set \" , len(X))" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "### Insert columns of 1's\n", 107 | "In order to rewrite $X\\theta+b$ to $X\\theta$ hence we don't need to worry about the bias term " 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 67, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "X=np.insert(X, 0, values=1.0, axis=1)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": {}, 122 | "source": [ 123 | "### Loss function " 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "$$L(\\theta) = \\frac{1}{m} \\sum_{i=1}^m ||y^{(i)}-x^{(i)}\\theta^T||^2$$\n", 131 | "\n", 132 | "In a vectorized notation we have \n", 133 | "\n", 134 | "$$L(\\theta) = \\frac{1}{m} \\left(y-X\\theta \\right)^T \\left(y-X\\theta \\right)$$" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": 68, 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "def ssd(X,y, theta):\n", 144 | " return (y-X.dot(theta)).T (y-X.dot(theta))" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": {}, 150 | "source": [ 151 | "### Normal Equations" 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "Note the loss function can be written \n", 159 | "\n", 160 | "$$L(\\theta) = \\frac{1}{m} \\left(y^Ty-\\theta^TX^Ty-yX\\theta+\\theta^T X^TX \\theta \\right)$$\n", 161 | "\n", 162 | "By taking the gradient with respect to the parameters \n", 163 | "\n", 164 | "$$L(\\theta) = \\frac{1}{m} \\left(-2X^Ty +X^TX \\theta \\right)$$\n", 165 | "\n", 166 | "Equating that to 0 \n", 167 | "\n", 168 | "$$X^TX \\theta = X^Ty$$\n", 169 | "\n", 170 | "From linear algebra if $X$ has linearly indpendent columns then $X^TX$ is symmetric and invertible \n", 171 | "\n", 172 | "$$\\theta = (X^TX)^{-1} X^Ty$$\n", 173 | "\n" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "metadata": {}, 179 | "source": [ 180 | "### Evaluating the parameters " 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 69, 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "theta = inv((X.T).dot(X)).dot(X.T).dot(y)" 190 | ] 191 | }, 192 | { 193 | "cell_type": "markdown", 194 | "metadata": {}, 195 | "source": [ 196 | "### Predict the values of y " 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": {}, 202 | "source": [ 203 | "Note that \n", 204 | "\n", 205 | "$$\\hat{y} = X \\theta$$" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": 70, 211 | "metadata": { 212 | "scrolled": true 213 | }, 214 | "outputs": [], 215 | "source": [ 216 | "yhat = X.dot(theta)" 217 | ] 218 | }, 219 | { 220 | "cell_type": "markdown", 221 | "metadata": {}, 222 | "source": [ 223 | "### Plot the regression solution " 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": 71, 229 | "metadata": {}, 230 | "outputs": [ 231 | { 232 | "data": { 233 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG8ZJREFUeJzt3X+MHOd93/H3hydZEWUVJqkzQ/HHnQzICKT8wdhX1q2D\nAI3tSNE/svOHQeOkqKgQRqSgykECRMr9URUBC9utayhtJeOECmV5F6lCE8OCoSQQBQOGgETy0ZD1\ny2ZFR3eUCFo8i3FllYEqHr/9Y2Z1e8v9Mbs7uzsz+3kBi92dndl9Zvbuu898n2eeRxGBmZlV16ZR\nF8DMzAbLgd7MrOIc6M3MKs6B3sys4hzozcwqzoHezKziHOjNzCrOgd7MrOIc6M3MKu6yURcA4Jpr\nronp6elRF8PMrFSOHz/+s4iY7LRex0Av6ZeA7wFXpOv/r4j4t5IeAH4PWE1X/ZOIeCrd5n7gTmAN\n+DcR8TftPmN6epqlpaVORTEzszqSVrKsl6VG/x7wmxHxrqTLgWcl/VX62jci4j82fPANwH7gRuBa\n4Jikj0fEWvbim5lZXjrm6CPxbvr08vTWbiS0W4HHI+K9iHgdOAns67ukZmbWk0yNsZImJL0AnAWe\njojn0pfukfSipEclbUmX7QTeqNv8zXRZ43sekLQkaWl1dbXxZTMzy0mmQB8RaxGxF9gF7JP0q8DD\nwMeAvcAZ4OvdfHBEzEfETETMTE52bEswM7MeddW9MiJ+DnwXuDki3kp/AC4Cj7CenjkN7K7bbFe6\nzMzMRqBjoJc0Kekj6eMrgc8BP5a0o261LwAvp4+fBPZLukLSdcD1wPP5FtvMrGAWF2F6GjZtSu4X\nF0ddog9k6XWzAzgiaYLkh+GJiPiOpKOS9pI0zC4Dvw8QEa9IegJ4FbgA3O0eN2ZWaYuLcOAAnD+f\nPF9ZSZ4DzM6OrlwpFWEqwZmZmXA/ejMrrenpJLg3mpqC5eWBfayk4xEx02k9D4FgZtavU6e6Wz5k\nDvRmZv3as6e75UPmQG9m1q/Dh2Hz5o3LNm9OlheAA72ZWb9mZ2F+PsnJS8n9/HwhGmKhIKNXmpmV\n3uxsYQJ7I9fozcwqzoHezKziHOjNzCrOgd7MrOIc6M3MKs6B3sys4hzozcwqzoHezKziHOjNzCrO\ngd7MrOIc6M3MKs6B3sys4hzozcwqzoHebFQKPJm0VYuHKTYbhYJPJm3V4hq92SjMza0H+Zrz55Pl\nZjlzoDcbhYJPJm3V4kBvNgoFn0zaqsWB3mwUCj6ZtFVLx0Av6ZckPS/ph5JekfTv0uVbJT0t6bX0\nfkvdNvdLOinphKSbBrkDZqVU8MmkrVoUEe1XkARcFRHvSroceBa4F/gd4FxEfEXSfcCWiPhjSTcA\njwH7gGuBY8DHI2Kt1WfMzMzE0tJSPntkZjYmJB2PiJlO63Ws0Ufi3fTp5ektgFuBI+nyI8Dn08e3\nAo9HxHsR8TpwkiTom5nZCGTK0UuakPQCcBZ4OiKeA7ZHxJl0lZ8C29PHO4E36jZ/M11mZmYjkCnQ\nR8RaROwFdgH7JP1qw+tBUsvPTNIBSUuSllZXV7vZ1MzMutBVr5uI+DnwXeBm4C1JOwDS+7PpaqeB\n3XWb7UqXNb7XfETMRMTM5ORkL2U3M7MMsvS6mZT0kfTxlcDngB8DTwJ3pKvdAXw7ffwksF/SFZKu\nA64Hns+74GZmlk2WsW52AEckTZD8MDwREd+R9LfAE5LuBFaALwJExCuSngBeBS4Ad7frcWNmZoPV\nsXvlMLh7pZlZ93LrXmlmZuXmQG9mVnEO9GZmFedAb2ZWcQ70VhyeWs9sIDyVoBWDp9YzGxjX6K0Y\nPLWe2cA40FsxeGo9s4FxoLdiGPTUes7/2xhzoLdiGOTUerX8/8oKRKzn/x3sbUw40FsxDHJqPef/\nbcx5rBurvk2bkpp8IwkuXhx+ecxy4rFuzGoGnf83KzgHequ+Qeb/zUrAgd6qb5D5f7MS8JWxNh5m\nZx3YbWy5Rm+WhfvhW4m5Rm/WicfhsZJzjd6sE/fDtwyKfNLnGr1ZJx6Hxzoo+kmfa/RmnYxjP/wi\nV08LqOgnfQ70Zp2MWz98jw3UtaKf9DnQm3Uybv3wi149LaCin/Q50JtlMTsLy8vJ2DjLy9UN8lD8\n6mkBFf2kz4HezDZqVQ3dutV5+xaKftLXMdBL2i3pu5JelfSKpHvT5Q9IOi3phfR2S90290s6KemE\npJsGuQNmlrNm1dMPfQjeecd5+zaKfNKXpUZ/AfjDiLgB+BRwt6Qb0te+ERF709tTAOlr+4EbgZuB\nhyRNDKDsZjYIzaqnV18N77+/cT3n7UujY6CPiDMR8YP08S+AHwE722xyK/B4RLwXEa8DJ4F9eRTW\nzIaksXp67lzz9Zy3L4WucvSSpoFfA55LF90j6UVJj0raki7bCbxRt9mbtP9hMLOiK3q3Emsrc6CX\n9GHgL4AvR8Q7wMPAx4C9wBng6918sKQDkpYkLa2urnazqZn1qtcLoYrercTayhToJV1OEuQXI+Iv\nASLirYhYi4iLwCOsp2dOA7vrNt+VLtsgIuYjYiYiZiYnJ/vZBzPLop8LoYrercTa6jhnrCQBR4Bz\nEfHluuU7IuJM+vgPgH8WEfsl3Qj8OUngvxZ4Brg+ItZafYbnjDUbgunpJLg3mppK8vBWOlnnjM0y\nqNmngduBlyS9kC77E+BLkvYCASwDvw8QEa9IegJ4laTHzt3tgryZDYkvhBpbHQN9RDwLqMlLT7XZ\n5jDg5J1ZkezZ07xG7wbVyvOVsWbjwg2qY8uB3mxcuEF1bHniEbNx4knSx5Jr9GZmFedAb2ZWcQ70\nZmYV50BvZlZxDvRmZhXnQG9mVnEO9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRmZhXnQG9mVnEO9GZm\nFedAb2ZWcQ70ZpbJ4mIy7eymTcl9ljnFrRg8Hr2ZdbS4CAcOwPnzyfOVleQ5eHj7MnCN3swu0Vh7\nv/fe9SBfc/48zM2NonTWLdfozWyDZrX3Vk6dGk6ZrD+u0ZvZBnNzl9beW9mzZ7BlsXw40NvwuDWv\nFLLW0jdvhsOHB1sWy4cDvQ1HLR+wsgIR6615DvYjcegQXHYZSMn9oUPrr7WqpW/bBlNTyTZTUzA/\n74bYsnCgt+Fwa14hLC7C1VfDww/D2lqybG0teV4L9ocPJ7X1eps3w4MPwvIyXLyY3DvIl0fHQC9p\nt6TvSnpV0iuS7k2Xb5X0tKTX0vstddvcL+mkpBOSbhrkDlgJLC7C2283f82teUNz6BDcfju8+27z\n1+fnk/vZ2eSxa+/VoYhov4K0A9gRET+QdDVwHPg88K+AcxHxFUn3AVsi4o8l3QA8BuwDrgWOAR+P\niLVWnzEzMxNLS0u57JAV0PR0664bU1NJ9dAGanExCfId/t07vm7FIul4RMx0Wq9jjT4izkTED9LH\nvwB+BOwEbgWOpKsdIQn+pMsfj4j3IuJ14CRJ0Ldx1a7W7ta8oZib6xzEJyaGUxYbvq5y9JKmgV8D\nngO2R8SZ9KWfAtvTxzuBN+o2ezNdZuOqXeue8wFDkSVDVrvS1aonc6CX9GHgL4AvR8Q79a9Fkv/p\n6qRP0gFJS5KWVldXu9nUyqZd654NRbv+7hIcPAgPPTS88thwZQr0ki4nCfKLEfGX6eK30vx9LY9/\nNl1+Gthdt/mudNkGETEfETMRMTM5Odlr+a0M3Lo3cs1+a2sB/uJFB/mqy9LrRsB/A34UEf+p7qUn\ngTvSx3cA365bvl/SFZKuA64Hns+vyFZKs7PumzdCzX5rjx51gB8XWWr0nwZuB35T0gvp7RbgK8Dn\nJL0GfDZ9TkS8AjwBvAr8NXB3ux43NkZ8ZexI+bd2fHUc1CwingXU4uXPtNjmMODuFLbO49yajYyv\njLXhaDZSlq+MbavdMAVm3XCgt+Fo1b/PV8ZuUMtuSe2HKTDrhgO9DUer/n2dxrkdo7x+/bhvrdSG\nKTDrhgO9DUervvTtroyt2oiXDT9ai4ee7TiLU6M1d2uwHnQc62YYPNbNmFhcTHLyp04lNfnDh9s3\nxLYaI6eM4+M0NEYv8iUO8Ajnuaqrt5mYgAsXBlFAK6OsY9040FtxbdrUfIAWKekjWCbT0yyu/Avm\n+PecYg+buMhaDzN5+gpWq5fboGY2ZoqUE+81r19Aiyuf5gCPsMI0waaug/zEhIO89c6B3tYVLSfe\nS16/oOYmvpopTdM4i9PCQvJVXLjgIG+9c6C3dUXr616hMXJOrXUewNWzONmgOEdv66qUEy+YVu3K\nExPJoc3SNm3WyDl6616FcuJF0yoLdeSIa+82eA70tq7fnHiRGnILpkJZKCuh7vt3WXXVok43fd1r\nPGhZR7OzPhQ2Gq7R20a9jmVbtIbcnPlkxcrMNXrLR6sBWtoN3FISPlmxsnON3vIxMdHd8hIZ6smK\nTx1sAFyjt3y0Gm2rAqNwDW2EZZ862IC4Rm/5mJrqbnnBtKtID63XacXbOWx0HOgtHyUerqDTyA9D\n2zVPzmID4kBv+ShZR/H6afpuu619RXpou+YL1mxAPASCjY3acPhZOwINfeSHxhw9JKcOBf7BtNHy\nEAhmdQ4dgttv766359Ar0iU7K7LycK8bq7zFRfjmN5uP19bKyJoXfPmsDYBr9FZ5c3PZg7wr0lZF\nrtFb5WXttOIZnKyqOtboJT0q6aykl+uWPSDptKQX0tstda/dL+mkpBOSbhpUwc2y6pRrH8tp+nwF\n7ljJkrr578DNTZZ/IyL2prenACTdAOwHbky3eUhS+a+Bt1Jr1g9eSoL7WE7TV7QpI23gOgb6iPge\ncC7j+90KPB4R70XE68BJYF8f5TPrW7POLEePjllwr+crcMdOP42x90h6MU3tbEmX7QTeqFvnzXSZ\n2Uj1OvpyJfkK3LHTa6B/GPgYsBc4A3y92zeQdEDSkqSl1dXVHothVVd/BetllyXPrU++Anfs9BTo\nI+KtiFiLiIvAI6ynZ04Du+tW3ZUua/Ye8xExExEzk5OTvRTDKqrWTijBww+vD4C5tpY87znYuwEy\nUeJxiaw3PQV6STvqnn4BqPXIeRLYL+kKSdcB1wPP91fEAnCAGJr6dsJW5uf7fONaA+Tttye/JuP2\nnfoK3PETEW1vwGMk6Zn3SXLudwJHgZeAF0mC+4669eeAnwAngN/u9P4RwSc/+ckorIWFiM2bI5Lw\nkNw2b06WW64WFiImJjYe6la3rk1NtX/DIn6nCwtJuaXkvmjls5EDliJDjPWgZp1MTzevXk5NJa16\n1pf6gcakbFewTkwkXSK7smlT5zcv0nfqAc4sAw9q1otmKZqi9VCoQBqpcYjg2u9o1jpHbdKlrmRp\naCxSrxN3gbQ8Zan2D/pWiNRNqxTNtm3NT/WnpopTxpKc0h88mGQhsqRmmt0mJpL36EmzY1eE77SV\nVgdKGnXJrEDImLpxjb6mVQ0KitNDoZ9a3ojPBD772aTHTLeZwokJWFjI4QrW+gZISE4n6hWt14m7\nQFqesvwaDPpWiBp9uxpUURrFeq3ljfhMYGGhtxr8QItYlO+0lZKfvdlwkLFGP/IgH0UJ9K16ZRTp\ndL7XMg5i37oIlJ06vDT+ZtWKNvYxreg/RjZyDvTdKkMNqtcy5p3vbVGOg5/50QfdI+vz6Vnz8o5l\nZt1xoO9FGWpQvZQx7xp9k/c7yH8OuHjJRxw82LlG33MDa9WV4e/RRsqB3tblfbYixQJfiileD7GW\n3l9oGsQnJlp3ePnMZ/LdzUopwxmmjVzWQO9eN+Mgp0vePxiDJta4jUVWmCbY9MF9M2trzT9+YQGO\nHcth36rK/egtR74y1jJpdqFmFj1dxWqtr+SVkrGWzfCVsZajxUW4447ugzz0eBWruR+95cqB3tqq\n1eRrQwW38+EPJzV4GNN5WPPkoYQtRw70tkHjBbT33putJi/BN7+ZpGkixnAe1rx5KGHL0WWjLoCN\nXqsRJNuNCV9PgrvucgzK3eysD6rlwoF+zDU2smZvmw9ATE0l2QTHI7PicqAfc8168V0qCeo1m/m/\nzG+7n9mf/dkgi2ZmOXGOfkzUz8NaGwu+1ZwqjbbxM6ZYRlxkimXm+T1mz/2XQRfZzHLiGv0YaEzP\n1HrQZJnVabPO82DcyyyPbXxhz9RgCmtmuXONfgy0S89EXDo0e+351BTM3/UDZjd/e+MK7uZnVioO\n9BXTzWyINREbe/EdPZosW16G2Yd+3d38qqgCU1Jadh4CoUIOHUr6std/pZs3w5VXwttvt96uSHNi\n2xB44vHK8BAIY2Zx8dIgD61nQ6xxFmYMecC0seNAXxFzc60bVc+d2zhdam2YAmdhxlSrXF6nHJ+V\nlgN9ybRKrbb7H92zJwnmy8vrwxN8kIN3kB8/HjBt7DjQl0gttbqykgTqlZXk+eJi6/9RKUNqxg1z\n48UDpo2fTjOTAI8CZ4GX65ZtBZ4GXkvvt9S9dj9wEjgB3JRl9pOhzTBVsqnZGou7bVvrGQGbTUgk\nZZimb9QzGZXsO6kMH/dKIK+pBIHfAD7REOi/BtyXPr4P+Gr6+Abgh8AVwHXAT4CJTp8xlEA/6oDW\npYMHs0+qXZvju6f/3bznk+1Gyb4Ts6LJLdAn78V0Q6A/AexIH+8ATsR6bf7+uvX+Bvjnnd5/KIF+\nlAGtSwsL2YP8JbvQbbRv9UG1X49B7WC7GcML+J2YFVHWQN9rjn57RJxJH/8U2J4+3gm8Ubfem+my\nS0g6IGlJ0tLq6mqPxehCiXoatOtB02hDarVdEr+VYTfM1ZexlQJ+J2Zl1ndjbPqr0vVVVxExHxEz\nETEzOTnZbzE6K2BPg1560Gzb1uYi1V76Rw+7YS7LcJnu/WGWq14D/VuSdgCk92fT5aeB3XXr7UqX\njV7Behr02oPmwQeTbpEXLzbpHtnLWcuwZzLqVFt37w+z/GXJ73Bpjv4/sLEx9mvp4xvZ2Bj79xSl\nMTaiUD0N2jUZ9NyDZtDtEHkcv065eTfEmmVGjr1uHgPOAO+T5NzvBLYBz5B0rzwGbK1bf46kt80J\n4LezFGJogb5AOrWB9hRTB9mLJa/3LkpPmwL96Jv1KrdAP4xbFQN9pzgysMr3oAJYngUedZAtyo+N\nWZ8c6EegvtdgY429MY5kijWjDoj1RtENc1BK1NXWrJ2sgd5DIOSksddgNPRDauz80rENtJeukoNU\nwF5LPStRV1uzPDjQ96nWRfK22zr3GmyMI7WBxpr2oCnaULIF67XUlyr9aHXDYxqNLQf6PmS59qde\nV3GkaLXOYXfDHKQq/WhlVbQzRBsqzzDVh+np7EG+6wl8Wr25p4PKx+JicnZ06lTyC3z4cDl/tLLy\n31MleYapHPVyBSs0TLLdbeV3HGudw9Q2b1ZBRTtDtKG6bNQFKLrG6TVrZ7yQVARb1einpvqsJNY2\nHKdapw1Oqz/WqrdLGOAafUft2kRbVboXFnKqJI5brdMGx2eIY82Bvk6zFE27M96Btk/20kPCvSqs\nlSo1plvX3BibakzRQFLhufJKePvtS9cfaBtWq8K0+8fsZRszK7WsjbEO9KlWnRK2bYN//Mchx89e\neki4V4XZ2HGvmy61StGcOzeCM95eeki4V4WZtTBWgb5dCrvdxZJDbxPt5crNcb3aMy9u37AKG5tA\n3+nCwJF0SmgVXHopTJV6VQw76PqqUau6LCOfDfo2jNErswxYONTBIjsNX9lLYYo02mWvRjGEsEez\ntJIi4+iVY9MYu2lT8wm3pSQlk7tOl9i78bS5URyXof9xmOXDjbENhprCzpIKcONpc632f2VlcGkc\nt29YxY1NoB9qCjvLEMMOLs212/9B5c6r1L5h1sTYBPqhXhiYpbY+zsGlXWNrs+NSbxBj8vuqUau6\nLIn8Qd+qMpXghrkEszTuVaHxtFtZGls7HccyTl9oNgC4MXbImg1BUM/DESS6aWx1g7VZW26MHbZm\nefkapwLWddMIPc7pLbMcOdD3q5ZvbjUwveQhhut10wjt3LlZLhzo+5Fl0tii9aIZ9aX+3dbSPSa/\nWd8c6PvRLl0DxUszFOFSf9fSzYaur8ZYScvAL4A14EJEzEjaCvxPYBpYBr4YEf/Q7n1K2xjb6opK\nyGEuwQFw46ZZpQyzMfZfRsTeug+7D3gmIq4HnkmfV1OrtEwtcBYpyEN+V+OOOv1jZl0ZROrmVuBI\n+vgI8PkBfEYxlK1XSB5X4xYh/WNmXek30AdwTNJxSQfSZdsj4kz6+KfA9mYbSjogaUnS0urqap/F\nGJGy5Zvz+GHKMryDmRVKvzn6nRFxWtJHgaeBe4AnI+Ijdev8Q0Rsafc+pc3Rl1GnUTU78UiPZoWR\nNUd/WT8fEhGn0/uzkr4F7APekrQjIs5I2gGc7eczLGezs/2dcezZ07xBt2jdSM3sAz2nbiRdJenq\n2mPgt4CXgSeBO9LV7gC+3W8hrUDK1i5hZn3V6LcD35JUe58/j4i/lvR94AlJdwIrwBf7L6YVRu1s\noJ/0j5kNlQc1MzMrKQ9qZmZmgAO9mVnlOdCbmVWcA72ZWcU50JuZVVwhet1IWiXpitnoGuBnQy5O\nUflYrPOxWOdjsW4cj8VUREx2WqkQgb4VSUtZug6NAx+LdT4W63ws1vlYtObUjZlZxTnQm5lVXNED\n/fyoC1AgPhbrfCzW+Vis87FoodA5ejMz61/Ra/RmZtankQd6SVslPS3ptfS+6SQlkh6VdFbSy71s\nXwZdHIubJZ2QdFLSfXXLH5B0WtIL6e2W4ZW+f632q+51Sfqz9PUXJX0i67Zl0+exWJb0Uvo3UPrR\nAjMci1+R9LeS3pP0R91sOzYiYqQ34GvAfenj+4CvtljvN4BPAC/3sn0Zbln2BZgAfgJ8DPgQ8EPg\nhvS1B4A/GvV+9LjvLferbp1bgL8CBHwKeC7rtmW69XMs0teWgWtGvR9DPBYfBf4pcLj+779qfxf9\n3EZeoyfjZOIR8T3gXK/bl0SWfdkHnIyIv4+I/wc8nm5Xdln261bgf0Ti74CPpLOYVe2Y9HMsqqbj\nsYiIsxHxfeD9brcdF0UI9JkmEx/g9kWSZV92Am/UPX8zXVZzT3oq/2jJ0lid9qvdOlm2LZN+jgVA\nAMckHZd0YGClHI5+vtuq/V30rK85Y7OSdAz45SYvzdU/iYiQ1HM3oH63H4YBH4uHgT8l+Uf/U+Dr\nwL/upZxWar8eEaclfRR4WtKP0zNiG1NDCfQR8dlWr0nqdzLxUk1GnsOxOA3srnu+K11GRLxV916P\nAN/Jp9RD0XK/MqxzeYZty6SfY0FE1O7PSvoWSQqjrIE+y7EYxLaVUoTUTb+TiVdpMvIs+/J94HpJ\n10n6ELA/3Y6GHO0XSCZrL4uW+1XnSeB30x4nnwL+T5rqyrJtmfR8LCRdJelqAElXAb9Fuf4OGvXz\n3Vbt76J3o24NBrYBzwCvAceArenya4Gn6tZ7DDhD0uDyJnBnu+3LeOviWNwC/G+SHgVzdcuPAi8B\nL5L8Qe8Y9T51uf+X7BdwF3BX+ljAf01ffwmY6XRMynrr9ViQ9DD5YXp7ZUyOxS+nMeEd4Ofp439S\nxb+LXm++MtbMrOKKkLoxM7MBcqA3M6s4B3ozs4pzoDczqzgHejOzinOgNzOrOAd6M7OKc6A3M6u4\n/w8qA+56b+0HgwAAAABJRU5ErkJggg==\n", 234 | "text/plain": [ 235 | "" 236 | ] 237 | }, 238 | "metadata": {}, 239 | "output_type": "display_data" 240 | } 241 | ], 242 | "source": [ 243 | "plt.scatter(X[:, 1], y, color='red')\n", 244 | "plt.scatter(X[:, 1], yhat, color='blue')\n", 245 | "\n", 246 | "plt.show()" 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": null, 252 | "metadata": {}, 253 | "outputs": [], 254 | "source": [] 255 | } 256 | ], 257 | "metadata": { 258 | "kernelspec": { 259 | "display_name": "Python 3", 260 | "language": "python", 261 | "name": "python3" 262 | }, 263 | "language_info": { 264 | "codemirror_mode": { 265 | "name": "ipython", 266 | "version": 3 267 | }, 268 | "file_extension": ".py", 269 | "mimetype": "text/x-python", 270 | "name": "python", 271 | "nbconvert_exporter": "python", 272 | "pygments_lexer": "ipython3", 273 | "version": "3.5.4" 274 | } 275 | }, 276 | "nbformat": 4, 277 | "nbformat_minor": 2 278 | } 279 | -------------------------------------------------------------------------------- /Locally Weighted Regression .ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import matplotlib.pyplot as plt\n", 10 | "import numpy as np\n", 11 | "from sklearn import datasets\n", 12 | "from numpy.linalg import inv" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": {}, 18 | "source": [ 19 | "Here we discuss the concept of locally weighted regression where a model is generated for each query point $x_q$ depending on closer neighbors. " 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "metadata": {}, 25 | "source": [ 26 | "### Generating the data " 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 11, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "# Load the diabetes dataset\n", 36 | "diabetes = datasets.load_diabetes()\n", 37 | "\n", 38 | "\n", 39 | "# Use only one feature\n", 40 | "diabetes_X = diabetes.data[:, np.newaxis, 2]\n", 41 | "\n", 42 | "#points \n", 43 | "X = diabetes_X[0:50]\n", 44 | "\n", 45 | "# target values \n", 46 | "y = diabetes.target[0:50]\n" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "### Plot the data " 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 12, 59 | "metadata": { 60 | "scrolled": true 61 | }, 62 | "outputs": [ 63 | { 64 | "data": { 65 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF0hJREFUeJzt3W2MHVd9x/HvzyYENqSKnSzG8cNuIhlVDi8MuXUjUSGV\nAgl549AXyNWSWmrUBSlKQwVSHfyiqZAlQKSIqiWSo0Y18ZLUKqBYKFDZUaQoEk1Yozw54MY0tmPL\nsRcMDZElN7H/fTGz2evNvXvnPs7T7yON7twzM3vPzLX/98w5/5lRRGBmZtW1LO8KmJnZcDnQm5lV\nnAO9mVnFOdCbmVWcA72ZWcU50JuZVZwDvZlZxTnQm5lVnAO9mVnFvSvvCgBcc801MTk5mXc1zMxK\n5eDBg7+OiPFO63UM9JLeAzwJXJ6u/x8R8feS7gX+GphLV/1KRDyWbnMPcAdwAfibiPjPpT5jcnKS\n2dnZTlUxM7Mmko5lWS9Li/488PGIeEPSZcBTkn6cLvtWRHxz0QdvBLYCNwDXAgckfTAiLmSvvpmZ\nDUrHPvpIvJG+vSydlroT2hbgkYg4HxGvAEeAzX3X1MzMepJpMFbScknPAmeA/RHxdLroLknPS3pQ\n0oq0bA3watPmJ9KyxX9zWtKspNm5ubnFi83MbEAyBfqIuBARm4C1wGZJHwLuB64HNgGngPu6+eCI\n2BURjYhojI93HEswM7MedZVeGRG/A54AbomI0+kPwEXgARa6Z04C65o2W5uWmZlZDjoGeknjkq5K\n598LfBL4paTVTat9Bngxnd8HbJV0uaTrgA3AM4OttplZwczMwOQkLFuWvM7M5F2jt2XJulkN7Ja0\nnOSHYW9E/EjSQ5I2kQzMHgU+DxARhyTtBV4C3gLudMaNmVXazAxMT8O5c8n7Y8eS9wBTU/nVK6Ui\nPEqw0WiE8+jNrLQmJ5PgvtjEBBw9OrSPlXQwIhqd1vMtEMzM+nX8eHflI+ZAb2bWr/XruysfMQd6\nM7N+7dwJY2OXlo2NJeUF4EBvZtavqSnYtSvpk5eS1127CjEQCwW5e6WZWelNTRUmsC/mFr2ZWcU5\n0JuZVZwDvZlZxTnQm5lVnAO9mVnFOdCbmVWcA72ZWcU50JuZVZwDvZlZxTnQm5lVnAO9mVnFOdCb\nmVWcA72ZWcU50JvlpcAPk7Zq8W2KzfJQ8IdJW7W4RW+Whx07FoL8vHPnknKzAXOgN8tDwR8mbdXi\nQG+Wh4I/TNqqxYHeLA8Ff5i0VUvHQC/pPZKekfScpEOS/iEtXylpv6SX09cVTdvcI+mIpMOSbh7m\nDpiVUsEfJm3VoohYegVJwBUR8Yaky4CngLuBPwfORsTXJG0HVkTE30naCDwMbAauBQ4AH4yIC+0+\no9FoxOzs7GD2yMysJiQdjIhGp/U6tugj8Ub69rJ0CmALsDst3w3cls5vAR6JiPMR8QpwhCTom5lZ\nDjL10UtaLulZ4AywPyKeBlZFxKl0ldeAVen8GuDVps1PpGVmZpaDTIE+Ii5ExCZgLbBZ0ocWLQ+S\nVn5mkqYlzUqanZub62ZTMzPrQldZNxHxO+AJ4BbgtKTVAOnrmXS1k8C6ps3WpmWL/9auiGhERGN8\nfLyXupuZWQZZsm7GJV2Vzr8X+CTwS2AfsC1dbRvwaDq/D9gq6XJJ1wEbgGcGXXEzM8smy71uVgO7\nJS0n+WHYGxE/kvRTYK+kO4BjwGcBIuKQpL3AS8BbwJ1LZdyYmdlwdUyvHAWnV5qZdW9g6ZVmZlZu\nDvRmZhXnQG9mVnEO9GZmFedAb8XhR+uZDYUfJWjF4EfrmQ2NW/RWDH60ntnQONBbMfjRemZD40Bv\nxTDsR+u5/99qzIHeimGYj9ab7/8/dgwiFvr/HeytJhzorRiG+Wg99/9bzfleN1Z9y5YlLfnFJLh4\ncfT1MRsQ3+vGbN6w+//NCs6B3qpvmP3/ZiXgQG/VN8z+f7MS8JWxVg9TUw7sVltu0Ztl4Tx8KzG3\n6M068X14rOTcojfrxHn4VnIO9Gad+D48VnIO9Gad1DEP32MSleJAb9ZJ3fLwfW+gynGgN+ukbnn4\nHpOoHN/rxswu5XsDlYbvdWNmvWk39rBypfvtS6pjoJe0TtITkl6SdEjS3Wn5vZJOSno2nW5t2uYe\nSUckHZZ08zB3wMwGrNWYxLvfDa+/7n77ksrSon8L+FJEbARuAu6UtDFd9q2I2JROjwGky7YCNwC3\nAN+RtHwIdTezYWg1JnHllfDmm5eu53770ugY6CPiVET8PJ3/PfALYM0Sm2wBHomI8xHxCnAE2DyI\nyprZiExNwdGjSZ/80aNw9mzr9XwtQSl01UcvaRL4MPB0WnSXpOclPShpRVq2Bni1abMTLP3DYGZF\nV8drCSokc6CX9D7g+8AXI+J14H7gemATcAq4r5sPljQtaVbS7NzcXDebmlmver0Qqm7XElRMpkAv\n6TKSID8TET8AiIjTEXEhIi4CD7DQPXMSWNe0+dq07BIRsSsiGhHRGB8f72cfzCyLfi6Eqtu1BBXT\nMY9ekoDdwNmI+GJT+eqIOJXO/y3wxxGxVdINwPdIAv+1wOPAhoi40O4znEdvNgKTk0lwX2xiIumH\nt9LJmkef5TbFHwVuB16Q9Gxa9hXgLyRtAgI4CnweICIOSdoLvESSsXPnUkHezEbEN2errY6BPiKe\nAtRi0WNLbLMTcOedWZGsX9+6Re8B1crzlbFmdeEB1dpyoDerCw+o1pYfJWhWJ35Iei25RW9mVnEO\n9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRmZhXnQG9mVnEO9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRm\nZhXnQG9mVnEO9GZmFedAb2ZWcQ70ZmYV50BvZlZxDvRmZhXnQG9mVnEO9DY6MzMwOQnLliWvMzN5\n18isFvxwcBuNmRmYnoZz55L3x44l78EPqzYbMrfobTTuvnshyM87dw527MinPmY10jHQS1on6QlJ\nL0k6JOnutHylpP2SXk5fVzRtc4+kI5IOS7p5mDtgJTAzA7/5Tetlx4+Pti5mNZSlRf8W8KWI2Ajc\nBNwpaSOwHXg8IjYAj6fvSZdtBW4AbgG+I2n5MCpvJbFUq339+tHVw6ymOgb6iDgVET9P538P/AJY\nA2wBdqer7QZuS+e3AI9ExPmIeAU4AmwedMWtRJZqte/cObp6mNVUV330kiaBDwNPA6si4lS66DVg\nVTq/Bni1abMTaZnVVbtW+9VXeyDWbAQyB3pJ7wO+D3wxIl5vXhYRAUQ3HyxpWtKspNm5ubluNrWy\n2bkTxsYuLRsbg29/O5/6mNVMpkAv6TKSID8TET9Ii09LWp0uXw2cSctPAuuaNl+bll0iInZFRCMi\nGuPj473W38pgagp27YKJCZCS11273Jo3G5EsWTcC/hX4RUT8Y9OifcC2dH4b8GhT+VZJl0u6DtgA\nPDO4KlspTU3B0aNw8WLy6iBvNjJZWvQfBW4HPi7p2XS6Ffga8ElJLwOfSN8TEYeAvcBLwE+AOyPi\nwlBqb+XiK2PNctHxytiIeApQm8V/1mabnYDTKWyBr4w1y42vjLXR2LHDV8aa5cSB3kajXS69r4w1\nGzoHehuNdrn0na6Mdb++Wd8c6G002uXSL3Vl7Hy//rFjELHQr1/WYO8fLcuJA72NRi+59FXq16/a\nj5aVipKLWvPVaDRidnY272pY0SxblgTFxaQkH79MJieT4L7YxERyXYFZDyQdjIhGp/XcordLFal7\nodd+/SLyYLTlyIHeFhSte6GXfv2iqtKPlpWOA70tKFqfeJXukVOlHy0rHffR24Iq9YkX0cxM8qN5\n/HjSkt+5s5w/WlYYWfvo/XBwW7B+fesBQ3cvDMbUlAO75cJdN7ag3+6FIg3kmtnbHOhtQT994kUb\nyDWzt7mP3gbDeeJmI+c8ehutVkF+qXIzGxkHehuM5cu7K7fWPM5hQ+CsGxuMC20eItau3N7JD2ex\nIXGL3gZjYqK7cnunol2wZpXhQG+D4Ss/++f74diQONDbYFTpdgV58f1wbEgc6G1wpqaSVMqLF5NX\nB/nu+KzIhsSB3qwofFZkQ+KsG7Mi8f1wbAjcojczqzgHejOziusY6CU9KOmMpBebyu6VdFLSs+l0\na9OyeyQdkXRY0s3DqriZ9cFX4NZKlj76fwP+GfjuovJvRcQ3mwskbQS2AjcA1wIHJH0wInx5pFlR\n+Arc2unYoo+IJ4GzGf/eFuCRiDgfEa8AR4DNfdTPzAbNV+DWTj999HdJej7t2lmRlq0BXm1a50Ra\nZmZF4Stwa6fXQH8/cD2wCTgF3NftH5A0LWlW0uzc3FyP1TCzrvkK3NrpKdBHxOmIuBARF4EHWOie\nOQmsa1p1bVrW6m/siohGRDTGx8d7qYZZdzwAmfAVuLXTU6CXtLrp7WeA+YycfcBWSZdLug7YADzT\nXxULwAGi/Fo96vD225MrUOv2nfoK3PqJiCUn4GGS7pk3Sfrc7wAeAl4AnicJ7qub1t8B/Ao4DHy6\n09+PCG688cYorD17IsbGIpLwkExjY0m5lcfExKXf4eKpiN/pnj1JvaXktWj1s9wBs5EhxvqZsZ34\nWajVsGxZEtKXUqTvdHEKJCTdK255WxM/M7YXrbpoipah4G6k3mQZaCxS1olTIG2AHOjnterDnZ6G\nlStbr59HhkK7OjrYd9ZqAHKxImWdFK2BYaXmQD+vXQsKipOh0E8rr+5nAs0DkJAMQjYrWtaJUyBt\ngBzo57VrKZ09W5wMhV5beT4TSMw/GCUCHnqoGN9pO06BtEHKMmI77KkQWTftsjImJvKu2YJe6ziM\nfXNGyPD5GFsHZMy6cYt+XhlaUL3WcdD9vT5DGA0/mtEGxIF+XhkuIum1joPu73VGyGjUfVzFBsZ5\n9HUw6JzsdjnpUtL6tP45j94ycB69LRj02YozQobPZ002QA70dTHI/t4yjGeUnfPobYAc6K17ZRjP\nKDufNdkAOdBbb5wRMlw+a7IBcqA3KyKfNdkAOdBbZ07zy4fPmmxA3pV3BazgFqf5zV8cBQ48ZiXh\nFr0tzWl+ZqXnQG9Lc5qfWek50NvSnOZnVnoO9LY0p/mZlZ4DvS3NaX7V5EyqWnHWjXU2NeXAXiXO\npKodt+jN6saZVLXjQG9WN86kqh0HerO6cSZV7TjQmwfm6saZVLXTMdBLelDSGUkvNpWtlLRf0svp\n64qmZfdIOiLpsKSbh1XxnjigvVPez3/1dzJ6zqSqn05PDwc+BnwEeLGp7BvA9nR+O/D1dH4j8Bxw\nOXAd8CtgeafPuPHGG4f/uPQ9eyLGxiKScJZMY2NJeZ1NTFx6TOaniYnhf7a/E7O+ALPRIb5GROcW\nfUQ8CZxdVLwF2J3O7wZuayp/JCLOR8QrwBFgc0+/QINWl0yDblvIeQzMzdfxc5+rx3dilrNe++hX\nRcSpdP41YFU6vwZ4tWm9E2nZO0ialjQraXZubq7HanShDpkGvXTDjHpgrrmO7VTpOzErgL4HY9PT\nh+hhu10R0YiIxvj4eL/V6KwOmQa9nLWMemCuVR0Xq9J3YlYAvQb605JWA6SvZ9Lyk8C6pvXWpmX5\nq0OmQS9nLaMemOvUWq/ad2JWAL0G+n3AtnR+G/BoU/lWSZdLug7YADzTXxUHpA6ZBr2etWR9ktEg\nMmSWqksVvxOzIug0Wgs8DJwC3iTpc78DuBp4HHgZOACsbFp/B0m2zWHg01lGhEeSdVMHw8xiGdTf\nLkqmzZ49SWaRlLw608dKiIxZNx1XGMXkQD9Awwpgg0zDzDvIFuXHxqxPWQO9knXz1Wg0YnZ2Nu9q\nFM/MTDJ4efx40uWxc2d+3RrLliUhcTEp6fIpk8nJ1lk/ExNJ15VZSUg6GBGNTuv5FghFlfcVq4tV\nKWupDqm2Zk0c6IuqaBd4VSlrqUo/Wt3w7SZqy4G+qIrW6qxS1lKVfrSyKtoZoo2U++iLyv3Iw1Wk\n8Y9R8L+nSnIffdnVsdU5SlmvHaiKop0h2kg50BdVlbpKLH91HZcwwIG+2OrW6rTh8RlirTnQF1Uv\nGRLOqrB2fIZYax6MLaL5DInm9MqxsaX/Y/ayjZmVWtbBWAf6IuolQ8JZFWa146ybMuslQ8JZFWbW\nhgN9EfWSIeGsiv54fMMqzIE+T+2CSy8ZElXKqhh10PVVo1Z1WW5xOeyplrcp7nSr3F5u5Zv37X8H\nIY9bCA/yFsxmI4RvU5yzTpfYe/C0tTyOS5VuwWy14sHYPGXpCvDgaWvt9v/YseF143h8wyrOgX4Y\nstxi2MGltaX2f1h951Ua3zBrwYF+GLK01uscXJYabG11XJoN4578vmrUqi5LR/6wp8oMxs4PhrYa\n2Gs1uFeFwdNuZRls7XQcpdyqb1YkeDB2xFrdgqCZb0eQ6Gaw1QPWZkvyYOyoteqXn+eugAXdDELX\nuXvLbIAc6Ps139/cquUJSZ+vbzG8oJtBaPedmw2EA30/mtMo2ylaFk3el/p320r3PfnN+uZA34+l\numugeN0MRbjU3610s5HrazBW0lHg98AF4K2IaEhaCfw7MAkcBT4bEb9d6u+UdjC23RWVkASwoj1w\n2oObZpUyysHYP42ITU0fth14PCI2AI+n76upXbfMfOAsUpCHwV2Nm3f3j5l1ZRhdN1uA3en8buC2\nIXxGMZQtK2QQV+MWofvHzLrSb6AP4ICkg5Km07JVEXEqnX8NWNVqQ0nTkmYlzc7NzfVZjZyUrb95\nED9MWW7vYGaF0m8f/ZqIOCnp/cB+4C5gX0Rc1bTObyNixVJ/p7R99GXU6a6anfhOj2aFkbWP/l39\nfEhEnExfz0j6IbAZOC1pdUSckrQaONPPZ9iATU31d8axfn3rAd2ipZGa2dt67rqRdIWkK+fngU8B\nLwL7gG3patuAR/utpBVI2cYlzKyvFv0q4IeS5v/O9yLiJ5J+BuyVdAdwDPhs/9W0wpg/G+in+8fM\nRso3NTMzKynf1MzMzAAHejOzynOgNzOrOAd6M7OKc6A3M6u4QmTdSJojScVc7Brg1yOuTlH5WCzw\nsVjgY7GgjsdiIiLGO61UiEDfjqTZLKlDdeBjscDHYoGPxQIfi/bcdWNmVnEO9GZmFVf0QL8r7woU\niI/FAh+LBT4WC3ws2ih0H72ZmfWv6C16MzPrU+6BXtJKSfslvZy+tnxIiaQHJZ2R9GIv25dBF8fi\nFkmHJR2RtL2p/F5JJyU9m063jq72/Wu3X03LJemf0uXPS/pI1m3Lps9jcVTSC+m/gdLfLTDDsfhD\nST+VdF7Sl7vZtjYiItcJ+AawPZ3fDny9zXofAz4CvNjL9mWYsuwLsBz4FXA98G7gOWBjuuxe4Mt5\n70eP+952v5rWuRX4MSDgJuDprNuWaernWKTLjgLX5L0fIzwW7wf+CNjZ/O+/av8u+plyb9GT8WHi\nEfEkcLbX7Usiy75sBo5ExP9ExP8Bj6TblV2W/doCfDcS/wVclT7FrGrHpJ9jUTUdj0VEnImInwFv\ndrttXRQh0Gd6mPgQty+SLPuyBni16f2JtGzeXemp/IMl68bqtF9LrZNl2zLp51gABHBA0kFJ00Or\n5Wj0891W7d9Fz/p6ZmxWkg4AH2ixaEfzm4gIST2nAfW7/SgM+VjcD3yV5D/6V4H7gL/qpZ5Wan8S\nESclvR/YL+mX6Rmx1dRIAn1EfKLdMkn9Pky8VA8jH8CxOAmsa3q/Ni0jIk43/a0HgB8NptYj0Xa/\nMqxzWYZty6SfY0FEzL+ekfRDki6Msgb6LMdiGNtWShG6bvp9mHiVHkaeZV9+BmyQdJ2kdwNb0+1Y\n1Ef7GZKHtZdF2/1qsg/4yzTj5Cbgf9OurizblknPx0LSFZKuBJB0BfApyvXvYLF+vtuq/bvoXd6j\nwcDVwOPAy8ABYGVafi3wWNN6DwOnSAZcTgB3LLV9GacujsWtwH+TZBTsaCp/CHgBeJ7kH/TqvPep\ny/1/x34BXwC+kM4L+Jd0+QtAo9MxKevU67EgyTB5Lp0O1eRYfCCNCa8Dv0vn/6CK/y56nXxlrJlZ\nxRWh68bMzIbIgd7MrOIc6M3MKs6B3sys4hzozcwqzoHezKziHOjNzCrOgd7MrOL+HwLGhCgHTK97\nAAAAAElFTkSuQmCC\n", 66 | "text/plain": [ 67 | "" 68 | ] 69 | }, 70 | "metadata": {}, 71 | "output_type": "display_data" 72 | } 73 | ], 74 | "source": [ 75 | "plt.scatter(X, y, color='red')\n", 76 | "plt.show()" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "### Insert a column of 1's for the biases " 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 13, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "X=np.insert(X, 0, values=1.0, axis=1)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "### Loss function " 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "$$\\epsilon(\\theta) = \\sum_{i=1}^m \\omega_i (y^{(i)}-x^{(i)}\\theta^T)^2 $$\n", 107 | "\n", 108 | "This can be vectorized as \n", 109 | "\n", 110 | "$$\\epsilon(\\theta) = \\left(Wy-WX\\theta \\right)^T \\left(y-X\\theta\\right)$$\n", 111 | "\n", 112 | "This can be expanded to \n", 113 | "\n", 114 | "$$\\epsilon(\\theta) = y^TWy-\\theta^TX^TWy-y^TWX\\theta +\\theta^TX^TWX\\theta$$\n", 115 | "\n", 116 | "Take the partial derivative \n", 117 | "\n", 118 | "$$\\frac{ \\partial \\epsilon}{\\partial \\theta} =-2X^TWy+2X^TWX\\theta$$\n", 119 | "\n", 120 | "By equating to zero we deduce \n", 121 | "\n", 122 | "$$\\theta = \\left( X^TWX\\right)^{-1} X^TWy$$\n" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": {}, 128 | "source": [ 129 | "### Similarity of query point and training point " 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "We choose the weights as \n", 137 | "\n", 138 | "$$\\omega_i = \\exp \\left(-(x_i-x_q)^T {\\sum}^{-1} (x_i-x_q)\\right)$$" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "### Initialize the initial values of the weights $\\omega$" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 14, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "sig = 0.5\n", 155 | "cov = np.identity(X.shape[1])*sig**2" 156 | ] 157 | }, 158 | { 159 | "cell_type": "markdown", 160 | "metadata": {}, 161 | "source": [ 162 | "### Retrieve the weights matrix of each query point " 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": 15, 168 | "metadata": {}, 169 | "outputs": [], 170 | "source": [ 171 | "def get_Weight(xq):\n", 172 | " m = X.shape[0]\n", 173 | " W = np.zeros((m,m))\n", 174 | " for i in range(0,m):\n", 175 | " W[i,i] = np.exp(-0.5*((X[i]-xq).T).dot(inv(cov)).dot((X[i]-xq))) \n", 176 | " return W" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "### Predicting a new query point " 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "metadata": {}, 189 | "source": [ 190 | "Note that we need to keep the whole training set in the memory to retrieve the prediction of a point. Also there is no parameters for the model hence called non-parametric. " 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 19, 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "def predict(Xte):\n", 200 | " m = Xte.shape[0]\n", 201 | " yhat = np.zeros(m)\n", 202 | " for i in range(0,Xte.shape[0]):\n", 203 | " W = get_Weight(Xte[i])\n", 204 | " yi = inv((X.T).dot(W).dot(X)).dot(X.T).dot(W).dot(y)\n", 205 | " yhat[i] = Xte[i].dot(yi)\n", 206 | " return yhat" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 20, 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "yhat = predict(X)" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": 21, 221 | "metadata": {}, 222 | "outputs": [ 223 | { 224 | "data": { 225 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHA9JREFUeJzt3X2MJPV95/H3Z4dd4sVYZpfxZlnYGSxhRZA/NvaE+JQo\n0sVxIPyDfX9YWGNuI5A3PIjDp4sUYP4hikZyfOezcrqD06Ajt8d2IEiJZc5ycgJkKbKUgAeLZ3uP\ndZgFVgs7sHEwXomHmW/+qGqmZ7a7q/q5Hj4vqdXV1VXdVdUz3/719/ekiMDMzKpr26QPwMzMRsuB\n3sys4hzozcwqzoHezKziHOjNzCrOgd7MrOIc6M3MKs6B3sys4hzozcwq7pxJHwDAhRdeGLOzs5M+\nDDOzUnnqqafejIjprO0KEehnZ2dZXl6e9GGYmZWKpON5tnPqxsys4jIDvaRfkvSkpGckvSDpT9L1\nd0s6Ienp9HZNyz53Sjom6aikq0Z5AmZm1l2e1M27wO9ExDuStgM/kPS36XPfioj/0rqxpMuB64Ar\ngIuAxyR9KiLWhnngZmaWT2aJPhLvpA+3p7duYxtfCzwUEe9GxMvAMeDKgY/UzMz6kitHL2lK0tPA\nKeDRiHgifeo2Sc9Kul/SBem6fcCrLbu/lq7b+pqHJC1LWl5dXR3gFMzMrJtcgT4i1iLiAHAxcKWk\nXwXuBT4JHABOAt/s5Y0jYiki5iJibno6s3WQmZn1qadWNxHxM+D7wNUR8Ub6BbAO3MdGeuYEcEnL\nbhen68zMbALytLqZlvTxdPkjwOeBn0ja27LZF4Hn0+VHgOsknSvpUuAy4MnhHraZWbE0GjA7C9u2\nJfeNxqSPaEOeVjd7gcOSpki+GB6OiO9KekDSAZKK2RXgDwEi4gVJDwMvAh8At7rFjZlVWaMBhw7B\nmTPJ4+PHk8cA8/OTO64mFWFy8Lm5uXDPWDMrq9nZJLhvNTMDKyuje19JT0XEXNZ27hlrZjagV17p\nbf24OdCbmQ1o//7e1o+bA72Z2YAWF2Hnzs3rdu5M1heBA72Z2YDm52FpKcnJS8n90lIxKmKhIMMU\nm5mV3fx8cQL7Vi7Rm5lVnAO9mVnFOdCbmVWcA72ZWcU50JuZVZwDvZlZxTnQm5lVnAO9mVnFOdCb\nmVWcA72ZWcU50JuZVZwDvZlZxTnQm5lVnAO92YQUeTJpqxYPU2w2AUWfTNqqxSV6swlYWNgI8k1n\nziTrzYbNgd5sAoo+mbRViwO92QQUfTJpqxYHerMJKPpk0lYtmYFe0i9JelLSM5JekPQn6fpdkh6V\n9FJ6f0HLPndKOibpqKSrRnkCZmVU9MmkrVoUEd03kAScFxHvSNoO/AC4Hfh3wOmI+LqkO4ALIuKP\nJV0OPAhcCVwEPAZ8KiLWOr3H3NxcLC8vD+eMzMxqQtJTETGXtV1miT4S76QPt6e3AK4FDqfrDwNf\nSJevBR6KiHcj4mXgGEnQNzOzCciVo5c0Jelp4BTwaEQ8AeyJiJPpJq8De9LlfcCrLbu/lq7b+pqH\nJC1LWl5dXe37BMzMrLtcgT4i1iLiAHAxcKWkX93yfJCU8nOLiKWImIuIuenp6V52NTOzHvTU6iYi\nfgZ8H7gaeEPSXoD0/lS62QngkpbdLk7XmZnZBORpdTMt6ePp8keAzwM/AR4BDqabHQS+ky4/Alwn\n6VxJlwKXAU8O+8DNzCyfPGPd7AUOS5oi+WJ4OCK+K+kfgIcl3QgcB74EEBEvSHoYeBH4ALi1W4sb\nMzMbrczmlePg5pVmZr0bWvNKMzMrNwd6M7OKc6A3M6s4B3orDM+4ZDYanmHKCsEzLpmNjkv0Vgie\ncclsdBzorRA845LZ6DjQWyGMesYl5/+tzhzorRBGOeNSM/9//DhEbOT/HeytLhzorRBGOeOS8/9W\ndx4CwSpv27akJL+VBOvr4z8es2HxEAhmqVHn/82KzoHeKm+U+X+zMnCgt8obZf7frAzcM9ZqYX7e\ngd3qyyV6sxzcDt/KzIHeLIPb4VsuBS4NONCbZXA7fMtU8NKAA71ZBo/DY5kKXhpwoDfLUMd2+AXO\nQhRTwUsDDvRmGerWDr/gWYhiKnhpwIHeLEPd2uEXPAtRTAUvDXisGzPbxGMD9anRSL4NX3klKckv\nLo68NOCxbsysL52yDbt2OW/f1fw8rKwk34YrK4X6yZcZ6CVdIun7kl6U9IKk29P1d0s6Ienp9HZN\nyz53Sjom6aikq0Z5AmY2XO2yEDt2wNtvO29fVnlK9B8A/ykiLgc+C9wq6fL0uW9FxIH09j2A9Lnr\ngCuAq4F7JE2N4NjNbATa1Umcfz68//7m7Zy3L4/MQB8RJyPiR+nyz4EfA/u67HIt8FBEvBsRLwPH\ngCuHcbBmNh5bsxCnT7ffriCtBy1DTzl6SbPArwFPpKtuk/SspPslXZCu2we82rLba3T/YjCzgit4\n60HLkDvQS/oo8NfA1yLibeBe4JPAAeAk8M1e3ljSIUnLkpZXV1d72dXM+tRvR6iCtx60DLkCvaTt\nJEG+ERF/AxARb0TEWkSsA/exkZ45AVzSsvvF6bpNImIpIuYiYm56enqQczCzHAbpCFW3vgRVk9mO\nXpKAw8DpiPhay/q9EXEyXf6PwG9ExHWSrgD+kiTwXwQ8DlwWEWud3sPt6M1Gb3Y2Ce5bzcwkeXgr\nn7zt6PNMPPKbwPXAc5KeTtfdBXxZ0gEggBXgDwEi4gVJDwMvkrTYubVbkDez8Sj4cCw2QpmBPiJ+\nAKjNU9/rss8i4OydWYHs39++RO8K1epzz1izmnCFan050JvVhCtU68uTg5vViCdJryeX6M3MKs6B\n3sys4hzozcwqzoHezKziHOjNzCrOgd7MrOIc6M3MKs6B3sys4hzozcwqzoHezKziHOjNzCrOgd7M\nrOIc6M3MKs6B3szy6XdmcZs4D1NsZtmaM4ufOZM8bs4sDh73uARcojezs20tvd9++0aQbzpzBhYW\nJnF01iOX6M1ss3al9048s3gpuERvZpstLJxdeu/EM4uXggO9jY3r8koibyndM4uXhgO9jUUzG3D8\nOERs1OU52E/ILbfAOecks4Sfc07yuKlTKX33bs8sXlIO9DYWrssriEYDzj8f7r0X1taSdWtryeNm\nsF9cTErrrXbuhD//c1hZgfX15N5BvjQyA72kSyR9X9KLkl6QdHu6fpekRyW9lN5f0LLPnZKOSToq\n6apRnoAVX6MBb73V/jnX5Y3RLbfA9dfDO++0f35pKbmfn0+WXXqvDEVE9w2kvcDeiPiRpPOBp4Av\nAH8AnI6Ir0u6A7ggIv5Y0uXAg8CVwEXAY8CnImKt03vMzc3F8vLyUE7Iimd2tnPDjZmZpHBoI9Zo\nJEE+4/8983krFElPRcRc1naZJfqIOBkRP0qXfw78GNgHXAscTjc7TBL8Sdc/FBHvRsTLwDGSoG81\n1a3U7rq8MVlYyA7iU1PjORYbu55y9JJmgV8DngD2RMTJ9KnXgT3p8j7g1ZbdXkvXWU11q9tzNmBM\n8uTImj1drXJyB3pJHwX+GvhaRLzd+lwk+Z+efvNJOiRpWdLy6upqL7tayXSr27Mx6dbeXYKbb4Z7\n7hnf8dhY5Qr0kraTBPlGRPxNuvqNNH/fzOOfStefAC5p2f3idN0mEbEUEXMRMTc9Pd3v8VsJuG6v\nANp92zYD/Pq6g3zF5Wl1I+B/AT+OiP/a8tQjwMF0+SDwnZb110k6V9KlwGXAk8M7ZCuj+Xm3zJuo\ndt+2DzzgAF8TeUr0vwlcD/yOpKfT2zXA14HPS3oJ+N30MRHxAvAw8CLwd8Ct3VrcWH24Z+yE+du2\ntjIHNYuIHwDq8PTnOuyzCLg9hX3Io9yaTY57xtpYtBsnyz1jM3QbpsCsBw70NhadWve5Z+wWzfyW\n1H2YArMeONDbWHRq3Zc1ym2V8vqZ59I68lsnzWEKzHrgQG9j0aktfbeesVUa8bLtudzwAY0L/0P3\nWZy2WnO7Butd5lg34+Cxbuqh0Uhy8q+8kpTkFxe7V8R2GiOnjOPjdDwXVljh0vwvNDUFH3wwtOOy\ncss71o2nErSxmZ/vrYVNlfL6Hc+FHmdo8jAF1genbmyTIuXE+83rF9H+Xe2HBt5Pzm+tqSkPU2B9\nc6C3DxUtJ95PXr+oFrmLnfxi07qd/IJF7tq84dZZnI4cST6MDz5wkLe+OdDbh4rW1r1KY+TMn/7v\nLPFVZlhBrDPDCkt8lXke3NjIszjZiLgy1j60bVv7IculJO7YADrVxk5NJRc3T+202RZDm3jE6qNK\nOfHC6ZSHOnzYpXcbOQd6+9CgOfEiVeQWTpXyUFY6bl5pH2rGnF7aujd50LIcem1fajYkLtHbJv2O\nZFu0ityh888VKzEHehuKTsOzdBu2pTTG2O7U3yc2Cg70NhRTU72tL5Ux/VwpWj8Gqw4HehuKTmNt\nlWYMrm5F6TGNxVD59JdNjAO9DcXMTG/rCyWrKD2mdqdVGtvHisWB3oaidMMVtM7e9JWvdC9Kj+nk\n3I/BRsWB3oaiVM3Eb7ll8+xNnTSL0mM6udJ9WVppeAgEq4dGI5nY46238u8zgYHvex2z3+rN49Gb\nNTUacMMN8N57+feZUFHafapsFJy6sepbWMgf5AufdzLrnUv0Vn15m614Yg+rqMwSvaT7JZ2S9HzL\nurslnZD0dHq7puW5OyUdk3RU0lWjOnCz3LKardRw9ib3wK2XPKmb/w1c3Wb9tyLiQHr7HoCky4Hr\ngCvSfe6RVIW+kVZmi4uwY8fZ67dvT2ZwqtnsTe6BWz+ZgT4i/h44nfP1rgUeioh3I+Jl4Bhw5QDH\nZza4+Xm4//5kmr6m3bvhL/6ilnl498Ctn0EqY2+T9Gya2rkgXbcPeLVlm9fSdWaTNT8Pb76ZFGEj\nkuUaBnlwD9w66jfQ3wt8EjgAnAS+2esLSDokaVnS8urqap+HYZXX2oP1nHOSxzYQ98Ctn74CfUS8\nERFrEbEO3MdGeuYEcEnLphen69q9xlJEzEXE3PT0dD+HYVW3tQfr2lryuM9g7wrIhHvg1k9fgV7S\n3paHXwSaLXIeAa6TdK6kS4HLgCcHO8TJc4AYs+YFv/fe9s8vLfX1klsrIK+/PvmhULfPtFTDVdhw\nRETXG/AgSXrmfZKc+43AA8BzwLMkwX1vy/YLwE+Bo8DvZ71+RPCZz3wmiurIkYidO5uJ3eS2c2ey\n3obsyJGI3bs3X+xOtx7NzHR/uSJ+pkeOJMctJfdFOz6bPGA5csRYj3WTYXa2/SxJExgGpZqag7sc\nP54UL/P8PU5NJU0ie7BtW/ZLF+kz3ToHLyTpFZe8rVXesW48BEKGorVQqEQaaesQwc1v0ryFjuas\n4z3IU9FYpFYnbgJpw+RA36JdEC1SC4XSd3S55Zbk4uYZIridAXqwtquA3KpIrU6KVsCwksuT3xn1\nrQg5+k65+JtvLk6OvlOeeWYme9+J53s/97l8ufcRJtCb1wCS61CEz7STQT5rqw9y5ugnHuSjIIG+\n2z/WxINkamtwat6k7vtNvEL5yJH+g/zu3SM50KJ8pp1M/DOzUsgb6F0Zm+pUWSfB+vr4j6edfiuG\nJ16h3OkA2mlWyM7M1H7WDU9CYllcGdujIuXiO+m3o8so8r1tK4U79WLN+0YzM/DAA0mgX1mpfVSb\nn08uw/q6L4cNxoE+VYbegv12dBn2l1jbSuE/eJfGvT9r34s1641uvtnBvY1KtLCyYsiT3xn1rQg5\n+oji5237Nex8b8f6DF4+e+XUVPsDgKSC1tpyjt7ywJWx1mooX2Lpi4i19pXCrLX/BhjaAdSHW91Y\nHnkDvStjLZ+WrpqzvMxxZs/aZIYVVrh088o+erFaORoH2OS5MtaGp9GAgwc/7Kq5yF3s5BebNtnJ\nL1jkrrP37aMXq5WjcYCVhwO9ddcsybf0ZJ3nQZb4KjOsINaZYYUlvsr8R/9vUoKHWs7DOkxlaBxg\n5XHOpA/ACmZr4+133jl70BWSYD/PgxsrJPifD7jVzJA0L6Pb0dswuERvG+34pGSQ9tZ2k2+9RYMv\nM8vLbGONWV6mwZc37y/BTTc5Cg2Z29HbsLhEX3dbx8PdUgPY4Msc4j7OcB4Ax5nlEPcBSanePVjN\nis+tbuouY3iCji1sdr/DypsfHd1xmVkmt7qxzVrTM81hCnKMQfMK7Zt5vHLaQd6sLBzo66B1zALY\naEHTnNWpi/16rf16N/MzKw0H+jpoN11RU8TZwb75eGaGxZtecTO/CvI4OvXiQF8ljQZceGESqKVk\nudHIHj0yYvNIaS0jSM7f81t9DaRmxVX6mcqsZ66MrYpGA264Ad57b/P67dvhYx+Dt97qvG+RZsW2\nkZv4/AQ2NK6MrZuFhbODPMD77yf3nSZMdR6mdjwfbf040FdFt//S06c3BrKHjWEKnIepJY+jUz8O\n9GXTqRat23/p/v0b3SwjktEkwxN91JXH0amfzEAv6X5JpyQ937Jul6RHJb2U3l/Q8tydko5JOirp\nqlEdeD9K39KgWy3a4iLs2HH2Ptu3Z/4Hl/66WE/6nanMSixrwHrgt4FPA8+3rPsGcEe6fAfwZ+ny\n5cAzwLnApcBPgams9xjHxCOlm7HnyJGI3bs3Dnb37s2P281G0W6fjBOc9HXxfCRm/WOYM0wBs1sC\n/VFgb7q8FziaLt8J3Nmy3f8D/k3W648j0Jdqxp4jRyK2b29/wG2ndlLfbzXJ6zLpLxmzsssb6PvN\n0e+JiJPp8uvAnnR5H/Bqy3avpesmrlQtDRYWNlrL5NGSn+81DTOJ69I8xq985ex+XGfOJKdvZsMz\ncGVs+q3Sc2N8SYckLUtaXl1dHfQwMpWqpUEvUbalFq2fjjDjvi5bR2Nop5BfvmYl1m+gf0PSXoD0\n/lS6/gRwSct2F6frzhIRSxExFxFz09PTfR5GfoVsadBPC5rduzvWorUb6SCrhDzu69JtNIamQn75\nmpVZnvwOZ+fo/zObK2O/kS5fwebK2H+iIJWxEQWr+OuWoO6Uo9+xo+tBS/2l8PNel2Fcv07H6By9\nWe8YVmUs8CBwEnifJOd+I7AbeBx4CXgM2NWy/QJJa5ujwO/nOYhxBfpCyaoF7aMFzSgrVodVcdrp\nGJvH6SBvlt/QAv04brUM9P0Wv7sYZSuWYX2JFKWlTaF+3Zn1KW+gd8/YUclq/jKCWtBRdoQZVuuc\nInTW8eiNVjt5vg1GfatMib5ZTGyWzLsVW3MUbYtU6ixVP4QMVToXqzecuhmTbsE9K5J0ieRFSXEU\n9XgGMYKsWSkUqeBgw+FAPw7tot+QerAWsdRZlUBRxGs7alX6orYNeQO9Jx4ZRI7JtTfpYWaHbduS\nf8etJFhfz/+WdrZmjr61Pf/OndUe2MuTjVSTJx4Zpk4Vq332YM2jVD15S6YIFcLjVqohQGzoHOiz\ndGuikRV1WybZ7jWSFLInb4U0h+dfX6/HsPwuONSbA32WbuMKtIvGrcG9ZZLtXiNJHUudNjouONSb\nA32rdimabr9520XjAYL7VnUrddrouOBQbw70TZ1SNLt2td+++Zt3RNG4n1mfPFOUdeOCQ32dM+kD\nKIxOKZqPfCT5jbu1icYIf/NubRXS/M6Bzv+c/exjZvXgEn1TpxTN6dNj/83bz3DD/exjZvVQr0Df\nLbfRrVnCmH/z9tMUzs3nzKyT+gT6rJGsJtAsodd5R7o1hatS87lJ1DW4fsMqLU/32VHfxjIEQp5+\n72Ps458170iv3dWr0sV9EudRlWtn9YPHutlizCNZZX1n5Jl3pNfvnCqMRTOJcWjqOPaNVUPeQF+f\nsW7GONhHnrFUPJZNe5O4Lv4srKw81s1WY8zB52kBU6Wc+jB1Ov+I0eXO/VlY1dUn0I+xa2CeFjB1\n7pLereKz3XVpGtVMUHX+LKwm8uR3Rn0r7Xj0W7TOQTLgvCOVlafis9frOKzjqttnYeWHc/Tj1S4v\n36rq453n1UtViXPnZt05Rz9m7fLyTR5AakMvHbucOzcbDgf6ATXzzZ0mmpI8gFSrXoK3c+dmw+FA\nP4DWzradFK30OekeoL0Ebw+tazYcHr1yAN3SNVC80mcRRrhsvs/CQpKu2b8/uUad3n9+3oHdbFAD\nVcZKWgF+DqwBH0TEnKRdwF8Bs8AK8KWI+Odur1PWythOlYWQlD67BbBJ8ATRZtUyzsrYfxsRB1re\n7A7g8Yi4DHg8fVxJndIyzcBZpCAPwxvhctLpHzPrzShy9NcCh9Plw8AXRvAehVC2ysJhtGLJGgTU\nzIpn0EAfwGOSnpKUZnvZExEn0+XXgT3tdpR0SNKypOXV1dUBD2MyylZZOIwvJk9wYlY+g+bo90XE\nCUmfAB4FbgMeiYiPt2zzzxFxQbfXKWuOvowajfwVoe24E5NZceTN0Q/U6iYiTqT3pyR9G7gSeEPS\n3og4KWkvcGqQ97DhGrQVy/797St0i9aM1Mw29J26kXSepPOby8DvAc8DjwAH080OAt8Z9CCtOMpW\nL2Fmg5Xo9wDfltR8nb+MiL+T9EPgYUk3AseBLw1+mFYUvbaDN7PJ86BmZmYl5UHNzMwMcKA3M6s8\nB3ozs4pzoDczqzgHejOzinOgNzOrOAd6M7OKK0Q7ekmrJJ2rtroQeHPMh1NUvhYbfC02+FpsqOO1\nmImI6ayNChHoO5G0nKczQB34Wmzwtdjga7HB16Izp27MzCrOgd7MrOKKHuiXJn0ABeJrscHXYoOv\nxQZfiw4KnaM3M7PBFb1Eb2ZmA5p4oJe0S9Kjkl5K79tOOyjpfkmnJD3fz/5l0MO1uFrSUUnHJN3R\nsv5uSSckPZ3erhnf0Q+u03m1PC9J/y19/llJn867b9kMeC1WJD2X/g2UfvzvHNfiVyT9g6R3Jf1R\nL/vWRkRM9AZ8A7gjXb4D+LMO2/028Gng+X72L8Mtz7kAU8BPgU8CO4BngMvT5+4G/mjS59HnuXc8\nr5ZtrgH+FhDwWeCJvPuW6TbItUifWwEunPR5jPFafAL4dWCx9e+/an8Xg9wmXqIHrgUOp8uHgS+0\n2ygi/h443e/+JZHnXK4EjkXEP0XEe8BD6X5ll+e8rgX+TyT+Efh4Oi9x1a7JINeiajKvRUSciogf\nAu/3um9dFCHQ74mIk+ny6yRTFI5z/yLJcy77gFdbHr+Wrmu6Lf0pf3/J0lhZ59Vtmzz7lskg1wIg\ngMckPSXp0MiOcjwG+Wyr9nfRt0HmjM1N0mPAL7d5aqH1QUSEpL6bAQ26/ziM+FrcC/wpyT/6nwLf\nBG7o5zit1H4rIk5I+gTwqKSfpL+IrabGEugj4nc7PSfpDUl7I+Jk+tPzVI8vP+j+YzWEa3ECuKTl\n8cXpOiLijZbXug/47nCOeiw6nleObbbn2LdMBrkWRETz/pSkb5OkMMoa6PNci1HsWylFSN08AhxM\nlw8C3xnz/kWS51x+CFwm6VJJO4Dr0v3YkqP9IvB8m/2LquN5tXgE+Pdpi5PPAv+Sprry7FsmfV8L\nSedJOh9A0nnA71Guv4OtBvlsq/Z30b9J1wYDu4HHgZeAx4Bd6fqLgO+1bPcgcJKkwuU14MZu+5fx\n1sO1uAb4/yQtChZa1j8APAc8S/IHvXfS59Tj+Z91XsBNwE3psoD/kT7/HDCXdU3Keuv3WpC0MHkm\nvb1Qk2vxy2lMeBv4Wbr8sSr+XfR7c89YM7OKK0LqxszMRsiB3sys4hzozcwqzoHezKziHOjNzCrO\ngd7MrOIc6M3MKs6B3sys4v4V77khkuQqD/kAAAAASUVORK5CYII=\n", 226 | "text/plain": [ 227 | "" 228 | ] 229 | }, 230 | "metadata": {}, 231 | "output_type": "display_data" 232 | } 233 | ], 234 | "source": [ 235 | "plt.scatter(X[:,1], yhat, color='red')\n", 236 | "plt.scatter(X[:,1], y, color='blue')\n", 237 | "plt.show()" 238 | ] 239 | } 240 | ], 241 | "metadata": { 242 | "kernelspec": { 243 | "display_name": "Python 3", 244 | "language": "python", 245 | "name": "python3" 246 | }, 247 | "language_info": { 248 | "codemirror_mode": { 249 | "name": "ipython", 250 | "version": 3 251 | }, 252 | "file_extension": ".py", 253 | "mimetype": "text/x-python", 254 | "name": "python", 255 | "nbconvert_exporter": "python", 256 | "pygments_lexer": "ipython3", 257 | "version": "3.5.4" 258 | } 259 | }, 260 | "nbformat": 4, 261 | "nbformat_minor": 2 262 | } 263 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Machine-Learning 2 | 3 | This repository implements most of the machine learning algorithm as ipython notebooks. The implemented algorithms so far are 4 | 5 | ## Regression 6 | * Linear regression using normal equations 7 | * Linear regression using SGD 8 | * Locally Weighted Regress (LOWESS) 9 | 10 | ## Classification 11 | * Logistic regression classifier 12 | * softmax classifier 13 | 14 | ## Optimization methods 15 | * Newton 16 | 17 | ## Deep Learning 18 | * MNIST training Pytorch 19 | * Pretrained model resnet for ImageNet Pytorch 20 | * Transfere Learning Pytorch 21 | * Multi-input CNN Pytorch 22 | -------------------------------------------------------------------------------- /README.md~: -------------------------------------------------------------------------------- 1 | # Machine-Learning 2 | 3 | This repository implements most of the machine learning algorithm as ipython notebooks. The implemented algorithms so far are 4 | 5 | ## Regression 6 | * Linear regression using normal equations 7 | * Linear regression using SGD 8 | * Locally Weighted Regress (LOWESS) 9 | 10 | ## Classification 11 | * Logistic regression classifier 12 | * softmax classifier 13 | 14 | ## Optimization methods 15 | * Newton 16 | * RMSP 17 | * Adam 18 | * Momentum Update 19 | -------------------------------------------------------------------------------- /dog.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zaidalyafeai/Machine-Learning/fb90c0fd0b96d2812a1ca8f23965bb91135ac442/dog.jpeg -------------------------------------------------------------------------------- /imagenet_class_index.json: -------------------------------------------------------------------------------- 1 | {"0": ["n01440764", "tench"], "1": ["n01443537", "goldfish"], "2": ["n01484850", "great_white_shark"], "3": ["n01491361", "tiger_shark"], "4": ["n01494475", "hammerhead"], "5": ["n01496331", "electric_ray"], "6": ["n01498041", "stingray"], "7": ["n01514668", "cock"], "8": ["n01514859", "hen"], "9": ["n01518878", "ostrich"], "10": ["n01530575", "brambling"], "11": ["n01531178", "goldfinch"], "12": ["n01532829", "house_finch"], "13": ["n01534433", "junco"], "14": ["n01537544", "indigo_bunting"], "15": ["n01558993", "robin"], "16": ["n01560419", "bulbul"], "17": ["n01580077", "jay"], "18": ["n01582220", "magpie"], "19": ["n01592084", "chickadee"], "20": ["n01601694", "water_ouzel"], "21": ["n01608432", "kite"], "22": ["n01614925", "bald_eagle"], "23": ["n01616318", "vulture"], "24": ["n01622779", "great_grey_owl"], "25": ["n01629819", "European_fire_salamander"], "26": ["n01630670", "common_newt"], "27": ["n01631663", "eft"], "28": ["n01632458", "spotted_salamander"], "29": ["n01632777", "axolotl"], "30": ["n01641577", "bullfrog"], "31": ["n01644373", "tree_frog"], "32": ["n01644900", "tailed_frog"], "33": ["n01664065", "loggerhead"], "34": ["n01665541", "leatherback_turtle"], "35": ["n01667114", "mud_turtle"], "36": ["n01667778", "terrapin"], "37": ["n01669191", "box_turtle"], "38": ["n01675722", "banded_gecko"], "39": ["n01677366", "common_iguana"], "40": ["n01682714", "American_chameleon"], "41": ["n01685808", "whiptail"], "42": ["n01687978", "agama"], "43": ["n01688243", "frilled_lizard"], "44": ["n01689811", "alligator_lizard"], "45": ["n01692333", "Gila_monster"], "46": ["n01693334", "green_lizard"], "47": ["n01694178", "African_chameleon"], "48": ["n01695060", "Komodo_dragon"], "49": ["n01697457", "African_crocodile"], "50": ["n01698640", "American_alligator"], "51": ["n01704323", "triceratops"], "52": ["n01728572", "thunder_snake"], "53": ["n01728920", "ringneck_snake"], "54": ["n01729322", "hognose_snake"], "55": ["n01729977", "green_snake"], "56": ["n01734418", "king_snake"], "57": ["n01735189", "garter_snake"], "58": ["n01737021", "water_snake"], "59": ["n01739381", "vine_snake"], "60": ["n01740131", "night_snake"], "61": ["n01742172", "boa_constrictor"], "62": ["n01744401", "rock_python"], "63": ["n01748264", "Indian_cobra"], "64": ["n01749939", "green_mamba"], "65": ["n01751748", "sea_snake"], "66": ["n01753488", "horned_viper"], "67": ["n01755581", "diamondback"], "68": ["n01756291", "sidewinder"], "69": ["n01768244", "trilobite"], "70": ["n01770081", "harvestman"], "71": ["n01770393", "scorpion"], "72": ["n01773157", "black_and_gold_garden_spider"], "73": ["n01773549", "barn_spider"], "74": ["n01773797", "garden_spider"], "75": ["n01774384", "black_widow"], "76": ["n01774750", "tarantula"], "77": ["n01775062", "wolf_spider"], "78": ["n01776313", "tick"], "79": ["n01784675", "centipede"], "80": ["n01795545", "black_grouse"], "81": ["n01796340", "ptarmigan"], "82": ["n01797886", "ruffed_grouse"], "83": ["n01798484", "prairie_chicken"], "84": ["n01806143", "peacock"], "85": ["n01806567", "quail"], "86": ["n01807496", "partridge"], "87": ["n01817953", "African_grey"], "88": ["n01818515", "macaw"], "89": ["n01819313", "sulphur-crested_cockatoo"], "90": ["n01820546", "lorikeet"], "91": ["n01824575", "coucal"], "92": ["n01828970", "bee_eater"], "93": ["n01829413", "hornbill"], "94": ["n01833805", "hummingbird"], "95": ["n01843065", "jacamar"], "96": ["n01843383", "toucan"], "97": ["n01847000", "drake"], "98": ["n01855032", "red-breasted_merganser"], "99": ["n01855672", "goose"], "100": ["n01860187", "black_swan"], "101": ["n01871265", "tusker"], "102": ["n01872401", "echidna"], "103": ["n01873310", "platypus"], "104": ["n01877812", "wallaby"], "105": ["n01882714", "koala"], "106": ["n01883070", "wombat"], "107": ["n01910747", "jellyfish"], "108": ["n01914609", "sea_anemone"], "109": ["n01917289", "brain_coral"], "110": ["n01924916", "flatworm"], "111": ["n01930112", "nematode"], "112": ["n01943899", "conch"], "113": ["n01944390", "snail"], "114": ["n01945685", "slug"], "115": ["n01950731", "sea_slug"], "116": ["n01955084", "chiton"], "117": ["n01968897", "chambered_nautilus"], "118": ["n01978287", "Dungeness_crab"], "119": ["n01978455", "rock_crab"], "120": ["n01980166", "fiddler_crab"], "121": ["n01981276", "king_crab"], "122": ["n01983481", "American_lobster"], "123": ["n01984695", "spiny_lobster"], "124": ["n01985128", "crayfish"], "125": ["n01986214", "hermit_crab"], "126": ["n01990800", "isopod"], "127": ["n02002556", "white_stork"], "128": ["n02002724", "black_stork"], "129": ["n02006656", "spoonbill"], "130": ["n02007558", "flamingo"], "131": ["n02009229", "little_blue_heron"], "132": ["n02009912", "American_egret"], "133": ["n02011460", "bittern"], "134": ["n02012849", "crane"], "135": ["n02013706", "limpkin"], "136": ["n02017213", "European_gallinule"], "137": ["n02018207", "American_coot"], "138": ["n02018795", "bustard"], "139": ["n02025239", "ruddy_turnstone"], "140": ["n02027492", "red-backed_sandpiper"], "141": ["n02028035", "redshank"], "142": ["n02033041", "dowitcher"], "143": ["n02037110", "oystercatcher"], "144": ["n02051845", "pelican"], "145": ["n02056570", "king_penguin"], "146": ["n02058221", "albatross"], "147": ["n02066245", "grey_whale"], "148": ["n02071294", "killer_whale"], "149": ["n02074367", "dugong"], "150": ["n02077923", "sea_lion"], "151": ["n02085620", "Chihuahua"], "152": ["n02085782", "Japanese_spaniel"], "153": ["n02085936", "Maltese_dog"], "154": ["n02086079", "Pekinese"], "155": ["n02086240", "Shih-Tzu"], "156": ["n02086646", "Blenheim_spaniel"], "157": ["n02086910", "papillon"], "158": ["n02087046", "toy_terrier"], "159": ["n02087394", "Rhodesian_ridgeback"], "160": ["n02088094", "Afghan_hound"], "161": ["n02088238", "basset"], "162": ["n02088364", "beagle"], "163": ["n02088466", "bloodhound"], "164": ["n02088632", "bluetick"], "165": ["n02089078", "black-and-tan_coonhound"], "166": ["n02089867", "Walker_hound"], "167": ["n02089973", "English_foxhound"], "168": ["n02090379", "redbone"], "169": ["n02090622", "borzoi"], "170": ["n02090721", "Irish_wolfhound"], "171": ["n02091032", "Italian_greyhound"], "172": ["n02091134", "whippet"], "173": ["n02091244", "Ibizan_hound"], "174": ["n02091467", "Norwegian_elkhound"], "175": ["n02091635", "otterhound"], "176": ["n02091831", "Saluki"], "177": ["n02092002", "Scottish_deerhound"], "178": ["n02092339", "Weimaraner"], "179": ["n02093256", "Staffordshire_bullterrier"], "180": ["n02093428", "American_Staffordshire_terrier"], "181": ["n02093647", "Bedlington_terrier"], "182": ["n02093754", "Border_terrier"], "183": ["n02093859", "Kerry_blue_terrier"], "184": ["n02093991", "Irish_terrier"], "185": ["n02094114", "Norfolk_terrier"], "186": ["n02094258", "Norwich_terrier"], "187": ["n02094433", "Yorkshire_terrier"], "188": ["n02095314", "wire-haired_fox_terrier"], "189": ["n02095570", "Lakeland_terrier"], "190": ["n02095889", "Sealyham_terrier"], "191": ["n02096051", "Airedale"], "192": ["n02096177", "cairn"], "193": ["n02096294", "Australian_terrier"], "194": ["n02096437", "Dandie_Dinmont"], "195": ["n02096585", "Boston_bull"], "196": ["n02097047", "miniature_schnauzer"], "197": ["n02097130", "giant_schnauzer"], "198": ["n02097209", "standard_schnauzer"], "199": ["n02097298", "Scotch_terrier"], "200": ["n02097474", "Tibetan_terrier"], "201": ["n02097658", "silky_terrier"], "202": ["n02098105", "soft-coated_wheaten_terrier"], "203": ["n02098286", "West_Highland_white_terrier"], "204": ["n02098413", "Lhasa"], "205": ["n02099267", "flat-coated_retriever"], "206": ["n02099429", "curly-coated_retriever"], "207": ["n02099601", "golden_retriever"], "208": ["n02099712", "Labrador_retriever"], "209": ["n02099849", "Chesapeake_Bay_retriever"], "210": ["n02100236", "German_short-haired_pointer"], "211": ["n02100583", "vizsla"], "212": ["n02100735", "English_setter"], "213": ["n02100877", "Irish_setter"], "214": ["n02101006", "Gordon_setter"], "215": ["n02101388", "Brittany_spaniel"], "216": ["n02101556", "clumber"], "217": ["n02102040", "English_springer"], "218": ["n02102177", "Welsh_springer_spaniel"], "219": ["n02102318", "cocker_spaniel"], "220": ["n02102480", "Sussex_spaniel"], "221": ["n02102973", "Irish_water_spaniel"], "222": ["n02104029", "kuvasz"], "223": ["n02104365", "schipperke"], "224": ["n02105056", "groenendael"], "225": ["n02105162", "malinois"], "226": ["n02105251", "briard"], "227": ["n02105412", "kelpie"], "228": ["n02105505", "komondor"], "229": ["n02105641", "Old_English_sheepdog"], "230": ["n02105855", "Shetland_sheepdog"], "231": ["n02106030", "collie"], "232": ["n02106166", "Border_collie"], "233": ["n02106382", "Bouvier_des_Flandres"], "234": ["n02106550", "Rottweiler"], "235": ["n02106662", "German_shepherd"], "236": ["n02107142", "Doberman"], "237": ["n02107312", "miniature_pinscher"], "238": ["n02107574", "Greater_Swiss_Mountain_dog"], "239": ["n02107683", "Bernese_mountain_dog"], "240": ["n02107908", "Appenzeller"], "241": ["n02108000", "EntleBucher"], "242": ["n02108089", "boxer"], "243": ["n02108422", "bull_mastiff"], "244": ["n02108551", "Tibetan_mastiff"], "245": ["n02108915", "French_bulldog"], "246": ["n02109047", "Great_Dane"], "247": ["n02109525", "Saint_Bernard"], "248": ["n02109961", "Eskimo_dog"], "249": ["n02110063", "malamute"], "250": ["n02110185", "Siberian_husky"], "251": ["n02110341", "dalmatian"], "252": ["n02110627", "affenpinscher"], "253": ["n02110806", "basenji"], "254": ["n02110958", "pug"], "255": ["n02111129", "Leonberg"], "256": ["n02111277", "Newfoundland"], "257": ["n02111500", "Great_Pyrenees"], "258": ["n02111889", "Samoyed"], "259": ["n02112018", "Pomeranian"], "260": ["n02112137", "chow"], "261": ["n02112350", "keeshond"], "262": ["n02112706", "Brabancon_griffon"], "263": ["n02113023", "Pembroke"], "264": ["n02113186", "Cardigan"], "265": ["n02113624", "toy_poodle"], "266": ["n02113712", "miniature_poodle"], "267": ["n02113799", "standard_poodle"], "268": ["n02113978", "Mexican_hairless"], "269": ["n02114367", "timber_wolf"], "270": ["n02114548", "white_wolf"], "271": ["n02114712", "red_wolf"], "272": ["n02114855", "coyote"], "273": ["n02115641", "dingo"], "274": ["n02115913", "dhole"], "275": ["n02116738", "African_hunting_dog"], "276": ["n02117135", "hyena"], "277": ["n02119022", "red_fox"], "278": ["n02119789", "kit_fox"], "279": ["n02120079", "Arctic_fox"], "280": ["n02120505", "grey_fox"], "281": ["n02123045", "tabby"], "282": ["n02123159", "tiger_cat"], "283": ["n02123394", "Persian_cat"], "284": ["n02123597", "Siamese_cat"], "285": ["n02124075", "Egyptian_cat"], "286": ["n02125311", "cougar"], "287": ["n02127052", "lynx"], "288": ["n02128385", "leopard"], "289": ["n02128757", "snow_leopard"], "290": ["n02128925", "jaguar"], "291": ["n02129165", "lion"], "292": ["n02129604", "tiger"], "293": ["n02130308", "cheetah"], "294": ["n02132136", "brown_bear"], "295": ["n02133161", "American_black_bear"], "296": ["n02134084", "ice_bear"], "297": ["n02134418", "sloth_bear"], "298": ["n02137549", "mongoose"], "299": ["n02138441", "meerkat"], "300": ["n02165105", "tiger_beetle"], "301": ["n02165456", "ladybug"], "302": ["n02167151", "ground_beetle"], "303": ["n02168699", "long-horned_beetle"], "304": ["n02169497", "leaf_beetle"], "305": ["n02172182", "dung_beetle"], "306": ["n02174001", "rhinoceros_beetle"], "307": ["n02177972", "weevil"], "308": ["n02190166", "fly"], "309": ["n02206856", "bee"], "310": ["n02219486", "ant"], "311": ["n02226429", "grasshopper"], "312": ["n02229544", "cricket"], "313": ["n02231487", "walking_stick"], "314": ["n02233338", "cockroach"], "315": ["n02236044", "mantis"], "316": ["n02256656", "cicada"], "317": ["n02259212", "leafhopper"], "318": ["n02264363", "lacewing"], "319": ["n02268443", "dragonfly"], "320": ["n02268853", "damselfly"], "321": ["n02276258", "admiral"], "322": ["n02277742", "ringlet"], "323": ["n02279972", "monarch"], "324": ["n02280649", "cabbage_butterfly"], "325": ["n02281406", "sulphur_butterfly"], "326": ["n02281787", "lycaenid"], "327": ["n02317335", "starfish"], "328": ["n02319095", "sea_urchin"], "329": ["n02321529", "sea_cucumber"], "330": ["n02325366", "wood_rabbit"], "331": ["n02326432", "hare"], "332": ["n02328150", "Angora"], "333": ["n02342885", "hamster"], "334": ["n02346627", "porcupine"], "335": ["n02356798", "fox_squirrel"], "336": ["n02361337", "marmot"], "337": ["n02363005", "beaver"], "338": ["n02364673", "guinea_pig"], "339": ["n02389026", "sorrel"], "340": ["n02391049", "zebra"], "341": ["n02395406", "hog"], "342": ["n02396427", "wild_boar"], "343": ["n02397096", "warthog"], "344": ["n02398521", "hippopotamus"], "345": ["n02403003", "ox"], "346": ["n02408429", "water_buffalo"], "347": ["n02410509", "bison"], "348": ["n02412080", "ram"], "349": ["n02415577", "bighorn"], "350": ["n02417914", "ibex"], "351": ["n02422106", "hartebeest"], "352": ["n02422699", "impala"], "353": ["n02423022", "gazelle"], "354": ["n02437312", "Arabian_camel"], "355": ["n02437616", "llama"], "356": ["n02441942", "weasel"], "357": ["n02442845", "mink"], "358": ["n02443114", "polecat"], "359": ["n02443484", "black-footed_ferret"], "360": ["n02444819", "otter"], "361": ["n02445715", "skunk"], "362": ["n02447366", "badger"], "363": ["n02454379", "armadillo"], "364": ["n02457408", "three-toed_sloth"], "365": ["n02480495", "orangutan"], "366": ["n02480855", "gorilla"], "367": ["n02481823", "chimpanzee"], "368": ["n02483362", "gibbon"], "369": ["n02483708", "siamang"], "370": ["n02484975", "guenon"], "371": ["n02486261", "patas"], "372": ["n02486410", "baboon"], "373": ["n02487347", "macaque"], "374": ["n02488291", "langur"], "375": ["n02488702", "colobus"], "376": ["n02489166", "proboscis_monkey"], "377": ["n02490219", "marmoset"], "378": ["n02492035", "capuchin"], "379": ["n02492660", "howler_monkey"], "380": ["n02493509", "titi"], "381": ["n02493793", "spider_monkey"], "382": ["n02494079", "squirrel_monkey"], "383": ["n02497673", "Madagascar_cat"], "384": ["n02500267", "indri"], "385": ["n02504013", "Indian_elephant"], "386": ["n02504458", "African_elephant"], "387": ["n02509815", "lesser_panda"], "388": ["n02510455", "giant_panda"], "389": ["n02514041", "barracouta"], "390": ["n02526121", "eel"], "391": ["n02536864", "coho"], "392": ["n02606052", "rock_beauty"], "393": ["n02607072", "anemone_fish"], "394": ["n02640242", "sturgeon"], "395": ["n02641379", "gar"], "396": ["n02643566", "lionfish"], "397": ["n02655020", "puffer"], "398": ["n02666196", "abacus"], "399": ["n02667093", "abaya"], "400": ["n02669723", "academic_gown"], "401": ["n02672831", "accordion"], "402": ["n02676566", "acoustic_guitar"], "403": ["n02687172", "aircraft_carrier"], "404": ["n02690373", "airliner"], "405": ["n02692877", "airship"], "406": ["n02699494", "altar"], "407": ["n02701002", "ambulance"], "408": ["n02704792", "amphibian"], "409": ["n02708093", "analog_clock"], "410": ["n02727426", "apiary"], "411": ["n02730930", "apron"], "412": ["n02747177", "ashcan"], "413": ["n02749479", "assault_rifle"], "414": ["n02769748", "backpack"], "415": ["n02776631", "bakery"], "416": ["n02777292", "balance_beam"], "417": ["n02782093", "balloon"], "418": ["n02783161", "ballpoint"], "419": ["n02786058", "Band_Aid"], "420": ["n02787622", "banjo"], "421": ["n02788148", "bannister"], "422": ["n02790996", "barbell"], "423": ["n02791124", "barber_chair"], "424": ["n02791270", "barbershop"], "425": ["n02793495", "barn"], "426": ["n02794156", "barometer"], "427": ["n02795169", "barrel"], "428": ["n02797295", "barrow"], "429": ["n02799071", "baseball"], "430": ["n02802426", "basketball"], "431": ["n02804414", "bassinet"], "432": ["n02804610", "bassoon"], "433": ["n02807133", "bathing_cap"], "434": ["n02808304", "bath_towel"], "435": ["n02808440", "bathtub"], "436": ["n02814533", "beach_wagon"], "437": ["n02814860", "beacon"], "438": ["n02815834", "beaker"], "439": ["n02817516", "bearskin"], "440": ["n02823428", "beer_bottle"], "441": ["n02823750", "beer_glass"], "442": ["n02825657", "bell_cote"], "443": ["n02834397", "bib"], "444": ["n02835271", "bicycle-built-for-two"], "445": ["n02837789", "bikini"], "446": ["n02840245", "binder"], "447": ["n02841315", "binoculars"], "448": ["n02843684", "birdhouse"], "449": ["n02859443", "boathouse"], "450": ["n02860847", "bobsled"], "451": ["n02865351", "bolo_tie"], "452": ["n02869837", "bonnet"], "453": ["n02870880", "bookcase"], "454": ["n02871525", "bookshop"], "455": ["n02877765", "bottlecap"], "456": ["n02879718", "bow"], "457": ["n02883205", "bow_tie"], "458": ["n02892201", "brass"], "459": ["n02892767", "brassiere"], "460": ["n02894605", "breakwater"], "461": ["n02895154", "breastplate"], "462": ["n02906734", "broom"], "463": ["n02909870", "bucket"], "464": ["n02910353", "buckle"], "465": ["n02916936", "bulletproof_vest"], "466": ["n02917067", "bullet_train"], "467": ["n02927161", "butcher_shop"], "468": ["n02930766", "cab"], "469": ["n02939185", "caldron"], "470": ["n02948072", "candle"], "471": ["n02950826", "cannon"], "472": ["n02951358", "canoe"], "473": ["n02951585", "can_opener"], "474": ["n02963159", "cardigan"], "475": ["n02965783", "car_mirror"], "476": ["n02966193", "carousel"], "477": ["n02966687", "carpenter's_kit"], "478": ["n02971356", "carton"], "479": ["n02974003", "car_wheel"], "480": ["n02977058", "cash_machine"], "481": ["n02978881", "cassette"], "482": ["n02979186", "cassette_player"], "483": ["n02980441", "castle"], "484": ["n02981792", "catamaran"], "485": ["n02988304", "CD_player"], "486": ["n02992211", "cello"], "487": ["n02992529", "cellular_telephone"], "488": ["n02999410", "chain"], "489": ["n03000134", "chainlink_fence"], "490": ["n03000247", "chain_mail"], "491": ["n03000684", "chain_saw"], "492": ["n03014705", "chest"], "493": ["n03016953", "chiffonier"], "494": ["n03017168", "chime"], "495": ["n03018349", "china_cabinet"], "496": ["n03026506", "Christmas_stocking"], "497": ["n03028079", "church"], "498": ["n03032252", "cinema"], "499": ["n03041632", "cleaver"], "500": ["n03042490", "cliff_dwelling"], "501": ["n03045698", "cloak"], "502": ["n03047690", "clog"], "503": ["n03062245", "cocktail_shaker"], "504": ["n03063599", "coffee_mug"], "505": ["n03063689", "coffeepot"], "506": ["n03065424", "coil"], "507": ["n03075370", "combination_lock"], "508": ["n03085013", "computer_keyboard"], "509": ["n03089624", "confectionery"], "510": ["n03095699", "container_ship"], "511": ["n03100240", "convertible"], "512": ["n03109150", "corkscrew"], "513": ["n03110669", "cornet"], "514": ["n03124043", "cowboy_boot"], "515": ["n03124170", "cowboy_hat"], "516": ["n03125729", "cradle"], "517": ["n03126707", "crane"], "518": ["n03127747", "crash_helmet"], "519": ["n03127925", "crate"], "520": ["n03131574", "crib"], "521": ["n03133878", "Crock_Pot"], "522": ["n03134739", "croquet_ball"], "523": ["n03141823", "crutch"], "524": ["n03146219", "cuirass"], "525": ["n03160309", "dam"], "526": ["n03179701", "desk"], "527": ["n03180011", "desktop_computer"], "528": ["n03187595", "dial_telephone"], "529": ["n03188531", "diaper"], "530": ["n03196217", "digital_clock"], "531": ["n03197337", "digital_watch"], "532": ["n03201208", "dining_table"], "533": ["n03207743", "dishrag"], "534": ["n03207941", "dishwasher"], "535": ["n03208938", "disk_brake"], "536": ["n03216828", "dock"], "537": ["n03218198", "dogsled"], "538": ["n03220513", "dome"], "539": ["n03223299", "doormat"], "540": ["n03240683", "drilling_platform"], "541": ["n03249569", "drum"], "542": ["n03250847", "drumstick"], "543": ["n03255030", "dumbbell"], "544": ["n03259280", "Dutch_oven"], "545": ["n03271574", "electric_fan"], "546": ["n03272010", "electric_guitar"], "547": ["n03272562", "electric_locomotive"], "548": ["n03290653", "entertainment_center"], "549": ["n03291819", "envelope"], "550": ["n03297495", "espresso_maker"], "551": ["n03314780", "face_powder"], "552": ["n03325584", "feather_boa"], "553": ["n03337140", "file"], "554": ["n03344393", "fireboat"], "555": ["n03345487", "fire_engine"], "556": ["n03347037", "fire_screen"], "557": ["n03355925", "flagpole"], "558": ["n03372029", "flute"], "559": ["n03376595", "folding_chair"], "560": ["n03379051", "football_helmet"], "561": ["n03384352", "forklift"], "562": ["n03388043", "fountain"], "563": ["n03388183", "fountain_pen"], "564": ["n03388549", "four-poster"], "565": ["n03393912", "freight_car"], "566": ["n03394916", "French_horn"], "567": ["n03400231", "frying_pan"], "568": ["n03404251", "fur_coat"], "569": ["n03417042", "garbage_truck"], "570": ["n03424325", "gasmask"], "571": ["n03425413", "gas_pump"], "572": ["n03443371", "goblet"], "573": ["n03444034", "go-kart"], "574": ["n03445777", "golf_ball"], "575": ["n03445924", "golfcart"], "576": ["n03447447", "gondola"], "577": ["n03447721", "gong"], "578": ["n03450230", "gown"], "579": ["n03452741", "grand_piano"], "580": ["n03457902", "greenhouse"], "581": ["n03459775", "grille"], "582": ["n03461385", "grocery_store"], "583": ["n03467068", "guillotine"], "584": ["n03476684", "hair_slide"], "585": ["n03476991", "hair_spray"], "586": ["n03478589", "half_track"], "587": ["n03481172", "hammer"], "588": ["n03482405", "hamper"], "589": ["n03483316", "hand_blower"], "590": ["n03485407", "hand-held_computer"], "591": ["n03485794", "handkerchief"], "592": ["n03492542", "hard_disc"], "593": ["n03494278", "harmonica"], "594": ["n03495258", "harp"], "595": ["n03496892", "harvester"], "596": ["n03498962", "hatchet"], "597": ["n03527444", "holster"], "598": ["n03529860", "home_theater"], "599": ["n03530642", "honeycomb"], "600": ["n03532672", "hook"], "601": ["n03534580", "hoopskirt"], "602": ["n03535780", "horizontal_bar"], "603": ["n03538406", "horse_cart"], "604": ["n03544143", "hourglass"], "605": ["n03584254", "iPod"], "606": ["n03584829", "iron"], "607": ["n03590841", "jack-o'-lantern"], "608": ["n03594734", "jean"], "609": ["n03594945", "jeep"], "610": ["n03595614", "jersey"], "611": ["n03598930", "jigsaw_puzzle"], "612": ["n03599486", "jinrikisha"], "613": ["n03602883", "joystick"], "614": ["n03617480", "kimono"], "615": ["n03623198", "knee_pad"], "616": ["n03627232", "knot"], "617": ["n03630383", "lab_coat"], "618": ["n03633091", "ladle"], "619": ["n03637318", "lampshade"], "620": ["n03642806", "laptop"], "621": ["n03649909", "lawn_mower"], "622": ["n03657121", "lens_cap"], "623": ["n03658185", "letter_opener"], "624": ["n03661043", "library"], "625": ["n03662601", "lifeboat"], "626": ["n03666591", "lighter"], "627": ["n03670208", "limousine"], "628": ["n03673027", "liner"], "629": ["n03676483", "lipstick"], "630": ["n03680355", "Loafer"], "631": ["n03690938", "lotion"], "632": ["n03691459", "loudspeaker"], "633": ["n03692522", "loupe"], "634": ["n03697007", "lumbermill"], "635": ["n03706229", "magnetic_compass"], "636": ["n03709823", "mailbag"], "637": ["n03710193", "mailbox"], "638": ["n03710637", "maillot"], "639": ["n03710721", "maillot"], "640": ["n03717622", "manhole_cover"], "641": ["n03720891", "maraca"], "642": ["n03721384", "marimba"], "643": ["n03724870", "mask"], "644": ["n03729826", "matchstick"], "645": ["n03733131", "maypole"], "646": ["n03733281", "maze"], "647": ["n03733805", "measuring_cup"], "648": ["n03742115", "medicine_chest"], "649": ["n03743016", "megalith"], "650": ["n03759954", "microphone"], "651": ["n03761084", "microwave"], "652": ["n03763968", "military_uniform"], "653": ["n03764736", "milk_can"], "654": ["n03769881", "minibus"], "655": ["n03770439", "miniskirt"], "656": ["n03770679", "minivan"], "657": ["n03773504", "missile"], "658": ["n03775071", "mitten"], "659": ["n03775546", "mixing_bowl"], "660": ["n03776460", "mobile_home"], "661": ["n03777568", "Model_T"], "662": ["n03777754", "modem"], "663": ["n03781244", "monastery"], "664": ["n03782006", "monitor"], "665": ["n03785016", "moped"], "666": ["n03786901", "mortar"], "667": ["n03787032", "mortarboard"], "668": ["n03788195", "mosque"], "669": ["n03788365", "mosquito_net"], "670": ["n03791053", "motor_scooter"], "671": ["n03792782", "mountain_bike"], "672": ["n03792972", "mountain_tent"], "673": ["n03793489", "mouse"], "674": ["n03794056", "mousetrap"], "675": ["n03796401", "moving_van"], "676": ["n03803284", "muzzle"], "677": ["n03804744", "nail"], "678": ["n03814639", "neck_brace"], "679": ["n03814906", "necklace"], "680": ["n03825788", "nipple"], "681": ["n03832673", "notebook"], "682": ["n03837869", "obelisk"], "683": ["n03838899", "oboe"], "684": ["n03840681", "ocarina"], "685": ["n03841143", "odometer"], "686": ["n03843555", "oil_filter"], "687": ["n03854065", "organ"], "688": ["n03857828", "oscilloscope"], "689": ["n03866082", "overskirt"], "690": ["n03868242", "oxcart"], "691": ["n03868863", "oxygen_mask"], "692": ["n03871628", "packet"], "693": ["n03873416", "paddle"], "694": ["n03874293", "paddlewheel"], "695": ["n03874599", "padlock"], "696": ["n03876231", "paintbrush"], "697": ["n03877472", "pajama"], "698": ["n03877845", "palace"], "699": ["n03884397", "panpipe"], "700": ["n03887697", "paper_towel"], "701": ["n03888257", "parachute"], "702": ["n03888605", "parallel_bars"], "703": ["n03891251", "park_bench"], "704": ["n03891332", "parking_meter"], "705": ["n03895866", "passenger_car"], "706": ["n03899768", "patio"], "707": ["n03902125", "pay-phone"], "708": ["n03903868", "pedestal"], "709": ["n03908618", "pencil_box"], "710": ["n03908714", "pencil_sharpener"], "711": ["n03916031", "perfume"], "712": ["n03920288", "Petri_dish"], "713": ["n03924679", "photocopier"], "714": ["n03929660", "pick"], "715": ["n03929855", "pickelhaube"], "716": ["n03930313", "picket_fence"], "717": ["n03930630", "pickup"], "718": ["n03933933", "pier"], "719": ["n03935335", "piggy_bank"], "720": ["n03937543", "pill_bottle"], "721": ["n03938244", "pillow"], "722": ["n03942813", "ping-pong_ball"], "723": ["n03944341", "pinwheel"], "724": ["n03947888", "pirate"], "725": ["n03950228", "pitcher"], "726": ["n03954731", "plane"], "727": ["n03956157", "planetarium"], "728": ["n03958227", "plastic_bag"], "729": ["n03961711", "plate_rack"], "730": ["n03967562", "plow"], "731": ["n03970156", "plunger"], "732": ["n03976467", "Polaroid_camera"], "733": ["n03976657", "pole"], "734": ["n03977966", "police_van"], "735": ["n03980874", "poncho"], "736": ["n03982430", "pool_table"], "737": ["n03983396", "pop_bottle"], "738": ["n03991062", "pot"], "739": ["n03992509", "potter's_wheel"], "740": ["n03995372", "power_drill"], "741": ["n03998194", "prayer_rug"], "742": ["n04004767", "printer"], "743": ["n04005630", "prison"], "744": ["n04008634", "projectile"], "745": ["n04009552", "projector"], "746": ["n04019541", "puck"], "747": ["n04023962", "punching_bag"], "748": ["n04026417", "purse"], "749": ["n04033901", "quill"], "750": ["n04033995", "quilt"], "751": ["n04037443", "racer"], "752": ["n04039381", "racket"], "753": ["n04040759", "radiator"], "754": ["n04041544", "radio"], "755": ["n04044716", "radio_telescope"], "756": ["n04049303", "rain_barrel"], "757": ["n04065272", "recreational_vehicle"], "758": ["n04067472", "reel"], "759": ["n04069434", "reflex_camera"], "760": ["n04070727", "refrigerator"], "761": ["n04074963", "remote_control"], "762": ["n04081281", "restaurant"], "763": ["n04086273", "revolver"], "764": ["n04090263", "rifle"], "765": ["n04099969", "rocking_chair"], "766": ["n04111531", "rotisserie"], "767": ["n04116512", "rubber_eraser"], "768": ["n04118538", "rugby_ball"], "769": ["n04118776", "rule"], "770": ["n04120489", "running_shoe"], "771": ["n04125021", "safe"], "772": ["n04127249", "safety_pin"], "773": ["n04131690", "saltshaker"], "774": ["n04133789", "sandal"], "775": ["n04136333", "sarong"], "776": ["n04141076", "sax"], "777": ["n04141327", "scabbard"], "778": ["n04141975", "scale"], "779": ["n04146614", "school_bus"], "780": ["n04147183", "schooner"], "781": ["n04149813", "scoreboard"], "782": ["n04152593", "screen"], "783": ["n04153751", "screw"], "784": ["n04154565", "screwdriver"], "785": ["n04162706", "seat_belt"], "786": ["n04179913", "sewing_machine"], "787": ["n04192698", "shield"], "788": ["n04200800", "shoe_shop"], "789": ["n04201297", "shoji"], "790": ["n04204238", "shopping_basket"], "791": ["n04204347", "shopping_cart"], "792": ["n04208210", "shovel"], "793": ["n04209133", "shower_cap"], "794": ["n04209239", "shower_curtain"], "795": ["n04228054", "ski"], "796": ["n04229816", "ski_mask"], "797": ["n04235860", "sleeping_bag"], "798": ["n04238763", "slide_rule"], "799": ["n04239074", "sliding_door"], "800": ["n04243546", "slot"], "801": ["n04251144", "snorkel"], "802": ["n04252077", "snowmobile"], "803": ["n04252225", "snowplow"], "804": ["n04254120", "soap_dispenser"], "805": ["n04254680", "soccer_ball"], "806": ["n04254777", "sock"], "807": ["n04258138", "solar_dish"], "808": ["n04259630", "sombrero"], "809": ["n04263257", "soup_bowl"], "810": ["n04264628", "space_bar"], "811": ["n04265275", "space_heater"], "812": ["n04266014", "space_shuttle"], "813": ["n04270147", "spatula"], "814": ["n04273569", "speedboat"], "815": ["n04275548", "spider_web"], "816": ["n04277352", "spindle"], "817": ["n04285008", "sports_car"], "818": ["n04286575", "spotlight"], "819": ["n04296562", "stage"], "820": ["n04310018", "steam_locomotive"], "821": ["n04311004", "steel_arch_bridge"], "822": ["n04311174", "steel_drum"], "823": ["n04317175", "stethoscope"], "824": ["n04325704", "stole"], "825": ["n04326547", "stone_wall"], "826": ["n04328186", "stopwatch"], "827": ["n04330267", "stove"], "828": ["n04332243", "strainer"], "829": ["n04335435", "streetcar"], "830": ["n04336792", "stretcher"], "831": ["n04344873", "studio_couch"], "832": ["n04346328", "stupa"], "833": ["n04347754", "submarine"], "834": ["n04350905", "suit"], "835": ["n04355338", "sundial"], "836": ["n04355933", "sunglass"], "837": ["n04356056", "sunglasses"], "838": ["n04357314", "sunscreen"], "839": ["n04366367", "suspension_bridge"], "840": ["n04367480", "swab"], "841": ["n04370456", "sweatshirt"], "842": ["n04371430", "swimming_trunks"], "843": ["n04371774", "swing"], "844": ["n04372370", "switch"], "845": ["n04376876", "syringe"], "846": ["n04380533", "table_lamp"], "847": ["n04389033", "tank"], "848": ["n04392985", "tape_player"], "849": ["n04398044", "teapot"], "850": ["n04399382", "teddy"], "851": ["n04404412", "television"], "852": ["n04409515", "tennis_ball"], "853": ["n04417672", "thatch"], "854": ["n04418357", "theater_curtain"], "855": ["n04423845", "thimble"], "856": ["n04428191", "thresher"], "857": ["n04429376", "throne"], "858": ["n04435653", "tile_roof"], "859": ["n04442312", "toaster"], "860": ["n04443257", "tobacco_shop"], "861": ["n04447861", "toilet_seat"], "862": ["n04456115", "torch"], "863": ["n04458633", "totem_pole"], "864": ["n04461696", "tow_truck"], "865": ["n04462240", "toyshop"], "866": ["n04465501", "tractor"], "867": ["n04467665", "trailer_truck"], "868": ["n04476259", "tray"], "869": ["n04479046", "trench_coat"], "870": ["n04482393", "tricycle"], "871": ["n04483307", "trimaran"], "872": ["n04485082", "tripod"], "873": ["n04486054", "triumphal_arch"], "874": ["n04487081", "trolleybus"], "875": ["n04487394", "trombone"], "876": ["n04493381", "tub"], "877": ["n04501370", "turnstile"], "878": ["n04505470", "typewriter_keyboard"], "879": ["n04507155", "umbrella"], "880": ["n04509417", "unicycle"], "881": ["n04515003", "upright"], "882": ["n04517823", "vacuum"], "883": ["n04522168", "vase"], "884": ["n04523525", "vault"], "885": ["n04525038", "velvet"], "886": ["n04525305", "vending_machine"], "887": ["n04532106", "vestment"], "888": ["n04532670", "viaduct"], "889": ["n04536866", "violin"], "890": ["n04540053", "volleyball"], "891": ["n04542943", "waffle_iron"], "892": ["n04548280", "wall_clock"], "893": ["n04548362", "wallet"], "894": ["n04550184", "wardrobe"], "895": ["n04552348", "warplane"], "896": ["n04553703", "washbasin"], "897": ["n04554684", "washer"], "898": ["n04557648", "water_bottle"], "899": ["n04560804", "water_jug"], "900": ["n04562935", "water_tower"], "901": ["n04579145", "whiskey_jug"], "902": ["n04579432", "whistle"], "903": ["n04584207", "wig"], "904": ["n04589890", "window_screen"], "905": ["n04590129", "window_shade"], "906": ["n04591157", "Windsor_tie"], "907": ["n04591713", "wine_bottle"], "908": ["n04592741", "wing"], "909": ["n04596742", "wok"], "910": ["n04597913", "wooden_spoon"], "911": ["n04599235", "wool"], "912": ["n04604644", "worm_fence"], "913": ["n04606251", "wreck"], "914": ["n04612504", "yawl"], "915": ["n04613696", "yurt"], "916": ["n06359193", "web_site"], "917": ["n06596364", "comic_book"], "918": ["n06785654", "crossword_puzzle"], "919": ["n06794110", "street_sign"], "920": ["n06874185", "traffic_light"], "921": ["n07248320", "book_jacket"], "922": ["n07565083", "menu"], "923": ["n07579787", "plate"], "924": ["n07583066", "guacamole"], "925": ["n07584110", "consomme"], "926": ["n07590611", "hot_pot"], "927": ["n07613480", "trifle"], "928": ["n07614500", "ice_cream"], "929": ["n07615774", "ice_lolly"], "930": ["n07684084", "French_loaf"], "931": ["n07693725", "bagel"], "932": ["n07695742", "pretzel"], "933": ["n07697313", "cheeseburger"], "934": ["n07697537", "hotdog"], "935": ["n07711569", "mashed_potato"], "936": ["n07714571", "head_cabbage"], "937": ["n07714990", "broccoli"], "938": ["n07715103", "cauliflower"], "939": ["n07716358", "zucchini"], "940": ["n07716906", "spaghetti_squash"], "941": ["n07717410", "acorn_squash"], "942": ["n07717556", "butternut_squash"], "943": ["n07718472", "cucumber"], "944": ["n07718747", "artichoke"], "945": ["n07720875", "bell_pepper"], "946": ["n07730033", "cardoon"], "947": ["n07734744", "mushroom"], "948": ["n07742313", "Granny_Smith"], "949": ["n07745940", "strawberry"], "950": ["n07747607", "orange"], "951": ["n07749582", "lemon"], "952": ["n07753113", "fig"], "953": ["n07753275", "pineapple"], "954": ["n07753592", "banana"], "955": ["n07754684", "jackfruit"], "956": ["n07760859", "custard_apple"], "957": ["n07768694", "pomegranate"], "958": ["n07802026", "hay"], "959": ["n07831146", "carbonara"], "960": ["n07836838", "chocolate_sauce"], "961": ["n07860988", "dough"], "962": ["n07871810", "meat_loaf"], "963": ["n07873807", "pizza"], "964": ["n07875152", "potpie"], "965": ["n07880968", "burrito"], "966": ["n07892512", "red_wine"], "967": ["n07920052", "espresso"], "968": ["n07930864", "cup"], "969": ["n07932039", "eggnog"], "970": ["n09193705", "alp"], "971": ["n09229709", "bubble"], "972": ["n09246464", "cliff"], "973": ["n09256479", "coral_reef"], "974": ["n09288635", "geyser"], "975": ["n09332890", "lakeside"], "976": ["n09399592", "promontory"], "977": ["n09421951", "sandbar"], "978": ["n09428293", "seashore"], "979": ["n09468604", "valley"], "980": ["n09472597", "volcano"], "981": ["n09835506", "ballplayer"], "982": ["n10148035", "groom"], "983": ["n10565667", "scuba_diver"], "984": ["n11879895", "rapeseed"], "985": ["n11939491", "daisy"], "986": ["n12057211", "yellow_lady's_slipper"], "987": ["n12144580", "corn"], "988": ["n12267677", "acorn"], "989": ["n12620546", "hip"], "990": ["n12768682", "buckeye"], "991": ["n12985857", "coral_fungus"], "992": ["n12998815", "agaric"], "993": ["n13037406", "gyromitra"], "994": ["n13040303", "stinkhorn"], "995": ["n13044778", "earthstar"], "996": ["n13052670", "hen-of-the-woods"], "997": ["n13054560", "bolete"], "998": ["n13133613", "ear"], "999": ["n15075141", "toilet_tissue"]} -------------------------------------------------------------------------------- /softmax.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Import libraries " 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np \n", 17 | "import matplotlib.pyplot as plt\n", 18 | "import random\n", 19 | "import sklearn\n", 20 | "from sklearn import datasets\n", 21 | "from sklearn.datasets.samples_generator import make_blobs\n", 22 | "import math" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "### Generated random data\n", 30 | "\n", 31 | "We generate random data from $L^2$. The main problem is classifying each point to the correct plane. " 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "outputs": [ 39 | { 40 | "data": { 41 | "text/plain": [ 42 | "" 43 | ] 44 | }, 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "output_type": "execute_result" 48 | }, 49 | { 50 | "data": { 51 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd429XVwPHv1fSOnXhlOXs5ewBJgEAChEAg7NlBC5SW\nskqhUNq3LaWllLJKy6bMQhd7BxIIWWSTvXfiLCexHW/N+/7xs7wk2bItWZJ9Ps+Tp43mkYGj6/M7\n91yltUYIIUT8MEU7ACGEEC0jiVsIIeKMJG4hhIgzkriFECLOSOIWQog4I4lbCCHijCRuIYSIM5K4\nhRAizkjiFkKIOGOJxItmZmbqvn37RuKlhRCiQ1q1atUxrXVWKI+NSOLu27cvK1eujMRLCyFEh6SU\n2hvqY6VUIoQQcUYStxBCxBlJ3EIIEWckcQshRJyRxC2EEHEmIl0lQkRCyaY9rL7/NY4s3kBCdgYj\nfn4FA757NkqpaIcmRLuSxC3iQtHanXxy+u24Kx3g1VQdKmLJT//K8TXbOeWxn0Y7PCHalZRKRFxY\n/ovncJdXg7fuqD13RTVbn/2IyoPHohiZEO1PEreIC4WLNgS83WQ1c3jh+naORojoksQt4oIlOSHw\nHUphT09p32CEiDJJ3CIuDL7hfMwJNr/bTVYL3aeNjUJEQkSPJG4RF8bcfx1ZpwzDkpyAyWbBkpKI\ntUsyZ3/0ICarXGMXnYv8Gy/igiXBxoyvHuPo0k0ULtlEYk4GeRefijU5MdqhCdHuJHGLuKGUInvS\ncLInDY92KEJElSRuIaKsdMcBY1NRZhd6nDMes80a7ZBqVR46zpbnPuTY8q2k5/dh2C0Xkdq/R7TD\n6vQkcQsRJV6Ph0XXP8Ket+ajLCaUMmGyWZj+2Z/JnDAk2uFRtG4nn075GR6HC6/DxcGvVrP1+Y85\n64M/0OOscdEOr1ML6eKkUmqPUmq9UmqNUkpOSBAiDDY/9T573lmAp9qJu7waV1kljuOlfH7uvXic\nrmiHx6IbHsVVWonXYcSiXW7cldUs+N5DaK83ytF1bi3pKpmqtR6jtZ4QsWiE6EQ2PfkunkqH3+3a\n7aHgs+VRiKiOo6Sc4nW7At7nKq+ieP3udo5I1CftgEJEiaO4LODtXo8Hx9GSdo6moabndmkwyWCv\naAo1cWtgrlJqlVLqpkAPUErdpJRaqZRaefTo0fBFKEQHlT0pP/AdGrJPHRHx99dac3DuKpbe/ndW\n3PsCRevrVti2Lil0HTswYAa3pSWTMbxvxOMTwSmtdfMPUqqn1vqAUiobmAPcprVeEOzxEyZM0HJY\nsBBNK1q7k09Oux13RXXtbeYkO73OO5lpb90f0ff2uj18eclvOPz1WtwV1SizCZPNyuhfXcvoX38X\nMMbofnLaHXgcTjxVTkw2KyarmXM+eYjcKaMiGl9npJRaFWopOqTE3ejF7wfKtdaPBnuMJG4hQnN8\n9XZW/vJFCpdswtYlmWG3XMyIu6/EZDFH9H23vzKbpbf/vcGXBoA50caslc+RPqwPANXHTrDtxU84\numIr6cPyGPKTC0npnR3R2DqrliTuZtsBlVLJgElrXVbz/6cDD7QxRiEE0G3sIM79/C/t/r5b//GJ\nX9IG8Lo87P7f14z93XUAJGR2YdR917Z3eKIZofRx5wDv1ZwyYgH+pbWeHdGohBAR5Wvxa0x7vHiq\nne0cjWipZhO31noXMLodYhFCtJN+V0+lZPM+PFUN2xEtiXbyZk2OUlQiVNIOKEQnNPQns0jpm4M5\n0V57myU5gd4XTSJrYpBuFxEzZMu7EJ2QNSWRC5c/w/aXPmPXf+dhSU5gyI8uoO9lp8vhy3GgxV0l\noZCuEhErtNYULt5A4dLNJGan0+fS07GmyChYEXvC2lUiRLxyVzn4Ysa9HP92Ox6nC7PdxtLb/845\nn/6ZnMkyGlbEL6lxi7DyOJxUHi7C6/ZEOxRW3/8ax1ZsxV1RjXZ5cJdX4SqtZO6Fv8brckc7PCFa\nTRK3CAuvy82ynz3Nm90u5u3+3+FfWZew/tH/EolSXKi2v/RZwNY27fFw8MtvoxCREOEhpRIRFot/\n/Di7//t1bXuZp9rJmvtfR5lMjPj5FVGJyV3pv8EEAK1xnqho32CECCNZcYs2qz5awu7/zPPrCXZX\nVrP2j2/g9USnbJJz+siAt3ucbnKD3CdEPJDELdrsxPYDmOyBj9tyVzlwRWB1W7J5Lxv/+g5bnv+I\n6iAjUE969CdYUhJRprp/zS3JCeTfejFJPTLDHpMQ7UVKJaLNUvvmBN1CbbJasKYlh+29tNYs+elf\n2fH6HLTXizKbWH7nM5z64l0M+M7ZDR7bdWR/Llz+DGseeJ0jC9eTkJPByLuupN/VU5t8fSCsvcxa\naw7PX8uheWs4vmobpdsLsGWkkn/rxfS/9qwGXyxChEISt2izpB6Z9DhnPAe+WNkggZuT7Ay79eKw\nTrrb8/YCdr4x168ss/imx8iZMspvcl360DzO/Nf/Nfu6J7btZ+ltf+fQV6tRJhO9Z01i4t9uI6l7\ntzbF6yqvYvbZd1O8YbffaTff3PxXCj5fSXp+Httf+RztdtPvqqmMvOdq7BmpbXpf0bHJBhwRFq7y\nKhZ8/yEOzF6ByWbB63Qz6IbzOOWvt2Ayhy9xf3rGzziycL3f7SablbEP/IBR91zd4tesPFzEe/k/\nNC5Y+lbcFjOJORlcuuVVrMmt37DzzU//yvZXZgf9jQQFymxG17RPmuxWkrp34+K1L2JNTcLr9rD2\nT2+y+e/v4Sguo8vQPE5+5Mf0Ou+UVsckYpNswBHtzpqSyFnvPkDVkSIq9h8ldWBP7OkpYX8fZ0l5\nwNu9ThfOotJWvebmp97HXeWoTdpgnPvoLCln97+/YvCNM1v1ugA7/zkneNIG0NQmbTCm9lUdKWbr\ni58w4udXsOjGR9jz9oLa1fqJTXv56vLfM/Wt39H7fEnenZUU10RYJeZ0JXPCkIgkbYDeF0wKeCHU\nkpJIj7PHt+o1Dy9YGzC5uiuqObJoQ6teE4zatrvK/zDg5niqHOz7YDHl+wuNFstGJRZPlYMVv3i+\n1XGJ+CeJW8SV4Xdejj0jFZO17pdFc6KdzAlD6H7WuFa9Zmq/7gEPvzXZraT0yw3pNVzlVWx5/iPm\nf/dBvv3tK5TvO4JSiqxThrUqJnvXNIpW78AcpFvnxJZ9DTY3ed0eqo+WyI7QTkJKJSKuJGR24aLV\nz7P2wTfZ+/5iLEl2Bt94Pvm3X9rqTpD8Oy5lzzsL/Fa2JrOZwTec3+zzKw4c5eNTbsF5ogJ3RTUm\nm5UNj7/FtLfv55S/3sLsqXfhrqo2jtwOgSU5gaE3z8LeNRXtDfwkW0YKSim018uaB99k3UNv4q02\nfmtIyEpn8os/p8+sU0N7QxF3ZMUt4k5iTlcm/u02rtr3Hy7b8hoj774Ksy3wyjQUmeMGc/JjN6Os\nFpTFjMlmwZqWxLR37ie5V1azz196+1NUHSmuPQrM63ThqXQw/5o/0nX0AGYu+Tt9Lj2dxO7dSBvS\nG1MTsZrsVobePIue0yfQbfxgkntl+rULmpPs5N92CQBrHnidtX/4Z23SBmND1LzLf8+u/3zVmh+H\niAOSuEWnV7bnMKt/+yoma013h9Z43R5UCG2MWmv2f7QE7fEGvK9w8Qa6juzPtLfu5+oD/+Oyza9y\n6os/x5xkb/BYk9VCn0tP55L1L3HSX34MGL3k02c/TNrgnlhSErGmJWFOsNH3simM/vV3cVc7Wf/o\n/xpc3Kx9b7eH5Xc+E9VZMSJypFQiOr1vfvw4juOlaK+RfL0uD7g8fH3NH7n60FtNtzNq3aAbpbFA\nUxIHfm869m5dWP3bVyjdcYCUvrmM/d119LnkNL/HpuTlcMnGVzi+ahsVB47Rbdyg2l71iv2FEKSU\nAuAoLsNx7AQJWenB4xdxSRK36NQ8TheH5q2pTdr1eR0ujq3YSnYTR3kpk4nu08ZxcO4qvwSuPV5y\nTgs8E6X3+aeE3M6nlCJzwhAyJwxpcHtCdnrQGnjNM7GkJoX0HiK+SKlEdG5NrZgVAUsgjU186jZs\n6cmYEmzG00wmzEl2Jj9/J5ZEe9DnuasceBytP1HdmpzIwB9Mh0AXZU2KPpecyuan3+fjybfy+bn3\nsOfdhVI66SBkxS3iUtWRIra//gUVe46QPXk4fS+fgtlua/HrmO02sicN58jiDX4JXJlMZJ08tNnX\n6DKoF5dueoXNz3zAkQXrSemXS/7tl9JtzMCAjy9au5Nvbn6CYyu2glL0OGsck1/4ud92/VBMfPJW\nHMdK2fvewtquFWU2kXnSEI6u2Mq+D5fUjgco/GYjBZ+cyWkv/SKk1y7Zso+Nf32Hkk17yBw/hPw7\nLiW1b2jtkSKyZMu7iDsHv1rNl7P+D+314ql2YklJJCErnQuXPtWqem7Jpj18fOrteKqdeB0ulNmE\nyW7ljDd+RZ+L/evObVG+7wjvj7wRV1ll7W3KbCIhK53Ltr/e6u31VYXF7PvgG7wuNzmnjWDv+4tZ\n/+d/+x0kYU6yM3Phk3QbO6jJ1yuYvZx5l9+Px+FCe7yYrBZMdiszvnyUrJOa/zITLdeSLe9SKhFx\nxetyM++K+3FXVtcmJXd5FRUFhSy785lWvWZ6fl8u3fQKI+66ktypYxh0w/lcuOyZkJP2oa/X8Nm0\nu/hPjyv49Mw7mzxdZ9OT7/olU+3x4iqrZNe/Wt++l5idwZAfzWTYTy+i66gB7Pnf1wFP//E6XOz/\nZGmTr+X1eFj4g4dxVzpqS0Velxt3eRWLbnik1TGK8JFSiYgrhxeuD9x65/Kw550FTPnnfa3aiJPU\nvRvj/3h9wPsOfb2GNb9/jaL1uzEn2Oh9wURG//q7pPTOZvfb81n4g4drN+9UHS5i7kX/x6kv/JwB\n157t91qF32wMuLvRXVHNsRVbGPKj1s9FqS9Yr7gym5rteS/ZtBd3ZeCt+qXbCqg+doKEzC5tjlG0\nnqy4RVzxNnExL1A/c1tt+vt7fHHeLzk8fx3OojKqDh5n2wuf8O6w69j28md88+PH/WeJVDpYdsfT\nAVsBUwf1DDx/Wym2vzKbf3e/nLV/erPNhy0PvvF8v15xMOr2fa84o8nnmizmJtsMlVnSRrTJPwER\nV3JOG2n0WTemFLlTx4T1AATniXJW3vtCwAFUnkoni3/0KM7iwNMKPQ4XpTsO+N0+4s7LMSUEWPFq\njfZ4qT5SzNo/vcmC7z/UptiH/Ggm2ZOHY0kxaubKYsacaGfcg9cbs1ma0GVoHglZAVbUStFt7CCZ\nFR4DJHGLuGJNTeLkJ35qrCZrkrTJZsGamsjEJ28N63sdWbi+6dVlE9f1tduNrYv/yT/dxg7i9Jfv\nwdolGWtaEirAcCtPpYN97y+mdOfB1oQNGDsxz539MNPevp+ht1zEiLuvZNbKZxlxZ/MHNyulOPO/\nv8WaauzUBLAk2bFlpHD6a/e2OiYRPlLjFnFn6E0XkDGiHxsff4vyvUfInTKK4XdeHtJckZYwN9GD\n3RRlNtFt3OCgp+f0u/JM8i4+lWMrtjL7rLvQTv+at8lq5tjyLaQN6NGqGMAoi/ScPoGe00NqVGgg\n6+ShXLb9dba/MpuSzXvJHDeYAd+fHrFxvaJlQk7cSikzsBI4oLW+IHIhCdG8nMnDyZk8PKLvkTtl\nFCabBUI961gprCkJ2DJSOePfTR+XZrZZyTl1BPZuaVQdKgr0YiTkZLQ45nBKzM5g1L3XRDUGEVhL\nSiV3AJsjFYgQscZktTDt7ftrywVNPtZmoeeMCUx541dcvuONkDfT5N9+qf9FRKWwpiWRe8ao1oQt\nOoGQVtxKqV7ATOBB4OcRjUiIGNJ96liu3PtvNj/7IQe/WIlXeyles8vYqu7rvFAKS0oiU167r8Vt\nciPuvpLiDbvZ+85ClNWMwkja537xl1af1emudrL/w28o33uErmMG0OOscXKSfAcT0s5JpdTbwENA\nKnB3c6US2TkpOrLSHQf45uYnOPz1WgByzxjNpGd/RpdBvVr9mmW7D3F02WYSczLIPWN0qxNtyea9\nfHrmnXirnbirXZjtVlL65nD+/L9KN0iMa8nOyWYTt1LqAuB8rfVPlVJnEiRxK6VuAm4CyMvLG793\n794WB96evF7NxrWHWL/6IMnJNiaf2Y+sHPkXW4TO4zTaBNtyiEMwpTsOULRuFyl9cug2blBIbY5a\na94Z/H3Kdh1qMHfFZDNmfZ/5r6br7iK6wn3K+6nALKXU+UACkKaUekNr/d36D9JavwC8AMaKu4Ux\ntyuXy8Mj989lz84iHNVuzBYTH72zge/fdDJTzg48GEiIxiKRsN3VTr6+6gEOzl2FyWpBe7ykDujB\n9NkPk5TbtcnnFq/bRdXhIr9hWV6nm73vLsTrcjc4q1PEr2Z/H9Na36e17qW17gtcDXzVOGnHmzkf\nb2HX9uM4qo02LI/bi8vp4fXnl3OipCrK0YnObMXdz3Fwzio8VU5cpZW4K6op2bSXLy/5bbPPdZ6o\nCHpqj/Z4a39DEPGvU16x+HrOdlxO/913SsHKJfuiEJEQxmk521+Z7T+Eyu2heN2ugDsx6+s2flDQ\nU967DOnd6smDIva0KHFrrb/uCD3cblfg4fherXEF2k5do6LcyaJ5O/lq9jYKD5dFKjzRSbkrq9Hu\nwInXZLNQeeh4k8+3Jicy/sEbsNRvL1QKc5KdiU/dHs5QRZR1yoLX+Im9+eqzbbjdDRO4UopR43oG\nfM7KJXt5/onFKJNCezVaw5nTB/GdGyeEdT6G6LysqUkk5GRQWXDM7z6vw0XGiH7NvsbwOy4jbUAP\n1v7pX1TsO0LXcYMY+7vryBw/OBIhiyjplIn7wstHsmLxXsrKHLWrb7vdwqlT+9Ojl38fbklRJc89\nsdivvLJg7g6GDM/mpMl92iVuEVscJeUoBbYu4dkGrpTipL/8mEU3PFp7ag2AJSmBITdfGHI7X+8L\nJtH7gklhiUnEpg6fuL1ezZFDpdhsFrplGUN/0rok8McnL2TOJ1tYvbyApGQrZ58/lPETewd8jaWL\n9gQ8l9DhcPPFx1skcXcyRWt3suiGRyhevxswasunvXwP6UPz2vza/a+ehtluY+V9L1K28yAJWemM\nvOcq8u+4rM2vXZ/X7cFT5cCSkii/McahDn102ZoVBbz09BIcVW68Xk1Oj1RuuXsKPXq3bHfb22+s\n5qO3NwS8r0fvLjz091nhCFfEgcqDx3g3/4e4SuuOHkMpbOnJXL79n9i7pkUvuBC4KqpYfucz7Hxj\nLl63h+ReWZz8+M1hP6JNtJwcXQbs2Xmcpx9dQGlJNQ6HG5fLQ8G+Eh781Wyqq1rWFjVsZC72BP9f\nTiwWE6PGtX56m4g/m5/+AE/j+dxa46l2se3lz6ITVAt8edFv2PnGHDzVTrTbQ/mew8z/zp848EX0\nF1oidB02cX/63kb/lj8NLpeXZYv2tOi18kfl0qd/V2y2uh5Zk0mRkGjlvIsjO6FOxJajy7cEPlih\nysGx5VuiEFHoitbupHDpJjzVDeP3VDlY9et/RCkq0RodtsZdsK8kUFkaR7WbgwUnWvRaSil+cf/Z\nfPb+RubP2YHT4WbsSb24+OrRpGd0nt5YrTX795bgcnro0y8Di7V1Q5BigcfhZO+7iyhav4u0AT3o\nd9VUrCnN/7NMH5bH4flr/Y5JM9mtpOfH9rWOorU7g85AKdks+xfiSYdN3Hl9u3KwoBTd6Ow8e4KF\nXnktn3Nss5m56MpRXHRl5xy1uXvHcZ76y3zKSh0opTCZ4LqfnMLE05tvUYs1FQVH+XjSrThPVOAu\nr8KSnMCKe17g/PlPNNtyN+y2S9j+8me4Gydui5nBYTroN1JS+uQEva+57fThVLR+F6t/+yqFSzaS\nkJ3OyLuvYsD3zpGLpC3QYUslMy8djtXa8OMpBTa7mZNPi+2VUaypKHfy8G/ncKywAke1m+oqF5UV\nLl56agk7t/n3HMe6hT98mKrDRbjLjfEG7opqnMVlfHX5/TR3sb7LoF5Me/cBErLTsaQkYklOIKln\nJud8+hDJPcN7Ak+45Zw+ksScDL/j2CxJCYy679p2ieHYt9v4ZPJt7PvwG6oLSyjZsIcltzzJ8rue\nbZf37yg6bOLu3TeDn/1qKt2ykrHazFisJvoPyuQ3fz4Pu73D/qIREd/M34XH47/b1OX08On7G6MQ\nUes5Sys4smA9OsDnqSg4Sum2gmZfo+f0CVx98C1mLnySmd/8nSv3/Yfc02P/NzFlMjHjq8dIH9EP\nc5Ida5dkzAk28n9+GYOuP69dYlh+17O4K6obtNe6K6rZ8uyHVB6Mv0VAtHS4DOZyeVBKYbGYGD66\nO4+9cAlFxyqx2sykdUmIdnhxqfBwGU6H/ygAreHIwdIoRNR6Xqe79pDhxkxmM+7K6pBeR5lMdB09\nIJyhtYuU3tlcvPoFSrbso7qwhK6j+4dtA1Eoji7ZFPB2k83KkcUb6XfFGe0WSzzrMIl73+4iXn1u\nGbu3HwcFo8b24LqbJ9K1W1LtxhvROn36d8WeYKmdpuhjMin6DcyMUlStY++WRkqfbEq3+w9sUhYT\nGSP7RyGq9pc+NA/CsGGopSzJiTid/nN+lAJ7hhxEHKoOUSo5VljOg7/6nJ1bj+H1arwezbpvD/LA\nLz7F6Qg8tEeE7uTJfUhKstK4IcFqNTPzkvhqh1RKceoLd2FOstd1WNQMYpr0zM8wWcwcX72dRTc+\nyucz7mH9I//FUSwDxcJl8I3nBzzD02SzknvmmChEFJ86ROKe/eFmv55tr1dTWeli2aLYPoknHtjs\nFn77l/PIH9Uds1lhNit6983gngfOJrdnbO8UDCT3jNFcuPRp+l19Jl2G5pE3axIz5jxC/6umsu3l\nz/jk9DvY8ernHPxiFavvf4338n9IxYGj0Q67Qxj7+x+QNSkfS3ICJrsVS2oitvQUpn/6EKYgs8SF\nvw6x5f3+uz9l947AIy/PnD6IH/50YrvF0tE5HG68Hi+JSXWrpl3bj/HZ+5s4fLCUAYMzOe/i4eR0\nj79j4JwnyvlPjysbDHgCUGYTfa88gzPflKO/wkFrzbHlWyhcuonE3K7kzZqMJdHe/BM7uHAfXRbz\nsnNT2bPzuN+GG6vNHJcJJJY17shZunA3Lz21BJfTg9bGxqdv5u/mvj9Op9/AblGKsnUOfrkak8VM\n48uw2uNl/wdLohJTR6SUIuuUYWSdMizaocStuCiVVJQ7eeuN1dx903vcc/P7fPjWehz1atczLhqG\n1eb/a5bJpDhtavxcbKqqNA5q+OLjzezbUxztcJrldnl49dllOB2e2i9Nr0fjqHbzyjNLoxtcKyiT\ngmB7QGRviIghMb/irq5ycf/dn1B0vLJ2dvaHb61n1dJ9/Obh87BYjP7sH/50Iq89twyFQqOx2S3c\nds8ZpKXHx5b0DWsO8rc/zwfA4/FiUorRE3ry07tOx2SOze/XPbuKIEilbf/eYqqrXCQkhv9A3Ujw\nejyU7yvEVeZ/5qiymOl72ZQoRCVEYDGfuBd8uZOS4qoGx425nB4OHSjl22X7OflUYxfk5DP6M2FS\nH3ZtP4bVaqLfgG4xm/Aaq6py8bc/z/drt1u76gBfzt7GOTOHRimyplmtZrxBrpEoiJufP8DXV/+B\nA5+t8Ju7bk6yk9AtjQkP/yhKkQnhL+YT97fL9gXc/OGodrN43k7WrNjPoQOl9BvYjRkX5TN0ePB5\nDLHq26X7A97udHiY+8lWv8RdXubg0IETdO2WHNUe9d59M0hKtgXo74b80d0bTFOMZUdXbKHgs+V4\nKhtelEQpsifmc9YHf5CDdkVMifnEnZwS+GqzUrDu2wNojDMg9+wqYtG8Xdz7wDkMGNy+m0JKiqv4\n6O31rF5egM1m5szpgzh75lAsltBWnJUVzoBbygEqK+tO/PZ6vLz50krmz9mOxWrG7fYyeFgWt/xi\nStCfUySZTIrb7j2Dv/xuLl6PF6fTgz3BQmKSlevjqJPn0NxvA45qRWvK9x6RpC1iTsz/LjttxuCA\ns0W0Bq+X2ul/tRfFnm7fq/+lJ6r5zZ0fM2/2do4freDQgVLeeXMNT/zxq2YHFvkMHZGDyeR/9Usp\nGDG6e+3fP3hrPQu+3IHL5aWq0oXL6WHrxsLa2ng0DBicyWMvXMIV3xvL2ecP4fs3ncwjz15M18z4\n2a1qSU3CZA28hrGmJrVzNEI0L+YT9/DR3Tl75hCsVmNQlNVqxmIxYbEEvsx/sKCUinJHwPsiYfYH\nm6gsb7hidjo9bN9ylK0bC0N6jd59MxhzUi9s9rrSglKQkGjlkmtGA0bv6+cfbvYrG7ndXnZuO8aR\nQ9GbGZKSamf6hcP43k0nc9q0AdjibIhXvyumBOwasSQnMPRmOZZOxJ6Y+i/M4XBTWeGkS5eEBhe2\nrvz+OM44ZyCrVxRgNpno3rsLf/vT17jdgbezt+dFsdUrCnC7/cscjmo3G9cdYuiI0GruN995Gl/O\n3sbcT7dSWeFk+OjuXHrNaLJzjT50l9PjV0v2sVhMHCusIKd7/O1ijAWJOV057aVfsOj6R0ApvC43\nZpuVXuedzKDrZ0Q7PCH8xETidlS7ePW5ZSxfvBelFHa7mcu/O5ap5w6ufUxO9zRmzMoHjNVnahc7\njsKGiUyZFAOHZpHYji1oSUn+cxcALFZT0PsCMZlNnDNzaNAOEqvNTGqXBE4U+7eruV2eFh+ALBrq\nf/U0uk8dw+63FuAuq6TH9Alkjh/c/BOFiIKYKJU8+dB8Vizei9vlxeX0UF7m5F8vr2TJgl0BH6+U\n4pZfTCEh0Vq78caeYCEl1caNt01qz9A5e+aQgDV4pRSnnN43bO+jlOKya0c3KKeAcTLP+Il5ZHSV\nWmxbJeZ0Jf/Wixl137WStEVMi/qK+9CBE2zfXIjL1bDc4HR4eOfNtUyaEnjnY/9BmTz2/CUsmreT\nQwdK6TugK5Om9Gv3DR8TT+/LprWHWLpwD16vxmRWaA3X32KMlG2LHVuP8tHbGzi4v4SeeenMumIk\n114/gXfeXENlhRMUJCXbKNhbwnv/Wcv0C4ZGpbtECNG+oj5kauXSffzjb99QVenfjqUUXP7dMSya\ntwvthUlINiz6AAAfrklEQVRT+nLuRfntWgoJVcG+EjasPojNbmHCxN5t3rG5cslenn9iMU6Xx9id\nqMBmNXPLL6YwanxPnn5kAeu+PVg7ttZqNZHWJZEHnphJSqokbyHiTUuGTEW9VJKTmxq0h9lkUrz/\n3/UcKijl8MFSPn5nIw/c81lMztjulZfOjIvymTZjcJuTttfj5bXnluN0euq2lGujW+WVZ5eyZ+dx\n1n17oMHPweXycuJEFZ9/tLlN7y2EiH3NJm6lVIJSarlSaq1SaqNS6vfhDKB33wx65aVjbnyAqcUE\nigZztl0uD8cLy/lmfuDad0dReLg8aAdJZYWTZQv3+M0fB3C7vKxYLPPHhejoQllxO4BpWuvRwBhg\nhlIqrNvi7vrNWQwdkY3VaiIh0YrNbia7eyoet38Zx+HwsCrIFvGOwpZgwesN/FuI16tJSLT6fdHV\nPjfOeqiFEC3X7H/l2iiCl9f81VrzJ6yF8ZQ0O/f8/hyKiyopLakmt0cq/3p5FYcKTvjN2PZtlPjH\n377h4IET9B+UyYxZw8jMbv15dUZJQsdM0uvaLYmeeens3VXk9/ntdguTz+zPJ+/5n65us5uZeu6g\ndopSCBEtIV2cVEqZgVXAQOBprfW9TT0+HCfg7Np+jIf+7wu/nYIWq7HS9Ho0Xq/GbFZYrOZWDe4/\ncqiMV55ZwtZNhaCN7ds/vGUiPXuntyn2cDh84AS/vPVD/8MhrCbOv2Q4Gd2SePMfK/FqjcftxZ5g\nYejwHO741ZlBV+NCiNjVkouTLeoqUUqlA+8Bt2mtNzS67ybgJoC8vLzxe/e2vdb64f/W8eFbG9Ba\nozEuVmqtG4x49enTvysPPD4z5NeuKHdyz83vU1HuqEuOChITrfz56YtIz4juYKGNaw/x5EPzcFT7\n17JT0uw8/fqVHD1SxtKFe6iqdDFqfE+G5GejlEz8FyIeRezoMq11iVJqHjAD2NDovheAF8BYcbfk\ndYOZdeUoJk7px6ql+9Fak52byot/WxwwcRfsLaaq0tngLMSmLPxyB06nu+GKVhu7EL/6bCuXXhvd\nE6fLyxxBk3B1TetkVk4qF14+sj3DEkLEgGYTt1IqC3DVJO1E4Bzg4YhHViM7N5XzLja2uu/b7V/z\nrS/QhL1gdmw9GnDOt8vlZceWyJ3ofaKkim+X7cft8jJyXA9yewSeLzJwSBaeADNQAPoNiq+zHIUQ\n4RXKirs78FpNndsE/E9r/XFkwwos2OB+ZVIMzs/GnhD6xpzcHmlYLCa/AVEmkyIzp/UXOpuyYO4O\nXn9+OcpkjKP97+vfMm3GYK754Xi/1XW3rGROnTqAb+bvavAFY7ObueaH4yMSnxAiPjR7FUtrvU5r\nPVZrPUprPUJr/UB7BBaIUopb75lCQoKlbkaJ3ZhRcsOtLZtRMvXcwQEv4nm9msVf7+L1F5YFnPrX\nWkcOlfH6C8txuTw4HR5cNXNZvv58O2tXHgj4nOt+cgpXXTeerJwUEhKtDB+dy68ePJcBg7PCFpcQ\nIv5Efct7a5SWVLHwK2NGSf9B3Zh0Rv9WbYPftO4Qzzy6kMoKl9/uTZvdzOlnDeD7N50Slpjf+/da\nPnp7Q8BdoiPH9uDu350VlvcRQsSnuNry3hpp6YnMvHQEN942mWkzhrR6dkn+qO488txFBLoG6HR4\nWDB3J9VVAY60aoWKckfQrf3lZe138IMQIv7FXeJ2uTysWVHAN/N3UXSsos2vd6LEgTnI2ZBms6K4\nqLLN7wEwYkwP7An+lxSsVhNjT+4VlvcQQnQOsbFVMACtNfPn7uDTdzdyoqSKvH5dmXhaX95+YzVe\nDWiNx+Nl6rmDufaGCa3uX87omlh7bmVjHo8O25zrUeN60LtPOnt3F9fOGTFbTKSk2jn7/CFheQ8h\nROcQs4n7v699y5efba3tqNi2qZBtm/zPcJw/Zwd9B3bj1DMDz+1ujj3BypRzBrFg7na/7o0pZw0M\n23xvk9nEvX+YzucfbmLBXOPA35Mm5XHhFSNlhrYQokViMnGXlVYz55MtATfaNOZwuPn8g02tTtwA\n115vtNfNn7Mdk0nh9WqmnDWQa28I6TpByGw2MxdePlI2zQgh2iQmE/eu7cexWs0hJW6A0tK2Xdwz\nm01870cnceX3xlBcVEV6RmLAlbbb7cXldJOQaI341vKi45Xs211ERtck8vplyFZ2IUStmEzcqWl2\nvEHqzo2ZTIohw7P9bt+7q4ilC/fgcXuYMKkPg4ZlNZv87AlWcnv4J+zqKhf/fHE5SxfuQXs1Gd2S\n+e6NExh7cu/QPlALeDxeXn56KcsW7sZiNeP1aLJyUrjrt9Pompkc9vcTQsSfmOwq6TewG13SE/za\n9JSi4W3KqEVfctXoBo/73+vf8sdfzuaz9zfyxcdbePT3X/L8E4tC/jJo7LE/fMXShXtwu7x4PJpj\nheU88+hCNq071KrXa8qHb21g+eI9uFxeqipdOBxuDhac4LEHviISPfdCiPgTk4lbKcVdvzmrpmRh\nwWY3Y7ObGToih/Muzie1ix2b3cyocT34zcPnkduzbt7Hru3HmPPJFpxOD1qD1kYd/NvlBaxe3vID\nGHbvOM6encf9yjZOp4e33ljT5s/a2JyPN/vNUPF6NUePlLNvd3HY308IEX9islQCkNszjcdfvJSN\n6w5TfLySPv270qd/VwCuui74rI7F83YFPNbLUe3mv69/y+gJvYxj0UK0f08xisAlloP7SkJ+nVBo\nrakodwa8z2RWtT8HIUTnFpMrbh+T2cTIsT2YcvbAkBOWy+UJOkGw8FAZzzyyoEUxdMtKRgX5KaWH\nqcfbRynV4LeH+twuL737ZoT1/YQQ8SmmE3drjJ+YF3CHIhhlk/WrD3Jgf+gr5WEjc0lJtaMajYy1\n2y1ceMWINsUayNXXjcNWM0DLx2Yzc9KpeXTLkouTQogOmLhHju3BkHz/LhMfpRQ7tx4L+fVMJsV9\nf5xOr7x0bDYziUlWrDYz51+S36be8WDGntybm+8+ndweqQAkJVs57+J8brxtctjfSwgRn+JyOmBz\nPB4vt//gLcrL/OvFCYlWbr7rNMZMaPl8kIMFJygvddC7b3rIJ+20hdergx4O4XC4qShz0CUjUc6Y\nFKIDiNjRZfHCbDZx0ZWjeOuN1X4dGlariRFjerTqdXv06hKO8EIWKGk7nR7eeHE533y9G6WMw5Nn\nXTmKGbOGySYdITqJDpm4Ac6eOZR9e4pZumAPZouR0Gx2C7/43Vkt6ipprOhYBWtXHUCZFGNP6kWX\n9PY9VPj5vy5i7coDuFzGF5LT6eHdf63BajVx9vlD2zUWIUR0dMhSSX1Hj5Szc9tRUtMSGDYiB1Mb\nygrGqfPrUUqhFHi9cM314znrvPaZ7ld0rIJ7bn4fV4BRAKld7Pz91Stk1S1EnOr0pZL6snJSyArD\nGZJbNx7ho3c2+CXN/7yyiiH52fTqE/lWvUMHSrFYzQETd0WZE6fTg93e4f+RCtHpyVWtEH352Tac\nATb2uN1eFszd0S4xZOemBD0HMyHR6tdGKITomGR5Vs/B/SdYs7IAs9nEhEkN+6bLSqshQFXJ69WU\nnqhul/iyclIZkp/Nlo1HGmzBt9mN9kQpkwjROUjixthq/q+XVjLvi+1GC56Ct/65miu+N4ZzZ+UD\nMHpCL3ZsOeq36rYnWBg1vme7xXrrPVN4/onFbFhzELPFjNfj5eyZQ5l5afg3AwkhYpMkbmDdqoPM\nn7OjdsaJLzW/9cYaho/uTq8+GZxxzkDmfLSZkuKq2nKFxWoiMyuZkyb3abdYE5Ns/OzXUyktqaK4\nqIrs7qmtPixZCBGfOkSN+1hhOVs3Hml1yWLe59twONx+t3vcXhZ9tROAxEQrv39sJtNmDCY9I5GM\nbknMmJXPb/5yHlZr+9eW09IT6dO/qyRtITqhuF5xV5Q7efqR+WzbfBSLxYTb5WHSGf35wc2ntGg3\nYUVF4Il8Xq+mstJV+/eUNDvfufEkvnPjSW2O3ef4UeOk+q6ZSVKjFkKEJK4T99OPzGfrxsKaI8WM\nAsfSBbtJTbNz5ffHhfw64yfmsWfH8YD169ZsjQ/F7h3HefKhrykpqkRrY5DUtTdMYOq5gyPyfkKI\njiNuSyXHCsvZtvmoX3uc0+lh7qdb8XhCO68S4IxzBpLeNRGLte7HYbOZ6d0nndETwn/hsaSokgfv\nm03x8craEbROp4dXn13Gwi/bp7VQCBG/4jZxHz9aEXTrutvtxVHtX7MOJjHRyv2PzuT8S4aT0z2V\nHr26cOk1o7n3D9MjMsBp7mdbA26iAXjjHyvkiDIhRJPitlTSvVcX3C7/DTEASUnWgKe0NyU5xcZl\n147hsmvHhCO8Jm3bdDTofY5qN2WlDtK6JEQ8DiFEfGp2OamU6q2UmqeU2qSU2qiUuqM9AmtOWpcE\nJp3R3//QAbuZS64ZHXQcaizo3Te9yfvtdtkBKYQILpQ6gBu4S2udD0wEblFK5Uc2rND84OZTOOeC\nodgTLJgtJlLT7Fz9g/Yb+tRaF142wu8Ee5/ho7pjT5AWPyFEcC2eDqiU+gB4Sms9J9hj2ns6oMdj\n1LQTEq0xvdKub9XSfTz1lwV4vXU//8zsZO5/9HxS06RMIkRnE7HpgEqpvsBYYFnLw4ocs9lEUnLk\nT6QJp/ET83jxf9ewaN4ujhdWMGR4NvmjusfNF48QInpCTtxKqRTgHeBnWuvSAPffBNwEkJeXF7YA\nOzKLxcyZ5wyKdhhCiDgTUq+bUsqKkbTf1Fq/G+gxWusXtNYTtNYTsrKywhmjEKLVPEABsBpYBxwn\n4JhLNMblrND3P4joaXbFrYx92C8Bm7XWj0c+JCFEeLiBVUA1dQm5GMgF6l/ALwK21TwOILPm/s54\nkVwDh4EDGF96mUAesfazCKVUcirwPWC9UmpNzW2/0lp/GrmwIqO4qJJ9u4vp2i2J3n0jf2KNENFV\nQMOkTc3/Pwz0BFKAE8D6Ro85BlQCJwGBrrm4gN1AYc3fs4B+QHxdZwpsI8ZvJb6fx36Mn9dJxNLn\nazZxa60XEfifXtzweLy8/NQSli3ag8VqxuPxktsjjbt+M430rknRDk+ICDlC4NKHFyM5pwB7AjxG\nYyT8YqBro/s8wErAQV3J5RDGqv0kgqcUd83jjmGsXnsCsbZ4KqVh0gbjM7owEviAaAQVUNxueW+J\nD/67juWL9+JyeamqdOF0eCjYW8Ljf5wX7dCEiKBg6y1V777yII/xAhUBbj8MOGlYJ9c1tx0O8lou\nYAWwCygBjmLU23cHCzxKigj8RacxYo4dHT5xa6354uMtfpP/vF7NoQMn2L+nOEqRCRFp3Qn8n7jC\nKG8A2IM81wQE2k8QLLl5a+4LZA/GCr1xyWYfdXX1WGAm+JddbO1m7viJ26upqjdTuz6z2URxUWU7\nRyREe+kJpNIw6ZiAPoCvRNiXwGnADHQLcHtTdd5g9xUSuJMFjNJErMgicOI2AT3aOZamdfjEbTKb\nyMpJCXif2+WRi5SiAzNh7JcbhtFJ0gsYh5GsfTKB/jWPNdf8b1LN8wKlh55BbjfV3BdIU5fIYuny\nWQIwGOOz+OIyYdT5Yytxx+10wJa46rpxvPDkYpyOunKJzWbm5NP6kCEXJ0WH5iuLNLW3ojdGYirD\nuHCYRPCEmgIMBOrPjdcYF+5SgzwnF+PiXqASS6BVfTR1x7hoWohxIbYrkEZsfcF0ksTtO8z3f69/\nS+HhchKTrEy/YCgXXTUqypEJESvMQNNTK+v0xPgi8NW0u9J0CSUPo5ukirrkbcJI9sFq7NGUgBFz\nqEoxLrSWYXyePkB2BOKq0ykSNxjJ+6TJffB6vJgicDiCEJ2LDWMlHQoLMAGjM+M4xqq+O8bqPd4V\nY3TI+L6QXMBmjD74vhF7106TuH0kaQsRDSYgp+ZPR7IN/xKQF9iLcU0hMim20yVuIUQscmH0eft2\nY/oumsZiKcXHi7GyDkRhlE4i0/wgiVsIEWWBdmMexqihn0yszQmp49vIFKzVMXLpVRK3EKIVqjE6\nS4owklc2xgq5NUm2EP/dmGBskz+IcbEvFimM0s8R/GO3EskavhR8hRAt5MRYIR/FWC375pCsonVj\nYVuzGzNWDMJI0L5dl2aMpD2KSLYQyopbCNFCBRjJuj6NUeo4SssvQDZVx47lGjcYKXQ8xpTFMoxW\nwm5Eek0siVuITsOFMWfaN6GvF63bAFNM4Lqut+a+libu7jVxNV51N7UbM5YojB74UPvg204StxAd\nlgsjwdowyhsraHjKTQlGYhzYwtcNttlG0boVshnjS2Q/dStVjVEz79KK1+v4JHEL0eFUApuoG9ma\nCCRTl8h9vBgr3Z41jwlVbwLXpRXG6jlUboyDC0qoqwcnY2y/zyTwhU6N8fm8NY/tnJfpJHEL0aH4\njiurX4OuJHi/MRi7GXu14D3SMU682U1dwtUYw6wCjYINZjP+ZZdyjC3kgb4AyoANGL89+FrxhtL0\nHJZIcmGMrD1S8/f2OwlIErcQHcohWt7Z0ZpVax5Gci2iboJeS2ZWO2ue27hW7jtabWCj13NhHHjs\nafT4TRgTD4MNuIoUD3XnedY/Ceg4Ru95ZFOrJG4hOpRyWp64M1v5XlbqLkRWADsxyh5mjHJHH4J/\nKThoul3OTcPEfZjgF0T3A/khRx0ehTTcMAR1x5wdwignRU7nLBAJ0WG1pO5rwpg/3dZf7SsxVp/H\nMVaiTozTbdYRfFdhYhP3+XYj1l9d++ragQQ6Yi3SGp9N6eOlPQ6HkBW3EB1KLkbdtTkJwBiavyjp\nxKjhujBq2xn4r5R341/C8GL0NpcSuDPEgnFRtHEboC9pL6v532yMLxffST6N3wfav0wCrTsJKHxk\nxS1Eh2LDOL0mgeD/eZswasjNJe1jwBKM4U97MS4Mfot/8iwJ8nyNkbyDGYBRK6+/61BjJHJvzf8v\nBNZjlGSCnbzTktnZ4dKDlp8EFD6SuIXocFKBicBJGLXW+qe6+8arNlfX9rXq+ZIoGAm7DP8VfbD5\nJCaaXn0qjC6M04HTCNwDrjFW7VUYOxS7UPd5kjF+a4jGKVa+k4BMNDz2rR/t0XsupRIhOiSFkdAG\nYqwAj2Ik4G40XVo4grG6riRwDVpjXCgcUO+23gSeSw2hXfhUGKmoqon7KzC+cMZhfKlooj81sCdG\nKec4RjzdaI8yCUjiFqITSCS0csIejKTdXFdK4/tzMVbFh2lY/x5Fy1KMHaO9rjFNw/7wWEpbVkI/\nCSh8YuknIISIGjehJW0werbrU8AQjPY/Xztgc4OWfCUQD0ZpwVzz/O0BYkjEOLC3vkqM+rm1Jp7O\nVfWVxC2EwOj/bm4Mqe8CYv8g9ycQ2urzBMYFR1+C9p0S3xOji2Uvdd0lqcBwGu7Q3IxR+vHFpDBW\n951nrokkbiEExso1WF81GPXyrhgll7aMWnUBa/HvTNmJcbGxL0bNvAKjXtx4C30BdfX6+tYBk2nZ\n7s34JYlbCIGRNBPx38ziO90mXDsTCwm+A3IfRp+4Gf/SiE8Bgcs5GuMiYXYYYox9zRaGlFIvK6UK\nlVIb2iMgIUS0jMRYTfta20wYpYrBYXyPKoLX0QNdmGys8QEOPr7t5p1DKBX9V4EZEY5DCBF1icAk\nYATGkVxjMNrvwvmLeRrByxmh1KibekznqXE3m7i11guI/YPfhBBhoTBq2T2o2+wSTpkYtevGr2sm\ntJbF/vinLd90wsgdzhtrpMYthAjCd2iBByMpNjXprwCjvS8ZY7Z3sN2MJoxV/HaMi4waYxU+uInn\n1JeCsYNyJ0Z3im/mSWSn8cWasCVupdRNwE0AeXnRmB0ghAifCoyWvfrjVwfhf8BBGcacbN9skRKM\nsaYj8e/39rFhtPj5LlK2dFWfAoxu4XM6lrB1rWutX9BaT9BaT8jKitaJFEKItvNgDJPyXUj01PzZ\nhv9AqS019zU+Em0zTbcXQsMZKqIlOtd2IyFECAL1SVNz2956f3cRfBa2p4n7RFuF0g74b4zZjkOU\nUgVKqRsiH5YQInqaatlr6uzK+jTRW027qDvIYRv+MTswvlRaelJQ7Gi2xq21vqY9AhFCxIqmEm79\nyYLWmr+XBniclfCOW9UYzW2+g3mzMeahNI61CuM0Hg91ifkQxkHG6Rijak9Qt2btRzxe2JSuEiFE\nI0eauK9x48FQjHq4b263CSOZ1p8v0lYa41DgY9Ql46MYFz9HNHqfbfhvxPHV3BOpG1fr23K/C2PT\nUXztuJTELYSox0XwudiBKqvJGIc2HMToMEnG6AFvyzyTxorwP+PRCxRjJHNfM4RvVR5MFYFPld+N\nJG4hRBxr7rJXoPutGCNZI+UIgc+a9GDMAA+li62pmnsoW+1ji3SVCCHqMWPUggMlORvGirq9NddW\n6OPb9RnsvmCaO3sz9kjiFkI0MhRjFe2bKeI7U7FxPbm9BDso2FxzX32DMWKv/3gTxmdKxj9+3zmR\n8UVKJUKIRhIw6tZHMerWiRgJMlpnPHbDWEkXU1cyMWPMUmlcJkkETsHoJCmp+XtP6uaJb8Golyvq\nTruPvw2DkriF6PTcGHXkcoxVaS5GasglGucp+lMYq/1jGDVtML5Isgj8G4AVo/ulcQeMFWMrvrvm\njz3I82OfJG4hOrVKjHY+X9+zCaPLYiyxNW1PYSTqcKyOLcR76pMatxCd2iaMFkBfq50XYzW6MWoR\nGcoxeqx3YpRrRH3x/bUjhGgDJ0aCDKQao++5qY6LSoykaie8s7t3AAeo+zIpwCjZDA7je8Q3SdxC\ndFpe6k5TDyTY7V5gA8bFQl8itWGcmNP4cN+WKqFh0va9n69fO1i7X+cipRIhOi07RsINxELw1fZO\njKRdf+RrFcZQp1B7roM5TPDJhIfa+NodhyRuITothdHfHOgosKEELktojO3tgZJrFW0f5Rpoh6RP\nsIOCOx8plQjRqWUAE4D91M0aySN4R4km+DhUhVE3b4ssjD7rxgnchP9mm85LErcQnV4yxgo7FCaM\nEkqgQVSatrcQZta8Rhl1XxCmmhjjaxBUJEniFkK00ECMdsH6K28TxnmUwWrmoTJhXOQ8SMPNNj2Q\nym4d+UkIIVooE2MnYwpGecQG9Mc4TDgcyjB2clZh1LUbzx4RsuIWQrRCt5o/4VYAbK/3dzfGIQjH\nMQ5nECBfY0KImOGiYdKur5Dgm4U6H0ncQogY0dTpNWCsugVI4hZCxA1JVz5S4xZCxIjmtrPH4tzs\nY8AejAupSRiHMkR+W758hQkhYoQVo9UwkH60fQ5KuB3AaIssw7iIWgqsx+iIiSxZcQshYkhvIA1j\nJnglxip2AJAazaAC8GLMbGm8i9SLcYE1m0hOMpTELYSIMV0wNuGAsZI9UfMnjdgZ6xpo56iPB3AQ\nyd8QJHELIWLUPoyVty9ZmzGOHkuLWkR1LASf2aKpO2g5MqTGLYSIQccxknb90bFOYA2xMSXQTuAv\nEIUxuCuyBytL4hZCxKC9BF/RHm3PQJowHKMcYsZIpWaMAVzDIv7OUioRQsQgR5DbPU3c197swESM\nQyV87YDptEcdXhK3ECIGpWGce9mYmdjqMFFE4zi1kEolSqkZSqmtSqkdSqlfRjooIURn1xf/9KQw\nShNy7mSziVspZQaeBs4D8oFrlFL5kQ5MCNGZJQNjqbsAqDDmco8ldloCoyeUUsnJwA6t9S4ApdR/\ngIuATZEMTAjR2aUB46k7gFgStk8opZKeGAfS+RTU3NaAUuompdRKpdTKo0dj5aqvECL+KSRpNxS2\ndkCt9Qta6wla6wlZWbE4DEYIITqGUBL3AYwBAj69am4TQggRBaEk7hXAIKVUP6WUDbga+DCyYQkh\nhAim2YuTWmu3UupW4HOMJsqXtdYbIx6ZEEKIgELagKO1/hT4NMKxCCGECIHSWjf/qJa+qFJHMYYN\nREMmxrEUHVln+IzQOT6nfMaOo62fs4/WOqTOjogk7mhSSq3UWk+IdhyR1Bk+I3SOzymfseNoz88p\n0wGFECLOSOIWQog40xET9wvRDqAddIbPCJ3jc8pn7Dja7XN2uBq3EEJ0dB1xxS2EEB1ah0zcSqlH\nlFJblFLrlFLvKaXSox1TuCmlrlBKbVRKeZVSHeqKfWeY/66UelkpVaiU2hDtWCJFKdVbKTVPKbWp\n5t/VO6IdU7gppRKUUsuVUmtrPuPv2+N9O2TiBuYAI7TWo4BtwH1RjicSNgCXAguiHUg4daL5768C\nM6IdRIS5gbu01vkYZ3zd0gH/WTqAaVrr0cAYYIZSamKk37RDJm6t9Rdaa99R0EsxBmN1KFrrzVrr\nrdGOIwJq579rrZ2Ab/57h6K1XgAURTuOSNJaH9Jaf1vz/8uAzQQYCR3PtKG85q/Wmj8Rv3DYIRN3\nI9cDn0U7CBGykOa/i/iilOqLcXzNsuhGEn5KKbNSag1QCMzRWkf8M8btYcFKqblAboC7fq21/qDm\nMb/G+HXtzfaMLVxC+YxCxDqlVArwDvAzrXVptOMJN621BxhTcy3tPaXUCK11RK9dxG3i1lqf3dT9\nSqkfABcAZ+k47Xls7jN2UDL/vQNRSlkxkvabWut3ox1PJGmtS5RS8zCuXUQ0cXfIUolSagZwDzBL\na10Z7XhEi8j89w5CKaWAl4DNWuvHox1PJCilsnxda0qpROAcYEuk37dDJm7gKSAVmKOUWqOUei7a\nAYWbUuoSpVQBMAn4RCn1ebRjCoeai8q++e+bgf91xPnvSql/A0uAIUqpAqXUDdGOKQJOBb4HTKv5\n73CNUur8aAcVZt2BeUqpdRiLjjla648j/aayc1IIIeJMR11xCyFEhyWJWwgh4owkbiGEiDOSuIUQ\nIs5I4hZCiDgjiVsIIeKMJG4hhIgzkriFECLO/D+zFSyZ80FeSgAAAABJRU5ErkJggg==\n", 52 | "text/plain": [ 53 | "" 54 | ] 55 | }, 56 | "metadata": {}, 57 | "output_type": "display_data" 58 | } 59 | ], 60 | "source": [ 61 | "%matplotlib inline\n", 62 | "\n", 63 | "X, y = make_blobs(n_samples=100, centers=3, n_features=2, random_state=0 , cluster_std = 0.5)\n", 64 | "plt.scatter(X[:,0], X[:,1], s=40, c=y, cmap=plt.cm.Spectral)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "### Convert labels to the binary format " 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 3, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "bry = []\n", 81 | "for i in y:\n", 82 | " if i == 0:\n", 83 | " bry.append([1.0 , 0.0 , 0.0])\n", 84 | " elif i== 1 :\n", 85 | " bry.append([0.0 , 1.0 , 0.0])\n", 86 | " else:\n", 87 | " bry.append([0.0 , 0.0 , 1.0])\n", 88 | "bry = np.array(bry)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "### Add 1 for bias " 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 4, 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "X=np.insert(X, 0, values=1.0, axis=1)" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "### Create the model hyper-parameters \n", 112 | "\n", 113 | "The network will have a linear model with no hidden layer. Then input is 3 neurons and the output is one neuron. " 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "Initialize random weights and biases" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 5, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "m = X.shape[0]\n", 130 | "n = X.shape[1]\n", 131 | "nclasses = 3\n", 132 | "\n", 133 | "W = np.random.randn(n,nclasses)" 134 | ] 135 | }, 136 | { 137 | "cell_type": "markdown", 138 | "metadata": {}, 139 | "source": [ 140 | "### The loss function \n", 141 | "\n", 142 | "The loss function is the cross intropy loss \n", 143 | "\n", 144 | "$$L = -\\sum_{k} y_k \\log(p_k) $$\n", 145 | "Where the soft max function \n", 146 | "$$p_k = \\frac{e^{o_k}}{\\sum_i e^{o_i}}$$" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 6, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def loss(X,y,W , a = 0):\n", 156 | " return -np.sum(np.log(soft_max(np.dot(X,W)))*y)+ a * np.sum(W**2)\n", 157 | " " 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 7, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "def soft_max(X):\n", 167 | " exp = np.exp(X)\n", 168 | " summ = np.sum(exp, axis=1)\n", 169 | " return exp/summ[:, np.newaxis]" 170 | ] 171 | }, 172 | { 173 | "cell_type": "markdown", 174 | "metadata": {}, 175 | "source": [ 176 | "### The math behind the optimization \n", 177 | "\n", 178 | "We are trying to optimize the function \n", 179 | "\n", 180 | "$$L(W) = -\\sum_{k} y_k \\log \\left( \\frac{e^{W_kx}}{\\sum_i e^{W_i x}} \\right)$$\n", 181 | "\n", 182 | "For some $m$ crossponding to the true class\n", 183 | "\n", 184 | "$$L(W_m) = -W_m x - \\log\\left(\\sum_i e^{W_i x} \\right)$$\n", 185 | "\n", 186 | "$$\\frac{\\partial L}{\\partial W_{mj}} = - x + x p_j$$\n", 187 | "$$\\frac{\\partial L}{\\partial W_{kj}} = x p_j$$\n", 188 | "\n", 189 | "cobining these we have \n", 190 | "\n", 191 | "$$\\frac{\\partial L}{\\partial W} = x^T (p-y)$$\n" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 8, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "def evaluate_gradient(X,y,W , a = 0):\n", 201 | " g = soft_max(np.dot(X,W))\n", 202 | " dW = np.dot(X.T, g-y)\n", 203 | " return dW + 2* a * W" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "metadata": {}, 209 | "source": [ 210 | "### Training \n", 211 | "For training we randomly choose a point from the training data and we repreeat the process for 10,000 iterations. Note that the learning rate is small to compenceate for the large data points. We could've normalized the data points to set a smaller learning rate. " 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 9, 217 | "metadata": {}, 218 | "outputs": [], 219 | "source": [ 220 | "def train(X , y, W , lr =1e-2 , itr = 1000):\n", 221 | " for j in range(0, itr):\n", 222 | " #calculate the analytical gradient \n", 223 | " dW = evaluate_gradient(X,y,W)\n", 224 | " W = W - dW * lr \n", 225 | " if(j%100 == 0):\n", 226 | " print(\"Loss is \", loss(X,y,W))\n", 227 | " return W" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": 10, 233 | "metadata": { 234 | "scrolled": true 235 | }, 236 | "outputs": [ 237 | { 238 | "name": "stdout", 239 | "output_type": "stream", 240 | "text": [ 241 | "Loss is 119.810181191\n", 242 | "Loss is 3.23529828142\n", 243 | "Loss is 2.23063164356\n", 244 | "Loss is 1.79448358134\n", 245 | "Loss is 1.53514379891\n", 246 | "Loss is 1.35769916147\n", 247 | "Loss is 1.22614605822\n", 248 | "Loss is 1.12340842242\n", 249 | "Loss is 1.04019518341\n", 250 | "Loss is 0.970953938393\n" 251 | ] 252 | } 253 | ], 254 | "source": [ 255 | "What = train(X,bry,W)" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 11, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "def predict(X,W):\n", 265 | " g = soft_max(np.dot(X,W))\n", 266 | " yhat = np.argmax(g,axis=1)\n", 267 | " return yhat" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 12, 273 | "metadata": {}, 274 | "outputs": [], 275 | "source": [ 276 | "def accuracy(yhat , y):\n", 277 | " diff = y-yhat \n", 278 | " num_corr = len(y) - np.count_nonzero(diff)\n", 279 | " print(\"Accuracy is \" , float(num_corr)/len(y))" 280 | ] 281 | }, 282 | { 283 | "cell_type": "code", 284 | "execution_count": 13, 285 | "metadata": {}, 286 | "outputs": [ 287 | { 288 | "name": "stdout", 289 | "output_type": "stream", 290 | "text": [ 291 | "Accuracy is 1.0\n" 292 | ] 293 | } 294 | ], 295 | "source": [ 296 | "yhat = predict(X,What)\n", 297 | "accuracy(yhat , y)" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": 14, 303 | "metadata": {}, 304 | "outputs": [], 305 | "source": [ 306 | "def plot_decision_boundary(pred_func):\n", 307 | " # Set min and max values and give it some padding\n", 308 | " x_min, x_max = X[:, 1].min() - .5, X[:, 1].max() + .5\n", 309 | " y_min, y_max = X[:, 2].min() - .5, X[:, 2].max() + .5\n", 310 | " h = 0.01\n", 311 | " # Generate a grid of points with distance h between them\n", 312 | " xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))\n", 313 | " inp = np.c_[xx.ravel(), yy.ravel()]\n", 314 | " inp = np.insert(inp, 0, values=1.0, axis=1)\n", 315 | " # Predict the function value for the whole gid\n", 316 | " Z = pred_func(inp)\n", 317 | " Z = Z.reshape(xx.shape)\n", 318 | " \n", 319 | " # Plot the contour and training examples\n", 320 | " plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)\n", 321 | " plt.scatter(X[:, 1], X[:, 2], c=y, cmap=plt.cm.Spectral)" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": 15, 327 | "metadata": {}, 328 | "outputs": [ 329 | { 330 | "data": { 331 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvXuIbPt13/ldv/2qd/XzvM+91/IoA8LoyoywHSR5bjyJ\nfC17EhL7goUdHCZwESSRQwJBGWNCEAKFgJgoMYTLyDhg4zCa2MQkihyJkXEsZmzLQb6RrSiR9bj3\nPPp09+murvd+rvnjV1XdVbV3d1X3rtq1q9YH+p5zq6qr1+7T/a1V67fWdxEzQxAEQcgPKusABEEQ\nhPkQ4RYEQcgZItyCIAg5Q4RbEAQhZ4hwC4Ig5AwRbkEQhJwhwi0IgpAzRLgFQRByhgi3IAhCzjAX\n8aR2scaF2u1FPLVwDe6dHWYdgiAIV/ANt3nMzPuzPHYhwl2o3cYP/synFvHUwjX47CfbePPdv5p1\nGIIgXML//I3PfXfWx0qpZAN47WOVrEMQBCFFRLg3hMbnP5x1CIIgpIQI94bwzz9dzzoEQRBSQoR7\ng/jMp38u6xAEQUgBEe4N4snni1mHIAhCCohwbxgf/9BHsg5BEIQbIsItCIKQM0S4NxDJugUh34hw\nC4Ig5AwR7g1Fsm5ByC8i3IIgCDlDhHuDkWlKQcgnItwbTB6nKTliNE4DPPqui0dvuWieBWDmrMMS\nhKUiwr3hfODLr2UdwswwM5488nByHMB1GW6fcXwY4NlTP+vQBGGpiHBvOB/8xG7WIcxMtxPBdRkX\nE2zmwe39KLvABGHJiHALePebP5t1CDPR60aIq4owA72eCLewOYhwC7nx6zZMgGj6diLAMGLuEIQ1\nRYRbAJCPDpNqLX5hExFQrsiPsrA5yE+7ACAfHSamSbh734ZhaLEmAkwTuPfQhlKScQubw0J2Tgr5\npPH5D2Pr1V/POoxLKZYUXnyHA89lEAGWTaC4+okgrDGScQsj8pB1AwARwSko2I4S0U4Zz4vQ60WI\nIumNX2Uk4xbGePebPysb4RcMM+P0eYCzRogoApwCYe+WhUIhuzwqDBhPn3jwXAYIAAM7uwa2dqzM\nYhKSmeknhYi+Q0T/hYi+SkRfWXRQQnbkpcMkzxwdBmicatEGALfPePK2B8/LrqXx4IkHt6975DnS\nLZYnz0N02mFmMQnJzPMS/xeY+T3M/N6FRSOsBO//1MOsQ1hbwpDRboZT/ejMQOMkyCQm32e47nRp\nhBlonGYTk3A5UuMWpvjRL/541iGsLb7Hsb3ogM68syAKk79uKAn3SjKrcDOALxLRHxHR64sMSFgN\nXn7FyzqEtcS0KHb6EwBsZ7EHrUGgDbqeH/uDKVQdiG0nf91SWXK7VWTWf5X3M/N7APwYgL9FRD88\n+QAiep2IvkJEX/F7Z6kGKSyfnyp9NOsQ1hLTJJQrairrJgK2dxbXK9Dthnjr2y5OjgM0TkI8fezh\n4IkPZgYpwt4tcyomwwC2tqV/YRWZSbiZ+fHgz0MAvwngB2Ie8wYzv5eZ32sV89FWJlyO1LoXw607\nFupbxkgobYdw94EN21lMdsvMePbE1wePPLxNe7+0W/pAtFY3cfeBjVJZwXEIWzsGHr7kwDSl3XIV\nufInhYjKRFQd/h3ABwF8bdGBCdkjte7FQETY3bfwjncW8I53Onj4ooNicXEliX6fE825Wmfnh4/F\nosLd+zYevOhgd88S/5cVZpafltsAfo+I/hjAHwD498z8+cWGJawKefLrziPLGCCi0X+S7hTyxpUF\nLGb+FoCXlxCLsIJ88BO7+MWsgxBuhFMgEOkOg4sQ6RKJkD/kyFi4krz4dQvxEBHu3LNHxlz6NqBU\nUeKqmFPk5Va4ktc+VpGsO+cUi9qcq9MOEYZAqaTgZDhiL9wMEW5hJvLgHBhHGDKOD3102np7Trms\nsHfb2shuCcMgKY2sCfKSK8xEXpwDL8LMePyWh3brfOVZpxPh0Xddcb8Tco0ItzAzn/n0z8382F43\nwpNHLr77bReHBx78DAyUup0IQTAt0FEEdNqyo1LIL/K+SZiZJ58vzvS4VjPA0bNglOW2fEa75eHB\nizZse3m5gu8l9y97bgTAWFosgpAmknELc3HVbkpmxvOjINb97uR4uU5zejvO9O1EWNiUoiAsA/np\nFebiqlp3GGDkMz1Jr5dueSIIGM2zAM2zAGGMw12prGIPIZWS5cJCvpFSiTA3H/jya/hP7/ts7H3q\nkuqDmeII9VkjwPOj8wz++DDA/m1zbBM8EeH+C/ZYV0mprLB/y4pdLszMiCIt7GlMNDIzer0I3XaE\nfj9CGDBMi7C9Y6JUljKNcH1EuIW5uWyaUintfjc0LxpCBGyl5H7n+1FsOeboWYBiyRjLsg2DcPuu\nfeVznjUCnBwHiKJBrNsGtnfNaws4R4wnjz30e+NBBgHj4ImPWj1Ev8fwPC3mO7smKtVxMQ9DRhQx\nTFMWIgvjyPtF4Vpc5te9f9saWZeSGlqWGqjW0skyL7b3TXKdVVutps7ehyUevfklxOkNNtI0GkHi\nYgRm4KwRwXX14anvMQ4PfDQHhk9RxDh44uE733Lx9nc8fPdbLtot2UQjnCPCLVyLy/y6ldIj1i++\nw8H9hzZe+l4H27vpLZ3lhB7si7al83D6PGGV2Gk4WjYwL61m8otLHMPDW2Yt2p12BAyuJwyBw4MA\n/ZTPCIT8IsItXJur/LoNg+A4KraefBPKVSOxW6R8jY0tcb3ewPnS3GURhjr7niyvALL/URhHhFu4\nNln5dTuOQm1rXLx1Dd2AdY0+8aTVXYaBxP2QQ6KIcfTMx7f+ex9/9t/6ePy2C8+NUK1Nb7m5CqV0\nXTvJatX3zwW9349wdOjj4KmH1tn13xkI+USEW7gRWfl17+1buPfARn3LQH3bwP2HNnauWY7Z3bdi\nV4nt7l19OHnw2EPrwtb2fo/x+G0PlaoxslOdheGBqO2oaf/VAcWi0ttsDjw8fstDsxGi04pw+MzH\nd/7MhZfBdKqQDSLcwo344Cd2M/vahaLC3i0Le/vWjZzubIemFvXW6grVKwyZXDeK3S4TMdA6C3Hv\ngY279y3s7JrY3VegpBAH7xa2dkwYBmFre7oUpJTe/9jrRmg3pwU6ioCDx75k3huCCLdwY/Lu133w\nxJvqAGme6d7ry/C9BJFkwHUZRIRiSbcVbm3buPfAHvSIn3/U6grf87363cIwu9/eNbF/24LtEAwD\nqNQUHrzowLQI7VZy14zv81g5RVhfpI9buDF59uv2vSi2bY8ZaJwEuHMvuQfcSqiNgwDHmb6vUNCe\n2N1OhChiFIsqtiZPRKjW5m+fJNIHqsL6Ixm3kAp53QgfBMkHkMEV2avjKBRi6tiKgNpWfE6kFKFS\nNVCrm9c6SL1KzINA+4+fnviJ3TJC/hHhFlIhiw4TZka7FaJxEqDbvV5nhe1QYstfoXT1r8ed+zaq\ntfOadLGox+zjPFLcfoQnb7v49jf7eOvb7mjgZh4KRYVqPT4uyyI8e+rjrBHi5HmIt77totuZbSBJ\nT6P6ODzw0G5Jl8qqI6USIZf4PuPxWy6iQa81kW7ru/fQnqtv3DAI9W0DZ6fjQzjDw8CrUIqwf9vC\n/u3LO1pcN8Ljt73R14gixvGhNsfa3pm9G4aIcOu2jXo9xOlJiCBgOA7BMAmNkwvXwLo55dlTHy99\nr7q0O6bTDvHsqT/63HYrutb3UlgeknELqfHxD31kaV/r8MBDGJ4PyDADnsc4eT5/Frtz8TDQBKo1\nhYcvOleuN2NmnJ74+O63+vjOn/Vx9MyPdSkEhlORk58/mNq8xjYep2Dgzj0bD15wsH/bRq8TP6nJ\nQOLo/fAaDg/8sc8dfi+bZ/PbBwjLQTJuIXdEUfJ0YbsZYm9/vn7uyw4Dez1dQvBcBkMfOu7f0u2H\nB0989Lrngtk8C9HtRHj40nSm6rrxp4bMuoTiFC/Piq++hoQ7+PIhIrfPsW3jw+/lLO86hOUjGbeQ\nKsvMuuNIszLb7YR48ki3CjLrJ3f7jEdveTh44qIbk+WGIce27FlWsno+fuzjre+46HWv3xJSrZux\nAq0UpnrUL0IKid+0xL5zIXPkn0bIHUoRnEK8GFUq6flcHx8FiaLWaScbXcUJsLaITfhCDAQ+8PSx\nB9+/nnhXqmrkyAjoLFspfXh6WSZv27o8NAkRUE/ojBGyR4RbSJ1lZN237lijYRZA/2lZhJ299MQm\nccDmMii+v7tUMrB/24RxyeuKtnu9Xl2ZSPuO33/Bxt6+rtm/+A4HhSsmSokId+/ZUMb4YFClZsiW\noBVm5p9yIjIAfAXAY2b+icWFJKwDL7/i4Y9/5+oFBtfFtvUwS7sVwvcYTmGYcabXBWEY2rFvHghA\nLWFUvlrTyxJazRDHh9OHlcA1Xywu4DgKzpz7NG1H4aXBYFAYJg8GCavDPP86Pw/g64sKRFgvLvPr\nTgulCLW6id19C5WqkfqWmK2d+coulqVb6C7rRiEiFIvxv3ZEQKGYTfsdEaFcuf5gkLBcZvoXIqIH\nAH4cwP+52HCEdSKv05RD6lvmzOL9wks2Xvieq0sTAGDZ4/XoIUolZ+tXwcwIQ5bBmQ1h1p+S/wPA\nPwBQXWAswprxo1/8cfwi/mXWYVwbIsLunoXtHRO+x4g4wtlpNNZNQgTcujN/lnrrjoWzRoCzRgiO\n9BLjnT0LxpwLlXUveTDY1qPH7bd3TdS30n8HIqwOVwo3Ef0EgENm/iMieuWSx70O4HUAKFT3Uwsw\na1QQwXJDRAbBd2Zw1hfGuGwjfF4472JRKBQYbp/R6YQwDEKlYsC8pNUvCSLC1raFre3znnNmRhAw\nlMLME4tnp8HYxGQ0WIEmXSHrzSz/su8D8JeJ6EMACgBqRPSrzDzm5cnMbwB4AwBqt9+Z//drzKie\n9FFpuqOOsNBUeH63gsiUGuCsXLYRPo8QEQpFQiGhTn1dup0QR898BIPD0HJZ4dZtC+qKDPz0JGFf\n5kkgwr3GXPnTx8z/kJkfMPNLAH4awP8zKdrrSKHjo9x0QQyowYfpR9g56GQdWu7Iu1/3onHdCAdP\nfAQBdN84A512hKdPvEs/j5lHm+knCWQ9ZW54/6cezr1JSl6SEyifuVATmQwBsPwQhh8itNIb9Fh3\n8uzXvQzOTuNbA90+w/eixPo5EcE040U6aY+msDq8/1MPQT/4Q9f63Lne7zHz72xKD7dKMP5hAJSQ\n5agwQunMRaXRh+lKynORrHZT5gEvoXebCPCv8NTe3Z+eyCTStwuryTDDvq5oA5JxJ9IrWzAa7vQr\nGxGCmAzI6frYfqbLKMRAhYBexcbZXlEONLF+te40KRYV3P70pA+zHjS6jErVhFKE58cBAp9h2YTd\nPRPFkrwjXDXSTF5EuBPo1B2U2j4QRlADb2MmoBEnxBFj+1lnrLRCDBTbHvplC27petvHhdUkDBn9\nXgSl9EHlTdvu6tsmmmfhWL2aSG+7ucpaFgBKZQOlsgj1qrKId5si3AAoYhQ6PlQYwSuY8B0DbCgc\nPaii1HTh9AKEpkKn7iCwp39BnH58WUQxUGx5ItwDPvPpn8Pf/Oi/yjqMG9E48XHy/Dw7Vgq498CG\nPeeY+UVMk/DgRRsnxwG63QiGItS3DNS20hHjIGD0uhEMAyiW0rUFEJJZZHlw44XbcgPsPmkD0Fky\nE+AWTZzeLoMVobNVQGcr4yDXhCefL2Ydwo3o9SKcPB9vvwtD4MkjDy++w7mRIFqWwu276Xu7nBz7\naJwOXmgGBlL3Hthz+5kIs/HyKx5qn/iZhX+dzRZuZuwcTJc4nF6AYstDr+bM9DReIf7bGBHQqy7O\naCmPfPxDH8Evfi6f05TNRnz3RxQB/T6jmJHPSBLdbjiaqAQwWmf2NIUXGmGcZR++b7Rwm14Eiuke\nUQyU5xBuVoTTW2VsH54fTjJB17eLG/0tXmmGfdDaHvZqEUt0CiQgSlhZliXNxvRwDrC6LzR5JKtu\nqQ1XlfR+2dyyhcOHNRQ7PihiuCUTvrO+317DD1E97cPuhwhNhdaWA2/GWn7WWTczo3ES4HROf49K\nVaHfi9ntyEh9kjINoqRdloRr7bmcF8/Tvi4EoFyd7aA1L2Td3rq+yjIDgW2AFQET2VJEQLcy/4Fi\nNDjAXHcMP8T+oxaI9VCSGUSwngU42y3O/C5l0X7dl3HWCMdGxWf196hWDTQbITyPx0ymdvfMuc2h\nlkGlaqDfiynvLOGF5uS5P/JQIQKeHwfYu2Ve2/1wFVhW/XoWVi9NWCZEOL1dRkRarBn6T88x0Z1R\ngDaR6kl/JNpDFAP1kz5i35vHsAy/7iROTxI2rp9cPjRFinD/oY29WyZKZYVqTeHeAxv1FV2oW60Z\ncBwa614lAvZumTObWF0H143GjK+Y9cfxYYDwioGiVeTlVzx84MuvrYxoAxuecQP6YPHZCzUU2z6M\nMIJbMOEVTRmauQSnHyD2u8MMI4hmtgN4/6ce4vf+3tupxnYVzIwooVYdzjDsSoPlDbV6unEtAiK9\n2KHTjtBuazfDWt1YeEdJuxlfWweATifMTdZ9k5H0RZOP72CKWP0AtZPewKpV12Z7VRvdDShxpEVo\nKBgxJ3UEIDJmF4Us/LqJCKall/NOErcrMu8QESpVA5WqDOjMStb161nYKOE23QC7T9uj9j8VRKg/\n70GFjM52IdvgckR728HWYXesjTIioF+y9JnBHGRR697dM3F4MF4uIQL29k1EEaPVDNHtRDBMPQgj\nPc/zUa4aekFETNa9yhOeeRDsIRv1E1k71bXZiygGqo2+PqESZqJfttHcKYydDfRLFs72S3M/Vxa1\n7krVxJ17FmyHoBTgFAh37lsoFBUeveXh+VGAbidC6yzE47c8tM7EMGweCgU16NA5v21ofLWKnSUf\n+PJruRJtYMMybssN42uzAIwwQqhWNxtYNbr1ArpVB2YQITQIPCiR2L0A1dMeTC9CYCu0tov6zGDF\niPP3OD3xEfg8likyA0eHAcpVY6EHeuvG7r6FSs1Apx0OyjUKlrVaeWLexPoiq/cblQYRQzEjUjR2\nyBhY6dRmhQGKxrxbhg6JwxKK0Q9hHbRxert8qV9L4/Mfxtarv77oaK+k047p0QYAAlxXBlbmxXHU\nSpaZ8izYQ9ZLuJlRP+6h1NabQyJFONstol/RNdTWdgHWxIh7REC3as9dm10mRhCh3OjDdkP4tkKn\nXog1u8qa2vPe1PIJxfr2o0uE+59/ur4Slq80WUcbwnq6Usg36yDYQ9bqx3HrqIti29M9xgwYIWPr\nqAu7p2uUXtFCY7+EwKBRz3an5qC5u7rmR6YXYv/tJspND7YbotTysfe4BTvBkTBLTD9+w4TpR1f2\ndzc+/+FFhDQz7VYA142/zzRJNsrklGEP9jqJNrBGGTeFEYptf6qGTQxUGn00rBKMIIJbsnBYtkZ+\nIqver1173hsbdiHoa6ofdXH0sJZlaFNEBsGI8eyYLFnFkWXW7fajqS6TIaYJ3L1viSFTzlg3oZ5k\nbYTbCBlMmOoaIQB2P8Dtt5t67RiAds1Ba6ewPNFmht0PYPgRAseYy8PEThh2Mf1Id8IMSjx2L0D5\nrA8jYLhFE50tZ+l1+9aWg9pJf6oU1d6arUc+K7/uRsLOR5AW7aSdj8Lqse6CPWRthDs043+5GBhl\nrEMBLDddhKZaytCNCiPsPmnDCM7LCJ5j4uROeSS6lxHnpQJg7IKKTRf1C5m55YUotT0c3a8iSvi+\nLIJuzYGK9DucIe26M7N/S1Z+3b6fvPMx0RFQWCk2RbCHrE0qwYrQ2tK9xaPbBn9OyqNioHKWUNBM\nma2jLkxfrz8bfjj9ANXT/tWfDP3uIJq4AAbQrdhaWZj1ENFEOUWFvLRrHEGE9nYBBy/VcfiwhoOX\n6mjvzLdz8+Mf+sgCA4ynmGS4xLjRZhthsaxr/XoW1ibjBqDLAyah0nChQobvGHB6CWvFooRV7WkS\nMZzudKmDAJTaHlozHIp2thyYgT6UZNKf2y+aOBt8runFp4QE3Z6HLA5eiRCt4KBFEvVtE81mOOZh\nop0CjZV0/dt0NlGoJ8mfcDOjfOaicuaCIoZX0CIW2gZAhF7VQa/qjB576+0mzAlHMoYuVyyay37l\naUYXPRDhbL+M1nYE0w8RmgbCC4MMkaESv06UU9FZtl+3aRIevuDg5CRArxNCGYStbROVqmTbq4QI\n9jm5E+76cQ/Ftjc6AHN6AfYft3D4sDZdzyXC2V4J2886o/rvcFv7MloAWRF824DljU9sDkfE5yEy\nFbyYenVkKniOAbs//jX0oWB+/VeW7WFiWoRbty0Asth51RDBniZXKYUKIpRa3viOSAxb/uLruW7J\nwvN7FfRLFnxLoVu1cPSgurQBlsZ+CTzw9AD0n5FBaO5c/4XD9EJsPetg/+0mtg86aG4V4DmGfm4M\nfMUVodT0YK1gv/csZOnXLawGm1q/noVcZdymHya2/Dk9H7tPAu2RYSm0dgrwijp78h0Tp3eyudTA\nMXD4Qg3FlgfLC+E5BnpV59qTmlZfOxyOts/4EZyej5M7ZTCA3YMOwIAZMoyuD6fno7FfGk2P5oks\n/LqF7BGxvppcCXdoxtdzGVrAaOCxbLghrIPOlR4ZyyIyFDoplS0mx8pHAznHPXiOET+sc9xDv2yt\n/LDRJFn4dQvZsEprwfLAlcJNRAUAvwvAGTz+/2bmf7TowOIILQNuwYTdD6Y8MeJa/urHPRy+kL1w\np4ntxneRmL7eWB8nzTTnZhpBWBYi2Ndjlhq3C+BHmPllAO8B8CoRZbbP5/R2Gb2KBR74QPuXWEUa\nQQREESi82isjLyR1ijAl30c8GDvPIfK2eT1ZxT2OeeLKjJuZGUB78L/DY/fMVJCVbo8722PtN6II\nt797FuuRwQTceasFivQ4fGurgM6WM3fJwPRCOD0fERH6ZWvkPZ0F7boeK5/sUgG0YZY5UUqJoA9o\ns4z5JnzwE7sr4RwopMMq73HMEzPVuInIAPBHAP4HAL/EzL8f85jXAbwOAIXqfpoxJgWlTaIQ75HB\ng/+oQaZNw003hNnrzcyoPe+h1PJGplT15z2c3C7Dy6h23qnZqJ2MT12SDhWmF6Jdd1A9c0e+LF7B\nROPW/JtpVol3v/mzePPdv5p1GMINkHdO6TKTcDNzCOA9RLQF4DeJ6PuY+WsTj3kDwBsAULv9zqVm\n5N2aA4oGwjyMB4ARu6bM1d4ZM2TdTi8Yaz8cdrPsPOvg4MX6TF4jaWP6UWxnjQJQ6AU4elhDZ6sA\n0wsRmSrRwyVPvPaximTdOUUEezHM1VXCzA0i+hKAVwF87arHLw0idLZ1GUSFjMgg3P3OWfxD59gt\nWRxk2nE4/WAhHSsqjPSQUEJpgxVdOSnJiuAXctUwdCWrsiVHmA0R7MUyS1fJPgB/INpFAH8JwD9Z\neGTX4YJHhm8p2N60H0k45xh44qNTfk9huiG2DzujZQRewUDjVnkqYw4tQ09jutOTkrO68OWRVdmS\nI1yOCPZymOV99F0AXyKiNwH8IYAvMPO/W2xYN6e5U5xy1YsIaM7hw92r2FPPAegyRbHtQgXpGFVR\nGGHvaUu39GHoIR5i93Erthvm5HYZgaXGtqx36s7cY/R54zOf/rmsQxBiGE44imgvj1m6St4E8P1L\niCVVvJKFkztl1E76ML0QoaXQ3C7CLc8ubm7JRL9kodD1RyWTobAWOwGcfguHD6o37tgotTyAx7N7\nAqAihtObLslEpsLRgypML4QxcEHchGXHWfl1C/GIUGfHehVCJ/CKFo7v3yALJULjVgnFtoeto96U\nsFLEKLW8G09FGkE0NVA0ui9hjyOIEDgm8ulEcn2W7RwoTCOCnT3rK9yDdWGmH8FzDATXtXElAg9a\nD6c6OViXNDo3DNV3TETkxYq378i0o7AaiGCvDmsh3EYQoXbcRaEXgAH0yhacXgB1oYPEKwzWhV3D\nr+OytWh+CvsIexULlVMFCqJRVh+RFm0R7mkk614eMpK+muReuCli7D1qQQ18OvR2Ge02dVGi7X6A\nSqOP9vb8dVLfMRBYCpYXjU8sku4hvzFEOL5fQfW0j2JHb7rpVm3tp50zYyhhPRDBXm1yL9zFlgti\nnqo/T6IYKDW9awk3iPD8bgVbR10UurqqHFgKjf1Sast42VBo7pXQ3Evl6daerDbCrzsykp4Pci/c\nlhsmHuxNkjRMMwtsKJzeqQCDzH7KT5t1B4gRRPAcE8EiShzMKHR8FDo+IoPQrTlLWwixakiHSbqI\nYOeL3At3YBuIyL9SvBnQntQXsHs+as/7g12OCq3twtULBxRNzd4YQYTdJy2oC0ZXbtHE6e3r1dTj\nL4Cx+7Q9eqFi6DbCs90iemmUa3KIeJjcHBHsfJJ74e5WbVQaLvhCuYQv/Klwvi6stXPetmf3fOwc\ndEaCr/wIW0ddnEU8txBuP+vACMbLNU4vQPnMTW2BQrHtj727GC5J2HreQ79iX3ujTp4RD5PrIx0i\n+Sb3UxtsKBzfr8AtmNrjA7pL4/BhFe3tAjoVC83dIo4e1MaGVCbdBIHB8oWT/lze3SqMpsbPh89V\nbnrXvq5JCp34dkGGPnjdVESA5kMmHNeD3GfcgPbvOLlXORfcQXmivZ1c/zW9+E0yFDHqR12c7Zdm\nK3MM/VNjtT49QxMmGn2p6ftS+zK5Q/y6Z0PEer1YC+EeMUc9OTQVVMxUIkGXJXrVYLRs+DIigxAa\nCuaEb4muqae3oLdbc8ZG70dfhwjemjkBzovUuuOR+vX6srG/8a2dArafdeN3NEJbus4i3CDC6a0S\ndp+29eIG6Jp6aCi0ttI7NPSK5tiShOHXPrmb4gFoTpFa9ziSXa8/Gyvc/bKNTs1HueknW7fOiF8w\ncfiwhlLLgxGE8AoWemUr9UUL7Z0iujUHTj9ApAhu0ZwW7Yly0aYgft0i2JvExgo3ALR2Sig1z6aE\nmwnoVefLliNTob2dTgfJVV+nF9OyqIII9ePzASG3aKY6ILTqbLJftwj25rEZv9UJsCKc3i7rDekD\nX+toMMbuFa432GL4IcpnLkpn6fl1Xwkz9p60UOgGo7F/pxdgP8HPe13ZNL9u6RDZXDY64wYAt2zh\n2Qs1FDryLw4pAAAXBklEQVQ+KGK4Jeva04jl0z5qjb7WSgLqJz2c7hXRnzN7n5dCx4cKp8f+KWIU\nO35shr6ObMI0pRw4CsCGZ9xDIkOhW3PQ2SpcW7RNN0S10QcNDigV6wGZ7eMeVLjYzNv0o9hxfuLk\ntsd15eMf+kjWISyE93/qIT7w5ddEtAUAknGnRrETv1iYATgdf6Fj6b5txPqFM+n7hPwiGbYQhwj3\nHJheiMppH7YbwrcV2luF823ql5SSF93f4ZZMhKYC+dHY2H9kqCl/lk3g5Vc8/PHv5Ls8JLVr4TKk\nVDIjlhtg73ELxY4PM4hQ6AbYfdqG09Xe3/2yFTvBSMDil/gS4fheBd2KNVog3KtYOL5f2bi2QAD4\nqdJHsw7h2siBozALknHPSO15b8wrZGjyVD/u4fAFC37BRKfmoNx0z0sWBJztFJfSkseGwtmtMs4W\n/pXyQd6ybhFrYR42Rrjtro9Kw4URRnCLJtpbhbkE1XbjD/mMINItd0Ro7RbRq9godDyAgF7FRmhJ\njTkLfqr0UfwiVn+9mQi2cB02QriLTRf1Cxmz6XsotX0cPagm7pOcJFIEI5wuZE+WRwLHQNu5WVua\nCiLUnvfg9PSEZHvbmXsgSNAHe7/3997OOowp5MBRuCnrX+NmHhNt4LzHuXLan/lp2nUH0YRIRwR0\nak6qdWTDD3Hr7SaKHR9GxDCDCFtHPWw/baX2NTaFH/3ij2cdwhgvv+JJS5+QCmufcZsxDoDA+XTh\nrHTqDlTIqDTdkb1qr2KPLWdIg+rzHojPO1GGfxZ6Icx+gGDDnQDn5QNffg3/6X2fzTwGQUiTtVeB\nyKDEXZOhOUemPKhht7cLMPwIoUlgI/03LIVekNg+WGp5aIpwz0WWft0i2MKiuFJ5iOghEX2JiP6U\niP6EiH5+GYGlRWQouCUTk3l3RED7GmvFWBECx1iIaAO6lp543+Z19qXCu9/82aV+PWnpExbNLOlb\nAODvM/N/JqIqgD8ioi8w858uOLbUOL1VxvazDpx+MJqTae4U4C66v/oatLYL2DruxWbd/Q1dCnxT\nluHXLQeOwjK5UriZ+SmAp4O/t4jo6wDuA8iNcLMinNytQAURVMgILJW6V3Za9GoOnK6PYne8/t7c\ncq7toyIszq9bMmshC+YqmBLRSwC+H8DvLyKYRROZClEOSsSNOxW0vBCllgdW0g+eBmn7dYtgC1ky\ns4wRUQXAvwHwd5m5GXP/6wBeB4BCdT+1ADeV0DbQ2l1/m9JlkkbWLYItrAIzCTcRWdCi/WvM/Btx\nj2HmNwC8AQC12+/cHPd+ITfcJOvOjWCbBaC8DRiWnujtN4HehBECKcAuAWDA6wG8pIUfq4JhAVGY\n6+u+UriJiAB8BsDXmflTiw9JEBbHvBvhcyPYAGDYQG0fRIOOJyJwoQaQAXRP9G12GajsnLtZlgG0\nTwCvk/y8pgMoAwg8IJp99mHlKFSB0pa+diLA7QCd51lHdS1mybjfB+CvA/gvRPTVwW3/OzN/bnFh\nZQNFjHKjj2LXR6QInZqjbVE30GFvXZmlw+TlVzzUPvEzS4knVUp1TJoIk1LgQhnonepMu7Kjhf3C\nw7iyAzT6Ogsd/2SgdhswTIymztzu1WKnTEApLfSrgl0CSltj185OSf8lh+I9S1fJ72HxltKZQxFj\n/1ELKoxG4/GW20XHdaTWvGYkeZjkVrCHGBYoLslgaDG1LplbsEtAf8JWobJ3/pwXxS5wAbc9/Rxk\nANV9wLTOhb5zojPbrCnWz9+JDCBS+no6J7jUUH8FWX+vkhkpttwx0Qb0+rFKc4lLf4WlMOlhMvQQ\nybVoA0DggeOWQxOAMBj8JS4Hi7mdFGAVpl4IiJQuOcRR2wdMG0QKpJR+bHlHl1qyRl0idZfdt6Lk\noDluORS6wZhoD2FoS9f+Ejy1heWR++w6jt4ZYBdxUYSZo0EmzYDfAzBdTtH3dcdvIsJ52jwBxfwu\nGGZCxk9a6NvunBeTMr4Lto3p+JinS0Q5QNRoQGiqxDdLobH2laKNI89bchIJfaD5DOy7YGZwFALd\nhv4Y3t9rgjnS9zNrYe81Bxn5BaIQiKbfaTIz4HWnbgcZsdUGItIHm1nTbQCDax7CHA3KJPlDMu4B\nnbqDYnt84S9DC7rvrMAPniDMQuABzYPk+3tnugXQHhzMeV0gTDhEbB+Da7cAEIhIC10UTbcXAvo5\n4srrHOmvlzVRAJw9BYp1sOXoF6peEwhmt3ZeJUS4BwS2gcZ+CVvHPf32CUBgKZze2cy9jevOZz/Z\nBjbV4jz0gN4MHR+BCzSeAk4FbJiA3x8cNMbVFBnoNsDDzg3gXOjjDjKzIArm6yCxS0CxpktDXk+/\nYK1I77cI9wX6FRsHZQumF4IVyZj5GlNvZVxzzQtRAPQasz223wJCX/eOK0Nn8/3WyojdXBS3gGL1\n/EWoYAJOSb+QrcD1iHBPQoTAkW/LOvN3PiorlReG39cfl0E0SNpXtAWPFFCsjR1kEhEYCihUdIkl\nY0ShhI3jh+8ZQCvHE4B5xbBGveEAtMC3j1cigx3DtHVMNP6Om0iBreJKCLd0lQgbh5RJroFhDfqx\nr3neQwqo3Rm1DBKRHgiq30k1zFSIwthzLWae7r7JCMm4BUFIRhlA9db42Pt1piGdCkCYLj+QoY2x\nVqm7I/SBwAeb9kTf98C0awWQjFvYKP7jL+TPlyJTqrcGWfLkNKQ93/MY5tTI+cX7Vo7WIeD3R73u\nHIW6rBP6WUcGQDJuQVhvlKmFMfDmryUb9kBwY6YhnSoQzPEiGHjgKAJNjpcTVsuMaghHWrxJ6Y8V\nc0UU4RaEtYS04ZPl6B5rUrqfep5JQaUGnzvxzERgY85WWbejh1+YRi8EzBHgu8kDQKsAR6t3eAoR\nbmGDWOsyiV3WrWpDn2nDBixnwsa0rN/qT7oAJhG4CYd0c05DmgVtOUsK4EjPtzHrF5K4KUzhSkS4\nhY3g3qsrMHa9KMq7gF0alSHYsDAcU78IkdLDMbMKd9w0ZBQBHAL9GachrQJQ3R+rbzNHQPv5wPRq\nAqeipxWVoV84Oo0MMnICnLKOPQr1C8yK1LaHiHALQp5ReqLvojASqXh7V33nfM8/moas6r5mrwu4\nA7fBWShtx/tgl7aBswnhLtYHgy+Dx1tFcN0Bzg6WKJykWxSVqZdQMOt3Mu3n8eZaGSHCLWwEv/I/\nrc4vXapYhVj31biFCsx89VRjHBenIUlpgbVL2oek37xc0IbDNlO3T0oPjYv22O113dGxDArVsQ4Y\n/X0kcHlXhFsQhJSY0UuaORqVPq4NEVC/CwzbAg2AzV2gbyc/L4cAxcgMR7ocMoz/Yp/42Jck8Lyt\nhzdh4t3LGKa9Mh0wItzC2rPW9W2/B0D7TMeuLcPQQ7uvO0r4EqE3bX3ICejlwZMi5VQAUtNlmUJN\nj4HHdV90z8Dl7YkaN+sXga17Wqx7DV1HjrWFXfK0YlIHCV1yXwaIcAtrz9qWSYY0nwHVfbAyEHco\nCfCg1HBJXXrghjdSz0IF6LXGnQGt4nQfNqAzedOJP2x023rbfHFr8NR6s875JnqAS1taFN0O2ClP\nZLy83M6TfgtsOGPXOXrxWJFxd0CEW1hzNsIJMPSBxhNdT3Yq4EJl/P7WZaJNWnQvWJgOb+diVWfe\nw4PBKIjP7AmXD6j0W/qDFLB1f0r8iRS4WNfXwJE+CAUN/LNPdXfJsvB6gNHS1z78lnEEtI6WF8MM\niHALwroQ+kD3VB8YWkUArIUo6S1+aQcoDEojSUuEreK5cPdbuk1ubKflMBudoeuDo+SuluF6s9Gq\nteHOywzoNfS1Wo6uwS/zhWNGRLiFteZ/ra7O29ulMew9voziFlCYLEtMwhgTz9AHWsfgyu65AAfe\nfNloFMZ7kwSTwp+xVzeHK9VFMokIt7C2rPWh5E2ZKo0kMClefg84faT7x5kvP+w07FF7Hfy+zmK7\nJ+DK3oX1ZoMXh+7p+ecRAYWazu6Z9eetyvqzFUGEW1hbfvLPrUbr1upBSPLVHgkpALRPktsNrzJd\nsot6acJw0bA5EPHGU6B1pGvahqUz9l7jQgfLsOXQOBf38rauw8+zL3LNEeEW1paNLJPMBCeXLEJf\nt/b5l9TGZ6G8O902CAZKNX3gmDQI5JQBMqY/1ynp7pIVc+nLCvHjFoRNpHOih3IGDH2n9Wh352ai\nrczYQ0i99aZ4+edaTkLLIQYbeARgBuEmol8mokMi+toyAhKENPjsJ6Umeil+D2gegv0eOAx098nZ\nQTqGThwhccXZVS8IYZDsszLjlOgmMEvG/SsAXl1wHIKQKrJXcgYCF2geAo3HQPsoPSMnjkbbY8Zu\n5ujqRbtuG5MdJTw8BF2l9WYZc6VwM/PvApjDfV0QsmUjhm4WgTLP+6ljGdidlne19Spd8tj2MRC4\ng7Vf0UC0W1e32EUh0DwCh4H+XI70wWXz2bUuaV1J7XCSiF4H8DoAFKr7aT2tIMzND98zgJYcYs2M\nYeltOUPRDsPpDHxoMEXGwO400q59zcP4ARWOtNgqU3+Ec6xOC/r6XYAyMTpIFcZI7XCSmd9g5vcy\n83utYj2tpxWEuZEyyRwQAbU7oOFCYFK626R2G2N16kJdt+ipod3p4LGV3cufPwq0EF/nsDMKRLQT\nkK4SQdhkhm6AFxh6UMMund+YZHeqjCvKK8IikD5uYa1Y672Si0AZ8f4hROOCnDiBTnq6cZk4FT3M\nQ0p3x/TOzjNzq6BLLIG32kuIb8iVwk1Evw7gFQB7RPQIwD9i5s8sOjBBEJZA4iozHq9duy2w2pq2\nOw3c9HyqnbKum9Ng32T3dLrTpbQNFCrnU5Wqot8ZNJ8B1VtazIeX5LtA6zCd2FaMK4WbmT+8jEAE\nQcgApxK/gIExLtz9FmA6YLt4we40TG+lWKEGlOrngmwV9O7Hi/smSQGF6li8RAQGjQ5XL97HlqOf\nt39FC2IOkVKJsDZImWReSIthwl1TtI91B4pp69LEdfZXJn2xC6INDASZMb5v0rQH1rDjNXUiBVZJ\nW+0rItyCsKqIE+B14FghBJC87SX009+4bhiz7ZuMwtjSTuKkpX6WVEJcNaSrRBA2md7ZmGcJMJxw\nvMFS4Xm5TJAvvoCEPhD4MULNseZTzNFKe2rfBBFuYS1Y+72Si6LfAroNcBRqo6kwGBhNLfEdDDPg\ntqdeQGL3TbYOAb83MsXiMNCLHFrH5xOaADiK9AvCMvdVLhEplQjCpqEM7bQXRXo4ZrgTMks6JwDz\nYF8madHtnExPZQ73P9KgPn9xQKfxGHDKYMMEfE+7HK4pItxC7pH69hyUtnQP9MVFuM1nq+Fz3T3V\nHzRDb3jc9h2Osn8BWhJSKhFyj5RJZsQuAo5eWUZKf0AZQO1WNvGYg3Y9e3wB8dIHenKIZNxCrhEn\nwDlwqlNLCogIrAzd5pfULUJKTySCAa+PVBb5Vm/pLerDycvyts780+5YWVNEuAVhU0jq2WYkT1A6\nZaC8c67VFega8016uAtVvelmGA+R7hSp7ANnT67/vBuElEqEXCN7JefA68R0bgwIYnw9lAmUd8ZK\nK0RKTykmjsrPgFOZMqwiIt3PrSSXnAURbiG3yKHknPTbejVYNGiZG+6Z7CSMrTsTtechDMAqTd8u\nLA15eRNyy0/+ufV1f1sMDJw91S1zVlG30vVbyR0lSaUV4GYZt9sBG+a0TWwUrkZ3Sw4Q4RZyi5RJ\nronb0R9X4XWBYV/1RQjaTvW69JuAXQQb9njrXyslw6oNQIRbEIR4AhfwumCrNFhXxgAY6DZvvpmm\n+Uz7apuO9s1229IGOAci3EIu+ewn28BmzFpkS/s5YHXAdhmAHk2PPcicB8PSq9FGpZKybjdsHd00\n2o1BhFvIJbJXcon4/XQtXGt3AKIJ7+wiUNxarrlVjpGuEiF3SDdJjnFKU6INDNoBi9WMgsofItxC\n7vhnr8jm79xyaZ/2enpnLwIplQi5Q8okOWbS7e8iScsbssAqDhYSk+6u6beRyqh/SohwC4KwPPy+\nXoZgWqNyyWgxQtIg0LIp1oFi7Xz/pWnrDpizpxkHdo6USoRcIXsl14DmAdBvDZY3RLpLpfH05t0q\naUDGmGgDencllDmYJF0NJOMWBGHJ8Ln3NqBbAS0HCJC9O6DlxO+/VEp3vswyuLQERLgFQcgGZQz6\nuQfLigm6lJJlP3fCYBHHLW7IECmVCLlByiRrRmUPUOa486BV0IeCWRG4AEfxC4n77UxCikOEW8gF\n0ru9ZpACTCemn1tlK9zAaJUbc6QXEEeRniDNuoxzASmVCIKQAYTYYvLovgyJAqDxRI/mk7q8hTEj\nZsq4iehVIvoGEX2TiD626KAEYRLZK7lmcAiE0zVj5kj3Ta8Cob+Sog3MINxEZAD4JQA/BuBdAD5M\nRO9adGCCIKw57WNdihhs5eEo0oeDPdkjehWzlEp+AMA3mflbAEBE/xrAXwHwp4sMTBCGSH17TQk9\noPEYcCpgw9TZ7Yq02606swj3fQBvX/j/RwB+cDHhCMI0UiZZYzjSixWEuUitq4SIXieirxDRV3x5\nqyOkxN/5qPwsCcIkswj3YwAPL/z/g8FtYzDzG8z8XmZ+r1WspxWfIAiCMMEswv2HAN5JRN9DRDaA\nnwbwW4sNSxA0sldSEKa5ssbNzAER/W0Avw3AAPDLzPwnC49MEARBiGWmARxm/hyAzy04FkEYQ+rb\nghCPjLwLK4uUSQQhHhFuQRCEnCHCLawkn/3k6jixCcKqIcItrCSyV1IQkhHhFlYOGXEXhMsR4RZW\njn/2yupsGhGEVUSEW1g5pEwiCJcjwi0IgpAzaHq3WgpPSnQE4LupP3G67AE4zjqIlJFryg/reF1y\nTTfjRWben+WBCxHuPEBEX2Hm92YdR5rINeWHdbwuuablIaUSQRCEnCHCLQiCkDM2WbjfyDqABSDX\nlB/W8brkmpbExta4BUEQ8somZ9yCIAi5ZGOFm4j+KRH9VyJ6k4h+k4i2so4pDYjoNSL6EyKKiGjl\nTsPngYheJaJvENE3iehjWceTBkT0y0R0SERfyzqWtCCih0T0JSL608HP3s9nHdNNIaICEf0BEf3x\n4Jr+cdYxXWRjhRvAFwB8HzO/G8B/A/APM44nLb4G4K8B+N2sA7kJRGQA+CUAPwbgXQA+TETvyjaq\nVPgVAK9mHUTKBAD+PjO/C8APAfhba/Bv5QL4EWZ+GcB7ALxKRD+UcUwjNla4mfk/MvPQqf//g16C\nnHuY+evM/I2s40iBHwDwTWb+FjN7AP41gL+ScUw3hpl/F8BJ1nGkCTM/Zeb/PPh7C8DXAdzPNqqb\nwZqht7A1+FiZA8GNFe4J/jcA/yHrIIQx7gN4+8L/P0LOxWATIKKXAHw/gN/PNpKbQ0QGEX0VwCGA\nLzDzylzTTDsn8woRfRHAnZi7foGZ/+3gMb8A/Vbv15YZ202Y5boEYdkQUQXAvwHwd5m5mXU8N4WZ\nQwDvGZx//SYRfR8zr8TZxFoLNzP/xcvuJ6K/AeAnAPwvnKO+yKuua014DODhhf9/MLhNWEGIyIIW\n7V9j5t/IOp40YeYGEX0J+mxiJYR7Y0slRPQqgH8A4C8zczfreIQp/hDAO4noe4jIBvDTAH4r45iE\nGIiIAHwGwNeZ+VNZx5MGRLQ/7DQjoiKAvwTgv2Yb1TkbK9wA/gWAKoAvENFXiehfZh1QGhDRXyWi\nRwD+PIB/T0S/nXVM12FwcPy3Afw29GHX/8XMf5JtVDeHiH4dwP8L4H8kokdE9DezjikF3gfgrwP4\nkcHv0leJ6ENZB3VD7gL4EhG9CZ1EfIGZ/13GMY2QyUlBEIScsckZtyAIQi4R4RYEQcgZItyCIAg5\nQ4RbEAQhZ4hwC4Ig5AwRbkEQhJwhwi0IgpAzRLgFQRByxv8P5Mbmcdd79xUAAAAASUVORK5CYII=\n", 332 | "text/plain": [ 333 | "" 334 | ] 335 | }, 336 | "metadata": {}, 337 | "output_type": "display_data" 338 | } 339 | ], 340 | "source": [ 341 | "plot_decision_boundary(lambda x: predict(x,What))" 342 | ] 343 | }, 344 | { 345 | "cell_type": "markdown", 346 | "metadata": {}, 347 | "source": [ 348 | "### Concolusion \n", 349 | "* To lower the loss lower the learning rate\n", 350 | "* Use different learning rate for different optimization parameters aka (W,b)\n", 351 | "* Maybe if the data is normailzed we will need the same learning rate " 352 | ] 353 | } 354 | ], 355 | "metadata": { 356 | "kernelspec": { 357 | "display_name": "Python 3", 358 | "language": "python", 359 | "name": "python3" 360 | }, 361 | "language_info": { 362 | "codemirror_mode": { 363 | "name": "ipython", 364 | "version": 3 365 | }, 366 | "file_extension": ".py", 367 | "mimetype": "text/x-python", 368 | "name": "python", 369 | "nbconvert_exporter": "python", 370 | "pygments_lexer": "ipython3", 371 | "version": "3.5.4" 372 | } 373 | }, 374 | "nbformat": 4, 375 | "nbformat_minor": 2 376 | } 377 | --------------------------------------------------------------------------------