├── .gitignore ├── .travis.yml ├── Chapter11 ├── 11-2.ipynb └── 11-2.md ├── Chapter12 ├── 12-1-2.ipynb └── 12-1-2.md ├── Chapter6 ├── 6-2-2-1.ipynb ├── 6-2-2-1.md ├── 6-2-2-2.ipynb ├── 6-2-2-2.md ├── 6-2-2-3.ipynb ├── 6-2-2-3.md ├── 6-3-1.ipynb ├── 6-3-1.md ├── 6-3-2.ipynb ├── 6-3-2.md ├── __init__.py ├── deep-linear-network.ipynb ├── deep-linear-network.md └── mnist_loader.py ├── Chapter7 ├── 7-1-1.ipynb ├── 7-1-1.md ├── 7-1-2.ipynb ├── 7-1-2.md ├── 7-11.ipynb ├── 7-11.md ├── 7-12.ipynb ├── 7-12.md ├── 7-8.ipynb ├── 7-8.md ├── checkpoint.pt ├── early_stopping.py └── mnist_loader.py ├── Chapter8 ├── 8-3-1.ipynb ├── 8-3-1.md ├── 8-3-2.ipynb ├── 8-3-2.md ├── 8-5-1.ipynb ├── 8-5-1.md ├── 8-5-2.ipynb ├── 8-5-2.md ├── 8-5-3.ipynb ├── 8-5-3.md └── mnist_loader.py ├── Chapter9 ├── CNN.ipynb └── mnist_loader.py ├── README.md ├── SUMMARY.md ├── book.json ├── mnist.pkl.gz └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | __pycache__ 3 | .ipynb_checkpoints 4 | .idea 5 | node_modules 6 | _book 7 | _build -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '8.9.1' 4 | 5 | # S: Build Lifecycle 6 | install: 7 | - npm install gitbook-cli 8 | 9 | after_script: 10 | - gitbook install 11 | - gitbook build 12 | - cd ./_book 13 | - git init 14 | - git config usr.name "windmissing" 15 | - git config user.email "${USER_EMAIL}" 16 | - git add . 17 | - git commit -m "publish gitbook" 18 | - git push --force --quiet "https://${test_token}@github.com/windmissing/DeepLearningPractise" master:gh-pages 19 | 20 | branchs: 21 | only: 22 | master -------------------------------------------------------------------------------- /Chapter11/11-2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# 将模型写入文件" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "import torch \n", 15 | "torch.save(model.state_dict(), modelname)" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "# 从文件读入模型" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "model = CNN() # 需要先建立模型 \n", 30 | "model.load_state_dict(torch.load(modelname)) # 加载模型 " 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [] 39 | } 40 | ], 41 | "metadata": { 42 | "kernelspec": { 43 | "display_name": "Python 3", 44 | "language": "python", 45 | "name": "python3" 46 | }, 47 | "language_info": { 48 | "codemirror_mode": { 49 | "name": "ipython", 50 | "version": 3 51 | }, 52 | "file_extension": ".py", 53 | "mimetype": "text/x-python", 54 | "name": "python", 55 | "nbconvert_exporter": "python", 56 | "pygments_lexer": "ipython3", 57 | "version": "3.7.3" 58 | } 59 | }, 60 | "nbformat": 4, 61 | "nbformat_minor": 2 62 | } 63 | -------------------------------------------------------------------------------- /Chapter11/11-2.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter12/12-1-2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# 判断是否支持GPU" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "data": { 17 | "text/plain": [ 18 | "False" 19 | ] 20 | }, 21 | "execution_count": 1, 22 | "metadata": {}, 23 | "output_type": "execute_result" 24 | } 25 | ], 26 | "source": [ 27 | "import torch\n", 28 | "torch.cuda.is_available()" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 1, 34 | "metadata": {}, 35 | "outputs": [ 36 | { 37 | "data": { 38 | "text/plain": [ 39 | "device(type='cpu')" 40 | ] 41 | }, 42 | "execution_count": 1, 43 | "metadata": {}, 44 | "output_type": "execute_result" 45 | } 46 | ], 47 | "source": [ 48 | "import torch\n", 49 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", 50 | "device" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "metadata": {}, 56 | "source": [ 57 | "# 让模型支持GPU" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 3, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "# network、data、label要成为cuda类型\n", 67 | "def TensorAndCuda(img, label):\n", 68 | " img = torch.Tensor(img)\n", 69 | " label = torch.Tensor(label) \n", 70 | " if torch.cuda.is_available():\n", 71 | " img = img.cuda()\n", 72 | " label = label.cuda()\n", 73 | " network = network.cuda()\n", 74 | " else:\n", 75 | " img = Variable(img)\n", 76 | " label = Variable(label)" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "# 清除GPU的缓存" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 4, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "torch.cuda.empty_cache()" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "# GPU运行结果计算准确率" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "```python\n", 107 | "out = network(img)\n", 108 | "_, predicted = torch.max(out, 1)\n", 109 | "cpu_info = predicted.cpu()\n", 110 | "correct = (cpu_info == torch.Tensor(label[0])).sum().item()\n", 111 | "accuracy = correct / data[0].shape[0]\n", 112 | "print('validate accuracy: {:.4}'.format(accuracy))\n", 113 | "```" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [] 122 | } 123 | ], 124 | "metadata": { 125 | "kernelspec": { 126 | "display_name": "Python 3", 127 | "language": "python", 128 | "name": "python3" 129 | }, 130 | "language_info": { 131 | "codemirror_mode": { 132 | "name": "ipython", 133 | "version": 3 134 | }, 135 | "file_extension": ".py", 136 | "mimetype": "text/x-python", 137 | "name": "python", 138 | "nbconvert_exporter": "python", 139 | "pygments_lexer": "ipython3", 140 | "version": "3.7.3" 141 | } 142 | }, 143 | "nbformat": 4, 144 | "nbformat_minor": 2 145 | } 146 | -------------------------------------------------------------------------------- /Chapter12/12-1-2.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter6/6-2-2-1.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter6/6-2-2-2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch.nn as nn\n", 10 | "import torch.nn.functional as F\n", 11 | "from torch.autograd import Variable\n", 12 | "import torch as t" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": {}, 18 | "source": [ 19 | "# Sigmoid + MSE Cost" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 2, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "class Net(nn.Module):\n", 29 | " def __init__(self):\n", 30 | " super(Net, self).__init__()\n", 31 | " \n", 32 | " self.fc1 = nn.Linear(28*28, 30)\n", 33 | " self.fc2 = nn.Linear(30, 10)\n", 34 | " \n", 35 | " def forward(self, x):\n", 36 | " x = x.view(x.shape[0], -1)\n", 37 | " x = self.fc1(x)\n", 38 | " x = F.logsigmoid(self.fc2(x))\n", 39 | " return x" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 3, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "from mnist_loader import load_data_shared, vectorized_result\n", 49 | "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n", 50 | " seed=666,\n", 51 | " train_size=2000,\n", 52 | " vali_size=0,\n", 53 | " test_size=100)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 4, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "def predict(data, net):\n", 63 | " with t.no_grad():\n", 64 | " #for index in range(test_data[0].shape[0]):\n", 65 | " # get the inputs\n", 66 | " inputs, labels = t.Tensor(data[0]), data[1]\n", 67 | "\n", 68 | " # forward + backward + optimize\n", 69 | " outputs = net(inputs)\n", 70 | " _, predicted = t.max(outputs, 1)\n", 71 | "\n", 72 | " #print('Predicted: ', predicted)\n", 73 | " #print('target: ', t.Tensor(labels).int())\n", 74 | "\n", 75 | " correct = (predicted == t.Tensor(labels)).sum().item()\n", 76 | " accuracy = correct / data[0].shape[0]\n", 77 | " return accuracy" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 5, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "def fit(net, criterion, optimizer):\n", 87 | " loss_scores = []\n", 88 | " test_scores = []\n", 89 | " train_scores = []\n", 90 | " for epoch in range(100): # loop over the dataset multiple times\n", 91 | "\n", 92 | " # get the inputs\n", 93 | " inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n", 94 | " vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n", 95 | " # zero the parameter gradients\n", 96 | " optimizer.zero_grad()\n", 97 | "\n", 98 | " # forward + backward + optimize\n", 99 | " outputs = net(inputs)\n", 100 | " loss = criterion(outputs, vector_labels.float())\n", 101 | " loss.backward()\n", 102 | " optimizer.step()\n", 103 | "\n", 104 | " # print statistics\n", 105 | " loss_scores.append(loss.item())\n", 106 | " train_scores.append(predict(training_data, net))\n", 107 | " test_scores.append(predict(test_data, net))\n", 108 | " print('Finished Training')\n", 109 | " return loss_scores, train_scores, test_scores" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 6, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "name": "stdout", 119 | "output_type": "stream", 120 | "text": [ 121 | "Finished Training\n" 122 | ] 123 | } 124 | ], 125 | "source": [ 126 | "import torch.optim as optim\n", 127 | "net1 = Net()\n", 128 | "criterion = nn.MSELoss()\n", 129 | "optimizer = optim.SGD(net1.parameters(), lr = 1e-1)\n", 130 | "loss_scores, train_scores, test_scores = fit(net1, criterion, optimizer)" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 7, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "data": { 140 | "text/plain": [ 141 | "
" 142 | ] 143 | }, 144 | "metadata": {}, 145 | "output_type": "display_data" 146 | } 147 | ], 148 | "source": [ 149 | "import matplotlib.pyplot as plt\n", 150 | "\n", 151 | "plt.plot(loss_scores)\n", 152 | "plt.show()" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": 8, 158 | "metadata": {}, 159 | "outputs": [ 160 | { 161 | "data": { 162 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de3xU9Z3/8dcnd8iNBEIgCSHcr3INEQsq1dZLW6V2tUJrq9Uu2q61u7bd2u3+bGu3u9vL9qp1pd51W6XWWlQqtd4VhUy4CgEJGJKZBAgkIYSQy8x8f3+cCYQwISfJTCZz5vN8PPLIzDlnznyOg++c+Z7v+X7FGINSSinniot0AUoppcJLg14ppRxOg14ppRxOg14ppRxOg14ppRwuIdIFdDdq1ChTVFQU6TKUUiqqlJWVHTHG5ARbN+SCvqioCJfLFekylFIqqojIgZ7WadONUko5nK2gF5ErRGSPiFSIyF1B1l8kIptFxCsi13Zb9xMR2Ski5SLyaxGRUBWvlFKqd70GvYjEA/cBVwIzgZUiMrPbZlXATcDvu732I8ASYA4wG1gEXDzgqpVSStlmp42+BKgwxuwHEJGngOXArs4NjDGVgXX+bq81QAqQBAiQCBwacNVKKaVss9N0kw9Ud3nuDizrlTHmXeA1oDbws94YU959OxFZJSIuEXHV1dXZ2bVSSimb7AR9sDZ1WyOhichkYAZQgPXH4RIRueisnRmz2hhTbIwpzskJ2jtIKaVUP9kJejcwrsvzAqDG5v6vAd4zxjQbY5qBvwKL+1aiUkqpgbAT9KXAFBGZICJJwApgrc39VwEXi0iCiCRiXYg9q+lGKaViTYfPz/Pbali/8yA+f3iHi+/1YqwxxisitwPrgXjgYWPMThG5B3AZY9aKyCLgz0AWcJWI/MAYMwt4BrgE2IHV3POSMeb5cB2MUkoNttYOH21eqx9Kh8/P+55jlFbWs636GKMzkllUlM2ioixy0lMA8Pr8/GVrDQ++tZ+aY60ATBiVyq0XTeSaBfkkJ8SHvEYZahOPFBcXG70zVik11Pn8hgfe3McvX95Lu+/MDofxccK03HQONbVy9ER70NeXTMjmKxdPorXDx29f38cOzzFmjs3gxTuW0p/bjUSkzBhTHGzdkBsCQSmlhrrq+ha+sWYbmyrruXxWLiUTRgIQJzAtN515hSMYnpSAMYYPj5yg7EADTa3eU6+fN24EC8dnnXp+xewxvFNxlPqW9n6FfG806JVSKghjDJ7Gk+RlDiMuzgrf+hPtPPrOhzz8TiUA/3PdXD6zIL/HcBYRJuakMTEn7ZzvJSIsnTIqpPV3pUGvlFJB3PPCLh55p5LMYYkUj89iZFoSa7fV0Nrh57KZufy/T81kXPbwSJdpiwa9Ukp18/i7lTzyTiVXzc0jNSmeTZX1vLX3CFfPy+O2iycyeXR6pEvsEw16pZTq4vU9h/n+2p18bMZofnn9POIDzTbGmLC0nw8GDXqllMIK8tf2HOaOP2xlxtgMfrVi/qmQB6I25EGDXikV49q9fl7aeZD7X99HeW0T40cO56EbF5Ga7Jx4dM6RKKWUDcdbO9hc1Ujph/WUVtaztbqRNq+fSTmp/PTaOSyfl09SgrPmZNKgV0o5nt9veGRDJX8qc7P7YBN+Y93UNCsvg8+fP56lU0aybOroU90onUaDXinlaDWNJ7lzzVbe219P8fgs7rh0CouKspk3boSjmmfOJTaOUikVc9q9fp7d7OZH68rx+w0/uXYO1y0siOqLqv2lQa+UcpSWdi9Pbao+NWjYwvFZ/Pyzcxk/MjXSpUWMBr1SyjGqjrZww0MbqapvoaQomx995jyWTc2JybP4rjTolVKOsOfgcb7w0EbafX5+/+Xz+cjk8I0dE2006JVSUW9bdSM3PrKJpPg41tx6AVNzo2uIgnDToFdKRbWjzW3c+Mgm0lMS+L9bFlM4MjoGGhtMGvRKqaj245d209zqZc2tF2jI98BZt38ppWKKq7KeNS43t1w4QZtrzkGDXikVlbw+P//+3PvkZaZwxyVTIl3OkKZBr5SKSo+9e4DdB49z91WzYuYO1/6yFfQicoWI7BGRChG5K8j6i0Rks4h4ReTabusKReRvIlIuIrtEpCg0pSulnO5Em5d1O2qpPXby1DJjDI+/W8mPX9rNsmk5XD4rN3IFRole/wyKSDxwH/BxwA2UishaY8yuLptVATcB3wyyi8eBHxljXhaRNMAfZBullDql/kQ7j26o5LENlRw72UFivPDpefl8dtE47n21gjc+qGPZtBz+57q5MX8zlB12vu+UABXGmP0AIvIUsBw4FfTGmMrAujNCXERmAgnGmJcD2zWHpmyllBMZY3hyYxX/+WI5Jzt8XD4rl5Ulhby+p46nSqv4Y5mb5IQ4frh8FjcsHq8hb5OdoM8Hqrs8dwPn29z/VKBRRJ4FJgB/B+4yxvi6biQiq4BVAIWFhTZ3rZRykrrjbfzrM9t4bU8dF03N4e5PzTg1N+uyaaP52iWTeWnnQc6fMJLJo9MiXG10sRP0wf5kmj7s/0JgPlbzztNYTTwPnbEzY1YDqwGKi4vt7lsp5RAVh49z/QPv0dzm5QdXz+KLF5x9tj4yLZnPnz8+QhVGNztB7wbGdXleANTY3L8b2NKl2ec5YDHdgl4pFbuMMfzbn9/HZwwvfG0pU7Q/fMjZ6XVTCkwRkQkikgSsANba3H8pkCUiOYHnl9ClbV8ppf68xcOmD+v59hXTNeTDpNegN8Z4gduB9UA5sMYYs1NE7hGRqwFEZJGIuIHrgAdEZGfgtT6snjiviMgOrGag34XnUJRS0eZYSwf/ua6ceeNGcH3xuN5foPrF1l0Gxph1wLpuy+7u8rgUq0kn2GtfBuYMoEallEP97G97rK6UXypx7HytQ4HeGauUioj3Pcd4cuMBvnhBEbPzMyNdjqNp0CulIuI3r+4lPTmBOy+bGulSHE+DXik16PYeOs76nYe46SNFZKQkRrocx9OgV0oNuvvf2MewxHhuWjIh0qXEBA16pdSgcje0sHZrDStKxpGdmhTpcmKCBr1SalD97s39iMA/Xjgx0qXEDA16pdSgOdLcxlOl1VwzP5+8EcMiXU7M0KBXSg2KnTXHWLn6Pbx+w60XT4p0OTFFp2VRSoWV32948O39/HT9HkYMT+KRmxYxKUdHnxxMGvRKqbAxxnDXs9tZ43Jz+axc/uszc/QCbARo0Culwua3r+9jjcvN7R+dzDcum6oThUSIttErpcLixe21/HT9Hq6em6chH2Ea9EqpkNtW3cida7aycHwWP7l2joZ8hGnQK6VCqt3r5841WxmVlszqLywkJTE+0iXFPG2jV0qF1INv72df3QkeuWkRI9OSI12OQoNeKdUHPr/h95uqqDveRvH4LBaMzyIt+XSMuBta+PUre7l8Vi4fnT46gpWqrjTolVK2uBtauPPpbWyqrEcEjIE4gfMnjOTWiydy8dQc7nl+F4Jw91WzIl2u6kKDXinVq3U7avn2M9sxwP9cN5fLZuWytbqRTR/W80yZm5seKWXiqFT2HznBXVdOJ1+HNxhSxBgT6RrOUFxcbFwuV6TLUEoFeBpP8tGfvs6MvAzuXTmfcdnDz1jf7vXz3FYPD7yxj2FJ8Tz7lSUkJWg/j8EmImXGmOJg62x9GiJyhYjsEZEKEbkryPqLRGSziHhF5Nog6zNExCMi9/a9fKVUJP3y5Q9A4LefX3BWyAMkJcTx2eJxvPKNZTx/+1IN+SGo109EROKB+4ArgZnAShGZ2W2zKuAm4Pc97OaHwBv9L1MpFQl7Dx3nT5vdfHHxeFvNMdpffmiy86e3BKgwxuw3xrQDTwHLu25gjKk0xmwH/N1fLCILgVzgbyGoVyk1iH72tz0MT0rgqx+dHOlS1ADYCfp8oLrLc3dgWa9EJA74H+BbfS9NKRVJW6oaWL/zEKsumqgDkUU5O71ugn0Xs3sF96vAOmNM9bm+0onIKmAVQGFhoc1dK6VC6WhzG796ZS+Hm9oAKD/YxKi0JG5ZqvO6Rjs7Qe8GxnV5XgDU2Nz/BcCFIvJVIA1IEpFmY8wZF3SNMauB1WD1urG5b6VUiLy25zDf+uN2mk52MGFUKgDDkxL47idmkJqsvbCjnZ1PsBSYIiITAA+wAvicnZ0bYz7f+VhEbgKKu4e8UipyfH7DD1/YxaMbKpmWm84Tt5QwY2xGpMtSIdZrG70xxgvcDqwHyoE1xpidInKPiFwNICKLRMQNXAc8ICI7w1m0Uio0fvRiOY9uqORLS4r4y+1LNOQdSm+YUipGPfFuJf/vLzu5eckE7r6qe49pFW0GfMOUUspZXt9zmO8/v4tLp4/mu5+cEelyVJjpVRalYsixlg6eeK+S+1/fx7TcdH69cj7xcXqTk9Np0CvlIEea23i1/DBTctOYlZdJUkIcJ9t9bK1u5JXyQ/xhUxUn2n18dFoO//WZOdqjJkbop6yUQ7gbWrjhwY1UHm0BICUxjvHZqeyra8brN8THCZ+aM5bbLp6kF11jjAa9Ug5QcbiZLzy0kRNtXh750iJa232UVjawr66ZS2aMpqQomwWFWWQOT4x0qSoCNOiVinKbqxr4x8dciMBTqy5gZp51tn7leWMjXJkaKjTolYpSXp+fe1+r4DevVjA2M4XHby5hYk5apMtSQ5AGvVJRyN3Qwtf+sIUtVY1cMz+fHyyfRUaKNsuo4DTolYqw6voWnnjvAOnJCdyweDxZvYwU2XCinS88tIkjzW38euV8rp6bN0iVxrCmWlj/HfC29XMHAhd8FYqWnrl4y5OQnA4zlwd/WYho0CsVIeW1TfzvG/t4YXstAnj9hvvf2MfKkkJWlhQyKSf1rIk82r1+bn2yDE/jSf7wj+ezcHx2ZIqPNeVrYeefIXc29GdylSMVEJ9wZtAbA3//AaSN1qBXymk2fVjP/a9X8NqeOlKT4rll6QRuXjKBYyc7eOCNfTy6oZKH3v6QkalJFBdlsagom+KibGaOzeCuZ7ez6cN6frVinob8YHK7IH0sfOWd/r3+mZuhauOZy45Vw4nD0HIE2pohOXzXVzTolQqDlnYvLe0+ADp8fnZ6mig9UM+GiqPs8BwjOzWJb3x8Kl+8oOhUl8cxmSn8/Pp5fPPyabz5QR2llQ2UVtazfuchAJLi42j3+fmXj01l+Txbc/+oUPG4IH9h/1+fvxDe/xMcPwjpYwL7LLN+Gz/UboOiJQOvswca9EqF2JaqBm54cCMnAkHfKTFemJ2fyfevmsn1iwoZlhQf9PV5I4axoqSQFSXWJDyHmlpxBUI/Y1gid1yq0/oNqpZ6qN8P87/Q/33kB8Ya85TB9E9aj90uiEsAv9f6Q6JBr1R0aG7z8s9Pb2XE8CTuunKitVCEqaPTmDtuBCmJwcP9XHIzUvjknLF8co72i48Iz2br90DO6MfOsUK9a9B7NkPefGg+bIV+GGnQKxVC9zy/k+r6Fp5adQElE7QN3RE8ZYBYodxficMgd9bpQPd5oXYrLLjRaqfv3n4fYjpMsVIhsm5HLWtcbr66bLKGvJN4XJAzDVIGOD5Q/kKo2QJ+PxzeBR0t1rL8hdDkttrvw0TP6JXqpw0VR7j/jX20dfgB2FXbxNyCTL7+sSkRrkyFjDHWGf3UKwe+r/xicD0MR/eevhBbsBCa66zHXZt1QkzP6JXqozavjx+9uIvPPbiR/XUniI8T4uOECyaN5Ncr55MYr/9bOUZDJbQchfwFA99XZxu/22V9SxiWDVkTTrffh7GdXs/oVUx7fc9hXt51KOi6tOQE5hdmUVyURfbwJPYebqa0sp4n3zvA7oPHuWFxId/9xMwee88oBzh15h10hr6+GTUVktKtfXo2W8Evcrr9vvO9wkCDXsWsNaXV3PXsdlKTEkhOPPssvOmklwfe3A/A8KT4U/3i80cM46Ebi7l0Ru6g1qsiwFMGCSkwOgRz6sbFQf58+PBNOFoBM64+vS5/Iex4xmq/jwv9N0JbQS8iVwC/AuKBB40x/91t/UXAL4E5wApjzDOB5fOA+4EMwAf8yBjzdOjKV8qe7e5GahpbKS7KYlRaMg++tZ//eLGci6bm8MANC4Oelbd5fbzvacJVWY+n8SRzC0awqCibcdnDzhqaQDmU2wVj50F8iAaMyy+GD39uPe76LaFr+33OtNC8Vxe9Br2IxAP3AR8H3ECpiKw1xuzqslkVcBPwzW4vbwG+aIzZKyJ5QJmIrDfGNIakeqVseHazm289sx2f3wDWGbmn8SSfOG8Mv7x+PkkJwc+gkhPiWTg+i4XjswazXDVU+DqsO1YXfTl0++zaFz9vwdnL3a7IBD1QAlQYY/YDiMhTwHLgVNAbYyoD6/xdX2iM+aDL4xoROQzkABr0alA8tqGS763dyZLJI7njkilsrW7EdaCB5fPy+MZl03RibLsaq6y7Q3uTPQlGjAt/PeHibYPqTWB81jH72qyeMaHSeRafNQFSR55ePmoqJGdYF2nnfz507xdgJ+jzgeouz93A+X19IxEpAZKAfUHWrQJWARQWFvZ110qdobXDx7bqRv76/kEe3VDJx2fm8puV80lJjOf8iSO5NdIFRhtj4JFPwrGq3rcdORm+Fr6LimH33v3w9++dfi5xUFASuv2nj7H+GxUuPnN5XJx1Q1aYLsjaCfpgpzymL28iImOBJ4AbjTH+7uuNMauB1QDFxcV92rdSAMYYXt19mAff+hDXgXo6fNY/o88WF/Cf15xHgnZ57L+mGivkP/I1mPaJnrfb9RfY+L9w4gikjhq8+kKp6l3rbPvTv7WeD8sO/TeUm9dbF3i7u+yHkJga2vcKsBP0bqDrkRYANXbfQEQygBeBfzfGvNe38pQ6t5rGk7y99wgPvf0hew4dJ3/EMG5eOoGSomwWjs9ixPBzT+KhbPAE+nfPvObczRjGWEHvKYOplw9ObaHUeXPU5I/D+I+E7316+iM4dm7Y3tJO0JcCU0RkAuABVgCfs7NzEUkC/gw8boz5Y7+rVKqLQ02t/Gz9HjbsO4qn8SQAU3PT+MX1c/nUnDy9YSnUPGUQnwRjZp97u7x5VlOH2xWdQd9YBSfqQtsmP0T0GvTGGK+I3A6sx+pe+bAxZqeI3AO4jDFrRWQRVqBnAVeJyA+MMbOAzwIXASNF5KbALm8yxmwNx8Eo53vp/Vq+8+wOTnb4uHR6Ll++cAKLApNyxOmF1fBwl8GY8yAh+dzbJaVa/c3DeONPWHXWPZBRKocoW/3ojTHrgHXdlt3d5XEpVpNO99c9CTw5wBpVDDnc1MqfNnto9551KYeKumae31bDefmZ/HLFPCblhG9GHhXg91kDcc2/wd72+Quttnpj+jflXiR5yiA+2Zou0GH0zlg1ZKzfeZC7/rSdhpaOoOsT44V/+ugkvn7p1B77vqsQq9sNHSfsn+XmL4TNj1ldMUdOCm9toeYps9rJQ3Vz1BCiQa8i4khzG67KegL3MPHGnjqedlUzOz+DP952ARNHBT9b1+aZQdbXsV46t3O7oivofR1QsxWKvxTpSsJCg16FzIk2L1uqGnEdqCc1KYHPLhpH5rAzz46q61v43Vv7ebq0mrYuzTMi8NVlk/jnj+nZ+pDidkHKCMieaG/7nOlWF0GPC+ZeH97aQunwLvCedGT7PGjQqwEyxvDW3iM88OY+3ttfj89vELGaaH/1yl4+v7iQxRNGsqWqgU2V9ZRWNhAn8Jn5BVxfMo7UJOufYOawRMZkBulbrCLLU3Z6lEU74uLDeuNP2Dj4Qixo0Kt+MMZQXX+S9/Yf5fH3Knnf00RuRjK3XTyRkgkjWVA4gqr6Fv73jf387s39PPDGfuIEZuVlsuqiidx4QZGGejRoP2Gd6Z7rJqlg8hdY/em9bb331Bkq3GUwfCRkFUW6krDQoFe2Nba08+OXdvNK+WEOH28DYOKoVH7yD3NYPj+P5ITTI0DOysvkNyvn863LpuFubGFOwQjSkvWfW1Sp2QrG3/ex2AuKYUM7HHw/evqk9/WbS5TR//OULe9UHOEba7ZxpLmNT5w3lpIJ2SwqymbK6LRzXiAtHDmcwpHDB7FSFTL9bc7o3N7jio6gb22yehfNuibSlYSNBr06Q8OJdsoONLC1upGTHdZEG0eb23huaw0Tc1J57sYlzM7PjHCValB4XDBifN/HrcnIh7Qx1oXc86NgCLmaLYBxbPs8aNBHtTavj1+8vJe/lwefCq+r5IQ45o4bwaKiLOYWjCAl0WpmORkY6bE0cKG04nAzAPFxwrDANiLwxQvG850rZ+i0edGk7gP40y1WW3l/NFT2b7JqESs0y9fCvdv6996JKXDdo/Z7+3S1/ruw92X727ces36HYl7YIUqDPkp9cOg4d/xhC7sPHueiqTmk99L+3dTawfNba/j9xuBDzaanJLBwfBbXzM9nUVE2cwoyT/0xUFFqz4twcDvMXE7wQWh7kTsTzr+tf+99wT9BQpLV/aqvjA/Kn4cP1sPir/Tttb4OKH3Iuqjalwk8Rs+E4dl9e68ookEfZY6d7OCxDZXc+1oFGSkJPHxTMZdMtzd3qc9v2HPwOLtqm/D5rT7s8XFxzM7PYOrodL0ZyWncLuuM+LOPD/57Fy2xfvrr5zOt+vuqsz/8Rd+E867t//s7jAZ9FDDGUFXfwu83VvF/G6tobvNyxawx/Mc1sxmVZr/7WnycMDMvg5l5GWGsVg0Zns0DC9tIyl/Qv774Du8P318a9EOQz28or7UmpS490ICrsp5DTW3ECXxqTh63XTxJw1qdW1MNHK+xJp2ORvnFVvPNiaNnTrnXG4f3h+8vDfoIaff6T/Vq8fsN5QebcFU2UFpZz5aqRprbvADkZaaweOJIiouyWTY1h3HZ2lVR2dDXMWqGms66PWUw9TL7r/O4rD8SDu0P318a9BHw3v6jrHrcRVOr94zlIjAtN51Pz89jUVE2xUXZ5I8YFqEqVVRzuyAuMXqH3B0bmMSkL0Hf2gR1e2DWZ8JbWxTSoB9kr+4+xFee3My47OHccek4JHDmMXFUKgsKs8gc7rwhUlUEeAKThSRG6VATyWmQM+P0NIZ2dPaHj4abtAaZBn2IeX1+Ko+ewOs/u1vZ9upj/NufdzBjbAaP3VxCdqrOZ6rCoHOykLkrI13JwOQvgN0v2J/EpLO5Ks+5/eH7S4N+gDqH5i2trMd1wGpfb2n39bh9yYRsHrqxmPQUPXNXYVK3B9qbo7d9vlNBMWx5wv4kJp4yyJ7k6P7w/aVB3w8n232scVXzp81udtY04fMb4gSmj8nguoUFzB034tRdpV0lxsexdMoovRFJhZdTuhieGjNns/2gL7owvDVFKQ16mzr7sq/dWsMjGyqpP9HO3IJMvrpsEsVF2SwoHKFn6Wpo8LggJdM6u41mOTMgcbh1PHOuO/e2xzxwvDb6v8WEia2gF5ErgF8B8cCDxpj/7rb+IuCXwBxghTHmmS7rbgT+PfD0P4wxj4Wi8MGyYd8RnnzvAKWVDdQFhua9ZPpovrJsEouK9CuiGoI6h9yNi/KZuuITrElM7Nwh23nRNlrvGwizXoNeROKB+4CPA26gVETWGmN2ddmsCrgJ+Ga312YD3wOKAQOUBV7bEJryw6e1w8fP1u/hwbc/JCc9mSWTrL7sH5k0kok5weczVSri2lvg0C648M5IVxIa+Qtg4wPgbbfGzumJpwzik2BMlHYnDTM7Z/QlQIUxZj+AiDwFLAdOBb0xpjKwzt/ttZcDLxtj6gPrXwauAP4w4MrDaO+h43wtMGCYjtqowubABjhcHnzd+CUwenrf91m7zRoUzClntvnF4PsNvPUzSDvHmE57X7a6k0bLjFaDzE7Q5wPVXZ67gfNt7j/Ya/O7byQiq4BVAIWFhTZ3HR5lBxr40iObSEqI45GbFvHR6aMjWo9yKJ8X/u86q3dMMAWL4Mt/7/t+TzVhOKSLYeFiSEiBN37c+7YXfiP89UQpO0EfrAOr3bFHbb3WGLMaWA1QXFzcj3FNQ+PtvUdY9YSL0enJPPnl8ynI0uEGVJjU7bZC/pM/h+mfOnPdWz+Dskd7b64IxlMGIwohzSEnKOlj4FsVVpPUuYhAas7g1BSF7AS9GxjX5XkBUGNz/25gWbfXvm7ztYPq1d2HuO2JzUzMSeXxW0oYnR6ldxSq6NB55j1xGaR3a5IYvwQ2rYZDO/reRdJd5rw7Q5PTrR/Vb3Yuy5cCU0RkgogkASuAtTb3vx64TESyRCQLuCywbEhp8/r4zrM7mDQ6jadWLdaQV+HndsGwrOAzKHV2EXT3cZje5sNwrMo57fMqZHoNemOMF7gdK6DLgTXGmJ0ico+IXA0gIotExA1cBzwgIjsDr60Hfoj1x6IUuKfzwuxQ8uxmD4ea2vi3T0xnxHAdlkANAs9m62w92K39GfnWhce+jsfulBulVMjZ6kdvjFkHrOu27O4uj0uxmmWCvfZh4OEB1BhWPr/hgTf2Macgk6WT+zgJslL90dYMdeUw46rg60Wss/K+DOgF1rcEiYexcwdeo3KUKL+jYuDW7ail8mgLX1026dRIkkqFVe1WMP5zn3nnL4CjFXCyD7eceMqseV6TtBOBOlNMB70xht++vo9JOalcNnNMpMtRsaLzTs9zBf2piTc229un3x9oDtL2eXW2mA761/fUUV7bxG0XT9KJsdXg8ZRZU92da4q8vPmA2A/6+n3QdkzHelFBxWzQ+/2GX72yl7zMFJbPO+seLqXCx1PW+5l3SiaMmmq/nd7OtwQVs2I26P9YVs3W6kbuvGwaSQkx+59BDbamWmjy2Avk/IVWgBsb9xB6XJCUbv1xUKqbmEy4+hPt/Ndfd1NSlM0/LNCzeTWI+jJpd8FCaDkCjVX29ps3D+J0TCZ1tpgM+p+8tJvmVi8//PRs7WmjBpfHBXEJMGZO79t2Nu/01nzT0QoH39f2edWjmAv6sgMNPFVazc1LJzBtjN5WrQaZpwxyZ9ubtDt3ljWgV28XZA/uAH+Hts+rHsXcDFP3vLCLMRkpfP3SKZEuRfWm7gN46GO9D2gVTfwdsOjL9raNT7Rufnr3PmtM9p6YwOjgGvSqBzEV9B8cOs626ka+d9VMUpNj6tCj075XofUYXO3XbI0AABAgSURBVHC7NamEE0gczL/B/vaX/Qj2rOt9u6zxkJHX/7qUo8VU2j23xUN8nPCpOfo/RFTwuCA9Dy7/UaQriZxxi6wfpQYgZtrojTH8ZWsNSyaPIiddZ6GJCp4y50ygoVQExUzQlx1owNN4kk/P07P5qNBSD/X7tSeJUiEQM0H/3FYPKYlxXDZLx7SJCp09TfQCo1IDFhNB3+Hz8+L2Wj4+cwxpehE2OnhcgATGfFFKDURMBP2bH9TR0NKhzTbRxFMGOdN1CjmlQiAmgv65rTWMGJ7IhVN08uCoYIw1xovT5j5VKkIcH/RtXh+vlB/iytljdfCyaNHwIZys1/Z5pULE8cm3paqRlnYfH52mZ/NR49SFWO1xo1QoOD7o36k4QnycsHjSOSZ5UEOL2wUJw2D0zEhXopQj2Ap6EblCRPaISIWI3BVkfbKIPB1Yv1FEigLLE0XkMRHZISLlIvKd0Jbfu7crjjC3IJOMlMTBfmvVX51D7sZrDymlQqHXoBeReOA+4EpgJrBSRLqfat0CNBhjJgO/AH4cWH4dkGyMOQ9YCNza+UdgMDS1drCtupGlk0cN1luqgfK2Q+02bZ9XKoTsnDKVABXGmP0AIvIUsBzY1WWb5cD3A4+fAe4Va6B3A6SKSAIwDGgHmkJTejetx+DNn8L0q6DwfADe23cUv4ElGvQD43oYju4bnPdqbQRfmwa9UiFkJ+jzgeouz93A+T1tY4zxisgxYCRW6C8HaoHhwL8YY+q7v4GIrAJWARQWFvbxEAL8PtjwG8jIPxX0b1ccYVhiPPMLs/q3TwXNdfDCv1ijR8YNUvNXRj4ULR2c91IqBtgJ+mBTMHWfxLKnbUoAH5AHZAFvicjfO78dnNrQmNXAaoDi4mIbE2QG0XljTdvxU4verjjC+ROztVvlQHROffeF56BoSWRrUUr1i50EdAPjujwvAGp62ibQTJMJ1AOfA14yxnQYYw4D7wDh6TMXn2jNxtNmtQzVNJ5kf90JbZ8fKE+ZNYZ63rxIV6KU6ic7QV8KTBGRCSKSBKwA1nbbZi1wY+DxtcCrxhgDVAGXiCUVWAzsDk3pQSSnQ1szYHWrBFg6RYN+QDwuq5tjUmqkK1FK9VOvQW+M8QK3A+uBcmCNMWaniNwjIlcHNnsIGCkiFcCdQGcXzPuANOB9rD8Yjxhjtof4GE5LTj/VdPNOxRFGpSUxLVfHSuk3vz8wJrxeGFUqmtnqqGyMWQes67bs7i6PW7G6UnZ/XXOw5WGTlAZtxzHG8HbFUZZMHoXV+Uf1S/1+qzeTBr1SUc1ZVymTM6C9maMn2jnS3MbcghGRrii6eVzWb538Q6mo5rCgT4O2JqrqWwAYP3J4hAuKcp4ySEy1hgtWSkUthwW91UZfrUEfGm6XNfFHXHykK1FKDYADg76ZqqNW0BdkadD3m7cNDu7QMeGVcgBnBX3gYmxVfQu5GcmkJOqZaL8d3AH+Dr0Qq5QDOCvokzPA14bn6DEKs/VsfkA674jVMeGVinoOC3qrz/zR+nrGadAPjNsFaWMgQ+fZVSraOWvA7+Q0AFqON1CYbWPSCm87NB88/TwtFxKSe97eGGiqAeMbYKFRwF1qdavU+xCUinoOC3rrjD6VVntNN099DipePv18wsVwY/fRHbrY+AC89O0BFhlFFt4U6QqUUiHgyKBPo8Ve0B/eBYUfgfmfh90vQsUr1ll+QlLw7T98AzLHwbKzJtlynrgEmP7JSFehlAoBZwV9UiDo5WTvQe/3wfGDMHcFzL/BGrRrzzo49D7kLzh7e2OsdutJl1jbK6VUlHDkxdishDZy0s/R1g5wos5qa08faz3v7F3S2duku2NuOHFYhwNQSkUdhwW9dTG2MNXX+2BmTYEh9Tt7lWQWWBdj3a7g23eO+6L9ypVSUcZhQW+d0ecP8/a+bfegF7FCvKczek8ZxCdD7uwQFKqUUoPHUUFvEq3JMUYnd/S+8fFa63d6l37i+Qvh6F442XD29u4yGDun5wu1Sik1RDkq6I+e9HHCJDMqsb33jZtqrJ4lqTmnl3U2y9RsOXNbnxdqt2qzjVIqKjkq6KvqW2hmGFkJbb1vfLzWuvMzrst/gs7eNu5uzTd15dDRosMBKKWikqOCvrq+hWYzjAxp7X3jJs/Zt/enZMKoqWe3058a9yVIt0ullBriHBX0VUetM/rhpqX3jZtqIWPs2cvzi60eNsacXuZ2wbBsyJ4YumKVUmqQOCvo61toj08lvuNE7xsfrz3zQmyngoVWH/vGqtPLOifI1nFflFJRyFbQi8gVIrJHRCpE5Kz7/0UkWUSeDqzfKCJFXdbNEZF3RWSniOwQkZTQlX+mqvoWTGBM+nNqbYL25uAjM3ZecO1srmk7DofL9UKsUipq9Rr0IhIP3AdcCcwEVopI96EhbwEajDGTgV8APw68NgF4ErjNGDMLWAbY6PvYP9X1LUhKeu9B370PfVe5s63+8p1BX7MVMHpHrFIqatkZ66YEqDDG7AcQkaeA5cCuLtssB74fePwMcK9Yt6ZeBmw3xmwDMMYcDVHdZ2nz+qhtaiUxOxMaewn644GgTw/SRh+fCGPnwvY1UL8fGqut5Xl6IVYpFZ3sNN3kA9VdnrsDy4JuY4zxAseAkcBUwIjIehHZLCL/GuwNRGSViLhExFVXV9fXYwDgWEsH5+VnkpY+wjqj73oxtbumwM1SwS7GAhTfDOm5cKwaBGsQs9SR/apLKaUizc4ZfbArkN1TtKdtEoClwCKgBXhFRMqMMa+csaExq4HVAMXFxedI6J6Nzkhh7e1L4a2N8IEXvK2QOCz4xqfO6HuYPWneSutHKaUcwM4ZvRsY1+V5AVDT0zaBdvlMoD6w/A1jzBFjTAuwDghvG0hyhvW7rbnnbZpqrO6SiWG7LqyUUkOGnaAvBaaIyAQRSQJWAN2nYVoL3Bh4fC3wqjHGAOuBOSIyPPAH4GLObNsPvSRrBEvamnrepqlW50JVSsWMXptujDFeEbkdK7TjgYeNMTtF5B7AZYxZCzwEPCEiFVhn8isCr20QkZ9j/bEwwDpjzIthOhZLYARL2s9xRn+8RoNeKRUzbM0wZYxZh9Xs0nXZ3V0etwLX9fDaJ7G6WA6OzqA/VxfLploYO29w6lFKqQhz1J2xwKnJR3oMem+7NVOUntErpWKEA4O+l4uxzQet38H60CullAM5L+h7uxh7qg9991sBlFLKmZwX9L210Xf2oe/pZimllHIY5wV9UiogPfe6aTrH8AdKKeVAzgt6Eeusvqcz+qYaSEiBYVmDW5dSSkWI84Iezh30xwM3S+nY8kqpGGGrH33U6R709ftPTyRS90HPY9wopZQDOTPou04+4vfB7y6Bkw2n18//QmTqUkqpCHBm0Cenn74Ye+QDK+Qv/jZMXGYtG3NepCpTSqlB59ygPx7oL+92Wb9nXws5UyNXk1JKRYjzL8Z6yiA5E0ZOjmxNSikVIQ4O+kDTjccF+fMhzpmHqpRSvXFm+iWlWUMgtJ+AQ7sgXyf2VkrFLmcGfXI6YKDqXTA+yF8Y6YqUUipiHBz0wP7Xrd8a9EqpGObsoN/3OmSOg/TciJajlFKR5OygP7RDz+aVUjHP2UEPUKAXYpVSsc2ZQd85+QjoGb1SKubZCnoRuUJE9ohIhYjcFWR9sog8HVi/UUSKuq0vFJFmEflmaMruRecZvcTD2LmD8pZKKTVU9Rr0IhIP3AdcCcwEVorIzG6b3QI0GGMmA78Aftxt/S+Avw68XJs6g370zMBEJEopFbvsnNGXABXGmP3GmHbgKWB5t22WA48FHj8DXCpiDfguIp8G9gM7Q1OyDZ1BX6DNNkopZSfo84HqLs/dgWVBtzHGeIFjwEgRSQW+DfzgXG8gIqtExCUirrq6Oru19yxxGFz6PSi5deD7UkqpKGcn6INNxWRsbvMD4BfGmB4mcA1saMxqY0yxMaY4JyfHRkk2XHgn5HZvYVJKqdhjZ5hiNzCuy/MCoKaHbdwikgBkAvXA+cC1IvITYATgF5FWY8y9A65cKaWULXaCvhSYIiITAA+wAvhct23WAjcC7wLXAq8aYwxwYecGIvJ9oFlDXimlBlevQW+M8YrI7cB6IB542BizU0TuAVzGmLXAQ8ATIlKBdSa/IpxFK6WUsk+sE++ho7i42LhcrkiXoZRSUUVEyowxQYcCcOadsUoppU7RoFdKKYfToFdKKYfToFdKKYcbchdjRaQOODCAXYwCjoSonGgRi8cMsXncsXjMEJvH3ddjHm+MCXrH6ZAL+oESEVdPV56dKhaPGWLzuGPxmCE2jzuUx6xNN0op5XAa9Eop5XBODPrVkS4gAmLxmCE2jzsWjxli87hDdsyOa6NXSil1Jiee0SullOpCg14ppRzOMUHf2wTmTiEi40TkNREpF5GdIvL1wPJsEXlZRPYGfmdFutZQE5F4EdkiIi8Enk8ITEa/NzA5fVKkaww1ERkhIs+IyO7AZ36B0z9rEfmXwL/t90XkDyKS4sTPWkQeFpHDIvJ+l2VBP1ux/DqQb9tFZEFf3ssRQW9zAnOn8ALfMMbMABYD/xQ41ruAV4wxU4BXAs+d5utAeZfnP8aawWwK0IA1Sb3T/Ap4yRgzHZiLdfyO/axFJB+4Ayg2xszGGhp9Bc78rB8Frui2rKfP9kpgSuBnFXB/X97IEUGPvQnMHcEYU2uM2Rx4fBzrf/x8zpyg/THg05GpMDxEpAD4JPBg4LkAl2BNRg/OPOYM4CKs+R4wxrQbYxpx+GeNNU/GsMBsdcOBWhz4WRtj3sSav6Ornj7b5cDjxvIeMEJExtp9L6cEvZ0JzB1HRIqA+cBGINcYUwvWHwNgdOQqC4tfAv8K+APPRwKNgcnowZmf+USgDngk0GT1oIik4uDP2hjjAX4GVGEF/DGgDOd/1p16+mwHlHFOCXo7E5g7ioikAX8C/tkY0xTpesJJRD4FHDbGlHVdHGRTp33mCcAC4H5jzHzgBA5qpgkm0Ca9HJgA5AGpWM0W3Tnts+7NgP69OyXo7Uxg7hgikogV8v9njHk2sPhQ51e5wO/DkaovDJYAV4tIJVaz3CVYZ/gjAl/vwZmfuRtwG2M2Bp4/gxX8Tv6sPwZ8aIypM8Z0AM8CH8H5n3Wnnj7bAWWcU4L+1ATmgavxK7AmLHecQNv0Q0C5MebnXVZ1TtBO4PdfBru2cDHGfMcYU2CMKcL6bF81xnweeA1rMnpw2DEDGGMOAtUiMi2w6FJgFw7+rLGabBaLyPDAv/XOY3b0Z91FT5/tWuCLgd43i4FjnU08thhjHPEDfAL4ANgHfDfS9YTxOJdifWXbDmwN/HwCq836FWBv4Hd2pGsN0/EvA14IPJ4IbAIqgD8CyZGuLwzHOw9wBT7v54Asp3/WwA+A3cD7wBNAshM/a+APWNchOrDO2G/p6bPFarq5L5BvO7B6Jdl+Lx0CQSmlHM4pTTdKKaV6oEGvlFIOp0GvlFIOp0GvlFIOp0GvlFIOp0GvlFIOp0GvlFIO9/8BkbYrW00tPd8AAAAASUVORK5CYII=\n", 163 | "text/plain": [ 164 | "
" 165 | ] 166 | }, 167 | "metadata": { 168 | "needs_background": "light" 169 | }, 170 | "output_type": "display_data" 171 | } 172 | ], 173 | "source": [ 174 | "import matplotlib.pyplot as plt\n", 175 | "plt.plot(train_scores)\n", 176 | "plt.plot(test_scores)\n", 177 | "plt.show()" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "# Sigmoid Unit + Cross Entropy Cost" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 9, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "def fit2(net, criterion, optimizer):\n", 194 | " loss_scores = []\n", 195 | " test_scores = []\n", 196 | " train_scores = []\n", 197 | " for epoch in range(100): # loop over the dataset multiple times\n", 198 | "\n", 199 | " # get the inputs\n", 200 | " inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n", 201 | " vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n", 202 | " # zero the parameter gradients\n", 203 | " optimizer.zero_grad()\n", 204 | "\n", 205 | " # forward + backward + optimize\n", 206 | " outputs = net(inputs)\n", 207 | " loss = criterion(outputs, labels.long())\n", 208 | " loss.backward()\n", 209 | " optimizer.step()\n", 210 | "\n", 211 | " # print statistics\n", 212 | " loss_scores.append(loss.item())\n", 213 | " train_scores.append(predict(training_data, net))\n", 214 | " test_scores.append(predict(test_data, net))\n", 215 | " print('Finished Training')\n", 216 | " return loss_scores, train_scores, test_scores" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": 10, 222 | "metadata": {}, 223 | "outputs": [ 224 | { 225 | "name": "stdout", 226 | "output_type": "stream", 227 | "text": [ 228 | "Finished Training\n" 229 | ] 230 | } 231 | ], 232 | "source": [ 233 | "import torch.optim as optim\n", 234 | "net2 = Net()\n", 235 | "criterion = nn.CrossEntropyLoss()\n", 236 | "optimizer = optim.SGD(net2.parameters(), lr = 1e-1)\n", 237 | "loss_scores2, train_scores2, test_scores2 = fit2(net2, criterion, optimizer)" 238 | ] 239 | }, 240 | { 241 | "cell_type": "code", 242 | "execution_count": 11, 243 | "metadata": {}, 244 | "outputs": [ 245 | { 246 | "data": { 247 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAeKUlEQVR4nO3de3hc9Z3f8fd3ZnS/W5Kvki3ZCIwwvoAgEBKaJU5rcrHThhDyJE1o05JNQ0I2293Csy1Pl3SfZxO2ubVsGpYkm02zIYRmEy8hccJtuSwGi2BsfBc2tuWbZNmWLOs6M9/+MSMxNpI1tiWP5szn9TzzzJxzfjPne55jf+an3zlzjrk7IiKS/UKZLkBERCaHAl1EJCAU6CIiAaFAFxEJCAW6iEhARDK14pqaGm9oaMjU6kVEstIrr7xy1N1rx1qWsUBvaGigtbU1U6sXEclKZrZ3vGUachERCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkILIu0FvfPMZXf7MdXfZXROR0WRfom9q7+c4zb9B1aijTpYiITCtZF+iNtSUAvHn0VIYrERGZXrIv0KsTgb5bgS4icpqsC/S6qiIiIVMPXUTkDFkX6JFwiPkzinmzS4EuIpIq6wIdoKGmhN2dCnQRkVTZGejVJezt6tOpiyIiKbIy0BtrS+gfjnGkZzDTpYiITBvZGeijZ7r0ZrgSEZHpIysDvaGmGIA3j/ZluBIRkekjKwN9bkUR+ZGQznQREUmRlYEeChkN1cU600VEJEVWBjokznRRD11E5C1ZG+iNtSXs6+ojFtepiyIikM2BXl3CUCzOwRP9mS5FRGRayNpAb6hJnLq4R9d0EREBsjjQG5OBrnF0EZGErA30mWUFFOeH1UMXEUnK2kA3MxqqSxToIiJJWRvokBh20XXRRUQS0gp0M1tlZjvMrM3M7h6nza1mttXMtpjZ309umWNrrClh//F+hmPxi7E6EZFpLTJRAzMLAw8A7wPagQ1mttbdt6a0aQLuAW5w9+NmNnOqCk7VWFNCLO7s7erjkpmlF2OVIiLTVjo99GuBNnff7e5DwMPAmjPa/EfgAXc/DuDuHZNb5tgum10GwI7DJy/G6kREprV0An0esD9luj05L9WlwKVm9oKZrTezVZNV4NlcMrOUcMjYfrjnYqxORGRam3DIBbAx5p35e/sI0AS8B6gDnjOzJe5+4rQPMrsDuANg/vz551zsmQrzwjTWlLBdPXQRkbR66O1Afcp0HXBwjDa/dPdhd98D7CAR8Kdx9wfdvcXdW2pra8+35tNcNrtMPXQREdIL9A1Ak5k1mlk+cBuw9ow2vwD+AMDMakgMweyezELHc/nsMvYf66d3MHoxViciMm1NGOjuHgXuBNYB24BH3H2Lmd1nZquTzdYBXWa2FXga+BN375qqolNdNrsc0IFREZF0xtBx98eBx8+Yd2/Kawe+nHxcVItTznS5ekHVxV69iMi0kdW/FAWoqyqitCCicXQRyXlZH+hmxqWzSnWmi4jkvKwPdIDFc8rZfqiHxMiPiEhuCkagzy6jZyDK4Z6BTJciIpIxAQn0xJku2w9p2EVEclcgAv2yWYkzXTSOLiK5LBCBXlGcx9yKQp3pIiI5LRCBDolLAOjHRSKSywIT6IvnlNPW0ctQVDe7EJHcFJxAn11GNO60dfRmuhQRkYwITKBfMbcCgNcPdme4EhGRzAhMoC+sKaEkP8zrBxToIpKbAhPooZBxxdwKNivQRSRHBSbQAa6sq2DrwR6iMR0YFZHcE6xAn1fBYDTOLh0YFZEcFKxAr0scGNWwi4jkokAFemN1CaUFETa3K9BFJPcEKtBDIaN5brl66CKSkwIV6ABL51Ww7VAPwzowKiI5JnCBfmVd8sDoER0YFZHcErxAn5f8xaiGXUQkxwQu0BtGDowq0EUkxwQu0BO/GC1nkwJdRHJM4AIdYGmdDoyKSO4JZKAvmVfBUDTOziO64YWI5I5ABvqyukoAXtuvYRcRyR2BDPQF1cVUFeexcf/xTJciInLRpBXoZrbKzHaYWZuZ3T3G8tvNrNPMNiYf/2HyS02fmbFifhWv7juRyTJERC6qCQPdzMLAA8DNQDPwcTNrHqPpT919efLx0CTXec5W1Feyq6OX7v7hTJciInJRpNNDvxZoc/fd7j4EPAysmdqyLtyK+VUAbGpXL11EckM6gT4P2J8y3Z6cd6aPmNkmM3vUzOrH+iAzu8PMWs2stbOz8zzKTd/S+grM0LCLiOSMdALdxpjnZ0z/I9Dg7kuBJ4AfjvVB7v6gu7e4e0ttbe25VXqOygvzuKS2lI37FegikhvSCfR2ILXHXQccTG3g7l3uPpic/Bvg6skp78KsmF/Jq/uO437m94+ISPCkE+gbgCYzazSzfOA2YG1qAzObkzK5Gtg2eSWevxXzqzjeN8zerr5MlyIiMuUiEzVw96iZ3QmsA8LA9919i5ndB7S6+1rgi2a2GogCx4Dbp7DmtK2Yn/iB0av7j9NQU5LhakREptaEgQ7g7o8Dj58x796U1/cA90xuaReuaWYZJflhXt13gn+9oi7T5YiITKlA/lJ0RDhkLKuv1IFREckJgQ50gOX1lWw92MPAcCzTpYiITKnAB/qK+VVE464bXohI4OVAoCcOjP5+ry7UJSLBFvhAryktYGFNCRveVKCLSLAFPtABWhqqeGXvMeJx/cBIRIIrRwJ9Bsf7htl9tDfTpYiITJmcCPRrGmYAaNhFRAItJwK9obqYmtJ8Nrx5LNOliIhMmZwIdDOjZcEMWtVDF5EAy4lAh8SB0X3H+jjSM5DpUkREpkTOBPrIOLp66SISVDkT6M1zyynKC2scXUQCK2cCPS8cYsX8Slr3KtBFJJhyJtAhcT761oM9nBwYznQpIiKTLqcC/ZqGKuKuG0eLSDDlVKCvmF9FOGS8vEfDLiISPDkV6KUFEa6cV8GLu7syXYqIyKTLqUAHuH5RNa/tP0HfUDTTpYiITKqcC/TrFlYTjbvORxeRwMm5QG9ZUEUkZBp2EZHAyblALymIsKy+kvUKdBEJmJwLdIDrFs5gU3s3vYMaRxeR4MjJQL9+YQ2xuNOqywCISIDkZKBfvaCKvLDG0UUkWHIy0Ivywyyvr2T9Gwp0EQmOnAx0gOsXVrP5QLeu6yIigZFWoJvZKjPbYWZtZnb3WdrdYmZuZi2TV+LUuG5RNXFHl9MVkcCYMNDNLAw8ANwMNAMfN7PmMdqVAV8EXprsIqfCVfOryI+EeKFNwy4iEgzp9NCvBdrcfbe7DwEPA2vGaPcV4GtAVtzjrTAvzLUNM3huV2emSxERmRTpBPo8YH/KdHty3igzWwHUu/tjZ/sgM7vDzFrNrLWzM/NBeuOlNew80svh7qz4DhIROat0At3GmOejC81CwDeAP57og9z9QXdvcfeW2tra9KucIu9uStSgXrqIBEE6gd4O1KdM1wEHU6bLgCXAM2b2JnAdsDYbDowunl1GbVkBz+46mulSREQuWDqBvgFoMrNGM8sHbgPWjix09253r3H3BndvANYDq929dUoqnkRmxrubanh+VyfxuE/8BhGRaWzCQHf3KHAnsA7YBjzi7lvM7D4zWz3VBU61G5tqOd43zOsHuzNdiojIBYmk08jdHwceP2PeveO0fc+Fl3XxvKupBoDndh1laV1lhqsRETl/OftL0RE1pQVcMbecZ3fqwKiIZLecD3RInO3yyt7jupyuiGQ1BTqJ89GjcdfFukQkqynQSVxOtzg/zDM7OzJdiojIeVOgAwWRMO+6pIantnXgrtMXRSQ7KdCTVjbP4mD3AFsP9WS6FBGR86JAT7pp8UzM4ImtGnYRkeykQE+qKS1gRX0lT2w7kulSRETOiwI9xcrmWWw+0K2rL4pIVlKgp3jf5bMAeHK7eukikn0U6CkumVnK/BnFPLFVgS4i2UeBnsLMWHn5LF54o4u+If1qVESyiwL9DCubZzIUjfOcrpEuIllGgX6GaxpmUF4Y4bdbNOwiItlFgX6GvHCIlc2z+N3WwwxF45kuR0QkbQr0MXxo6Vx6BqK616iIZBUF+hhuuKSGiqI8Htt0KNOliIikTYE+hvxIiFVXzOZ3W48wMBzLdDkiImlRoI/jg8vm0DsY5ZkdGnYRkeygQB/H9QurqS7J57FNBzNdiohIWhTo44iEQ6xaMpsnt3XoR0YikhUU6GfxgaVz6B+O8fR2DbuIyPSnQD+LdzRWU1tWwNrXDmS6FBGRCSnQzyIcMtYsm8tT2zvo6h3MdDkiImelQJ/ArdfUMxxz/uFV9dJFZHpToE/g0lllLK+v5JHW/bqBtIhMa2kFupmtMrMdZtZmZnePsfwPzWyzmW00s+fNrHnyS82cW1vq2Xmkl03t3ZkuRURkXBMGupmFgQeAm4Fm4ONjBPbfu/uV7r4c+Brw9UmvNIM+uGwOhXkhHmndn+lSRETGlU4P/Vqgzd13u/sQ8DCwJrWBu/ekTJYAgRqbKC/M4/1L5rB240H6h3QpABGZntIJ9HlAate0PTnvNGb2eTN7g0QP/YtjfZCZ3WFmrWbW2tmZXed233pNPScHo/xmiy7YJSLTUzqBbmPMe1sP3N0fcPdFwH8B/utYH+TuD7p7i7u31NbWnlulGfaOxhksqC7mJy9r2EVEpqd0Ar0dqE+ZrgPOdoGTh4EPX0hR05GZ8Yl3zOflPcfYerBn4jeIiFxk6QT6BqDJzBrNLB+4DVib2sDMmlImPwDsmrwSp4+PtcynOD/MD17Yk+lSRETeZsJAd/cocCewDtgGPOLuW8zsPjNbnWx2p5ltMbONwJeBT09ZxRlUUZzHLVfX8cuNBzmqX46KyDQTSaeRuz8OPH7GvHtTXt81yXVNW7e/s4G/e3EvP16/j7tWNk38BhGRi0S/FD1HC2tLuWnxTH60fi+DUZ3CKCLThwL9PPz7Gxo52jvIY6/pFEYRmT4U6OfhhkuquXRWKQ89v0fXdxGRaUOBfh7MjM/euIhth3r43dYjmS5HRARQoJ+3Ncvn0lBdzDef2KVeuohMCwr08xQJh/jCTU1sPdTDb9VLF5FpQIF+AdYsn0tjTQnfUi9dRKYBBfoFSPTSL2HroR7WbVEvXUQyS4F+gVYvS/TSv/nETmJx9dJFJHMU6BcoEg7xR++7lO2HT/Iz3QBDRDJIgT4JPrR0Di0Lqvir3+7g5MBwpssRkRylQJ8EZsa9H2rmaO8Q//uptkyXIyI5SoE+SZbWVfLRq+v4/gt72HP0VKbLEZEcpECfRH+y6jLywyH+4ldbM12KiOQgBfokmllWyBfe28QT2zp4fLMu3CUiF5cCfZJ95l2NLJlXzn/7xescOzWU6XJEJIco0CdZXjjE/bcso2dgmP++dkumyxGRHKJAnwKXzynnCzc1sfa1g6zbcjjT5YhIjlCgT5HPvWcRzXPK+bN/eF33HxWRi0KBPkXywiG+/rFlnBwY5ksPb9RlAURkyinQp9Di2eXct+YKnm87yref3JXpckQk4BToU+zWlno+clUd335qF8/u7Mx0OSISYAr0KWZm/I8PL+HSmWV86acb2X+sL9MliUhAKdAvgqL8MH/9yauIxuL8u7/dQHefLuAlIpNPgX6RLKot5cFPtbC36xR3/KiVwWgs0yWJSMAo0C+i6xZWc/8ty3hpzzH+9NFNxHXmi4hMokimC8g1H14xjwMn+rl/3Q7KCiN8Zc0SzCzTZYlIAKTVQzezVWa2w8zazOzuMZZ/2cy2mtkmM3vSzBZMfqnB8Z/es4jP3riQ/7t+H/f+cotuMC0ik2LCHrqZhYEHgPcB7cAGM1vr7qnXiH0VaHH3PjP7HPA14GNTUXAQmBl337wYBx58djdm8Oerr1BPXUQuSDpDLtcCbe6+G8DMHgbWAKOB7u5Pp7RfD3xyMosMIjPjnpsXA4lQ7x2M8pf/Zin5ER3WEJHzk06gzwNS737cDrzjLO0/A/x6rAVmdgdwB8D8+fPTLDG4RkK9JD/CN57YSefJQf76E1dRVpiX6dJEJAul0x0caxxgzEFfM/sk0ALcP9Zyd3/Q3VvcvaW2tjb9KgPMzLhrZRP337KUF9/o4tbvrufgif5MlyUiWSidQG8H6lOm64CDZzYys5XAnwGr3V2XFzxHH22p5/u3X8P+Y3188H89zwttRzNdkohkmXQCfQPQZGaNZpYP3AasTW1gZiuA75II847JLzM33HhpLb+88waqS/L5t997iQeebtO56iKStgkD3d2jwJ3AOmAb8Ii7bzGz+8xsdbLZ/UAp8DMz22hma8f5OJnAotpSfvH5G/jA0rncv24Hn/7ByxzuHsh0WSKSBSxT50C3tLR4a2trRtadDdydH7+0j7/41TbywsZXPryE1cvm6tRGkRxnZq+4e8tYy3SO3DRlZnzyugX8+q53c8nMUu56eCN3/OgVDuiAqYiMQ4E+zTXUlPCzP3wn99y8mOd3HWXl//wnvvPMGwxF45kuTUSmGQV6FgiHjM/+i0X87ss38q6mGr76m+38q28+y683H9JlA0RklAI9i9RVFfM3n2rhB7dfQyRkfO7Hv+cj3/ln1u/uynRpIjINKNCz0B8snsmv73o3X/3IlRw40c9tD67n1u++yAttR9VjF8lhOsslyw0Mx/jJy/v4P//0Bkd6BllWX8ln3tXIzUtmkxfW97VI0JztLBcFekAMDMd49JV2HnpuN2929TG3opBPXr+Aj15dT21ZQabLE5FJokDPIfG489T2Dr73/B5e3N1FXtj4l82z+dg19dxwSQ3hkM5jF8lmZwt03bEoYEIhY2XzLFY2z6Kt4yQPv7yfR3/fzq82H2JmWQFrls9lzfJ5XDG3XD9SEgkY9dBzwMBwjKe3d/DzVw/wzI4OhmPOgupibl4yh5uXzObKeRWE1HMXyQoacpFRx08NsW7LYX61+RD//EYXsbgzs6yA914+k5sWz+Kdi6opKdAfbiLTlQJdxnT81BBPbu/gyW1HeHZnJ6eGYuSFjasXVPHuplreuaiaK+dVENHZMiLThgJdJjQYjbFhz3Gea+vk2Z1H2XaoB4CS/DDXNM7gmoYZtCyoYll9JYV54QxXK5K7FOhyzo72DvLS7mO8uPso63cfo62jF4C8sNE8p5zl9ZUsn1/JlfMqWVhTojF4kYtEgS4X7PipIV7Ze5wNe4+xcd8JNh/opm8oBiR68VfMraB5bjnNc8ppnlvOJTNL1ZMXmQI6bVEuWFVJ/ujpkADRWJxdHb1sPtDN6we62Xygm59u2E//cCLkQ5a4UuRls8pomlnKopmlLKpNPIryFfQiU0GBLuclEg5x+ZxyLp9Tzq0tiVvOxuLO3q5TbDt0kh1HTrLz8El2HD7Jui2HSb2T3tyKQhprS2isKaGhuoQF1SUsqC6mvqpYYS9yARToMmnCIWNhbSkLa0v5AHNG5w9GY7x5tI+2jl52d/ay++gpdnf28o+vHaK7f/i0z6gpLaB+RhF1VcXMqyxiXlUR8yoLmVtZxJyKIsoLI/pBlMg4FOgy5QoiYS6bXcZls8vetuxE3xB7u/p4s+sU7cf72X+sj33H+tjUfoLfvH6I4djpx3hK8sPMrihkdkUhs8qTj7ICZpYXMqu8gJllhdSWFWj8XnKSAl0yqrI4n8rifJbVV75tWTzudJwc5GB3PwdPJB6Huwc53NPPoe4BXtp9jCM9A0Tjbz+wX1YQoaasgJrSfGpKC6guzWdGSWJ6RknKI7n+/IjOtZfsp0CXaSsUstHe+FXzq8ZsE487x/qG6OgZpOPkAB0nBznaO0jnycSjq3eIto5e1u8e5ET/MOOd1FWSH6aqJJ+q4nwqi/OoKMqjsjiPyqJ8KooS0+VFeZQXRRKvCxPTZQURnbIp04YCXbJaKGTUlBZQU1pAM+VnbRuNxTnRP0xX7xDHTg1xvG+IrlNDnDg1xPG+YY73DXGib4gT/cO0H++nu3+Y7v5hYmP8BZCqtCBCWeHII4+ywkhyXuJ1SX6E0sIIpQVhSgoilBQklhfnh5PPienCvJCOD8gFUaBLzoiEQ6Phny53p3cwyom+YXoGEgHf0z9Mz0B09PnkwDAnk9OnhqIcO5U4LtA7GKV3IDp6KudEzKA4L0xxMuyL80eewxTlJZ+T84rywhSlPBfmhSjKC1OYfBSd9hyiIPmcH9aXRpAp0EXOwsySPe288/6MaCzOqaEYvYNRTg1GR58TjxinhhLz+odi9A3FODUYpS/5ui+5rPPk4Oi8geHE/An+cBhne6AgEkoEfyRMQV5o9LkgEqIgEh5dXhAJkZ98FIw+h8eczg+HyI8Y+eHw6HtS5+VFjPxwiLyR+eGQhqqmgAJdZIpFwiEqikJUFJ3/l8KZ3J2hWJyBoTj9yYAfGE68Hhh9nD49GI2ftmww+ta8oWicgeE4p4aiHO+LnzZ/KBZnMNn+fL5ExhMOGXlhIy8Z8HnhEHmRxHReKPE6EkosiyTbjbSPjLwOnb4sEg6RF0o8R1KWR0bmhRJtR9YdDr3V7q15ifWOvG+86fDI5ybnTYe/fBToIlnIzJK96TAVTN4XxUSisUTYjwT9UPSt6cFojOGYJ5fFGIr6aJvhWOIx8r7oaLvTl0VjPjov9fVQNPFXTqJNnGg88f5o/K120ZgTjcffdqrrxRIeDXsjbDYa+CFLzgsn54eML628lA8tmzvpNaQV6Ga2CvgWEAYecve/PGP5jcA3gaXAbe7+6GQXKiKZF0n2jkum8W1q3Z1Y3InGffSLIRr30fCPxp1oLBH8sbgzHI8nnmOJ55E2seSXQ9yd4VjiPbHkZyfeG09+VmLeyGfF4yPvf+sRjftb892pLJ6aL+EJA93MwsADwPuAdmCDma11960pzfYBtwP/eSqKFBFJl5klhkfC5NwPzNLpoV8LtLn7bgAzexhYA4wGuru/mVwWn4IaRUQkDen8PG4esD9luj05T0REppF0An2sQ7fnddTBzO4ws1Yza+3s7DyfjxARkXGkE+jtQH3KdB1w8HxW5u4PunuLu7fU1taez0eIiMg40gn0DUCTmTWaWT5wG7B2assSEZFzNWGgu3sUuBNYB2wDHnH3LWZ2n5mtBjCza8ysHfgo8F0z2zKVRYuIyNuldR66uz8OPH7GvHtTXm8gMRQjIiIZootAi4gEhPl4F4ie6hWbdQJ7z/PtNcDRSSwnW+TidufiNkNubncubjOc+3YvcPcxzyrJWKBfCDNrdfeWTNdxseXidufiNkNubncubjNM7nZryEVEJCAU6CIiAZGtgf5gpgvIkFzc7lzcZsjN7c7FbYZJ3O6sHEMXEZG3y9YeuoiInEGBLiISEFkX6Ga2ysx2mFmbmd2d6XqmgpnVm9nTZrbNzLaY2V3J+TPM7Hdmtiv5XJXpWiebmYXN7FUzeyw53WhmLyW3+afJ6wkFiplVmtmjZrY9uc+vz5F9/UfJf9+vm9lPzKwwaPvbzL5vZh1m9nrKvDH3rSV8O5ltm8zsqnNdX1YFesrdk24GmoGPm1lzZquaElHgj939cuA64PPJ7bwbeNLdm4Ank9NBcxeJawaN+CrwjeQ2Hwc+k5Gqpta3gN+4+2JgGYntD/S+NrN5wBeBFndfQuL2lrcRvP39t8CqM+aNt29vBpqSjzuA75zryrIq0Em5e5K7DwEjd08KFHc/5O6/T74+SeI/+DwS2/rDZLMfAh/OTIVTw8zqgA8ADyWnDbgJGLlHbRC3uRy4EfgegLsPufsJAr6vkyJAkZlFgGLgEAHb3+7+LHDsjNnj7ds1wN95wnqg0szmnMv6si3Qc+7uSWbWAKwAXgJmufshSIQ+MDNzlU2JbwJ/CozcyrAaOJG84icEc38vBDqBHySHmh4ysxICvq/d/QDwVyTuR3wI6AZeIfj7G8bftxecb9kW6JN296RsYGalwP8DvuTuPZmuZyqZ2QeBDnd/JXX2GE2Dtr8jwFXAd9x9BXCKgA2vjCU5brwGaATmAiUkhhzOFLT9fTYX/O892wJ90u6eNN2ZWR6JMP+xu/88OfvIyJ9gyeeOTNU3BW4AVpvZmySG0m4i0WOvTP5JDsHc3+1Au7u/lJx+lETAB3lfA6wE9rh7p7sPAz8H3knw9zeMv28vON+yLdBz4u5JybHj7wHb3P3rKYvWAp9Ovv408MuLXdtUcfd73L3O3RtI7Nen3P0TwNPALclmgdpmAHc/DOw3s8uSs94LbCXA+zppH3CdmRUn/72PbHeg93fSePt2LfCp5Nku1wHdI0MzaXP3rHoA7wd2Am8Af5bpeqZoG99F4k+tTcDG5OP9JMaUnwR2JZ9nZLrWKdr+9wCPJV8vBF4G2oCfAQWZrm8Ktnc50Jrc378AqnJhXwN/DmwHXgd+BBQEbX8DPyFxjGCYRA/8M+PtWxJDLg8ks20ziTOAzml9+um/iEhAZNuQi4iIjEOBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJiP8PiSlsGqZTtScAAAAASUVORK5CYII=\n", 248 | "text/plain": [ 249 | "
" 250 | ] 251 | }, 252 | "metadata": { 253 | "needs_background": "light" 254 | }, 255 | "output_type": "display_data" 256 | } 257 | ], 258 | "source": [ 259 | "import matplotlib.pyplot as plt\n", 260 | "\n", 261 | "plt.plot(loss_scores)\n", 262 | "plt.show()" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": 12, 268 | "metadata": {}, 269 | "outputs": [ 270 | { 271 | "data": { 272 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXwUVbr/8c+TnYQEAklYsgCBAAk7hEVERUEFF0BEBfcZFWdGrzp6Z0Ydr844d37X2cfdQUVxVBYVkEEUFVERZQm7hC0LJCFAdrInvZzfHx0xxoQ00Emnu5/368VLqup01VMWfKmcPlVHjDEopZTyfH7uLkAppZRraKArpZSX0EBXSikvoYGulFJeQgNdKaW8RIC7DhwVFWX69u3rrsMrpZRH2rZtW5ExJrq5bW4L9L59+5KWluauwyullEcSkSMtbdMuF6WU8hIa6Eop5SU00JVSyktooCullJfQQFdKKS+hga6UUl5CA10ppbyE28ahK6VUh5OzCTLWtf1xBk2D2DEu360GulJKAexcDKvuBbsVkLY9VnhPDXSllHI5Y+DrZ+CTx6HfRXDDmxAS4e6qzooGulLKd9nt8Mn/wDfPwZBr4Jp/QUCwu6s6axroSinfZK2H9++BPctg3N0w7Snwa5txIuW1FvYfq2D/8XL2Hatg9uhYxvbt5vLjaKArpXxPXSUsuxUy18GUx2HSgyDn3m9usxsOF1c1Cm9HgB8tqznVpkunQFL7RGqgK6WU04oy4J3boayZlxPa6h2/ZjwLo289o93a7IYjxVUcKakmp7ianJJqjhRXk1NSRU5JNbUWOwD+fkJiVBij+0Ry4/gEUnpFMLhXOD0jQhAX/OPRHA10pZT3OboN3roOEBh5U/N33wOnQeJFre6qvNbCzpwyth0pZXtOKTtyyqiss57aHhLoR0K3UBK6hXFBUjSDeoaT0iuCATGdCQn0d+FJtU4DXSnlXTI+haW3QlgU3LICuvc/bXO73XC0rOZU98jx8hqq621U1dnIK63mwIkKjAE/gUE9I5g5sjcj4rvSLyqMPt1CiQ4PbrM77jOlga6U8h67l8HKn0NMMtz0HoT3OLWpzmojPb/81J12Tkk1RRX1FFXWYbUbwHEj3z0smM7B/oQGBdAjIoTpQ3sxpk8kI+K7EB4S6K4zc4oGulLKO3z9HHz8W6p6ncdXqc+Qua2C3JITHCl29HEfO1lDQ24T360TA6I7k9wzgqjwYOIjQ0nuFc7AHuGEBXtuLHpu5Uop32WzcnLHSvILCjh2spbAEzu5oOx91tjH80D2z6jPPgRA97AgErqHMrZvJAndYknpHcHohEhiIkLcfAJtQwNdKeUxSqvq+XBnNv2/uI/x9ZvoAiQ3bPssfCb7hz7M33t1JTGqMwndQ+nswXfbZ8O3zlYp5TEsNjtfHixk/YECDhdVc6SkiorSQhYE/o1Uv4Ns6P8Q4SNnkRjdmYiwMC4J78El7i7azTTQlVJta+8KyPrCqaYGKK6sI7vIMc67zmJnuL8wqVMgnYMDSO66h8i6fGT2Qi4YOrtt6/ZATgW6iEwDngb8gVeMMU812Z4ALAK6NrR52BizxsW1KqU8iTHw5V9g/R8hpAv4N/+OFANY7YZ6q506qw1jN/QFBgX4EdzZnyB/P8e7D61Ap3C47l2nxo/7olYDXUT8geeBS4E8YKuIrDLGpDdq9hiwzBjzooikAGuAvm1Qr1LKE9ht8OGvYesrMHwuzHwO/L8f8ldntfHutjxW7zrGztwyaiw2AMb368bs0bFMH9aLiA4+RLAjcuYOfRyQYYzJAhCRJcBMoHGgG+C79012AfJdWaRSyo1KsmD1L6Gq2PnP1FdCaTZMvA+m/v7US69qLY4gf2F9BvknaxncM5wbxsYzpuHdJj27eOfok/biTKDHArmNlvOA8U3a/A74WET+CwgDpja3IxGZD8wHSEhIONNalVLt7dguePNasFmgz/ln9tlJD8CY28kuqmLdvhNsOFTEluwSaiw2xvSJ5E9zhjNpQFSHecrSGzgT6M393zZNlucBrxtj/iYi5wH/FpGhxhj7Dz5kzAJgAUBqamrTfSilXM1a5+jLbsovAPyb+etvszi6SwByvoGltzj6v2//AKIHOX3Ywoo61uw5xvLnN7IrtwyA/tFh3DA2nsuG9OC8xO4a5G3AmUDPA+IbLcfx4y6VO4BpAMaYb0QkBIgCClxRpFLqDFnrYNV/we6lzW8P6QJzFsKARj9Mb1sEa34Ftrrv10Unw83vQZfYFg9Va3G88+RwUTVbD5ew4VAR6cfKAUjuFcGjVwzmimG9iIsMdcWZqdNwJtC3Akki0g84CswFbmzSJgeYArwuIslACFDoykKVUk6qq4ClN0PW5zD2Lojo/eM23y6Ht2+AWS/CsOu+H43S7yJInOxoE9gJRsyFTpE/+rjVZmfVrnz+9UUWBwsqTv0QEOgvjE6I5FeXD+KSwTEk9/LMqdw8VauBboyxisi9wFocQxIXGmP2isiTQJoxZhXwEPCyiPwSR3fM7cY093OeUqpNVRbAW3Pg+LeOsB7Z9N6rwdg7YMlNsPwu2P4GHN7Q7GiUxuqtdjILK9meU8orG7LJLqoiuVcED0wZSJ/uocR3C2VwT89+F4qnE3flbmpqqklLS3PLsZXySiVZ8O/ZUHEcrl8EAy8/fXtLLayYD+nv/2g0CkBNvY3N2cV8daiIrzOLOVRQgcXmyIvkXhE8MDWJS5N74OenfeHtSUS2GWNSm9um/5Qq5Q2O7YI354DdArf9B+LHtv6ZwBCY8zqUZEJUEgDGGHbmlvHW5hxW786n1mInyN+P1L6R3HlBIsm9IkjuGc6AmM76pWYHpIGulKfL+sLRfRLSBW5ffUajUfDzg6gkKuusvL/zKG9tyiH9WDlhQf5cMyqOaUN7Mq5vNzoFte/MO+rsaKAr5cm+XQ4r7oZu/VsdjdJUcWUd23PK+PxAAe/vzKeyzsrgnuH876yhzBoV63NvKvQGesWU8lSbFzger0+YAPMWNzsapSmrzc5bm3NY9PVhsoqqAAgO8OPK4b24eUIfRsV31a4UD6aBrlRHUl8Nn/0BynJaaVfpGJY4+Cq49hXHEMNWbMoq5ner9rL/eAXj+nY79cj90Ngu7T6ZsWobGuhKdRTVJfD29ZCXBjEpzc9U39h59zpGpjT3xGcje/JO8vS6g3y6r4DYrp146ebRXD6kp96JeyENdKU6gpN5jiGHpYfh+jcgZcY57a6yzsqmzGIWb8lh3f4CunQK5KFLB3LnBYn6BacX00BXyt1qyuDVy6GuHG5ZDn0nndVusgorWbv3BOsPFLD9SClWu6FraCD/fdlAbpvYt8PPWK/OnQa6Uu624W9QfhTu/BTimn1epEXZRVWs3HGUNXuOcaigEoAhvSO468JELhgQxeg+kdo/7kM00JVyp9IjsPklGDHP6TCvtdhYseMoy9Jy2ZFThohjYoibxqdw2ZCe9O7a+hekyjtpoCvlTp/9AcQPLnms1aYlVfX8+5sjvPHNYYqr6hnUI5xHpg9m5shYnRhCARroSrnP0e2w5x244KHTPhCUXVTFq19l8U5aHnVWOxcPimb+hf2ZkNhNR6qoH9BAV6q9GAPZX0J1kWN500sQGgXnP9CkmSGrqIoNBwtZf6CQLw8VEujnxzWjYrnzgn4k9Qh3Q/HKE2igK9Vedr4N7//ih+uufgZCIiitqmdjZhEbDhax4VAh+SdrAejTPZR7Jg/g1ol9iAnXbhV1ehroSrWH754AjR0Ds16k3mZn3aEK1mcHs+3zz8ksdDyGHxESwMT+Ufzi4iguSIqiT/cwNxeuPIkGulLtYdPzUHEM5izki9Ju/P4/e8kqrKJraCBjEiKZPTqO8/p3Z3hsFwL8/Vrfn1LN0EBXqq1VFsBX/6Q6cRr3fxHEJ+lb6Ns9lFdvS+WSwTH6xaZyGQ10pdqY9bP/Qyw1zDp4GXl+Rfx62iDumNSP4AB94Ee5lga6Umcr6wv4/CmwW3+0yeCYwq2spp6Yir28aZ1C8tDRvDE9WceMqzbjVKCLyDTgaRyTRL9ijHmqyfZ/ABc3LIYCMcaYrq4sVKkOxVIDK38BxgbRg79fbTccO1nD8ZO11FrsiEBW2GSGzfwTtw3u78aClS9oNdBFxB94HrgUyAO2isgqY0z6d22MMb9s1P6/gFFtUKtSHcemF6E8zzF/Z78LyS+rYcGXWSzdmkuNxcakAVHMGNGbKckxdO8c7O5qlY9w5g59HJBhjMkCEJElwEwgvYX284AnXFOeUh1QVRF89Q8YOI2TPc7jhQ/38frGw9iNYcYIx8M/yb0i3F2l8kHOBHoskNtoOQ8Y31xDEekD9AM+a2H7fGA+QEJCwhkVqlSH8cWfMfVVrOg+n9//ZT3ltRauGRnLg5cNJC4y1N3VKR/mTKA3N6bKtNB2LvCuMcbW3EZjzAJgAUBqampL+1Cq4yrKwL71VT4IvJQH19cyaUAUj16RTEpvvSNX7udMoOcB8Y2W44D8FtrOBe4516KU6igKKmp5Jy2PHTml2E4e47elj9HTHsACvxv41y1juCylh44jVx2GM4G+FUgSkX7AURyhfWPTRiIyCIgEvnFphUq1s3qrna8zi3hnWx5rvz2O1W6YEnWSP9f8jnApZ/2of/DOlTN14gjV4bQa6MYYq4jcC6zFMWxxoTFmr4g8CaQZY1Y1NJ0HLDHGaFeK6liMgX3/cTx632ITw6GCSpaX9uOt7M5U1Frp0imQ2yf25Sd9i4n94F4I9oObP2R6bx3EpTomcVf+pqammrS0NLccW/kQuw0+eAi2veZU8xqCWNznD/SZcA3nD4gi5PB6WHYLhEXDLSugu44lV+4lItuMMc1Ob6VPiirvZamF5Xc67s4n/RIm3veDzTtzS/nzRwdIP1ZOfLdO/Hx8NNP2P8JPcx6FkeGQHuB43W1MMtz0HoT3cNOJKOUcDXTlPSoLHI/i15U7lgv3w/E9MO0pmPDzU80Kymt56sP9LN9xlJ4RIfx2zvlcMyrW8ZbDCath6S3fv7e87wUw920I0VEsquPTQFfeoSQL/j0byvO/n87NPwjmLMQMmc2u3DI+P1DAhkNF7Mwtw1+EX0zuzz0XDyAsuNFfg+BwuHEZfPQbxztarvgrBOiTnsozaKArz3dsF7x5rSOAb/8A4scCUFFrYeWOo7z19Ab2H69ABIbHduFnFyVy3Zh4+ka1MHlEQBBc9Y92PAGlXEMDXXVcBz6EtY+CsZ++XcVxCIvGMu8dDtlj2fhlFl8eKmRLdgl1VjtDYyP4v9nDmD60J11Dg9qndqXcQANddVwbn4H6Kkic/IPV1fU2ckursdjs2OyGys7DecnMZvOzh6m3ZQEwIKYzN45P4JpRsQyP0xd/Kt+gga46ppJsyPkapjwOFzwEwMlqCy98nsFr2w9Tb7UT4CeEBvkTHhLIgJjO/GRwOMk9Ixif2I1eXTq5+QSUan8a6Kpj2r0MEKxD5rAlo4i1e4+zcme+40VYo2J58FJ9EZZSTWmgq47HGGw73yY3YgyznjtAWbWFkEA/Lh4Uw31TkvTVtEq1QANddRjGGI4UV/Pluv9wa9lhnrVM4/whUVw9vDcXDowiNEj/uCp1Ovo3RLnd5qxiVu7MZ8OhQvJKa/h/gUupCwjhnl88SGKsPp2plLM00JXbfJNZzD8/Pcjm7BI6BwdwXv/u/Pz83sz9Mg2/wbM0zJU6Qxroql0ZY/gmq5h/fnqILdklxIQH8+yFNqYHbiPAT6AgD+rLYeQ8d5eqlMfRQFdtzmY3HC6uYm9+OW9+c4Qth0voERHME1encFP4DoLevxtsFhA/xwd6Dne8Q0UpdUY00FWbsNsNnx8sYOFXh0k7UkKtxfG0Z4+IYH4/Ywg3jI0nZOdrsPy/IX4czFsCod3cXLVSnk0DXblEvdVOXmk1uQXFRH3zR+qO7Seo3sr9Af507xpEWHAAYUH+hAYF4HcI2F8POd/AwOkwZyEE6Zhypc6VBro6J1abnTc3HeHvnxxEast4NeivJMsh9gcMok90KJFhgfidmmfcBrZG84effz9c8jj46x9DpVxB/yaps/Z1ZhG/X5XOgRMVzOxn5w+Vf6JzVS4VV75M8ug5OnmyUu3MqUAXkWnA0zjmFH3FGPNUM22uB34HGGCXMeZHE0mrDmzzAtjyL5jxHPQ5r/k22Rtgxc+wVZdgsdkZYTesECEozA//E/VIYCjc8h5d+l3YvrUrpQAnAl1E/IHngUuBPGCriKwyxqQ3apMEPAKcb4wpFZGYtipYuZgxsO5J+OrvENAJ/j0L5rwGg6/4Ybu9KzHv3UW+X08+qL2YToH+jOjThZReEY7hhn7+MHwu9Ehxz3kopZy6Qx8HZBhjsgBEZAkwE0hv1OYu4HljTCmAMabA1YWqFtRVQFnu98uRfZ3/gtFmhdX3w443YcztcPFvYfFcWHoTTPsT9J0EgD3zM+Tjx9huBnK//WF+Mm0Uc8Yl0CnI3+Wno5Q6e84EeizQKDHIA8Y3aTMQQEQ24uiW+Z0x5iOXVKhalr8T3poDVYXfr+sSDzcvh+iBp/9sfTW8+xM4+BFc9BuY/AiIwK2rYNmt8OGvTjX1Az6xjWZ5vydZPmcsMREhbXM+Sqlz4kygN/fNlmlmP0nAZCAO2CAiQ40xZT/Ykch8YD5AQkLCGRerGslcD0tvhk6RMPsV8A8ESw188jgsvAxuehfiUpv/bHUJvH0D5G11zJk57q7vtwV3hhuXUrz7Qz7Yls2m7GJMYCiXXH0DL4ztq190KtWBORPoeUB8o+U4IL+ZNpuMMRYgW0QO4Aj4rY0bGWMWAAsAUlNTm/6joJz17Xuw/G6ISoKb34OI3t9vSxjvmCx50dVw8aMQ0uWHnzUGNr3gmFT5utdhyKwfbD5ZbeGFLzJ4fWMgxgzktol9uOfiATp1m1IewJlA3wokiUg/4CgwF2g6gmUlMA94XUSicHTBZLmyUNVg00uOGekTJsK8xdCpyfRq3RLhjo/hrevg48ea30dwF0e3TL/vH68/UV7Lqp35PLc+wzGJxMhYHrxMJ5FQypO0GujGGKuI3AusxdE/vtAYs1dEngTSjDGrGrZdJiLpgA34lTGmuC0L9zmNR6MMvgqufQUCW5hmrXMM3PUZVBxrfnunSAgKY1duGQs3ZpN2uJSjZTUAXDQwmt9MG0xKb51EQilPI8a4p+cjNTXVpKWlueXYHZbNCuv/CNVFP952Mg8yP3OMRrny745hgmep3mrnmXWHeOHzDLp0CuS8/t0ZnRDJhMTuDI3t0voOlFJuIyLbjDHNfkGmT4p2JBmfOO7Aw6LBr8mlET+4+DG48L8do1HO0oHjFdy/ZAf7j1dw3Zg4/ufqFCJCAs+xcKVUR6CB3pHsWgyhUfDgPseoFRdbvj2PR1fsoXNwIK/cmsrUFJ1AQilvooHeUdSUwoEPIfUOl4d5rcXGk6vTeXtzDhMSu/HMvFHEhOtYcqW8jQZ6R/HtcrDVw4i5Lt1tRa2FO15PY8vhEn4+uT8PXTqQAH8/lx5DKdUxaKB3FLuWQHQy9Brhsl2WVtVz22tbSM8v55l5o5gxonfrH1JKeSy9VesIijMhb4tjHk0XPYlZUFHL3AWb2H+8gn/dMkbDXCkfoHfoHcGuxY5RLMOuP+ddGWNYtSufJ/+TTo3Fxuu3j2XigCgXFKmU6ug00N3NboddSyFxMkT0OqddZRVW8vv/pPPFwUJGxHXhT3OGM7inPiCklK/QQHcnux3WPgInc2DqE2e1i5PVFlbvyWfF9qOkHSklNMifJ65O4dbz+uLvpy/SUsqXaKC7i7UOVvwM9i6HCffAkNlnvIsPdh/jsZV7KK22kBTTmV9PG8S1o+Pooa+3VconaaC7Q22549W32V/ApU/CxPvO6MvQ0qp6/uf9b1m9+xgj4rrw+k+GMjyui77aVikfp4He3ioL4M1r4cRemPUijHR+6tWKWgtvfHOElzdkUVVn5VeXD+LuCxN1XLlSCtBAb18lWY53lVeegHlLYOBlTn2sotbC6xsP88pX2ZyssXDJ4Bh+dfkgknvpF55Kqe9poLeX0sPw6uVgtzimeYsf2+pHyhuC/NVGQX7/lCRGxHdt9bNKKd+jgd5eNi9wvK/l5xshetBpmxpjeHdbHn9cs4+yagtTk2O4b0oSw+M0yJVSLdNAbw82K+xZBoOmtRrm2UVVPLp8D99kFZPaJ5Inrh7CsDh9R7lSqnUa6O0hcx1UFcKIeT/aVFNvY+HGbHbllrH/eAU5JdWEBwfwx2uGMm9sAn46llwp5SQN9PawazF06gYDLv3B6oKKWu5alMauvJMkRocxLLYL16fGcV1qvI4lV0qdMQ30tlZTBvvXOKaOCwg6tfrA8Qp++vpWSqrqefnWVC7VySaUUufIqQHMIjJNRA6ISIaIPNzM9ttFpFBEdjb8utP1pXqo9JVgqzv1nvOjZTX8de0B5rz4NRabnWV3n6dhrpRyiVbv0EXEH3geuBTIA7aKyCpjTHqTpkuNMfe2QY2ebediTPRg1pf35q1PtrL+QAEGmDI4hidnDqV3107urlAp5SWc6XIZB2QYY7IARGQJMBNoGuiqiYpjBwnP3cSLAbfw50XbiOoczC8mD2DuuHjiIkPdXZ5Syss4E+ixQG6j5TxgfDPtrhWRC4GDwC+NMblNG4jIfGA+QEJCwplX60G255SSvehxZho/0qMu5/kZo7lsSA8C9TF9pVQbcSbQmxs3Z5os/wdYbIypE5GfAYuAS370IWMWAAsAUlNTm+7DK9jshpe+yGTFJ5/zUdDHlCTfyHNzr3Z3WUopH+BMoOcB8Y2W44D8xg2MMcWNFl8G/nTupXmeOquN+xfv5KO9x3m/+wr8LZ2Iuep37i5LKeUjnPn5fyuQJCL9RCQImAusatxARBpPtTMD2Oe6Ej1DTb2Nu97Yxkd7j/PsxBpGVH2FTHoAOke7uzSllI9o9Q7dGGMVkXuBtYA/sNAYs1dEngTSjDGrgPtEZAZgBUqA29uw5g6notbCHYvS2Hq4hKeuGcrVu38C4b0dE1copVQ7cerBImPMGmBNk3WPN/r9I8Ajri3NM1TVWbn9ta3szC3jnzeMZGbgFji6DWa+AEE6kkUp1X50yMU5qLXYuHNRGjtySnl23ihmjoyFtIUQ2ffUg0RKKdVeNNDPUp3Vxt3/3sam7GL+dv0IrhjWC8pyIXsDjLgR/PzdXaJSysdooJ8Fi83OfYt38MXBQv44axjXjIpzbNi9FDAw/Hq31qeU8k0a6GfIZjc8tGwXa/ee4PGrUrhxfMMDUsbAriWQMBG69XNvkUopn6SBfgbsdsOjy/ewalc+v542iJ9OahTcR7dD8SEY+eN3niulVHvQQD8Df//kIEvTcrlvShK/mDzghxt3LYaAEEiZ6Z7ilFI+TwPdSUWVdby8IYuZI3vzy6lJP9xorYNv34XBV0KIThenlHIPneDCSW98fZh6m537piQhIlCWA/tWOzaWHnZMAN3MFHNKKdVeNNCdUF1v5Y1NR7g0uQf9ozs7Jn1+81ooOvh9o8h+kHix+4pUSvk8DXQnLNuaS1m1hbsvSnSs2L7IEeZzFkL/KY51QWHgr/87lVLuownUCqvNzssbskntE8mYPt2grgI+/z/H8MQhs0Gae7uwUkq1P/1StBUf7DnG0bIa7r6ov2PFxqehqhAu+18Nc6VUh6KBfhrGGF7ekEX/6DCmDI6B8nz4+jnHnXncGHeXp5RSP6BdLqex9XApM068yI3hO/B7OgDqK8HYYOoT7i5NKaV+RAP9NL76dCUPBnyALeZC6NrwvpZB0xxvU1RKqQ5GA70FeSWVTM19lvKQGCJuXgaBndxdklJKnZb2obdg2wevMtwvC9vk32qYK6U8ggZ6M6qrK0nNfJbc4AFETrjF3eUopZRTNNCbsf/9vxFLIdUXPaETVSilPIZTgS4i00TkgIhkiMjDp2k3R0SMiKS6rsT2Zax1JB58le2Boxh43tXuLkcppZzWaqCLiD/wPDAdSAHmiUhKM+3CgfuAza4usj1lfL2SruYklSPucLyESymlPIQzd+jjgAxjTJYxph5YAjT30u8/AH8Gal1YX7ur3voWxSaCMZfMcXcpSil1RpwJ9Fggt9FyXsO6U0RkFBBvjFl9uh2JyHwRSRORtMLCwjMutq2VlxSQXL6R/dGXExaqI1uUUp7FmUBvrt/BnNoo4gf8A3iotR0ZYxYYY1KNManR0dHOV9lO9n3yOkFiJXrS7e4uRSmlzpgzgZ4HxDdajgPyGy2HA0OBz0XkMDABWOVpX4waY4g4+C6H/fqQNHyiu8tRSqkz5kygbwWSRKSfiAQBc4FV3200xpw0xkQZY/oaY/oCm4AZxpi0Nqm4jRxI30Gy7QDFSbMRPx3NqZTyPK0mlzHGCtwLrAX2AcuMMXtF5EkRmdHWBbaX/C9ew2aEgVN/6u5SlFLqrDj1LhdjzBpgTZN1j7fQdvK5l9W+austDD6xhkOdxzI4OsHd5Sil1FnRvgVg11cf0FuK8Bs5192lKKXUWdNAB2w73qaSTvS/UANdKeW5fD7QK8rLGFH+BQe7T8E/OMzd5Sil1Fnz+UDf//kSwqSWsHE3ubsUpZQ6Jz4f6J3Sl5EvMSSlXu7uUpRS6pz4dKCXHT9Ccs12snpfhZ+/viZXKeXZfDrQs9e/hr8YYs6/zd2lKKXUOfPpQI/KXM63foNJSh7h7lKUUuqc+WygFx3NJN56hMI+0/W950opr+Czgb53yzoAEkdPdXMlSinlGj4b6JUZ31BHEAnJY91dilJKuYRPBnpRZR09K/ZQGJ6MBAS7uxyllHIJnwz0T3bnMFSy6dRvgrtLUUopl3HqbYveZv+OrwgWK0GDz3d3KUop5TI+d4deXFlHwLFtAEjcODdXo5RSruNzgf5x+glGySHqO8dCRC93l6OUUi7jc4G+Zs8xUgMyCeyjd+dKKe/iU4FeVl1PZuYheppC7W5RSnkdnwr0z/YXMJxDjoV4DXSllHdxKtBFZJqIHBCRDBF5uJntPxORPSKyU0S+EpEU15d67j7ee4JJIVkY/yDoOczd5SillEu1Gugi4g88D7LKCc8AAAnvSURBVEwHUoB5zQT228aYYcaYkcCfgb+7vNJzVGux8eWhQi7olI30Ggn6QJFSyss4Mw59HJBhjMkCEJElwEwg/bsGxpjyRu3DAOPKIs9JfTWsfoDKwuM8Z04SX7Mfhs13d1VKKeVyzgR6LJDbaDkPGN+0kYjcAzwIBAGXNLcjEZkPzAdISEg401rPTuY62L0US3AiMX4CvUfCsDntc2yllGpHzvShN/du2R/dgRtjnjfG9Ad+AzzW3I6MMQuMManGmNTo6Ogzq/RsZazDBIUzy/r/eGnQK/jd+SnEjm6fYyulVDtyJtDzgPhGy3FA/mnaLwFmnUtRLmMMZK6jrMcETlTZuWxIT3dXpJRSbcaZQN8KJIlIPxEJAuYCqxo3EJGkRotXwndjA92sJAvKctjsN5JAf2HyoHb6qUAppdyg1T50Y4xVRO4F1gL+wEJjzF4ReRJIM8asAu4VkamABSgFOsYknRmOSSzeKEhkQmJ3IkIC3VyQUkq1HafetmiMWQOsabLu8Ua/v9/FdblG5jrqI/rwdUEX/nBhD3dXo5RSbcp7nxS11kP2BtJDx+InMG2ovohLKeXdvDfQczeDpYplpUlM7B9FdLg+SKSU8m7eG+iZ6zASwKqT/bl6hN6dK6W8n/cGesY6csKGUucfxrQhGuhKKe/nnYFeWQjHd/NhTQoXDYymS6iOblFKeT/vDPR9jmHyq6qHcvWI3m4uRiml2od3ThK9awnHQxLJsvVjarIOV1RK+Qbvu0MvyoC8LSypO5+pyT0JC/bOf7OUUqop7wv03Usw4sfbNRO0u0Up5VO86/bVboddS9kfOoY6YvTdLUopn+Jdd+hHNsLJHF6tGM+Vw3sRHODv7oqUUqrdeFeg71qCJSCM1fVjmD0q1t3VKKVUu/KeQK+vhvSVbAyaREy3SMb0iXR3RUop1a68J9APrYX6Sl4qG8usUbGINDfRklJKeS/vCfSMddQFdGarfRDXaHeLUsoHeUegGwOZn7FFhjMsvjv9osLcXZFSSrU77wj0wgNQfpQPqlOYPVrvzpVSvsk7Aj3TMdXcBtswputEFkopH+Ulgf4ZR/3j6B43QCeyUEr5LKcCXUSmicgBEckQkYeb2f6giKSLyG4RWScifVxfagsstZjDG/m4bggXDdQnQ5VSvqvVQBcRf+B5YDqQAswTkZQmzXYAqcaY4cC7wJ9dXWiLcr5GrDV8aR+uga6U8mnO3KGPAzKMMVnGmHpgCTCzcQNjzHpjTHXD4iYgzrVlnkbGOqwSSHrQMEbGd223wyqlVEfjTKDHArmNlvMa1rXkDuDD5jaIyHwRSRORtMLCQuerPA2T+Rk7GExqUjwB/t7xlYBSSp0NZxKwuUcuTbMNRW4GUoG/NLfdGLPAGJNqjEmNjnZB90j5MaQgnU/rh3KRvllRKeXjnAn0PCC+0XIckN+0kYhMBX4LzDDG1LmmvFZkfALABvsw7T9XSvk8ZwJ9K5AkIv1EJAiYC6xq3EBERgH/whHmBa4vswW7lpLvH4vpMYweESHtdlillOqIWg10Y4wVuBdYC+wDlhlj9orIkyIyo6HZX4DOwDsislNEVrWwO9cpPQJHvmJJ3UQuGhTT5odTSqmOzqkZi4wxa4A1TdY93uj3U11cV+t2LwPgPesk/qb950op5aFT0BkDuxZzIGQEdQFx+u5zpZTCUx/9z0uDkkwWVk5g9uhYAnW4olJKeegd+q63sfqF8IF1LCvGtN8zTEop1ZF53q2ttQ7z7Xt87j+epITeJPUId3dFSinVIXheoB/8CKk9yaKq87huTHzr7ZVSykd4XqDbLBwOHcY2/+FcNULffa6UUt/xuD70mkHXcPW74Uwb2oOIkEB3l6OUUh2Gx92hr917nIo6K3NS9ctQpZRqzOMCvXNwAJem9GBCv+7uLkUppToUj+tymZrSg6kpPdxdhlJKdTged4eulFKqeRroSinlJTTQlVLKS2igK6WUl9BAV0opL6GBrpRSXkIDXSmlvIQGulJKeQkxxrjnwCKFwJGz/HgUUOTCcjyFL563L54z+OZ5++I5w5mfdx9jTLPzbrot0M+FiKQZY1LdXUd788Xz9sVzBt88b188Z3DteWuXi1JKeQkNdKWU8hKeGugL3F2Am/jiefviOYNvnrcvnjO48Lw9sg9dKaXUj3nqHbpSSqkmNNCVUspLeFygi8g0ETkgIhki8rC762kLIhIvIutFZJ+I7BWR+xvWdxORT0TkUMN/I91dq6uJiL+I7BCR1Q3L/URkc8M5LxWRIHfX6Goi0lVE3hWR/Q3X/Dwfuda/bPjz/a2ILBaREG+73iKyUEQKROTbRuuavbbi8ExDtu0WkdFnejyPCnQR8QeeB6YDKcA8EUlxb1Vtwgo8ZIxJBiYA9zSc58PAOmNMErCuYdnb3A/sa7T8J+AfDedcCtzhlqra1tPAR8aYwcAIHOfv1ddaRGKB+4BUY8xQwB+Yi/dd79eBaU3WtXRtpwNJDb/mAy+e6cE8KtCBcUCGMSbLGFMPLAFmurkmlzPGHDPGbG/4fQWOv+CxOM51UUOzRcAs91TYNkQkDrgSeKVhWYBLgHcbmnjjOUcAFwKvAhhj6o0xZXj5tW4QAHQSkQAgFDiGl11vY8yXQEmT1S1d25nAG8ZhE9BVRHqdyfE8LdBjgdxGy3kN67yWiPQFRgGbgR7GmGPgCH0gxn2VtYl/Ar8G7A3L3YEyY4y1Ydkbr3ciUAi81tDV9IqIhOHl19oYcxT4K5CDI8hPAtvw/usNLV/bc843Twt0aWad1467FJHOwHvAA8aYcnfX05ZE5CqgwBizrfHqZpp62/UOAEYDLxpjRgFVeFn3SnMa+o1nAv2A3kAYji6Hprztep/OOf9597RAzwPiGy3HAfluqqVNiUggjjB/yxizvGH1ie9+BGv4b4G76msD5wMzROQwjq60S3DcsXdt+JEcvPN65wF5xpjNDcvv4gh4b77WAFOBbGNMoTHGAiwHJuL91xtavrbnnG+eFuhbgaSGb8KDcHyJssrNNblcQ9/xq8A+Y8zfG21aBdzW8PvbgPfbu7a2Yox5xBgTZ4zpi+O6fmaMuQlYD8xpaOZV5wxgjDkO5IrIoIZVU4B0vPhaN8gBJohIaMOf9+/O26uvd4OWru0q4NaG0S4TgJPfdc04zRjjUb+AK4CDQCbwW3fX00bnOAnHj1q7gZ0Nv67A0ae8DjjU8N9u7q61jc5/MrC64feJwBYgA3gHCHZ3fW1wviOBtIbrvRKI9IVrDfwe2A98C/wbCPa26w0sxvEdgQXHHfgdLV1bHF0uzzdk2x4cI4DO6Hj66L9SSnkJT+tyUUop1QINdKWU8hIa6Eop5SU00JVSyktooCullJfQQFdKKS+hga6UUl7i/wOnLTqWRuq4mgAAAABJRU5ErkJggg==\n", 273 | "text/plain": [ 274 | "
" 275 | ] 276 | }, 277 | "metadata": { 278 | "needs_background": "light" 279 | }, 280 | "output_type": "display_data" 281 | } 282 | ], 283 | "source": [ 284 | "import matplotlib.pyplot as plt\n", 285 | "plt.plot(train_scores2)\n", 286 | "plt.plot(test_scores2)\n", 287 | "plt.show()" 288 | ] 289 | }, 290 | { 291 | "cell_type": "markdown", 292 | "metadata": {}, 293 | "source": [ 294 | "# MSE VS Cross Entropy" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 13, 300 | "metadata": {}, 301 | "outputs": [ 302 | { 303 | "data": { 304 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhU5d3/8fc3ISEsAQTCGjCgYVdUIkIFt7rgilvFtdVHxbZu1WqLj9aqfbo8drH+LD8UxWrdUNFaRETFouLCEkRQlmjYIyBhCWHNej9/nLGGMEmGMDMnc+bzuq5cmXPmnjPfwwmfnNxzn/uYcw4REUl8KX4XICIi0aFAFxEJCAW6iEhAKNBFRAJCgS4iEhDN/Hrjjh07upycHL/eXkQkIS1YsGCzcy4r3HO+BXpOTg75+fl+vb2ISEIyszV1PacuFxGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCwrdx6CIiTc7qD2Hl+7F/n76joPuQqG9WgS4iUl0NH/wR3vs94ACL7ftldlGgi4hE3Z4S+OcN8OUMOHIMnPNXSG/pd1WNokAXkeS18Qt48UrYvg7O+hMcex1YjM/OY0iBLiLJafHLMPVmyGgLV0+Hnsf5XdFBU6CLSDDtLYVZv4XtRfs/V7YDVr0PPb8HP3gKMjvHvbxYUKCLSPBsWg4vXgFbV0FWv/DdKCNug5PvhtS0+NcXIwp0EQmWJf+E1270Ptj80VTIGeF3RXGjQBeRYKiqhJm/hk/+BtlD4ZKnoU03v6uKKwW6iCS+nZvg5WtgzYdw7PVwxu+gWbrfVcWdAl1EEs+KWfD2r6B8h7e8eytUVcAFj8HgS/2tzUcKdBFJHM7Bhw/Bv38D7Q+DHqGhhqlpcNyPocsR/tbnMwW6iMTWuvmw9pPobGvNR94VnQMvhPMegeato7PdgIgo0M1sFPAwkAo84Zz7Q63newJPA+1CbcY556ZHuVYRSSQ1z6ZddXS2mZoOZ/wehv0koa/ojJUGA93MUoHxwGlAETDfzKY655bWaHYP8JJzboKZDQCmAzkxqFdE/FBVAVXlkbev2AOv3wrLp3ln02f9CZo1P/g6UtOis52AiuQMfShQ6JxbCWBmk4HRQM1Ad0Cb0OO2wPpoFikiPvp8Crz+s+8+gIyUpepsOs4iCfTuwLoay0VA7UkP7gPeNrObgVbAqeE2ZGZjgbEAPXv2PNBaRSSeqirg7Xtg7qPQczj0PfPAXp8zIiZTxErdIgn0cL9aXa3ly4CnnHN/NrPhwDNmNsi5fTvOnHMTgYkAeXl5tbchItH01Uz4YorXl11byw5wwh3Qsv1367atho8ehvLd3nLxMtiwCIbdCKfdH6hL5IMqkkAvAnrUWM5m/y6Va4FRAM65T8wsA+gIbIpGkSJyAKqr4L0/wAcPesGdHmYkSOl6r397zDPQdbAX/q9c6/WTt8ry2jTLgIsmwREXx7d+abRIAn0+kGtmvYCvgUuBy2u1WQt8H3jKzPoDGUBxNAsVkQjs3gqvXg+FM+GoK+HsP0Fai/3bFeXDi1fBpNNh4AWwaDJ0HgRj/gHte8e/bomKBgPdOVdpZjcBb+ENSXzSObfEzB4A8p1zU4GfA4+b2W143TFXOxfu7zwRiZkNi7yQLl0P5zwEQ66p+8PI7Dy44QOYcg0seiHh79QjHvMrd/Py8lx+fr4v7y0SOJ89D9NugxbtvW6U7LzIXldVCZuWeldYaiRKQjCzBc65sAdYV4qKJLLKcpgxDvInQc5IuPjv0Dor8tenNoOuR8auPokrBbpIoipdDy/9EIrmw/duhu/f5wW0JC0dfZGmxDkofNe7aXF9KvfC7D97V2T+4GkYeH586pMmTYEu0lRU7IE3fg6fPRdZ+4594Oo3IKtvbOuShKFAF2kKtq32RqhsXAwn/hKGXE34a/pqaJWlLhbZh34aRPxWugEeP8UbcXLZi9B3lN8VSYJSoIv4bfodUL4Lxr4Pnfr5XY0ksBS/CxBJakunepfgn3SXwlwOmgJdxC97SmD6nd5FPcNv8rsaCQB1uYjEU+l62Lvde/zxI7BrE1w+WR9uSlTop0gkXgrehMmX73s7tuE3Qbej/atJAkWBLhIPe0th2u2Q1R9OvNNbl9YKDjvF37okUBToIvHw7v2wYwOMeRaydRcfiQ19KCoSa2vnwPxJ3r01FeYSQzpDF2mssp3w1dv79omH8/6D0LYHnHx3fOqSpKVAF2kM57yZDle823DblGbeFaDNw9wKTiSKFOgijbH4JS/Mv38v9D+v/rbN20Bm5/jUJUlNgS5yoHZtgbfugu55cPzPICXV74pEAH0oKnLg3vpv7+Kg8x5RmEuTokAXORAFM2DxZBhxG3Qe4Hc1IvtQoItEwjmY8yi8eAV0GgAj7/C7IpH9qA9dksPeUigrbbhd6y77z6tSvgtevxU+fxn6ngUXPAppGbGpU+QgKNAl+D57AabdBpV7Gm6b1c+7mrNjrre8ZYV3J6FNS+GUX8GI2yFFf9hK06RAl+CqLPdGo8x/AnJGwpGX1N++fDd88CBMPBkumOCNH3/1Bi/Ar5wCh58an7pFGkmBLsGxZQXM+h2U7fCWS9ZA8XI4/lY45d7Ipqjtd7Z3wdCLV3rLXQfDJc/AIYfGrm6RKFGgSzAUzIBXxwIO2vf21jXPhEv+AQNGR76ddj3gv2bAzPuhuhJOux/SWsSkZJFoU6BL01W+C9Z83PBcKWs/gQ8fit7ZdLPmMOp3B7cNER8o0KXpevc3MHdCZG2PuhLO/pPOpiWpKdClaSrbAQufhX7nwMjb62+b1gqy+oJZfGoTaaIU6NI0LZoM5Tu8KzK7aw5xkUhoQK00Pc7BvInQ7RjIzvO7GpGEoUCXpmflLNj8JRx3g9+ViCQUBbo0PfMeh1ZZMPACvysRSSjqQxf/OedNRwtQuh4K3oQT7vCGD4pIxBTo4q8dG+Hla2Dtx9+ts1TI+y//ahJJUAp08c+aj+Hlq70hiifd5d2qDaBjH2jTzdfSRBKRAl2iq2IvzP4zbF1Rf7uqCiiYDu0Ohate080iRKJAgS7RU7LWm9hq/UJof1jDF/oMvNC7ujOjbXzqEwk4Bbp4qiph22roeHj97TZ+8d0HmDXt2ADT7/QmtLr0eW/WQhGJq4gC3cxGAQ8DqcATzrk/hGlzCXAf4IBFzrnLo1inxNLOYphyDayeDcf9GE7/H0hN27dNZTnMGAf5k+reTlY/GPNcw78URCQmGgx0M0sFxgOnAUXAfDOb6pxbWqNNLnAXcLxzbpuZdYpVwRJl6+Z73SR7tkL/82Duo7BhEfzgKcjs4rUpXe+1KZoPw2+C3NP3346leJfop7eMa/ki8p1IztCHAoXOuZUAZjYZGA0srdHmemC8c24bgHNuU7QLlTD2lMCbv4ANi79b13MYnPFbSG9V/2udg/wn4c1feiNKrn0Huh4Jn0+BqTfD+KGQGRppUroeXNWBzy0uInEVSaB3B9bVWC4CjqvVpg+AmX2E1y1zn3NuRu0NmdlYYCxAz549G1OvfOubJd5ddUrWQu4ZkJIKVeWw4ClYNw/GPAMdDgv/2oo9MO12WPS8d1u1Cx+Hlu295464GDr1hw//CpV7vXVdB3szHmb1jcuuiUjjRBLo4YYquDDbyQVOArKB2WY2yDlXss+LnJsITATIy8urvQ2J1Ldn0c3bwNVveGfl3yqcCa9c590Xc+RtkN5639c6BwufgY2L4cRfwonj9r/pceeBcNHjsd8PEYmqSAK9COhRYzkbWB+mzRznXAWwyswK8AJ+flSqFE9VBbx9j9fP3XP4vv3c3zr8VBj7vtfnPfO+8NvJaAeXvQh9R8W6YhGJo0gCfT6Qa2a9gK+BS4HaI1heAy4DnjKzjnhdMCujWWjScM77qm3Xpu8ukR/2Uzjtgf1HonzrkEPh+lneB53hpLeGtIzo1SwiTUKDge6cqzSzm4C38PrHn3TOLTGzB4B859zU0HOnm9lSoAq40zm3JZaFB1LpBpgwHPZsC/98Wku4aJLXz92QlBRo1TG69YlIkxbROHTn3HRgeq1199Z47IDbQ1/SWPlPeiNXTrgTUmqdfVsK9D8XOvXzpzYRafJ0pWhTUVkGC/4Ofc6AU+7xuxoRSUC6wUVTseQ12FUMQ8f6XYmIJCgFelMx7zHokAu9T/a7EhFJUAr0pqBoAXy9wDs7rz0mXEQkQkqPpmDeY5CeCUdd5nclIpLAFOh+27AIlvwTjrocmmf6XY2IJDAFup8+ex4mne7d4f57N/ldjYgkOA1b9ENlGcy4y5tbPGckXPx3aJ3ld1UikuAU6PG2/WtvnpWv8+H4W+GUeyFVh0FEDp6SJJ5WfeDNx1K5V3OLi0jUKdDjZfWH8I/zvTnKxzyrucVFJOoU6PHy4V+9ybKuexcy2vhdjYgEkEa5xMOWFVD4Dgy5RmEuIjGjQI+H+U9ASjPIu8bvSkQkwBTosVa2ExY+CwPO3//uQiIiUaRAj7XFk6GsFI67we9KRCTgFOix5BzMexy6HgXZx/pdjYgEnAI9llbOguLl3tm5md/ViEjAKdBjpbIM3hwHbXvAwAv9rkZEkoDGocfK7L/A5gK4YgqkZfhdjYgkAZ2hx8Km5TD7zzDoYsg9ze9qRCRJKNCjrboaXr8FmreGUX/wuxoRSSLqcom2z56DdXPh/AmaEldE4kpn6NHkHHzyN+g6GAbrdnIiEl8K9Gha9X5omOKPNUxRROJOgR5N8x6Hlh00TFFEfKFAj5aStVAwHY75kYYpiogvFOjRMv8JwODYa/2uRESSlAI9Gir2wKf/gH5nQ9tsv6sRkSSlYYuNVTgTFjztPd69BfZs04yKIuIrBXpjlK6Hl672+spbdvTWDboYDj3e17JEJLkp0A+Uc/DGHVBdCde+A+17+V2RiAigPvQDt2wqFLwBJ9+lMBeRJkWBfiD2lMD0O6HLkTDsRr+rERHZh7pcGvL2PVDwpve4bCfs2gyXvwSp+qcTkaZFqVSfZdPg40cgZyS07uSt6zMKuh3lb10iImEo0OuydztMvwM6D4Kr/gmpaX5XJCJSLwV6XWbeDzu/gUufU5iLSELQh6LhrPkE8id5syZ2H+J3NSIiEYko0M1slJkVmFmhmY2rp93FZubMLC96JcaZc15XS9uecPLdflcjIhKxBgPdzFKB8cCZwADgMjMbEKZdJnALMDfaRcbVqg/gmy/gpHHebeRERBJEJGfoQ4FC59xK51w5MBkYHabdb4AHgb1RrC/+5k305jQfdJHflYiIHJBIAr07sK7GclFo3X+Y2dFAD+fctPo2ZGZjzSzfzPKLi4sPuNiY05zmIpLAIgn0cPdSc/950iwFeAj4eUMbcs5NdM7lOefysrKa4A2UNae5iCSwSAK9COhRYzkbWF9jORMYBLxnZquBYcDUhPtgVHOai0iCiyTQ5wO5ZtbLzNKBS4Gp3z7pnNvunOvonMtxzuUAc4DznHP5Mak4Vj5/WXOai0hCazDQnXOVwE3AW8Ay4CXn3BIze8DMzot1gXHhnPdhaKeBmtNcRBJWRFeKOuemA9Nrrbu3jrYnHXxZcbZ2Dmz8HM59GCzcRwYiIk2frhQFmPcYZLSFIy7xuxIRkUZToJeuh6VT4eirIL2l39WIiDSaAj3/SXDVcOx1flciInJQkjvQK8tgwVPeHOe6nZyIJLjkDvQlr8GuYjhurN+ViIgctOQO9HmPQYdc6HWS35WIiBy05A30krXw9QIY8iNISd5/BhEJjuRNslWzve+9T/a3DhGRKEneQF8925smt9N+U7uLiCSk5Ax057wz9JwR6m4RkcBIzjTbuhJKiyBnpN+ViIhETXIG+upQ/3mvE/2tQ0QkipIz0FfNhtadoWOu35WIiERN8gW6c94Zes5IzawoIoGSfIG++UvY+Q30OsHvSkREoir5An3VB973XvpAVESCJfkCffVsaJMNh2gyLhEJlojuWJTQqqtgzgTYW+Itr3wP+p6t/nMRCZzgB/qKf8PbdwPmhXhKGgwY7XdVIiJRF/xAX/Y6pLeGO1dAWobf1YiIxEyw+9Crq2D5G5B7usJcRAIv2IG+bi7s3gz9z/G7EhGRmAt2oC97HVLTvTN0EZGAC26gOwfLpsFhp0DzTL+rERGJueAG+oZFsH0t9D/X70pEROIiuIG+7HWwFOhzpt+ViIjERXADffk0OPR4aNXB70pEROIimIFeXADFy6H/eX5XIiISN8EM9PwndUWoiCSd4AV62Q5Y+BwMPB8yO/tdjYhI3AQv0BdNhvIdMPQGvysREYmrYAW6czDvceh2NGTn+V2NiEhcBSvQV74Hmwu8s3NNjysiSSZYgT5vIrTsCIMu9LsSEZG4C06gb1sDBW/CkKuhWXO/qxERibvgBPoXUwDnBbqISBIKTqAvmwbdh0C7Hn5XIiLii2AEesk6WP+pJuISkaQWjEBf/ob3vZ8CXUSSV0SBbmajzKzAzArNbFyY5283s6VmttjM3jWzQ6Nfaj2WT4Os/tDx8Li+rYhIU9JgoJtZKjAeOBMYAFxmZgNqNVsI5DnnjgSmAA9Gu9A67doMaz7SbeZEJOlFcoY+FCh0zq10zpUDk4F9Zr1yzs1yzu0OLc4BsqNbZj0K3gRXrf5zEUl6kQR6d2BdjeWi0Lq6XAu8Ge4JMxtrZvlmll9cXBx5lfVZ9jq06wldjozO9kREElQkgR7uGnoXtqHZlUAe8MdwzzvnJjrn8pxzeVlZWZFXWZeyHbBylvdhqC71F5Ek1yyCNkVAzcHd2cD62o3M7FTgbuBE51xZdMprQMEMqCpX/7mICJGdoc8Hcs2sl5mlA5cCU2s2MLOjgceA85xzm6JfZl2VPQGH9IIew+L2liIiTVWDge6cqwRuAt4ClgEvOeeWmNkDZvbtPd7+CLQGXjazz8xsah2bi54Ni2DdHBh6PaQEYzi9iMjBiKTLBefcdGB6rXX31nh8apTratjciZDWEo66Iu5vLSLSFCXmqe2uLfD5y3DkGGjRzu9qRESahMQM9E+fhqoyGDrW70pERJqMxAv0qkqYPwlyRkLn2hesiogkr8QL9C/fhNIiOE43gRYRqSnxAr2qHHoOhz5n+l2JiEiTEtEolyZl0EXel4iI7CPxztBFRCQsBbqISEAo0EVEAkKBLiISEIn3oaiISIzM/qqYjwq3NPr1mRnN+OHwQ8nMSPvPup1llUyet5azj+xK17YtolFmnRToIpL0qqodD73zJX+bVUizFCMlpXH3VyivrOaVT4t47Moh5HbOpHDTTm54Jp8Vxbt49P0VPHLZMQw/rEOUq/+OORf2XhUxl5eX5/Lz8315bxEJb8vOMvZWVtfbJtWMzm2aYwl8U5m9FVVs2VUOQFlFFfe9vpQPvixmTF4P7h89kIy01EZtd+7KLdz4/EJ2l1dyzfE5PPXRajLSUhl3Zj8mvL+CNVt2M25UP64b2avR/35mtsA5lxfuOZ2hiwiVVdX88a0CHvtgZUTtT+6bxV/HHE3blmkNN25i5q/eyk+f+5TiHd/dhyc9NYXfX3gElw3teVDbPq53B6bdPIKfPreA8bNWMLhHOx698hi6tm3BqEFduPPlxfx2+jIcjrEnHHawu7IfnaGLJLnNO8u4+fmFfLJyC2PyejDk0EPqbb9++x7Gzyqka9sWPHrlEAZ0axOnSg+Oc46/f7Sa301fRo/2LRl7Qm9SQ2fJR/VsR5/OmVF7r/LKamZ/VcyI3I40b/bd2b5zjmfnrOG8wd0b/cuwvjN0BbpIAquudjz50So+Ktzc6G0sWV/K9j0V/O6CI7hoSHZEr1mwZhs/fW4B2/dUMKx3h7A3Ho5ETsdW3HlGX1qmH1hnwYrinTz0zpfsKquM+DUleypYuLaE0wZ05s+XDKZNRuL9dQEKdJFAKt1bwe0vLmLmsm/I7dSaFumN6/fNzGjGXWf2Z1D3tgf0uuIdZdz3+hLWbd3dqPd1Dpas305up0wevWoIvTq2iuh1M77YwB0vLybFvF8IkTLgrCO6cv3I3o3+0LMpUKCL+KSyqpq5q7ayt6Jqv+fSUlMY1rsD6c0O/HKQgo07uOGZfIq27eHus/tz9fdyEvJDytlfFXPLCwuprHLcdVZ/OrdpXm/7OSu38PjsVQzu0Y4JVxxDt3axHQbYFCnQRXyweWcZt7ywkI9X1D2u+eie7fj/VxxzQOOTpy5azy+nLKZ1RjPGX34MQ3u1j0a5vinatpufPPspn3+9PaL2lx/Xk1+fO2CfvulkokAXibPP1pXwk2cXsHVXOb86ZwBHZu/fnfHlNzv59b++oEV6akTjkyuqqvn99OU8+dEqjs05hPGXH0OnNhmx2oW4qqiqpmDjDqobyKOW6c04vFPrOFXVNGnYojRZeyuq+O0by5i57Bu/S4mq4h1ldGmbwSs/+V6dfdNHZrfjqB5tGfvMAq54Yg6dGwjnvRVVbNtdwTXH5/DfZ/UnLTU4M3ekpaYccB++7E+BLr6p+af2mYO6kJkRnB/Hdi3T+cmJh3FIq/R62x3eKZN/3Xg8E95bweadZfW2BTilXydGDeoarTIlYBLuf9DiohLeXbaJ207r43cpgbJpx15eW/g1FVXx6YKrqKrm6Y9XU1nlmHjVEE4f2CUu79sUZWak8YtR/fwuQwIg4QJ94doSHn73K84d3C3p+9KiJdyVc/HQv2sbxl9+NL2zdBxFoiHhAv30gZ359dQlvLVkI4d3Ojyi1+ytqKKswpufIjXVaN28/t2urKpmV9n+w8yC6NWFRfz2jWVkH9KCp28ZGddfkmmplpBD7USaqoQL9K5tWzC4RzveXrKRG09uONA/L9rO5Y/PYUeNK8ouycvmgdGDwk7A89m6Em587lO+LtkT1bqbslP7d+YvYxL3yjkR8SRcoAOcMbAzD84oYH3JnnovLKioquaXrywmIz2Vn53WBwPWbNnF05+sYemGUiZcMYQe7VsC3hwLz89by/1Tl9KpTXPuObs/KUlw9piV2Zyzj+ia0FfOiYgnIQN91MAuPDijgLeXbOTq43vV2W7Sh6tCwX0MZx7x3ciAkblZ3PbSZ5z7tw8ZcXhHALbtLuejwi2c2CeLhy89inYt6x+dICLS1CTkQNbeWa3J7dSaGUs21tlmzZZdPPTOl5w+oDOjBu07guLUAZ15/aYRDOzWhqUbSlm6oZSN2/dy26l9ePLqYxXmIpKQEvIMHWDUoC6Mn1XI1l3ltG+VTnW1Y1FRCWWhyfkfnvkV6akpPDB6UNgP3nI6tuK564bFu2wRkZhJ2EA/Y2AXHvl3ITOXfsMZA7vwsxcXMqugeJ82/3P+ILq0Dcal0SIiDUnYQB/YrQ3d27Xg2blr+NusQjZs38M9Z/f/z2T7bTLSdCmxiCSVhA10M2PUoC5M+nAVnds0Z/LY4Q3eaUVEJMgSNtABrh3RCwNuOPEwsjLrn0dZRCToEjrQu7VrwT3nDPC7DBGRJiEhhy2KiMj+FOgiIgGhQBcRCQgFuohIQEQU6GY2yswKzKzQzMaFeb65mb0Yen6umeVEu1AREalfg4FuZqnAeOBMYABwmZnVHlpyLbDNOXc48BDwv9EuVERE6hfJGfpQoNA5t9I5Vw5MBkbXajMaeDr0eArwfdOdC0RE4iqSQO8OrKuxXBRaF7aNc64S2A50iEaBIiISmUguLAp3pl37TsKRtMHMxgJjQ4s7zawggvcPpyOwuZGvTWTJuN/JuM+QnPudjPsMB77fh9b1RCSBXgT0qLGcDayvo02RmTUD2gJba2/IOTcRmBjBe9bLzPKdc3kHu51Ek4z7nYz7DMm538m4zxDd/Y6ky2U+kGtmvcwsHbgUmFqrzVTgR6HHFwP/ds7td4YuIiKx0+AZunOu0sxuAt4CUoEnnXNLzOwBIN85NxWYBDxjZoV4Z+aXxrJoERHZX0STcznnpgPTa627t8bjvcAPoltavQ662yZBJeN+J+M+Q3LudzLuM0Rxv009IyIiwaBL/0VEAkKBLiISEAkX6A3NKxMEZtbDzGaZ2TIzW2Jmt4bWtzezd8zsq9D3wN1zz8xSzWyhmU0LLfcKzQ/0VWi+oHS/a4w2M2tnZlPMbHnomA9PkmN9W+jn+wsze8HMMoJ2vM3sSTPbZGZf1FgX9tia5/+Fsm2xmR1zoO+XUIEe4bwyQVAJ/Nw51x8YBtwY2s9xwLvOuVzg3dBy0NwKLKux/L/AQ6F93oY3b1DQPAzMcM71Awbj7X+gj7WZdQduAfKcc4PwRtBdSvCO91PAqFrr6jq2ZwK5oa+xwIQDfbOECnQim1cm4TnnNjjnPg093oH3H7w7+86Z8zRwvj8VxoaZZQNnA0+Elg04BW9+IAjmPrcBTsAb+otzrtw5V0LAj3VIM6BF6GLElsAGAna8nXMfsP9FlnUd29HAP5xnDtDOzLoeyPslWqBHMq9MoISmIj4amAt0ds5tAC/0gU7+VRYTfwV+AVSHljsAJaH5gSCYx7s3UAz8PdTV9ISZtSLgx9o59zXwJ2AtXpBvBxYQ/OMNdR/bg863RAv0iOaMCQozaw28AvzMOVfqdz2xZGbnAJuccwtqrg7TNGjHuxlwDDDBOXc0sIuAda+EE+o3Hg30AroBrfC6HGoL2vGuz0H/vCdaoEcyr0wgmFkaXpg/55x7NbT6m2//BAt93+RXfTFwPHCema3G60o7Be+MvV3oT3II5vEuAoqcc3NDy1PwAj7IxxrgVGCVc67YOVcBvAp8j+Afb6j72B50viVaoEcyr0zCC/UdTwKWOef+UuOpmnPm/Aj4V7xrixXn3F3OuWznXA7ecf23c+4KYBbe/EAQsH0GcM5tBNaZWd/Qqu8DSwnwsQ5ZCwwzs5ahn/dv9zvQxzukrmM7FfhhaLTLMGD7t10zEXPOJdQXcBbwJbACuNvvemK0jyPw/tRaDHwW+joLr0/5XeCr0Pf2ftcao/0/CZgWetwbmAcUAi8Dzf2uLwb7exSQHzrerwGHJMOxBu4HlgNfAM8AzYN2vIEX8D4jqMA7A7+2rmOL1+UyPpRtn+ONADqg99Ol/yIiAZFoXS4iIlIHBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCD+D0czss0AAAAESURBVBDfFkhYcL7gAAAAAElFTkSuQmCC\n", 305 | "text/plain": [ 306 | "
" 307 | ] 308 | }, 309 | "metadata": { 310 | "needs_background": "light" 311 | }, 312 | "output_type": "display_data" 313 | } 314 | ], 315 | "source": [ 316 | "import matplotlib.pyplot as plt\n", 317 | "plt.plot(test_scores)\n", 318 | "plt.plot(test_scores2)\n", 319 | "plt.show()" 320 | ] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "metadata": {}, 325 | "source": [ 326 | "理论上，cross entropy只在初期优于MSE，但实际上差距很明显" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [] 335 | } 336 | ], 337 | "metadata": { 338 | "kernelspec": { 339 | "display_name": "Python 3", 340 | "language": "python", 341 | "name": "python3" 342 | }, 343 | "language_info": { 344 | "codemirror_mode": { 345 | "name": "ipython", 346 | "version": 3 347 | }, 348 | "file_extension": ".py", 349 | "mimetype": "text/x-python", 350 | "name": "python", 351 | "nbconvert_exporter": "python", 352 | "pygments_lexer": "ipython3", 353 | "version": "3.7.3" 354 | } 355 | }, 356 | "nbformat": 4, 357 | "nbformat_minor": 2 358 | } 359 | -------------------------------------------------------------------------------- /Chapter6/6-2-2-2.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter6/6-2-2-3.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter6/6-3-1.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter6/6-3-2.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter6/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/windmissing/DeepLearningPractise/0f324b5caa82a52bb44ba6749cd68572e0b36f6d/Chapter6/__init__.py -------------------------------------------------------------------------------- /Chapter6/deep-linear-network.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter6/mnist_loader.py: -------------------------------------------------------------------------------- 1 | #### Libraries 2 | # Standard library 3 | import pickle 4 | import gzip 5 | 6 | # Third-party libraries 7 | import numpy as np 8 | 9 | #### Load the MNIST data 10 | def get_part(data, seed, size): 11 | X, y = data 12 | n = X.shape[0] 13 | if n <= size:return data 14 | if seed: 15 | np.random.seed(seed) 16 | shuffle_indexes = np.random.permutation(n) 17 | part_indexes = shuffle_indexes[:size] 18 | return X[part_indexes], y[part_indexes] 19 | 20 | def vectorized_result(j): 21 | """Return a 10-dimensional unit vector with a 1.0 in the jth 22 | position and zeroes elsewhere. This is used to convert a digit 23 | (0...9) into a corresponding desired output from the neural 24 | network.""" 25 | e = np.zeros(10) 26 | e[j] = 1.0 27 | return e 28 | 29 | def vectorize(data): 30 | vector_y = [vectorized_result(y) for y in data[1]] 31 | return (data[0], np.array(vector_y)) 32 | #return (data[0], data[1]) 33 | 34 | def load_data_shared(filename, seed, train_size, vali_size, test_size): 35 | f = gzip.open(filename, 'rb') 36 | training_data, validation_data, test_data = pickle.load(f,encoding='bytes') 37 | 38 | return get_part(training_data, seed, train_size),\ 39 | get_part(validation_data, seed, vali_size),\ 40 | get_part(test_data, seed, test_size) -------------------------------------------------------------------------------- /Chapter7/7-1-1.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter7/7-1-2.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter7/7-11.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch.nn as nn\n", 10 | "import torch.nn.functional as F\n", 11 | "from torch.autograd import Variable\n", 12 | "import torch as t\n", 13 | "from early_stopping import EarlyStopping" 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "# ReLU + Sigmoid + Cross Entropy + L2 + early stopping" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "class Net(nn.Module):\n", 30 | " def __init__(self):\n", 31 | " super(Net, self).__init__()\n", 32 | " \n", 33 | " self.fc1 = nn.Linear(28*28, 300)\n", 34 | " self.fc2 = nn.Linear(300, 10)\n", 35 | " \n", 36 | " def forward(self, x):\n", 37 | " x = x.view(x.shape[0], -1)\n", 38 | " x = F.relu(self.fc1(x))\n", 39 | " x = F.logsigmoid(self.fc2(x))\n", 40 | " return x" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 3, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "from mnist_loader import load_data_shared, vectorized_result\n", 50 | "training_data1, validation_data1, _ = load_data_shared(filename=\"../mnist.pkl.gz\",\n", 51 | " seed=666,\n", 52 | " train_size=400,\n", 53 | " vali_size=100,\n", 54 | " test_size=0)\n", 55 | "training_data2, validation_data2, _ = load_data_shared(filename=\"../mnist.pkl.gz\",\n", 56 | " seed=666,\n", 57 | " train_size=400,\n", 58 | " vali_size=100,\n", 59 | " test_size=0)\n", 60 | "training_data3, validation_data3, _ = load_data_shared(filename=\"../mnist.pkl.gz\",\n", 61 | " seed=666,\n", 62 | " train_size=400,\n", 63 | " vali_size=100,\n", 64 | " test_size=0)\n", 65 | "_, _, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n", 66 | " seed=666,\n", 67 | " train_size=0,\n", 68 | " vali_size=0,\n", 69 | " test_size=100)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 4, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "def predict(data, net, criterion):\n", 79 | " with t.no_grad():\n", 80 | " #for index in range(test_data[0].shape[0]):\n", 81 | " # get the inputs\n", 82 | " inputs, labels = t.Tensor(data[0]), t.Tensor(data[1])\n", 83 | "\n", 84 | " # forward + backward + optimize\n", 85 | " outputs = net(inputs)\n", 86 | " _, predicted = t.max(outputs, 1)\n", 87 | " loss = criterion(outputs, labels.long())\n", 88 | "\n", 89 | " correct = (predicted == labels).sum().item()\n", 90 | " accuracy = correct / data[0].shape[0]\n", 91 | " return loss, accuracy" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 5, 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | "def fit(net, train_data, vali_data, criterion, optimizer, is_early_stopping):\n", 101 | " loss_scores = []\n", 102 | " validate_loss = []\n", 103 | " \n", 104 | " # initialize the early_stopping object\n", 105 | " early_stopping = EarlyStopping(patience=3, verbose=False)\n", 106 | " \n", 107 | " for epoch in range(1000): # loop over the dataset multiple times\n", 108 | "\n", 109 | " # get the inputs\n", 110 | " inputs, labels = t.Tensor(train_data[0]), t.Tensor(train_data[1])\n", 111 | " vector_labels = t.Tensor([vectorized_result(y) for y in train_data[1]])\n", 112 | " # zero the parameter gradients\n", 113 | " optimizer.zero_grad()\n", 114 | "\n", 115 | " # forward + backward + optimize\n", 116 | " outputs = net(inputs)\n", 117 | " loss = criterion(outputs, labels.long())\n", 118 | " loss.backward()\n", 119 | " optimizer.step()\n", 120 | "\n", 121 | " # print statistics\n", 122 | " loss_scores.append(loss.item())\n", 123 | " #train_scores.append(predict(training_data, net))\n", 124 | " one_vali_loss, _ = predict(vali_data, net, criterion)\n", 125 | " validate_loss.append(one_vali_loss)\n", 126 | " \n", 127 | " if is_early_stopping:\n", 128 | " early_stopping(one_vali_loss, net)\n", 129 | "\n", 130 | " if early_stopping.early_stop:\n", 131 | " print(\"Early stopping at \", epoch)\n", 132 | " break\n", 133 | " \n", 134 | " print('Finished Training')\n", 135 | " return net" 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "metadata": {}, 141 | "source": [ 142 | "# 子网络" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": 6, 148 | "metadata": {}, 149 | "outputs": [ 150 | { 151 | "name": "stdout", 152 | "output_type": "stream", 153 | "text": [ 154 | "EarlyStopping counter: 1 out of 3\n", 155 | "EarlyStopping counter: 1 out of 3\n", 156 | "EarlyStopping counter: 1 out of 3\n", 157 | "EarlyStopping counter: 1 out of 3\n", 158 | "EarlyStopping counter: 1 out of 3\n", 159 | "EarlyStopping counter: 1 out of 3\n", 160 | "EarlyStopping counter: 1 out of 3\n", 161 | "EarlyStopping counter: 1 out of 3\n", 162 | "EarlyStopping counter: 1 out of 3\n", 163 | "EarlyStopping counter: 1 out of 3\n", 164 | "EarlyStopping counter: 1 out of 3\n", 165 | "EarlyStopping counter: 1 out of 3\n", 166 | "EarlyStopping counter: 1 out of 3\n", 167 | "EarlyStopping counter: 1 out of 3\n", 168 | "EarlyStopping counter: 1 out of 3\n", 169 | "EarlyStopping counter: 1 out of 3\n", 170 | "EarlyStopping counter: 1 out of 3\n", 171 | "EarlyStopping counter: 1 out of 3\n", 172 | "EarlyStopping counter: 1 out of 3\n", 173 | "EarlyStopping counter: 1 out of 3\n", 174 | "EarlyStopping counter: 1 out of 3\n", 175 | "EarlyStopping counter: 1 out of 3\n", 176 | "EarlyStopping counter: 2 out of 3\n", 177 | "EarlyStopping counter: 1 out of 3\n", 178 | "EarlyStopping counter: 1 out of 3\n", 179 | "EarlyStopping counter: 1 out of 3\n", 180 | "EarlyStopping counter: 1 out of 3\n", 181 | "EarlyStopping counter: 1 out of 3\n", 182 | "EarlyStopping counter: 1 out of 3\n", 183 | "EarlyStopping counter: 2 out of 3\n", 184 | "EarlyStopping counter: 1 out of 3\n", 185 | "EarlyStopping counter: 2 out of 3\n", 186 | "EarlyStopping counter: 1 out of 3\n", 187 | "EarlyStopping counter: 2 out of 3\n", 188 | "EarlyStopping counter: 1 out of 3\n", 189 | "EarlyStopping counter: 1 out of 3\n", 190 | "EarlyStopping counter: 1 out of 3\n", 191 | "EarlyStopping counter: 2 out of 3\n", 192 | "EarlyStopping counter: 3 out of 3\n", 193 | "Early stopping at 696\n", 194 | "Finished Training\n" 195 | ] 196 | }, 197 | { 198 | "data": { 199 | "text/plain": [ 200 | "(tensor(0.4662), 0.84)" 201 | ] 202 | }, 203 | "execution_count": 6, 204 | "metadata": {}, 205 | "output_type": "execute_result" 206 | } 207 | ], 208 | "source": [ 209 | "import torch.optim as optim\n", 210 | "net1 = Net()\n", 211 | "criterion1 = nn.CrossEntropyLoss()\n", 212 | "optimizer1 = optim.SGD(net1.parameters(), lr = 1e-1, weight_decay=1e-2)\n", 213 | "net1 = fit(net1, training_data1, validation_data1, criterion1, optimizer1, True)\n", 214 | "predict(test_data, net1, criterion1)" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": 7, 220 | "metadata": {}, 221 | "outputs": [ 222 | { 223 | "name": "stdout", 224 | "output_type": "stream", 225 | "text": [ 226 | "EarlyStopping counter: 1 out of 3\n", 227 | "EarlyStopping counter: 1 out of 3\n", 228 | "EarlyStopping counter: 1 out of 3\n", 229 | "EarlyStopping counter: 1 out of 3\n", 230 | "EarlyStopping counter: 1 out of 3\n", 231 | "EarlyStopping counter: 1 out of 3\n", 232 | "EarlyStopping counter: 1 out of 3\n", 233 | "EarlyStopping counter: 1 out of 3\n", 234 | "EarlyStopping counter: 1 out of 3\n", 235 | "EarlyStopping counter: 1 out of 3\n", 236 | "EarlyStopping counter: 1 out of 3\n", 237 | "EarlyStopping counter: 1 out of 3\n", 238 | "EarlyStopping counter: 1 out of 3\n", 239 | "EarlyStopping counter: 1 out of 3\n", 240 | "EarlyStopping counter: 1 out of 3\n", 241 | "EarlyStopping counter: 1 out of 3\n", 242 | "EarlyStopping counter: 1 out of 3\n", 243 | "EarlyStopping counter: 1 out of 3\n", 244 | "EarlyStopping counter: 1 out of 3\n", 245 | "EarlyStopping counter: 1 out of 3\n", 246 | "EarlyStopping counter: 1 out of 3\n", 247 | "EarlyStopping counter: 1 out of 3\n", 248 | "EarlyStopping counter: 1 out of 3\n", 249 | "EarlyStopping counter: 1 out of 3\n", 250 | "EarlyStopping counter: 1 out of 3\n", 251 | "EarlyStopping counter: 1 out of 3\n", 252 | "EarlyStopping counter: 2 out of 3\n", 253 | "EarlyStopping counter: 1 out of 3\n", 254 | "EarlyStopping counter: 2 out of 3\n", 255 | "EarlyStopping counter: 1 out of 3\n", 256 | "EarlyStopping counter: 1 out of 3\n", 257 | "EarlyStopping counter: 2 out of 3\n", 258 | "EarlyStopping counter: 1 out of 3\n", 259 | "EarlyStopping counter: 1 out of 3\n", 260 | "EarlyStopping counter: 1 out of 3\n", 261 | "EarlyStopping counter: 1 out of 3\n", 262 | "EarlyStopping counter: 1 out of 3\n", 263 | "EarlyStopping counter: 2 out of 3\n", 264 | "EarlyStopping counter: 1 out of 3\n", 265 | "EarlyStopping counter: 2 out of 3\n", 266 | "EarlyStopping counter: 1 out of 3\n", 267 | "EarlyStopping counter: 2 out of 3\n", 268 | "EarlyStopping counter: 1 out of 3\n", 269 | "EarlyStopping counter: 2 out of 3\n", 270 | "EarlyStopping counter: 1 out of 3\n", 271 | "EarlyStopping counter: 1 out of 3\n", 272 | "EarlyStopping counter: 1 out of 3\n", 273 | "EarlyStopping counter: 1 out of 3\n", 274 | "EarlyStopping counter: 2 out of 3\n", 275 | "EarlyStopping counter: 3 out of 3\n", 276 | "Early stopping at 711\n", 277 | "Finished Training\n" 278 | ] 279 | }, 280 | { 281 | "data": { 282 | "text/plain": [ 283 | "(tensor(0.4531), 0.85)" 284 | ] 285 | }, 286 | "execution_count": 7, 287 | "metadata": {}, 288 | "output_type": "execute_result" 289 | } 290 | ], 291 | "source": [ 292 | "net2 = Net()\n", 293 | "criterion2 = nn.CrossEntropyLoss()\n", 294 | "optimizer2 = optim.SGD(net2.parameters(), lr = 1e-1, weight_decay=1e-2)\n", 295 | "net2 = fit(net2, training_data2, validation_data2, criterion2, optimizer2, True)\n", 296 | "predict(test_data, net2, criterion2)" 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": 8, 302 | "metadata": {}, 303 | "outputs": [ 304 | { 305 | "name": "stdout", 306 | "output_type": "stream", 307 | "text": [ 308 | "EarlyStopping counter: 1 out of 3\n", 309 | "EarlyStopping counter: 1 out of 3\n", 310 | "EarlyStopping counter: 1 out of 3\n", 311 | "EarlyStopping counter: 1 out of 3\n", 312 | "EarlyStopping counter: 1 out of 3\n", 313 | "EarlyStopping counter: 1 out of 3\n", 314 | "EarlyStopping counter: 1 out of 3\n", 315 | "EarlyStopping counter: 1 out of 3\n", 316 | "EarlyStopping counter: 1 out of 3\n", 317 | "EarlyStopping counter: 1 out of 3\n", 318 | "EarlyStopping counter: 1 out of 3\n", 319 | "EarlyStopping counter: 1 out of 3\n", 320 | "EarlyStopping counter: 1 out of 3\n", 321 | "EarlyStopping counter: 1 out of 3\n", 322 | "EarlyStopping counter: 1 out of 3\n", 323 | "EarlyStopping counter: 1 out of 3\n", 324 | "EarlyStopping counter: 1 out of 3\n", 325 | "EarlyStopping counter: 1 out of 3\n", 326 | "EarlyStopping counter: 1 out of 3\n", 327 | "EarlyStopping counter: 1 out of 3\n", 328 | "EarlyStopping counter: 1 out of 3\n", 329 | "EarlyStopping counter: 1 out of 3\n", 330 | "EarlyStopping counter: 1 out of 3\n", 331 | "EarlyStopping counter: 1 out of 3\n", 332 | "EarlyStopping counter: 1 out of 3\n", 333 | "EarlyStopping counter: 1 out of 3\n", 334 | "EarlyStopping counter: 1 out of 3\n", 335 | "EarlyStopping counter: 1 out of 3\n", 336 | "EarlyStopping counter: 1 out of 3\n", 337 | "EarlyStopping counter: 1 out of 3\n", 338 | "EarlyStopping counter: 1 out of 3\n", 339 | "EarlyStopping counter: 1 out of 3\n", 340 | "EarlyStopping counter: 2 out of 3\n", 341 | "EarlyStopping counter: 3 out of 3\n", 342 | "Early stopping at 716\n", 343 | "Finished Training\n" 344 | ] 345 | }, 346 | { 347 | "data": { 348 | "text/plain": [ 349 | "(tensor(0.4567), 0.84)" 350 | ] 351 | }, 352 | "execution_count": 8, 353 | "metadata": {}, 354 | "output_type": "execute_result" 355 | } 356 | ], 357 | "source": [ 358 | "net3 = Net()\n", 359 | "criterion3 = nn.CrossEntropyLoss()\n", 360 | "optimizer3 = optim.SGD(net3.parameters(), lr = 1e-1, weight_decay=1e-2)\n", 361 | "net3 = fit(net3, training_data3, validation_data3, criterion3, optimizer3, True)\n", 362 | "predict(test_data, net3, criterion3)" 363 | ] 364 | }, 365 | { 366 | "cell_type": "markdown", 367 | "metadata": {}, 368 | "source": [ 369 | "# Bagging" 370 | ] 371 | }, 372 | { 373 | "cell_type": "code", 374 | "execution_count": 9, 375 | "metadata": {}, 376 | "outputs": [], 377 | "source": [ 378 | "import numpy as np\n", 379 | "def predict2(data, nets):\n", 380 | " with t.no_grad():\n", 381 | " inputs, labels = t.Tensor(data[0]), t.Tensor(data[1])\n", 382 | " output = t.FloatTensor(data[0].shape[0], 10).zero_()\n", 383 | " for net in nets:\n", 384 | " output += net(inputs)\n", 385 | " _, predicted = t.max(output, 1)\n", 386 | "\n", 387 | " correct = (predicted == labels).sum().item()\n", 388 | " accuracy = correct / data[0].shape[0]\n", 389 | " return accuracy" 390 | ] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "execution_count": 10, 395 | "metadata": {}, 396 | "outputs": [ 397 | { 398 | "data": { 399 | "text/plain": [ 400 | "0.84" 401 | ] 402 | }, 403 | "execution_count": 10, 404 | "metadata": {}, 405 | "output_type": "execute_result" 406 | } 407 | ], 408 | "source": [ 409 | "predict2(test_data, [net1, net2, net3])" 410 | ] 411 | }, 412 | { 413 | "cell_type": "markdown", 414 | "metadata": {}, 415 | "source": [ 416 | "总的准确率比单个准确率低，这是为什么？ \n", 417 | "难道是相关性太大？" 418 | ] 419 | }, 420 | { 421 | "cell_type": "code", 422 | "execution_count": null, 423 | "metadata": {}, 424 | "outputs": [], 425 | "source": [] 426 | } 427 | ], 428 | "metadata": { 429 | "kernelspec": { 430 | "display_name": "Python 3", 431 | "language": "python", 432 | "name": "python3" 433 | }, 434 | "language_info": { 435 | "codemirror_mode": { 436 | "name": "ipython", 437 | "version": 3 438 | }, 439 | "file_extension": ".py", 440 | "mimetype": "text/x-python", 441 | "name": "python", 442 | "nbconvert_exporter": "python", 443 | "pygments_lexer": "ipython3", 444 | "version": "3.7.3" 445 | } 446 | }, 447 | "nbformat": 4, 448 | "nbformat_minor": 2 449 | } 450 | -------------------------------------------------------------------------------- /Chapter7/7-11.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter7/7-12.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter7/7-8.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter7/checkpoint.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/windmissing/DeepLearningPractise/0f324b5caa82a52bb44ba6749cd68572e0b36f6d/Chapter7/checkpoint.pt -------------------------------------------------------------------------------- /Chapter7/early_stopping.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | class EarlyStopping: 5 | """Early stops the training if validation loss doesn't improve after a given patience.""" 6 | def __init__(self, patience=7, verbose=False, delta=0): 7 | """ 8 | Args: 9 | patience (int): How long to wait after last time validation loss improved. 10 | Default: 7 11 | verbose (bool): If True, prints a message for each validation loss improvement. 12 | Default: False 13 | delta (float): Minimum change in the monitored quantity to qualify as an improvement. 14 | Default: 0 15 | """ 16 | self.patience = patience 17 | self.verbose = verbose 18 | self.counter = 0 19 | self.best_score = None 20 | self.early_stop = False 21 | self.val_loss_min = np.Inf 22 | self.delta = delta 23 | 24 | def __call__(self, val_loss, model): 25 | 26 | score = -val_loss 27 | 28 | if self.best_score is None: 29 | self.best_score = score 30 | self.save_checkpoint(val_loss, model) 31 | elif score < self.best_score + self.delta: 32 | self.counter += 1 33 | print(f'EarlyStopping counter: {self.counter} out of {self.patience}') 34 | if self.counter >= self.patience: 35 | self.early_stop = True 36 | else: 37 | self.best_score = score 38 | self.save_checkpoint(val_loss, model) 39 | self.counter = 0 40 | 41 | def save_checkpoint(self, val_loss, model): 42 | '''Saves model when validation loss decrease.''' 43 | if self.verbose: 44 | print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}). Saving model ...') 45 | torch.save(model.state_dict(), 'checkpoint.pt') 46 | self.val_loss_min = val_loss 47 | -------------------------------------------------------------------------------- /Chapter7/mnist_loader.py: -------------------------------------------------------------------------------- 1 | #### Libraries 2 | # Standard library 3 | import pickle 4 | import gzip 5 | 6 | # Third-party libraries 7 | import numpy as np 8 | 9 | #### Load the MNIST data 10 | def get_part(data, seed, size): 11 | X, y = data 12 | n = X.shape[0] 13 | if n <= size:return data 14 | if seed: 15 | np.random.seed(seed) 16 | shuffle_indexes = np.random.permutation(n) 17 | part_indexes = shuffle_indexes[:size] 18 | return X[part_indexes], y[part_indexes] 19 | 20 | def vectorized_result(j): 21 | """Return a 10-dimensional unit vector with a 1.0 in the jth 22 | position and zeroes elsewhere. This is used to convert a digit 23 | (0...9) into a corresponding desired output from the neural 24 | network.""" 25 | e = np.zeros(10) 26 | e[j] = 1.0 27 | return e 28 | 29 | def vectorize(data): 30 | vector_y = [vectorized_result(y) for y in data[1]] 31 | return (data[0], np.array(vector_y)) 32 | #return (data[0], data[1]) 33 | 34 | def load_data_shared(filename, seed, train_size, vali_size, test_size): 35 | f = gzip.open(filename, 'rb') 36 | training_data, validation_data, test_data = pickle.load(f,encoding='bytes') 37 | 38 | return get_part(training_data, seed, train_size),\ 39 | get_part(validation_data, seed, vali_size),\ 40 | get_part(test_data, seed, test_size) -------------------------------------------------------------------------------- /Chapter8/8-3-1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# 动态调整学习率" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "```python\n", 15 | " scheduler = lr_scheduler.StepLR(net,step_size=5,gamma = 0.9)\n", 16 | " for epoch in range (param['epoch']):\n", 17 | " scheduler.step()\n", 18 | "```" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": {}, 24 | "source": [ 25 | "各种学习率衰减算法的表现： \n", 26 | "https://www.jianshu.com/p/9643cba47655" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [] 35 | } 36 | ], 37 | "metadata": { 38 | "kernelspec": { 39 | "display_name": "Python 3", 40 | "language": "python", 41 | "name": "python3" 42 | }, 43 | "language_info": { 44 | "codemirror_mode": { 45 | "name": "ipython", 46 | "version": 3 47 | }, 48 | "file_extension": ".py", 49 | "mimetype": "text/x-python", 50 | "name": "python", 51 | "nbconvert_exporter": "python", 52 | "pygments_lexer": "ipython3", 53 | "version": "3.7.3" 54 | } 55 | }, 56 | "nbformat": 4, 57 | "nbformat_minor": 2 58 | } 59 | -------------------------------------------------------------------------------- /Chapter8/8-3-1.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter8/8-3-2.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter8/8-5-1.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter8/8-5-2.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter8/8-5-3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch.nn as nn\n", 10 | "import torch.nn.functional as F\n", 11 | "from torch.autograd import Variable\n", 12 | "import torch as t" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "class Net(nn.Module):\n", 22 | " def __init__(self):\n", 23 | " super(Net, self).__init__()\n", 24 | " \n", 25 | " self.fc1 = nn.Linear(28*28, 300)\n", 26 | " self.fc2 = nn.Linear(300, 10)\n", 27 | " \n", 28 | " def forward(self, x):\n", 29 | " x = x.view(x.shape[0], -1)\n", 30 | " x = F.relu(self.fc1(x))\n", 31 | " x = F.logsigmoid(self.fc2(x))\n", 32 | " return x" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 3, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "from mnist_loader import load_data_shared, vectorized_result\n", 42 | "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n", 43 | " seed=666,\n", 44 | " train_size=1000,\n", 45 | " vali_size=0,\n", 46 | " test_size=100)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 4, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "def predict(data, net):\n", 56 | " with t.no_grad():\n", 57 | " #for index in range(test_data[0].shape[0]):\n", 58 | " # get the inputs\n", 59 | " inputs, labels = t.Tensor(data[0]), data[1]\n", 60 | "\n", 61 | " # forward + backward + optimize\n", 62 | " outputs = net(inputs)\n", 63 | " _, predicted = t.max(outputs, 1)\n", 64 | "\n", 65 | " correct = (predicted == t.Tensor(labels)).sum().item()\n", 66 | " accuracy = correct / data[0].shape[0]\n", 67 | " return accuracy" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 5, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "def fit(net, criterion, optimizer):\n", 77 | " loss_scores = []\n", 78 | " test_scores = []\n", 79 | " train_scores = []\n", 80 | " for epoch in range(100): # loop over the dataset multiple times\n", 81 | "\n", 82 | " # get the inputs\n", 83 | " inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n", 84 | " vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n", 85 | " # zero the parameter gradients\n", 86 | " optimizer.zero_grad()\n", 87 | "\n", 88 | " # forward + backward + optimize\n", 89 | " outputs = net(inputs)\n", 90 | " loss = criterion(outputs, labels.long())\n", 91 | " loss.backward()\n", 92 | " optimizer.step()\n", 93 | "\n", 94 | " # print statistics\n", 95 | " loss_scores.append(loss.item())\n", 96 | " train_scores.append(predict(training_data, net))\n", 97 | " test_scores.append(predict(test_data, net))\n", 98 | " print('Finished Training')\n", 99 | " return loss_scores, train_scores, test_scores" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "# ReLU + Sigmoid + Cross Entropy + L2" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 6, 112 | "metadata": {}, 113 | "outputs": [ 114 | { 115 | "name": "stdout", 116 | "output_type": "stream", 117 | "text": [ 118 | "Finished Training\n" 119 | ] 120 | } 121 | ], 122 | "source": [ 123 | "import torch.optim as optim\n", 124 | "net1 = Net()\n", 125 | "criterion1 = nn.CrossEntropyLoss()\n", 126 | "optimizer1 = optim.SGD(net1.parameters(), lr = 1e-1, weight_decay=1e-2)\n", 127 | "loss_scores1, train_scores1, test_scores1 = fit(net1, criterion1, optimizer1)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": 7, 133 | "metadata": {}, 134 | "outputs": [ 135 | { 136 | "data": { 137 | "text/plain": [ 138 | "
" 139 | ] 140 | }, 141 | "metadata": {}, 142 | "output_type": "display_data" 143 | } 144 | ], 145 | "source": [ 146 | "import matplotlib.pyplot as plt\n", 147 | "\n", 148 | "plt.plot(loss_scores1)\n", 149 | "plt.show()" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 8, 155 | "metadata": {}, 156 | "outputs": [ 157 | { 158 | "data": { 159 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU1fnH8c+TnWwQSAiQENYAsqhAAPdWQMSq4FqFatXa2v4qXdRq9ddW/dnW2loX2lorVau2Vqq4IYIoCKK4lACyEwhrFkgCSQgJZJnM8/vjjhrChAwwyWRmnvfrlRe5957MPJcbvq/LmXPPEVXFGGNM8IsIdAHGGGP8wwLdGGNChAW6McaECAt0Y4wJERboxhgTIqIC9capqanat2/fQL29McYEpZUrV+5T1TRvxwIW6H379iU3NzdQb2+MMUFJRHa1dMy6XIwxJkRYoBtjTIiwQDfGmBBhgW6MMSHCAt0YY0KEBboxxoQIC3RjjAkRARuHbowxIUsVilZCfQ0ADY3K3qpaiioOU1R5mKEjRnPK4MF+f1sLdGOMaUJVEZFW27ndSlHlYXbur2F/dT37a+o5cLiBKNchJuX/hiH73/uybTTQ2/MF8Jn7lzD4Tr/XboFujOmQvgjMvVW1JMdF0zUhhi7x0UT6ELZfUKDqcAP7a+qpOFRPg8vttc2OfTX8d0c5/91RTnlNPSkJ0aTEx5AUF4Vw9PtV17nYvq+a2oYjX68X+/h7zCMMkt082ng1u5NGkpnSiYyUTvROiadXl05kdOnEuO4Dj/evwycW6MaYk1dXDe/9Cko3n/BL1De6OVjbwMFaF9V1LmobGmn0LKhW6fk6WZEt7M8GhkUKP46LJqZrBK5GN67DSuMh7yu6RYgQ1zmSTtERxEVHEh0ZQXSkELk/D3G74cqXuX3QJD9UfHws0I0xJ6diJ/rSNCjbTFXaGPByBy0iX4aeW+FwfSOHG1wcqm/0fN9IvefuOTJCSIyLolunOOJjIomJiqDRrbga3TQ0Hv+SmZERX713S10pMZERxEVHeLkXP059zoaJ90Nq9sm+0gmxQDfGtG7fVqgqPmKXohQU7CL1w1/R6HLxw/q7+HD3qcf1somxUQzsnsjA7okM6ZHEmL5dGdYrmahIG4B3IizQjTHHtvMjeO4SnN7mrwiQBeRrBk/2/A2TR45mRlqi15c43NBIeU095TX1RIiQnZ5Idvck0pNjffoA0vjGAt0Y0zJXPcy7He3Sm/VjHuLVlUVs3FNFVGQEp2V2Zmy/bow84+s80rlLoCs1WKAbE362vAuf/BkaXc52XDJc8ACkOeOiVZWyg3VsLa0m7tOZjN6Xx/8l3cdzb0F6cn++f/EArs7JJCkuOoAnYbyxQDcmXKjCh4/A+7+BlD7Q2RkV7S5Ygfup83mt3/38p2o4W0sOUlXrIlNKeS/mbyySsayLP4NfX5bB1aMziYtuaayICTQLdGPCQX0NvHkrbHgdRlxN4yUzWbajhhc/28Wmyk08Gf0oV225C02cTs2AMWR0iWds8avE7otiwq3PMrFL79bfwwScT4EuIpOBmTjDOJ9W1YeaHc8Cnge6eNrcrarz/VyrMeZEVO6G2dNh73qqz/0Vz8tU/v3YZxRVHiY1MZarzhvL4f5vo2vv5ZoNL8K2F7/62QsfBAvzoCGqxx7XKSKRwBbgAqAQWAFMU9WNTdrMAlar6pMiMhSYr6p9j/W6OTk5amuKGtPGdi6Hl6+n0dXAU6n38OjOvrjcypn9u3HdGX24YGg6MVGeIYKqsGcNNBxytmMSoOdpgavdeCUiK1U1x9sxX+7QxwL5qrrd82KzganAxiZtFEj2fN8ZOHLAqjGm/a14BhbcRU1Cb75Z8xMKintx41m9mTYuiwHehheKQK/T279O4ze+BHoGUNBkuxAY16zN/cC7IvIjIAGY6O2FROQW4BaArKys463VGOMLVz0suAtW/oOdXc9myp6byOrVg3euz6FXl06Brs60IV8C3duo/+b9NNOA51T1ERE5E/iniAxX1SNmrlHVWcAscLpcTqRgY0wzNfuhZJ3zvbppXPJ7Igs/5ZVO3+TnxVO49PRMHrriVDrF2OiUUOdLoBfy1ayPAJkc3aVyMzAZQFU/EZE4IBUo9UeRxhjvDuYtJfrVm4irL/9yXz0x3Fn/I/KSL+DBK/pxzZje9jRmmPAl0FcA2SLSDygCrgWmN2uzG5gAPCcipwBxQJk/CzXGOFSVBev3UvL+X7mu/Al2a3ce0nuI7pRI505RxHXvz7fPymFM3xQL8jDTaqCrqktEZgALcYYkPquqG0TkASBXVecCdwB/F5HbcLpjbtTWhs8YY3xXfwgW3Ud94Rp27q8h83AN34jYwbaUszh86d+Y1T/Lwtv4Ng7dM6Z8frN99zb5fiNwtn9LM8ZsK6tm9/Y8Tlv+Q1Kq8lirQ2ggkj7d0nCfNp0B590BEdY3bhz2pKgxHVBJVS0PL8xj9+pF/DX6caJo4DsNP8M14AJ+e9kIenWLD3SJpgOyQDemA6ltaOTvy7bz5AfbSG0s4724P0JSDw5e/gJPZQ776iEgY7ywQDemA1BV5q4p5vcLNlN8oJbJw3rwKM8Tu0vhxteJTekb6BJNELBANyaAVJVlW/fx+KItrN5dybBeyTx6zemc4cqFf8+HCfeChbnxkQW6MQHy/uYSZi7ayprCA/TqHMcfrjyVK0dnEuk6DH/9GaQOhjN/FOgyTRCxQDemnbndyu/f2cxTy7bTu2snfnfFCK4clflV//iyh50ZEm+cD1ExgS3WBBULdGPaUb3LzV1z1vDG58Vcd0YW9106jOjIJrMdfvSY83XadOhrI4HN8bFAN6adlFbVcscra/hw6z5+NmkQt54/8KuHgeoPwdwZsP5VGHYFXPxIYIs1QckC3Zg2VtvQyDMf7eCJJfk0NLr5w1Wn8s2cJtMjVRbAf74Fe9Y6H4Kec7szla0xx8kC3Zg2oKpsKalm/ro9zFlZSFHlYS4cls49F51C39SErxru+hj+cz246mDabBg8OXBFm6BngW6Mn727YS8PvbOZ7WU1iMDYvl15+OpTOWtA6pENc5+F+XdClz4w7SVIGxyYgk3IsEA3xk9qGxr53fxNPP/JLob0SOI3lw1n0rB0uifFHd1463sw7zYYOBGufAY6dWn/gk3IsUA3xg927Kvhhy+uYtOeKm4+px93TR5MbFQLk2bVH4K373DGmV/7kg1NNH5jgW7MSVqaV8qPXlpNVITw7I05jB+Sfuwf+PARqNwFN8yzMDd+ZYFuzAlSVWYt287v39nM4B7JzLp+NL27tjILYlkeLJ8Jp02Dfue2T6EmbFigG3MCtpQc5NfzNvLh1n1cPKInD199KvExnn9OjQ1Q8Bk01h/9g8v+CDEJcMGv27dgExZ8CnQRmQzMxFmx6GlVfajZ8ceA8z2b8UB3VbVPeUzIqaip57FFW3jxs93Ex0Ry36VDufGsvl89IFSzD17+Nuxa3vKLXDoTEtPap2ATVloNdBGJBJ4ALsBZMHqFiMz1rFIEgKre1qT9j4CRbVCrMQGjqsxbu4f7526g8nAD3xqXxU8nDqJrQpM+8D1rYfZ0qCmDix+F9GFHv1BsMqQPbb/CTVjx5Q59LJCvqtsBRGQ2MBXY2EL7acB9/inPmABzu6le/Hv25r5Fz8MN/Csmkj69E4jfFwmzm7Xdsxbiu8JNCyBjVEDKNeHNl0DPAAqabBcC47w1FJE+QD/g/RaO3wLcApCVlXVchRrT3rT2AMXPfpuM0qVU60B6dOtGry6diGjpqfyhU5y+8aRWRrkY00Z8CXRvv77aQttrgTmq2ujtoKrOAmYB5OTktPQaxgRc0bZ18NJ00hsKebbzDxl//S/ITEsMdFnGHJMvgV4INJlJiEyguIW21wK3nmxRxgTKwdoGFrzxIhdu+l/cRLBk3N+5cfKVRLR4W25Mx+FLoK8AskWkH1CEE9rTmzcSkcFACvCJXys0ph2UVNUyb00xVUse58eNL1AS15/ob73EBVk2v4oJHq0Guqq6RGQGsBBn2OKzqrpBRB4AclV1rqfpNGC2qlpXigkKlTV1FLx8J9HF/6WmzsVZ1HFKxG4q+32DXtOfdsaLGxNEJFD5m5OTo7m5uQF5bxPe1hZW8vzHu2hcN4fHI//E5sjBxCd1ITUxlvihk+DMGTYfuemwRGSlquZ4O2ZPipqwUVhxiIcWbGbe2j2kx9SxKPZFDqecypD/WQoRLUykZUwQsUA3Ia22oZFVuytYsrmU5z/ZRYTAjydkM6N2FjEry+GyORbmJmRYoJuQ9Nn2/Ty+aCu5u8ppaFQiBC49rRc/nzyEXoc2w9+fgTHftQeATEixQDchZff+Q/xuwSYWrN9Lz85xfOfsfozr35Wcvl1JjouGmv3w5o8gIQ0m/CrQ5RrjVxboJiSoKs9/vJMHF2wmUoQ7LhjEd8/tT6eYJt0pe9c5c60cLIFr/glxnQNXsDFtwALdBL3ymnrumrOGRZtKmTCkOw9eMYL05GbLvm18E17/gRPi31kAGaMDU6wxbcgC3QStA4caeHVVIU8t20ZFTcPRU9l+oWIXvHKT019+zb8gqUdgCjamjVmgm6Bx4FAD+WUH2VpSzX93lvP22j3UudyMzOrCMzeMYXhGC10oK58DFK76h4W5CWkW6KbDqzxUz6/nbeLVVYVf7kuMjeKq0ZlMH5fFsF7H6At31cPqf0L2hdCld8vtjAkBFuimQ5u/bg/3vrmeykMNfPecfpw1sBsD05LISOlEpC8TZm1+y1lwYszNbV+sMQFmgW46JFXlt29v4umPdjAiozMvfGccQ3slH/8LrXgWuvSBARP8X6QxHYwFuumQHlu0lac/2sENZ/bhV5cMJSoy4vhfpHQz7PoIJt4PESfw88YEGQt00+HMWraNPy3eyjU5vbl/yrCjR634KvdZiIyBkdf7t0BjOigLdNNhNDS6+euSbTy2aAsXn9qTB68YceJhvn8brJkNQ6dCQqp/CzWmg7JANx3C+qID3DVnLRv3VDH19F48fNVpvn3o6U3+Yphzk9PNcs7t/i3UmA7MAt0E1IbiA/zr0128nFtI14QY/nbdaCYPP46x4qpQtBIOVzrbxath6YOQdgpM+zek9G2Tuo3piHwKdBGZDMzEWbHoaVV9yEubbwL34ywgvUZVj1qmzpgvLM0r5fFFW/m8oJLYqAiuHdObuy4cQuf46ON7odxn4e1md+GnTIHLnoRYW9TZhJdWA11EIoEngAtwFoxeISJzVXVjkzbZwD3A2apaISLd26pgE/z++eku7ntzPVld4/nVJUO5alTm8Qc5OJNsLfo/6HsuTLjP2RcVCz1G2IpDJiz5coc+FshX1e0AIjIbmApsbNLme8ATqloBoKql/i7UBD9V5Y/v5vHEkm1MGNKdP08fSXzMSfT6vftLcB2GSx6H1IH+K9SYIOXLv6YMoKDJdiEwrlmbQQAishynW+Z+VX2n+QuJyC3ALQBZWVknUq8JMvmlB1mxs4L80mrWFFSSu6uCa3J689vLh5/Y2PIvbF8K616Gr/3cwtwYD18C3dv/XZuvLB0FZANfBzKBD0VkuKpWHvFDqrOAWeAsEn3c1ZoOq9HtrAr0xTDDtYWV/GlxPos2lQAQFx3BgLRE7rloCLec1//4hyO66mD3J9DoAhTeuRtS+tkoFmOa8CXQC4GmsxplAsVe2nyqqg3ADhHJwwn4FX6p0nRI1XUu3t9cyjvr97Bkcxkut5uU+BgS46LYXlZDclwUP52YzRUjM8lM6UTEiQ5DrCqG/1znjGb5ksB1cyA6rsUfMybc+BLoK4BsEekHFAHXAs1HsLwBTAOeE5FUnC6Y7f4s1HQsb6wu4pdvrKe6zkVaUiyXj8ogOS6a8po6Kg41cNXoTK4/ow9JcSfwYWdTBSucMK+vdkaudMt29id0g679T/5EjAkhrQa6qrpEZAawEKd//FlV3SAiDwC5qjrXc2ySiGwEGoE7VXV/WxZuAqOmzsWv3lzPa6uKGNM3hTsvHMLoPikn/hCQ2w3LH4fNb3s/vnctJPeC61+H9KEnXrgxYUBUA9OVnZOTo7m5uQF5b+M7V6Ob5dv2k7e3iq0l1Xy8bT97Dhxmxvhsfjx+4Ml9sFl30FkWbvM8yMjxvsZnci+44AGI73ri72NMCBGRlaqa4+2YPSlqvKp3uXljdRFPLM1n1/5DAKQlxTIoPZE/Xn0aZw7odnJvUL4DXpoG+/Lgwt/BGf9jY8eNOUkW6AaAdYUHmLe2mLKDdZQfqidv70H2HKhlREZnnvzWKM4akHpiD/9446qHf18D1SVw3Wsw4Hz/vK4xYc4CPcyt3FXBX97fypK8MmIiI0hLiqVrQgzDMzrz4BUj+PqgtBOf8bAln/zZuTOf/oqFuTF+ZIEeBlSV0oN1bC2pZmvpQfJLq9laWk1+aTXlNfWkxEdz54WDuf7MPiSf7KiU1pTvgA/+4My3MmhS276XMWHGAj3EbS+r5p7X1vHZjvIv93XuFE1290QmDU1neEZnLh+ZQUJsO/wqqMKCuyAiCiYfNb+bMeYkWaCHKFejm6c/2sFj720hJiqCn08ewmm9OzOweyJpibH+70bxxaa3YOu7cOGD0Dmj/d/fmBBngR5iXI1u5q3dw1+W5JNfWs2koen8+rLhpCcH+InKTW85QxTTR8DY7we2FmNClAV6iCitqmXB+r38Y/kOdu4/xOD0JP523SguHNYjMHfjX3C7YdkfYOnvIGM0XPMviLRfO2Pagv3L6uBKq2pZuGEvizaVUnm44cv9CTGRpCTE0DU+hk17qli5uwJVGJ6RzN+uG82koeknPneKv6jCG/8Da2fDadPhksds7hVj2pAFegekqizdUsZTH2zjsx3lqEL/1AR6d413juM8gr+xuIrymnp6denEbRMHcdHwHmSnJwW2+KY2z3PC/Lw74fxf2INDxrQxC/QOpN7l5v3NJTyxZBvrig6Q0aUTP50wiItG9CC7e2Jgu06OV101LPg5pA+Hr91tYW5MO7BAD7CGRjdL88qYv24PizaVcLDWRZ9u8fzhylO5bGQGMVEnMVdKIC39HVQVwdXPWZ+5Me3E/qW1k3qXm5W7KoiPiaRrQgyNbuXVVYXMXlFA2cE6OneKZvKwHlw0ogfnZaed3KRXgbZ3PXz6JIy6AXqPDXQ1xoQNC/R2sKagkrvmrCWv5OAR+0Vg/ODuTB+XxXmD0ogO5hD/wqFyePNW6NQFJt4f6GqMCSsW6G2ops7FzMVbefrD7XRPiuPxa04nMTaK8pp6al2NjB/SncyU+ECX6T8lG2H2NGeFoaufsylvjWlnFuhtoKq2gRc+3skzH+2g4lAD08dlcfdFQ9p+npT2VrQKavY531cVwsJfQmwi3Dgfeo8JbG3GhCGfAl1EJgMzcVYselpVH2p2/EbgYZwl6gD+oqpP+7HOoNDoVv6xfAczF2/lYK2LCUO6M2P8QEZmpQS6NP9qdMF7v4JP/3rk/i8eHEruFZi6jAlzrQa6iEQCTwAX4CwGvUJE5qrqxmZN/6OqM9qgxqCwpeQgd81Zy+cFlZw/OI07Jg1meIaXFXiC3aFymHMTbF/qPMJ/6jXOfhHoMQIiQ+x/IcYEEV/u0McC+aq6HUBEZgNTgeaBHrZezi3gF6+vIykumj9NG8mlp/YMrjHjx1K4EhbdB/U1zvaBQqithCl/gVHXB7Y2Y8wRfAn0DKCgyXYhMM5LuytF5DxgC3CbqhY0byAitwC3AGRlZR1/tR3Qx9v28b+vreOM/t2Yee3pdEuMDXRJ/vP5S/DWTyC+G6QPc/Yl94Kzf2LDEY3pgHwJdG+3ms1Xln4LeElV60TkB8DzwPijfkh1FjALnEWij7PWDqeg/BC3vriKvqkJPHndKJKC4UPP2ioo+AzUfex2296Hz/4Gfc+Fq5+HhJNcQ9QY0+Z8CfRCoHeT7UyguGkDVd3fZPPvwO9PvrSOrbrOxXefz8Wt8PS3c4IjzEs3O8MKy7f71n7cD2DSb6xf3Jgg4UugrwCyRaQfziiWa4HpTRuISE9V3ePZnAJs8muVHdA9r60jv6yaF74zlr6pCYEup3Wb58Nr34PoeLj235DU49jtY5MhNbt9ajPG+EWrga6qLhGZASzEGbb4rKpuEJEHgFxVnQv8WESmAC6gHLixDWsOuLfWFPPWmmJ+NmkQZw9MDXQ5rVv5vNMX3ut0uOZFWy3ImBAlqoHpys7JydHc3NyAvPfJKK2qZdLjy+iXmsAr3z+z48+5cqAQ/jLWedBn2myI7hToiowxJ0FEVqpqjrdjHTyNOhZV5eevrqW2oZFHrj6t44c5OFPYqhsu/ZOFuTEhLggSqeN46b8FLMkr4+7JQ+iflhjoclqX946zyMTX7oKUPoGuxhjTxizQfbRoYwn3vrmec7NT+faZfQNdTuvqD8GCOyFtCJwZtg/wGhNWbHIuH3ycv48f/nsVQ3sl89dvjQr8Wp2tcTfCOz+Hyt3ORFlRMYGuyBjTDizQW7FqdwXffSGXft0SeP6msR1/vPnhCpjzHefBoLN/Cn3PDnRFxph2YoF+DA2Nbn780mrSkmL5581jSUno4He6Xzw4VFngfAg6+oZAV2SMaUcW6Mfw1ppiCisO88wNOXRPjgt0OceWtwBe/Z4zkuXGeZB1RqArMsa0Mwv0Frjdyl+XbmNIjyTGD+ke6HJapgof/hHe/y30PA2ufRE6Zwa6KmNMAFigt+C9TSXkl1Yz89rTAz8VbnUpzP8ZVOw6+lh9DezfCiO+CVNsrLkx4cwC3QtV5+48q2s8F4/oGdhiilfD7G85C0v0Oxevk1+O+z6M+a6zyIQxJmxZoHvxybb9rCmo5DeXDW//p0H3rIEqzzxnlbvgvXshIQ1ufhd6ntq+tRhjgooFuhdPfrCNtKRYrhrdzn3RW9+DF686cl+fs+GbL0BCEEwCZowJKAv0ZooqD/Ph1n3cNnEQcdGR7ffGDYedfvJu2XDFU4BARCR0HwaRdpmMMa2zpGjmjdVFAFw+sp2nmP3wEajYCTe8BRmj2/e9jTEhweZyaUJVeX11ETl9UsjqFt9+b1y2BT56HE69Bvqd137va4wJKXaH3sSG4iryS6v57eXD/f/irnrY+SE01h99bPmfICbeWe7NGGNOkE+BLiKTgZk4KxY9raoPtdDuKuAVYIyqBt3qFa+tKiImMoJLRvTy7wtXl8LL34bdn7Tc5tKZkNiBH2AyxnR4rQa6iEQCTwAX4CwYvUJE5qrqxmbtkoAfA5+1RaFtzdXoZu6aYsYP6U7neD9OwFX8uWcc+X6Y8mfoMeLoNjFJkDrQf+9pjAlLvtyhjwXyVXU7gIjMBqYCG5u1+zXwB+Bnfq2wnXyUv4991XVc5uuHoSufh8IVcOGDEJfcZP9zzjFtdLbL8iA+Fb7zjrOmpzHGtBFfPhTNAAqabBd69n1JREYCvVV13rFeSERuEZFcEcktKys77mLb0uuri+jcKZrzh6S13nj/NmeI4ep/wtMTnW1XPbz1U2cxZrcLkno6XyOugluWWJgbY9qcL3fo3p4n/3JlaRGJAB4DbmzthVR1FjALnEWifSux7R2sbWDhhr1cOSqT2KhWxp6rwtt3QFQcXPYkzL8T/n4+dBsIRSudOcgn3OuMITfGmHbkS6AXAr2bbGcCxU22k4DhwFLPJFY9gLkiMiVYPhh9a80eahvcXJ3Tu/XGG16D7Uvgooedu+/MHKePvGQDXPmMs88YYwLAl0BfAWSLSD+gCLgWmP7FQVU9AHz5XLqILAV+FixhDvBybgGD0hM5LbPzsRvWHoB37oGep8OYm519KX3he+87x2yUijEmgFrtQ1dVFzADWAhsAl5W1Q0i8oCITGnrAtvalpKDfF5QyTdzeh97mlxXPcy73RmCeMljR3apRMVamBtjAs6nceiqOh+Y32zfvS20/frJl9V+Xl5RQFSEHPtR/+oyeOUG2LUczv8FZIxqvwKNMcZHYf2kaL3Lzeuri5h4SjrdEmO9N9qzFmZPh5oyuOLvcOo327dIY4zxUVgH+vubS9lfU881Y1r4MLS2Cl68GiQCblpgd+bGmA4trAP95dwC0pNjOTe7hbnGl/wWqkucDz0tzI0xHVzYzrZYWlXL0rxSrhyV6X1VouLP4b+znKXdLMyNMUEgbAP99dVFuBXvqxK5G2Hebc4j++N/2f7FGWPMCQjLLhdV5dVVhYzK6kL/tMSjG+Q+C8WrnAeFOnVp/wKNMeYEhOUd+rqiA2wpqeZKb3fnB0tg8a+h39dg+JXtX5wxxpygsAz0V1cWEhMVwSWnepn3/N1fgOswXPwoHOtBI2OM6WDCLtDrXI28uaaYSUPT6dyp2bzn25fCulfgnNtsfnJjTNAJu0BfsrmUykMNR38Y6qpzZlFM6Qfn3B6Y4owx5iSE3Yeic1YW0T0plnOzm817vnwm7M+H616D6LjAFGeMMSchrO7QKw/VszSvlMtHZhAZ0aR/vLHBCfRTpsDACYEr0BhjTkJYBfqHW/fhcisXDu9x5IHi1VBfbaNajDFBLawC/YMtZXTuFM1pmc3Glu/8yPmzz9ntX5QxxvhJ2AS6qrJsSxnnZKce2d0CzrS4qYMh0Yf1RI0xpoMKm0DfvPcgpQfr+NqgZqHd6ILdn0Jfuzs3xgQ3nwJdRCaLSJ6I5IvI3V6O/0BE1onI5yLykYgM9X+pJ2fZljKAowN97xqn/9y6W4wxQa7VQBeRSOAJ4CJgKDDNS2D/W1VHqOrpwB+AR/1e6Un6YEsZQ3okkZ7cbEjizuXOn33Paf+ijDHGj3y5Qx8L5KvqdlWtB2YDU5s2UNWqJpsJgPqvxJNXU+dixc7yo+/Owek/7zoAknocfcwYY4KILw8WZQAFTbYLgXHNG4nIrcDtQAww3tsLicgtwC0AWVlZx1vrCft0+34aGpXzmge6uxF2fQLDpnr/QWOMCSK+3KF7m6HqqDtwVX1CVQcAPwe8TiKuqrNUNUdVc9LS2m9EyQdbyugUHUlO35QjD5Ssh7oD0Me6W4wxwc+XQC8Emi66mQkUH6P9bOCykynK35ZtKePMAd2IjYo88sCX/ef2gagxJvj5EugrgJuNnPUAAAvKSURBVGwR6SciMcC1wNymDUQku8nmxcBW/5V4cnbuq2Hn/kOc523d0F3LoUsf6OxlXnRjjAkyrfahq6pLRGYAC4FI4FlV3SAiDwC5qjoXmCEiE4EGoAK4oS2LPh6LN5cCMH5I+pEH3G7Y9TEMvigAVRljjP/5NNuiqs4H5jfbd2+T73/i57r8ZvGmErK7J5LVLf7IA2Wb4HC5jT83xoSMkH5StKq2gf/uKGfCKelHH7T+c2NMiAnpQP8grwyXW5l4SvejD+76CJIznT50Y4wJASEd6O9vLiUlPpqRWc2GK6o6/ed9z7Z1Q40xISNkA93V6GZJXinnD+5+9OyK+7ZATZn1nxtjQkrIBvqq3ZVUHmpoof/cM/+5zd9ijAkhIRvoizeVEB0pnDeohfHniT2ga//2L8wYY9pIyAb6ok0ljOvXjaS46CMPqDojXKz/3BgTYkIy0Hfuq2FbWQ0TvI1uKd8O1Xut/9wYE3JCMtDf21gCwETrPzfGhJGQDPSFG/YytGcyvbvGH31w13JISIPUQe1fmDHGtKGQC/Syg3Ws3F3BpGFe7s6/6D/vc5b1nxtjQk7IBfqiTSWowoXDvKxAtOMDqCqEfl9r/8KMMaaNhVygL9ywl6yu8QzpkXTkAVcdvH0HpPSD078VmOKMMaYNhVSgH6xt4OP8/Vw4LB1p3qWyfCbsz4eLH4HoOO8vYIwxQSykAn1JXhn1jW4mNe9u2b8Nlv0Rhl0OAycEpjhjjGljIRXo727YS2piDKOaTsalCvPvhMgYuPB3gSvOGGPamE+BLiKTRSRPRPJF5G4vx28XkY0islZEFotIu89JW+dqZGleGRcMTT9yMq6dH8G2xXD+/0Jyz/Yuyxhj2k2rgS4ikcATwEXAUGCaiAxt1mw1kKOqpwJzgD/4u9DWfLR1H9V1rqO7W3KfgbgukHNTe5dkjDHtypc79LFAvqpuV9V6YDYwtWkDVV2iqoc8m58C7b7q8muriuiaEMPZA5pMxnWwBDa95Yxqie7U3iUZY0y78iXQM4CCJtuFnn0tuRlY4O2AiNwiIrkikltWVuZ7la2oPFTPextLmHp6L2KimpzS6hfA7YKc7/jtvYwxpqPyJdC9PVKpXhuKXAfkAA97O66qs1Q1R1Vz0tLSfK+yFW+t3UN9o5srRzX5j4G7EXKfcx4iSh3ot/cyxpiOypdALwR6N9nOBIqbNxKRicAvgCmqWuef8nwzZ2UhQ3okMaxX8lc7tyx0ngodc3N7lmKMMQHjS6CvALJFpJ+IxADXAnObNhCRkcBTOGFe6v8yW5ZfepA1BZVcNTrzyIeJcp9xFrEY/I32LMcYYwKm1UBXVRcwA1gIbAJeVtUNIvKAiEzxNHsYSAReEZHPRWRuCy/nd3NWFhEZIUw9vUm3/o4PIX8xjL4BIqNb/mFjjAkhUb40UtX5wPxm++5t8v1EP9flk0a38vrqQr4+KI20pFhn54pnYMFd0G0AjPleIMoyxpiACOonRT/eto+SqjquGp0JjS5466fw9u3Q/3z47mJI9N8Hr8YY09H5dIfeUS3csJf4mEjOH9Idti6Elf+AM2fABQ9ARGSgyzPGmHYVtHfoqsqSzWWcPTCVuOhI2JfnHPj63RbmxpiwFLSBvqWkmqLKw4wf4lkIunwHxKdCbNKxf9AYY0JU0Ab6+5ud0ZHnD/YEesUO6NovgBUZY0xgBW2gL9lcytCeyfTo7FmsonynsxqRMcaEqaAM9AOHGli5u+Kr7hZXvfNUqN2hG2PCWFAG+gdby2h0qzO6BaByN6jb7tCNMWEtKAN9yeZSuibEcHrvLs6Oih3On3aHbowJY0EX6I1uZWleKV8blPbVykTlnkC3O3RjTBgLukBfU1hJxaGGr7pbwLlDj06AxO4t/6AxxoS4oAv0pXllREYIX8tu8lh/+Q5I6Qvibep2Y4wJD0H36P+t5w9g/JDudI5vMotixQ7oZotYGGPCW9DdocdGRX71YSiA2w0VO507dGOMCWNBF+hHqd4Lrlob4WKMCXvBH+g2wsUYYwAfA11EJotInojki8jdXo6fJyKrRMQlIlf5v8xjsDHoxhgD+BDoIhIJPAFcBAwFponI0GbNdgM3Av/2d4GtKt8BEgmde7fe1hhjQpgvo1zGAvmquh1ARGYDU4GNXzRQ1Z2eY+42qPHYKnZAl962dqgxJuz50uWSARQ02S707DtuInKLiOSKSG5ZWdmJvMTRyndY/7kxxuBboHt7WkdP5M1UdZaq5qhqTlqan9b7tHnQjTEG8C3QC4GmHdSZQHHblHOcDlfC4Qq7QzfGGHwL9BVAtoj0E5EY4FpgbtuW5SMb4WKMMV9qNdBV1QXMABYCm4CXVXWDiDwgIlMARGSMiBQCVwNPiciGtiz6S2WehaHtDt0YY3yby0VV5wPzm+27t8n3K3C6YtqPqw6WPeyEedrgdn1rY4zpiIJucq4vLZ8J+/PhuldtyKIxxhCsj/7v3wbL/gjDLoeBEwNdjTHGdAjBF+iqMP9OiIyBC38X6GqMMabDCL5A3/gGbFsM438JyT0DXY0xxnQYwRfoMUkw+GIY891AV2KMMR1K8H0omj3R+TLGGHOE4LtDN8YY45UFujHGhAgLdGOMCREW6MYYEyIs0I0xJkRYoBtjTIiwQDfGmBBhgW6MMSFCVE9oNbmTf2ORMmDXCf54KrDPj+UEi3A873A8ZwjP8w7Hc4bjP+8+qup1Dc+ABfrJEJFcVc0JdB3tLRzPOxzPGcLzvMPxnMG/521dLsYYEyIs0I0xJkQEa6DPCnQBARKO5x2O5wzhed7heM7gx/MOyj50Y4wxRwvWO3RjjDHNWKAbY0yICLpAF5HJIpInIvkicneg62kLItJbRJaIyCYR2SAiP/Hs7yoi74nIVs+fKYGu1d9EJFJEVovIPM92PxH5zHPO/xGRmEDX6G8i0kVE5ojIZs81PzNMrvVtnt/v9SLykojEhdr1FpFnRaRURNY32ef12orjT55sWysio473/YIq0EUkEngCuAgYCkwTkaGBrapNuIA7VPUU4AzgVs953g0sVtVsYLFnO9T8BNjUZPv3wGOec64Abg5IVW1rJvCOqg4BTsM5/5C+1iKSAfwYyFHV4UAkcC2hd72fAyY329fStb0IyPZ83QI8ebxvFlSBDowF8lV1u6rWA7OBqQGuye9UdY+qrvJ8fxDnH3gGzrk+72n2PHBZYCpsGyKSCVwMPO3ZFmA8MMfTJBTPORk4D3gGQFXrVbWSEL/WHlFAJxGJAuKBPYTY9VbVZUB5s90tXdupwAvq+BToIiI9j+f9gi3QM4CCJtuFnn0hS0T6AiOBz4B0Vd0DTugD3QNXWZt4HLgLcHu2uwGVqurybIfi9e4PlAH/8HQ1PS0iCYT4tVbVIuCPwG6cID8ArCT0rze0fG1POt+CLdDFy76QHXcpIonAq8BPVbUq0PW0JRG5BChV1ZVNd3tpGmrXOwoYBTypqiOBGkKse8UbT7/xVKAf0AtIwOlyaC7UrvexnPTve7AFeiHQu8l2JlAcoFralIhE44T5i6r6mmd3yRf/BfP8WRqo+trA2cAUEdmJ05U2HueOvYvnv+QQmte7EChU1c8823NwAj6UrzXARGCHqpapagPwGnAWoX+9oeVre9L5FmyBvgLI9nwSHoPzIcrcANfkd56+42eATar6aJNDc4EbPN/fALzZ3rW1FVW9R1UzVbUvznV9X1W/BSwBrvI0C6lzBlDVvUCBiAz27JoAbCSEr7XHbuAMEYn3/L5/cd4hfb09Wrq2c4Fve0a7nAEc+KJrxmeqGlRfwDeALcA24BeBrqeNzvEcnP9qrQU+93x9A6dPeTGw1fNn10DX2kbn/3Vgnuf7/sB/gXzgFSA20PW1wfmeDuR6rvcbQEo4XGvg/4DNwHrgn0BsqF1v4CWczwgacO7Ab27p2uJ0uTzhybZ1OCOAjuv97NF/Y4wJEcHW5WKMMaYFFujGGBMiLNCNMSZEWKAbY0yIsEA3xpgQYYFujDEhwgLdGGNCxP8DX+CgLEisb10AAAAASUVORK5CYII=\n", 160 | "text/plain": [ 161 | "
" 162 | ] 163 | }, 164 | "metadata": { 165 | "needs_background": "light" 166 | }, 167 | "output_type": "display_data" 168 | } 169 | ], 170 | "source": [ 171 | "import matplotlib.pyplot as plt\n", 172 | "plt.plot(train_scores1)\n", 173 | "plt.plot(test_scores1)\n", 174 | "plt.show()" 175 | ] 176 | }, 177 | { 178 | "cell_type": "markdown", 179 | "metadata": {}, 180 | "source": [ 181 | "# ReLU + Sigmoid + Cross Entropy + L2 + Adam" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": 10, 187 | "metadata": {}, 188 | "outputs": [ 189 | { 190 | "name": "stdout", 191 | "output_type": "stream", 192 | "text": [ 193 | "Finished Training\n" 194 | ] 195 | } 196 | ], 197 | "source": [ 198 | "import torch.optim as optim\n", 199 | "net2 = Net()\n", 200 | "criterion2 = nn.CrossEntropyLoss()\n", 201 | "optimizer2 = optim.Adam(net2.parameters(), lr = 1e-3, weight_decay=1e-2)\n", 202 | "loss_scores2, train_scores2, test_scores2 = fit(net2, criterion2, optimizer2)" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 11, 208 | "metadata": {}, 209 | "outputs": [ 210 | { 211 | "data": { 212 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAdrElEQVR4nO3dd3hc9Z3v8fd3inqzrGJbcpFt3DDYGNkYHHpITFng0iF3KYE4BHJJQnZzSTbPpmx2s3fvBRJIQkIogYQSipeYEp4AoXe54gLYuEm2bMuWVSxZ1b/7x4xt2UiWbI10Zs58Xs8zj2bOOdJ8Dsd85sxvzpxjzjlERCTxBbwOICIisaFCFxHxCRW6iIhPqNBFRHxChS4i4hMhr564oKDAjRkzxqunFxFJSAsXLtzunCvsbp5nhT5mzBgqKiq8enoRkYRkZht6mqchFxERn1Chi4j4hApdRMQnVOgiIj6hQhcR8QkVuoiIT6jQRUR8IuEKfUt9Cz95dgXtnXu8jiIiElcSrtCXVNbx4NvrueuV1V5HERGJKwlX6HOnDuPiGaX8+tU1LNyw0+s4IiJxI+EKHeDH509heG46tz6xhKbWDq/jiIjEhYQs9Oy0MHdePp2Ntc387PlVXscREYkLCVnoALPK8vn6KeN47IONvLi82us4IiKeS9hCB7j1rAlMK83ln59axsYdzV7HERHxVEIXekoowK+umoEBNz+6iNaOTq8jiYh4JqELHWBkfga3XzadjzbV8+8aTxeRJJbwhQ5w1pRivnZyGQ+/u4EFSzd7HUdExBO+KHSA782dRPnoIdz29DLWbGv0Oo6IyKDzTaGHg5Hx9IyUIDf+aZGOTxeRpOObQgcYlpvGXVccx9qaXdw2/yOcc15HEhEZNL4qdICTxhfw3S9N5Nmlm3n43R6vpSoi4ju+K3SAb5w6jjMmFfGz51eytLLO6zgiIoPCl4UeCBi3XzqNouw0bn50EfXN7V5HEhEZcL4sdIAhmSncfdVxbKlv4Z+eWqrxdBHxPd8WOsCMUUO47exJvLRyKw+8vd7rOCIiA8rXhQ5w/RfK+OLkYv7zr6tYvqne6zgiIgPG94VuZvzXJceSn5nCLY8t1vHpIuJbvi90gPzMFO68fDrrdjTxk2dXeB1HRGRAJEWhA5w0roCbThvHExVVPLdM53sREf9JmkIH+PYXJzB9ZB4/fGY52xpavI4jIhJTSVXo4WCA2y+bRkt7J//76WU6lFFEfCWpCh1gXGEWt82dxKuf1PD4h5VexxERiZmkK3SAq08cw0njhvKz51ZSWatL14mIPyRloQcCxv+9dBoBM773lIZeRMQfei10MxtpZq+a2SozW2Fm3+pmGTOzu8xsjZktM7MZAxM3dkry0vn+OZN5d+0Onqyo8jqOiEi/9WUPvQP4rnNuMjAbuNnMphy0zNnAUdHbPOCemKYcIFfMHMmssnx+9vxKtjXqqBcRSWy9Frpzrto5tyh6vxFYBZQctNgFwMMu4j0gz8yGxzxtjAUCxs8vOoaWjj38ZMFKr+OIiPTLYY2hm9kY4Djg/YNmlQBdDxmp4vOlj5nNM7MKM6uoqak5vKQDZFxhFrecMZ7nP6rmbyu2eB1HROSI9bnQzSwLeBr4tnOu4eDZ3fzK5z5pdM7d65wrd86VFxYWHl7SATTvlHFMGpbNjxesYHdbp9dxRESOSJ8K3czCRMr8Eefc/G4WqQJGdnlcCiTM9+tTQgF+esFUNte38JvX1ngdR0TkiPTlKBcD7gdWOefu6GGxBcDV0aNdZgP1zrnqGOYccLPK8rlw+gh+9/pa1m9v8jqOiMhh68se+hzgH4EzzGxJ9HaOmd1oZjdGl3kBWAusAX4P3DQwcQfW98+ZTDho/Ntz+oBURBJPqLcFnHNv0f0YeddlHHBzrEJ5pTgnjW998Sj+44WPeWXVVs6cXOx1JBGRPkvKb4oeynVzyhhflMW/PbeS9s49XscREekzFfpBwsEAPzhnEut3NPPo+xu9jiMi0mcq9G6cPrGIE8cO5ZevrKahpd3rOCIifaJC74aZ8YNzJlPb1MbvXv/M6zgiIn2iQu/BMaW5XDh9BPe9uY7q+t1exxER6ZUK/RD+6csTcQ5u/9unXkcREemVCv0QSodkcM1Jo5m/qIrPanZ5HUdE5JBU6L248dRxpIWD3PXKaq+jiIgckgq9F0OzUrnmpDEsWLqZ1VsbvY4jItIjFXofzDt5LBnhIL/QXrqIxDEVeh8MyUzhujllPL+smo+3HHzmYBGR+KBC76MbTi4jOzXEL17SXrqIxCcVeh/lZaRw3ZwxvLhiC2u2aSxdROKPCv0wXDunjLRwgN++vtbrKCIin6NCPwz5mSlcMXMUzyzexOY6fXtUROKLCv0w3XByGQ64/611XkcRETmACv0wlQ7J4PxpI3jsg43UNbd5HUdEZB8V+hH4+qljaW7r5OF3N3gdRURkHxX6EZg0LIczJhXx4NvraGnv9DqOiAigQj9iXzt5LDub2/nLkk1eRxERAVToR2z22HwmDcvmwbfXE7lGtoiIt1ToR8jMuG7OGD7e0sh7a2u9jiMiokLvjwumlzAkI8yDb+sQRhHxngq9H9LCQa6cNYqXV22lsrbZ6zgikuRU6P30P2ePxsx4+N31XkcRkSSnQu+nEXnpzJ06jMc/rKS5rcPrOCKSxFToMXDNiWNobOnguaXVXkcRkSSmQo+BmWOGML4oi0c+2Oh1FBFJYir0GDAzrpo1iqWVdSzfVO91HBFJUir0GLl4RimpoQCPai9dRDyiQo+R3Iww5x07gr8s3sSuVn04KiKDT4UeQ1edMIqmtk4WLNnsdRQRSUIq9BiaMSqPScOyefQDnVZXRAafCj2GzIyrThjF8k0NfFSlD0dFZHCp0GPsguklpIYCPLmw0usoIpJkVOgxlpseZu7UYTyzeJMufiEig0qFPgAuKx9JQ0sHf1u51esoIpJEVOgD4MSxQynJS+fJCg27iMjg6bXQzewBM9tmZst7mH+amdWb2ZLo7V9jHzOxBALGpeWlvLVmO1U7dVpdERkcfdlD/wMwt5dl3nTOTY/eftr/WInvkuNLAXh6oa45KiKDo9dCd869Aegaa4epdEgGc8YV8OTCSvbs0TVHRWTgxWoM/UQzW2pmfzWzo2P0NxPepeWlVO3czXvrdngdRUSSQCwKfREw2jk3DbgbeKanBc1snplVmFlFTU1NDJ46vn356GFkp4Y07CIig6Lfhe6ca3DO7YrefwEIm1lBD8ve65wrd86VFxYW9vep415aOMi5xw7nr8uradIJu0RkgPW70M1smJlZ9P6s6N/UGEPUxceX0tzWyYvLt3gdRUR8LtTbAmb2GHAaUGBmVcCPgDCAc+63wCXAN8ysA9gNXOGc06eAUeWjhzB6aAZPL6ri4uiRLyIiA6HXQnfOXdnL/F8Bv4pZIp8xMy46rpRfvPIpm+p2U5KX7nUkEfEpfVN0EFw0owTn4L8XVXkdRUR8TIU+CEbmZ3BCWT5PL9qERqNEZKCo0AfJxceXsm57E4s21nkdRUR8SoU+SM45Zjhp4QDzNewiIgNEhT5IslJDzD16GM8u3azzpIvIgFChD6KLZpTS0NLB3z/e5nUUEfEhFfogmjO+gOKcVA27iMiAUKEPomDAuPC4El77pIbtu1q9jiMiPqNCH2QXHVdKxx7HgiWbvY4iIj6jQh9kE4dlM7Ukh/mLNewiIrGlQvfAxTNKWb6pgU+2NHodRUR8RIXugfOnjSAUMH04KiIxpUL3wNCsVE6fVMT8xZvo6NzjdRwR8QkVukcuOb6UmsZW3ljt/ys3icjgUKF75PSJReRnpvDUQg27iEhsqNA9khIKcOH0El5euY2dTW1exxERH1Che+iS40tp69zDs8t0TLqI9J8K3UNTRuQwZXiOhl1EJCZU6B675PhSllXV65h0Eek3FbrHLpg+gnDQeLKi0usoIpLgVOgeG5qVyllTipm/eBOtHTpPuogcORV6HLh85ihqm9p4aeVWr6OISAJToceBk8cXUJKXzp8/1LCLiBw5FXocCASMy8pH8ubq7VTWNnsdR0QSlAo9TlxaXkrA4Al9OCoiR0iFHidG5KVz6oRCnqio1Am7ROSIqNDjyOUzR7G1oZXXPtEJu0Tk8KnQ48iZk4sozE7lkfc3eB1FRBKQCj2OhIMBrpw1itc+rWHDjiav44hIglGhx5mvnDCKoBl/ek976SJyeFTocaY4J40vTx3Gnz+sZHebvjkqIn2nQo9DV88eTUNLBwuWbvI6iogkEBV6HJpVls+kYdk89M4GnHNexxGRBKFCj0Nmxj+eOJqV1Q0s2rjT6zgikiBU6HHqwuklZKeFeODt9V5HEZEEoUKPU5mpIa46YRR//aiajTt0fhcR6Z0KPY59dU4ZwYBx/1trvY4iIglAhR7HinPSuGB6CU9UVLGzqc3rOCIS51TocW7eKWPZ3d7JH/VFIxHpRa+FbmYPmNk2M1vew3wzs7vMbI2ZLTOzGbGPmbwmFGdz+sRCHnpnPS3t+qKRiPSsL3vofwDmHmL+2cBR0ds84J7+x5Ku5p0yjh1NbTy9qMrrKCISx3otdOfcG0DtIRa5AHjYRbwH5JnZ8FgFFJg9Np9pI/P47euf0a5zpYtID2Ixhl4CdL3MTlV02ueY2TwzqzCzipoanfO7r8yMb505nsra3czXXrqI9CAWhW7dTOv2++rOuXudc+XOufLCwsIYPHXyOH1iEdNKc7n772u0ly4i3YpFoVcBI7s8LgU2x+DvShdmxre/OIGqndpLF5HuxaLQFwBXR492mQ3UO+eqY/B35SCnTSzct5fe1qG9dBE5UF8OW3wMeBeYaGZVZna9md1oZjdGF3kBWAusAX4P3DRgaZOc9tJF5FBCvS3gnLuyl/kOuDlmieSQTptYyLSRefzyldVceFwJaeGg15FEJE7om6IJxsy4be4kqutbeODtdV7HEZE4okJPQCeOG8qZk4q459XP2LGr1es4IhInVOgJ6razJ9HU1sHdf1/jdRQRiRMq9AR1VHE2l88cxZ/e28D67U1exxGROKBCT2DfOesoUkIB/vOvH3sdRUTigAo9gRVlp3HTaeN4ccUW3vhUp1IQSXYq9AT3tVPGUlaQyY8WrKC1Q6fXFUlmKvQElxoK8pPzj2bd9ibufV2XqhNJZip0HzhlQiHnHjOcX726hspaXVBaJFmp0H3ih+dNJhgwfrRgBZEv74pIslGh+8Tw3HRuPWsCf/94G88s2eR1HBHxgArdR66bU0b56CH8eMFKtja0eB1HRAaZCt1HggHjvy45lpb2Tn4w/yMNvYgkGRW6z4wtzOKfvzyRVz7exvxFGnoRSSYqdB/aN/Ty7Aod9SKSRFToPhQMGHdcNh0c3PL4Yl2DVCRJqNB9atTQDH5+8TEs3ljHHS996nUcERkEKnQfO+/YEVw5axT3vPaZzvUikgRU6D73r+dNYUJxFrc+sYTq+t1exxGRAaRC97n0lCC/+coMWtr38PU/LqSlXSfwEvErFXoSGF+UzR2XTWNZVb2OTxfxMRV6kvjS0cO49awJzF+8ifvf0sWlRfxIhZ5Evnn6eOYePYz/eGEVL6/c6nUcEYkxFXoSCQSM2y+bxtSSXL752CIWb9zpdSQRiSEVepLJTA3xwLUzKc5J4/qHKlhbs8vrSCISIyr0JFSQlcpD183CgGse/EBnZhTxCRV6khpTkMn9186kdlcbV/7+PWoaW72OJCL9pEJPYtNH5vHgdbOormvhqt+/x45dKnWRRKZCT3KzyvJ54NqZVO5s5iv3va9SF0lgKnThxHFDue/qmazb3sSlv3uXzXU6RYBIIlKhCwBfOKqAP15/AjUNrVxyzzt8pqNfRBKOCl32mVWWz2PzZtPWuYfLfvsuSyrrvI4kIodBhS4HmFqSy5M3nkRGapDLf/cuzy7d7HUkEekjFbp8TllBJs/cNIdjSnL5X48t5pcvr9YJvUQSgApdujU0K5VHvnYCF80o4c6XP+XGPy2koaXd61gicggqdOlRaijI7ZdO41/OmczLq7Zx/t1vsXJzg9exRKQHKnQ5JDPja6eM5fF5s2lu6+R//OZt/vTeBg3BiMQhFbr0ycwx+Tx/y8nMKsvnh88s56t/+JBtjToHjEg8UaFLnxVmR07q9eN/mMI7n+3gy3e+wXPLNmtvXSRO9KnQzWyumX1iZmvM7LZu5l9rZjVmtiR6uyH2USUeBALGtXPKeP6WL1A6JINvPrqYGx6q0LdLReJAr4VuZkHg18DZwBTgSjOb0s2if3bOTY/e7otxTokz44uy+e+bTuKH507mnc92cNYdr3Pfm2tp69jjdTSRpNWXPfRZwBrn3FrnXBvwOHDBwMaSRBAKBrjh5LH87TunMKssn589v4q5v3yDVz/Z5nU0kaTUl0IvASq7PK6KTjvYxWa2zMyeMrOR3f0hM5tnZhVmVlFTU3MEcSUejczP4MHrZvHgtTPBwXUPfsjVD3zA8k31XkcTSSp9KXTrZtrBn4I9C4xxzh0LvAw81N0fcs7d65wrd86VFxYWHl5SiXunTyrixW+fwg/PncyyqjrOu/stbnpkIWu26URfIoOhL4VeBXTd4y4FDjjBh3Nuh3Nu74m0fw8cH5t4kmhSQpFhmDe+dzq3nDGe1z6p4aw7X+fmRxbpS0kiA6wvhf4hcJSZlZlZCnAFsKDrAmY2vMvD84FVsYsoiSgnLcytX5rIm987nW+cOo7XP63hnLve5Kt/+JB31mzXoY4iA8D68j+WmZ0D/AIIAg845/7dzH4KVDjnFpjZz4kUeQdQC3zDOffxof5meXm5q6io6PcKSGKob27noXfX89A769nR1MakYdl8dU4Z/zBtBOkpQa/jiSQMM1vonCvvdp5Xe0oq9OTU0t7JgiWbuf+tdXyytZHstBAXzyjlqhNGMaE42+t4InFPhS5xxznHB+tqeeT9jby4fAttnXs4piSXi2eUcP70EvIzU7yOKBKXVOgS13bsauWZJZt5emEVK6sbCAWMOeMLOO/Y4Xzp6GHkpoe9jigSN1TokjBWVTfwzJJNPL+smqqduwkHjZPGFfClo4s5a3IxRTlpXkcU8ZQKXRKOc44llXW88FE1f1u5lQ07mgGYWpLDaROKOG1iIdNH5hEK6vxyklxU6JLQnHOs3raLl1Zu5bVPtrFww072OMhODXHC2HzmjC9g9tihTCzOJhDo7ntwIv6hQhdfqW9u560123lrzXbe+Wz7vr333PQwM8cMYcboIcwYNYRppXk6JFJ851CFHhrsMCL9lZsR5txjh3PusZHvs1XtbOaDdbX7bi+vipwcLBgwJhZnc2xpLseW5jG1JIcJxdmkhVXy4k/aQxffqW1qY/HGnSzauJNlVfUsq6qnfnfkAtfBgDG+MItJw7OZOCybicXZTCjOpiQvXcM1khC0hy5JJT8zhTMnF3Pm5GIgMga/sbaZlZsbWFndwIrNDVSs38lfluw/JVFaOMDYgizGFWVRVpBJWUEGY4ZmMnpoJkMywpip7CX+qdDF98yM0dFyPvuY/acdamhp59Mtjazetos10dvSyjqeX7aZPV3euGanhhiZn8HI/HRKh2RQkpfOiLz06M808jNTVPgSF1TokrRy0sKUj8mnfEz+AdNbOzqprG1m/fZmNtQ2s3FHExtrm1lb08Qbn25nd3vnAcunBAMU56YyPCedopxUinPSKM5JpSg7jcLsVAqzUynISiUvPaxhHRlQKnSRg6SGgowvymZ80efPLeOco7apjer6FjbV7aa6bjfVDS1sqW+hur6FFZsbeGXVts+VPkTG74dmpjA0KzX6M4X8zBSGZqYwZO/PjMi0vIwU8jLChHWcvRwGFbrIYTCzSCFnpTK1JLfbZZxzNLZ2UNPYSk1jK9saW9mxq5Xtu1rZ3tjGjqY2djS1snFjM7VNbexq7ejx+bJTQ+RlhhmSkUJueuRnXkaYvPQwedFpeRmRW256mJz0MHnpKaSE9EKQjFToIjFmZuSkhclJCzOuMKvX5VvaO9nZ3EZtUxt1ze3saGqjrrmNnU3t7GyO3K/b3c7O5nYqa5up291O/e52DnWAWno4SG56eP8tI3zg4/QwOemh/ffT9r8g6LDOxKVCF/FYWjjI8Nx0huem9/l3Ovc4GlsixV7XHCn++t3tNEQf10dLf2/5V9Y2szw6v6nt88NBXaWGAvvKPVL2oQMe730ByOnyopCTFnnRyEoJ6XMCD6nQRRJQMGDRcfYURg89vN9t79xDw+79pV+/u52Glo59LwgNB0xvZ/uuNj6radr3+FDvDMwiw0Q5BxR/6IB3ADlpXeYftExGSlBHDPWDCl0kyYSDgX2fAxyuPXscu9o6Diz93R00tBz4YtD1BWLd9siLQWNLB829vDsIBozstBDZaaF9w1bZ0ReArvf3zw+RHZ2eHb2fzJ8fqNBFpM8Cgf2fD5QOOfzf3/vuoLEl8iKw9wVh7/BR1+mNLZEXjvU7mmhs6aCxpeOQHyDvlRYOREo+NVLyWWkhslPD++9H52VFXwSy9i6XGiYr+jgrNUQwAYeOVOgiMmj68+4AoKNzD02tnZF3BC37Sz9S+NGfrR0HPN7V2kFN467I/ej8vkgPB6MvBiEyoyWfGS3/zNRgZFrK/nkZe6elhshICe5bPjMlRFo4MChDSSp0EUkYoWCA3IwAuRlHfhWrvcNGu6Jlv7f8m1o7aWqNvENoau1kV+v+F4Sm1sjPTXW7993f1dpBW8eePj1nwCAzJVr6KSGuOmEUN5w89ojXoScqdBFJKl2HjfqrvXPPvoKPvAh00NzWse/Fobmtg6a2vfcjP5vaOik4wncovVGhi4gcoXAwsO9oo3iQvB8Hi4j4jApdRMQnVOgiIj6hQhcR8QkVuoiIT6jQRUR8QoUuIuITKnQREZ8wd6hzYQ7kE5vVABuO8NcLgO0xjJMoknG9k3GdITnXOxnXGQ5/vUc75wq7m+FZofeHmVU458q9zjHYknG9k3GdITnXOxnXGWK73hpyERHxCRW6iIhPJGqh3+t1AI8k43on4zpDcq53Mq4zxHC9E3IMXUREPi9R99BFROQgKnQREZ9IuEI3s7lm9omZrTGz27zOMxDMbKSZvWpmq8xshZl9Kzo938xeMrPV0Z9HcJne+GdmQTNbbGbPRR+Xmdn70fX+s5nFx9UEYsTM8szsKTP7OLrNT0yGbW1m34n++15uZo+ZWZoft7WZPWBm28xseZdp3W5fi7gr2m/LzGzG4TxXQhW6mQWBXwNnA1OAK81sirepBkQH8F3n3GRgNnBzdD1vA15xzh0FvBJ97EffAlZ1efx/gDuj670TuN6TVAPnl8CLzrlJwDQi6+7rbW1mJcAtQLlzbioQBK7An9v6D8Dcg6b1tH3PBo6K3uYB9xzOEyVUoQOzgDXOubXOuTbgceACjzPFnHOu2jm3KHq/kcj/4CVE1vWh6GIPARd6k3DgmFkpcC5wX/SxAWcAT0UX8dV6m1kOcApwP4Bzrs05V0cSbGsil8BMN7MQkAFU48Nt7Zx7A6g9aHJP2/cC4GEX8R6QZ2bD+/pciVboJUBll8dV0Wm+ZWZjgOOA94Fi51w1REofKPIu2YD5BfA9YO/l1IcCdc65juhjv23zsUAN8GB0mOk+M8vE59vaObcJ+H/ARiJFXg8sxN/buquetm+/Oi7RCt26mebb4y7NLAt4Gvi2c67B6zwDzczOA7Y55xZ2ndzNon7a5iFgBnCPc+44oAmfDa90JzpmfAFQBowAMokMNxzMT9u6L/r17z3RCr0KGNnlcSmw2aMsA8rMwkTK/BHn3Pzo5K17335Ff27zKt8AmQOcb2briQynnUFkjz0v+rYc/LfNq4Aq59z70cdPESl4v2/rLwLrnHM1zrl2YD5wEv7e1l31tH371XGJVugfAkdFPwlPIfIhygKPM8VcdNz4fmCVc+6OLrMWANdE718D/GWwsw0k59z3nXOlzrkxRLbt351zXwFeBS6JLuar9XbObQEqzWxidNKZwEp8vq2JDLXMNrOM6L/3vevt2219kJ627wLg6ujRLrOB+r1DM33inEuoG3AO8CnwGfAvXucZoHX8ApG3WcuAJdHbOUTGk18BVkd/5nuddQD/G5wGPBe9Pxb4AFgDPAmkep0vxus6HaiIbu9ngCHJsK2BnwAfA8uBPwKpftzWwGNEPidoJ7IHfn1P25fIkMuvo/32EZGjgPr8XPrqv4iITyTakIuIiPRAhS4i4hMqdBERn1Chi4j4hApdRMQnVOgiIj6hQhcR8Yn/D1p9sTU0kJKNAAAAAElFTkSuQmCC\n", 213 | "text/plain": [ 214 | "
" 215 | ] 216 | }, 217 | "metadata": { 218 | "needs_background": "light" 219 | }, 220 | "output_type": "display_data" 221 | } 222 | ], 223 | "source": [ 224 | "import matplotlib.pyplot as plt\n", 225 | "\n", 226 | "plt.plot(loss_scores2)\n", 227 | "plt.show()" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": 12, 233 | "metadata": {}, 234 | "outputs": [ 235 | { 236 | "data": { 237 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD7CAYAAAB68m/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZgV5Zn38e/dO/QG9AY0zSbNIhhQO7jvZgaMEaMmwawaE5NM1IxmkuhMJpM4M69JJhknueKbjG9i1CRq3DKiYeJMRAWNCigouzR70/QCDb0AvZ77/aMO2LQNfYBzOJxzfp/r6ouuOk9X3WXBz6efqnrK3B0REUl8afEuQEREokOBLiKSJBToIiJJQoEuIpIkFOgiIklCgS4ikiQGDHQze8DMGsxs5WE+NzP7qZlVm9k7ZnZG9MsUEZGBRNJDfxCYdYTPZwOV4a+bgZ8ff1kiInK0MgZq4O4LzWzsEZrMAR724Aml181siJmNcPcdR9pucXGxjx17pM2KiEhfb7755k53L+nvswEDPQLlwLZeyzXhdUcM9LFjx7J06dIo7F5EJHWY2ZbDfRaNi6LWz7p+5xMws5vNbKmZLW1sbIzCrkVE5IBoBHoNUNFreRRQ219Dd7/f3avcvaqkpN/fGERE5BhFI9DnAZ8N3+1yNtA80Pi5iIhE34Bj6Gb2KHAxUGxmNcA/AZkA7v4LYD5wBVAN7ANujFWxIiJyeJHc5XL9AJ878NWoVSQiIsdET4qKiCQJBbqISJKIxn3oIiJyGO1dPVQ3tFHX3E5Dawf1Le1cNqWUD4waEvV9KdBFRHrZ19lNQ0sQvG0d3Uf1s109TmNbB40t7dQ2t7OqtoX19a10hw59NKckP1uBLiKJZ39nD3s7jy4YW9u7WVXbzMrtLWxsbCMU01cfO63t3TS2dtDQ2nHUId4fMyjJy2byiAIumVTC1JGFlA8dRFlBNsV52WSmx2a0W4EuIkfUE3I279rLjj3t1Le009jWQXdP6H3t3KF5fxcNrR00tAbDC40tHbQeR0Bmphtji3JjFoAH5GanM2VEARdOzKa0IJuy/BxKC7IpyMnE+nsW/jDSzCjJz6YoN4uMGNfcHwW6iBzU3RNifUMbK7Y3s2p7Myu2N7NmRyv7u3oi+vmczDRK83Mozc9m8vB8LqwsobQgm7zsjH7nCDmc7Mx0Th1RQGVZHtkZ6cd2MClIgS6SAtyd2uZ2VtQ0s6q2mU079+LhYYyunhCNbR00tHTQ2NpBZ7j3nZuVztSRhcydWcHUkYVUDB1EaUEOJfnZZGf03/vMSDPsaLq0ElUKdJGTVEd3TxCw3UHAOtByYEijpT2iXnPT3q7wWHQzu/d1AZCeZlQMHXRwSCDdjOL8LGaOG0ZZQQ5TRuQzrbyQcUW5pKUpnBOJAl0kjtq7emho6aCupZ21dS3hHnQLtc372RMO4OORkWZMGp7PX506nGnlBUwrL2TKiAJyMjWMkYwU6CLHyN3p6H7v4mBnT4iGluCC4J59XQeHNLpDIRpbg+GM+pb28EXD4PvW9kMvGBblZjG1vJAzxwylND+bkvxsBmW9F7552RmUFQRj1LnZA//zzcpIi/kFRTl5KNBFegmFnC1N+1i5vZldbR0H1+/t7KGhVxg3tLbT0NJxSKAPJCs9jZL8bMoKsplQkse5pxRRFh6TLivIobI0jxGFORqDlmOmQJeUt2XXXhasbWDB2gaWb91z2Nvs8nMyKM3PpjQ/hzNGD6WsIIfCQe/d1paZFgR2aX42Q3OzSA+PP6eZUZyXFW6rsJbYUaBL0nN3lmzezcbGtoO96/qW8LBHczt1Le0AnFKSy5zTR3JaeSHTygsZUTjo4K12OZnphwx9iJyMFOiStNydP69p4KcvrGfF9uaD64cOzgzulS7I5pSSIqaNLOSyKaWMKcqNY7Uix0+BLknF3VlV28KCtQ3MX7GDtXWtjB42mB9cexrnV5ZQkpdN1mHuoRZJdAp0SXh7O7p5tXonC9Y28OK6BupbOjCD6aOG8OOPTWfOjJFxeQxb5ERToEtCcnde27iLxxZv40+r6ujsDpGfncEFE4u5dHIZF08qoTgvO95lipxQCnQ56e3v7OGRxVt5flUdofC0e/Wt7Wxr2k9+TgafqKpg9rThVI0dpuEUSWkKdDlp7d7byRNvbuP+hRvZ2dbJtPICCgdlAjCpLJ/bL5/IFaeN0FOPImEKdDlp7O/sYfWOZt7Y1MSCNQ28tXU3IYcLKou59dJKZo4bFu8SRU5qEQW6mc0CfgKkA7909+/3+XwM8ABQAjQBn3b3mijXKklofX0rv1+yjYXrG6lueO9FBtPKC7jlkgn81dThTCsvjG+RIgliwEA3s3TgPuBDQA2wxMzmufvqXs1+BDzs7g+Z2aXAPcBnYlGwJIc/r67n5y9v4M0tu8lMN849pZhZ00ZwWnkh00cVUlqQE+8SRRJOJD30mUC1u28EMLPHgDlA70A/Fbg9/P2LwH9Fs0hJHi3tXXxv3mqeequGsUWDuWv2ZK49c5TuSBGJgkgCvRzY1mu5BjirT5u3gWsJhmU+CuSbWZG774pKlZJwQiFn0669rKptoWV/MA1sZ3eIX72yibqWdm67dAK3XFqpu1JEoiiSQO9vNqG+r2z9O+BnZnYDsBDYDrxvhiMzuxm4GWD06NFHVaic3Lbu2sfSLU2s3N7CytpmVte29Puy3XHFuTz55XM4ffTQOFQpktwiCfQaoKLX8iigtncDd68FrgEwszzgWndvpg93vx+4H6Cqqiqm7/GWE2NVbTM/W1DNf6+sA4J3Sk4ZUcA1Z5QzrbyQqSMLKMl/bzilKDf74CyEIhJdkQT6EqDSzMYR9LznAp/s3cDMioEmdw8BdxHc8SJJamdbBy+ubaBm8TPUbt9CcUYa/zm1mOkVQyjJzyY9bRdMuBzyh8e+mO5OWPscdO6N7X5GfABGTI/tPlJNyw7Y8AIH3wSSSirOgpKJUd/sgIHu7t1mdgvwPMFtiw+4+yozuxtY6u7zgIuBe8zMCYZcvhr1SiVuek94tWBtA2/X7KGKtTyRfTdkhhttCH8dUDAKPjcPik6JXWHdHfDEDbBufuz2cUBaJnzsQZhyZez3lQp2bYAHr4TW2oHbJqMP/3tMAt08Tv93rKqq8qVLl8Zl3xKZfZ3d/Pb1LTz46mZqm9sxgw+MGsJlk0v5fPWt5LZuxD7/pyDsemuugd9/CtKz4HPPQfGE6BfX1Q6PfxbWPw+zfgCTPxz9fRzQ0wl/+BLULoNrfwVTr47dvlJB47vw0Ecg1A1zfwcF5fGu6MQbNASy84/pR83sTXev6u8zPSmaytpboGE1jD770NVdPTzw6iZ+uWgTTXs7OW9CEbd/aCIXTyoNxsM3LYRFrwVBOmz8+7c7pCII8oc+Ag9eAZd9Jwj3aHr7UdiwAK78D6i6Mbrb7s+nn4bfXQdPfh52bzo5QmjUB2HYuNjvxx02vgR7G49/Wz2d8OfvBd/f8EconXz825SD1ENPVfua4OE5UPcOXPqPcOHfAbCippk7Hl/O+oY2Lp5Uwq2XVnLmmF53pLjDr2fD7s1w23LIPMIDQA1r4eGroK0++vVbGnzkJ3DGZ6O/7cPpaIVHPgFbXj1x+zyS7EL4zNMwqt/OWnS4wwvfg1fujd4284bD556NyZBDKlAPXQ61d2cQ5jvXw/hLYME/09nVxS+4jp++sJ6ivCwe+vxMLppY8v6f3fgibH0NrvjRkcMcgt7XbcugtS76x5BdAHn91BdL2fnBbx67N53Y/fanowWeuBEevho+/RSM7vtoSBS4w/98G177GZx5I5x7a3S2mz8CsgZHZ1tyCAX6yaB2GZRNg/TMgdser7bGoNfctJE9V/+GZ1snMLb+W1yw6Pt49zqumHYb/3z1aRQODtcSCgUh3tEaLL9yb3DBM9KecVZubC+MnmhpaSfP8dw4P7iw+Ntr4K//FXKGRHf7G1+CN38NM2+G2T8EveD6pKdAj7fV8+Dxz8DkK+G6X0NGlMeae2utxx++ilDTZn5a+i/c92gP3aG1jC/6G35cnMPXdv4BysbDoNOD9qEemHcbLP/todv5yE8hQ4/qx13ByCDUH/oIPPu12OzjrK/ArHsU5glCgR5PoRC8+H9g0NDgXurHPwsffyg2Ydmyg64HPkx383ZuaP8G6xvG8vnzR/GxM0dRWZYPoYvhj7fDoh8Hdx9c+h145qvwzmNw4Tdg6jXBdtIzoSgGd63IsckfDl9+JbgNMNoyc/q/6C0nLQV6PK3+AzSugesegP274Y9fh8c+BR+8Kaq78Z4uWp/7Nul76/kbu4trr7mWq08vP3QelbQ0+PC9kJYBr/4E1v037HwXLvk2XPSNqNYjUZaRDWWnxrsKOQko0OMl1AMvfR9KpsCpHw0CNS0j+NW5+n+juisDzAfxo9J7uOdT1zNyyKD+G6alBRc70zLgjV/AZf8EF9wR1VpEJHYU6PGy8qmgB/yxh4IgBTjzhuCuk/1Nx7Xp1vZuHl28ledX1dPR3cPYolwuP/sM/vHc6aQNNI+KGcz6Ppx/+4l5dF9EokaBfqKEemDLX4IHKyDonZdNgylXHdpu6Jjg61h2EXIeX7qNHz6/jj37crjmjPP49NljmD6qEDuai1pmCnORBKRAP1GWPwLzbjl03dxH3uudH4fdezt5etl2Hlu8lfUNbVSNGcp3r5qpV7eJpBgF+omy8sngjoGrfxEsZw2G4acd1yZb2ru4Z/5annqzhs6eEDMqhvCTuTO4avrIo+uRi0hSUKCfCG2NwfwnF3w9ak/0/aV6J3/3xNvUt3Zw/cwKPnXWGKaMKIjKtkUkMSnQT4Q1z4CHYOpHj3tT79a38uBfNvPIG1sZr7f/iEgvCvQTYeUfoHgSlB7bvcJdPSHmLa/ld29s4a2te8hMN244dyzfmjWZQVnpUS5WRBKVAj3WWuuC2fku+tZRPz7d2R3iqbdquO/Famp272d8SS7/cMUUrjmjnKI8PXovIodSoMfa6mcAh2nXRNR8f2cPr1bvZMG6Bv68up6G1g6mjyrk7jlTuWRSqS52ishhKdBjbeXTUDoVSiYN2PSNjbv4wsNLaW3vJjcrnfMri7l+5mgumliiIBeRASnQY6m5Bra9HsyHMoC/bNjJTQ8upXzoIP7vp05l5rhhZGdofFxEIqdAj5WeLvjTXcH3Awy3vLJ+J194eAmjhw3md184O3jNm4jIUVKgx0J3Jzx5YzAl7l/9y2FfiNDe1cMvF23kpwuqGV+cy+++cJYudorIMVOgR1t3JzxxA6z7YzDJ1dlf6bfZC2vqufu51WzZtY/Z04bzrx89jWG5MXy5hYgkvYgmEjGzWWa2zsyqzezOfj4fbWYvmtkyM3vHzK6IfqkJ4rWfBWF+xY/6DfNQyPmX51Zz00NLyUxP47c3ncXPP32mwlxEjtuAPXQzSwfuAz4E1ABLzGyeu6/u1ezbwOPu/nMzOxWYD4yNQb0nv+oXYMR0mPnF933U0d3D1x9/m+fe2cHnzhnDt688lcz045+cS0QEIuuhzwSq3X2ju3cCjwFz+rRx4MBEIoVAbfRKTCBd+6FmMYy94H0fNe3t5LO/Wsxz7+zgrtmT+e5VUxXmIhJVkYyhlwPbei3XAH1nmPou8D9mdiuQC1weleoSzbbFwXzn4y48ZPWCtfV866kV7NnXyU/mzmDOjPI4FSgiySySLmJ/T7R4n+XrgQfdfRRwBfAbM3vfts3sZjNbamZLGxsbj77ak93mRWDpMPocIHjq866nV/D5B5dSlJvFM189X2EuIjETSQ+9BqjotTyK9w+p3ATMAnD318wsBygGGno3cvf7gfsBqqqq+v5PIfFtWgQjZ0BOAe7O7b9fzvOr6/jSReO540MT9aCQiMRUJD30JUClmY0zsyxgLjCvT5utwGUAZjYFyAGSsAt+BJ17YfubB8fP71+4kT+tquPvZ0/hrtlTFOYiEnMDBrq7dwO3AM8DawjuZlllZneb2YEXYn4d+KKZvQ08Ctzg7snXAz+Sra9DqAvGXcBfqnfygz+t5YrThvOFC8bFuzIRSRERPVjk7vMJbkXsve47vb5fDZwX3dISzOZFkJZBXeEMbv3FMsYV5/LD66ZrUi0ROWH0pGi0bFqEl5/JN5/dyP6uHn7/mbPJy9Z/XhE5cXQjdDR0tELtMt4dNIOF7zbyzb+exITS/HhXJSIpRoEeDVteA+/h3uoyplcM4TPnjI13RSKSgjQmEA2bF9JtGbzSPp4nrjmN9DSNm4vIiadAj4Lm1QtY2zOBz144hSkjCgb+ARGRGNCQy3Go2b2POx56mbzdq1k/aAa3XVYZ75JEJIUp0I/RM8u3c/m/v8z+6oWkm/Pxj32SnEw9PCQi8aNAPwb7Orv53rOrmTS8gH+raoH0bLLG9J2vTETkxFKgH4NHF2+jaW8n//jhKeTVvgYVMyEzJ95liUiKU6AfpY7uHu5fuIGzxw+jqhSoX/G+6XJFROJBgX6UnnyzhvqWDm69tBI2vxKs7OeFFiIiJ5oC/Sh09YT4+UsbOH30EM49pSiYvyVzMJSfGe/SREQU6EfjmeW11Ozez62XTggm3dq0CCrOggy94FlE4k+BfhQe/MsmJg/P55JJpdDWCI1rYJyGW0Tk5KBAj1B1Qxsrt7fwsaqKoHe+5cD4uS6IisjJQYEeoXlv12IGV35gRLCi+s+QlRe8ck5E5CSgQI+AuzNv+XbOGV9EWUEO7NkG7zwO066F9Mx4lyciAijQI7JiezObd+3jqukjgxWLfgzucOE34luYiEgvCvQIPLO8lsx0Y/a0EbB7Cyz7DZz5ORhSEe/SREQOUqAPoCfkPPt2LRdPKqVwcCYs/DewdDj/jniXJiJyCAX6AN7YtIuG1o5guKVpIyx/BKpuhMLyeJcmInKIiALdzGaZ2TozqzazO/v5/F4zWx7+etfM9kS/1PiYt7yWwVnpXD6lDF7+t+Ai6Pm3x7ssEZH3GfCNRWaWDtwHfAioAZaY2Tx3X32gjbvf3qv9rcDpMaj1hNvX2c0f39nBX08dzqCWTfDOY3DWVyB/eLxLExF5n0h66DOBanff6O6dwGPAnCO0vx54NBrFxdv8FXW0dnTziQ9WwMs/gIwcOP9v412WiEi/Ign0cmBbr+Wa8Lr3MbMxwDhgwfGXFn+PL9nG2KLBnJXXCCuegJlfhLzSeJclItKvSAK9v1fY+2HazgWedPeefjdkdrOZLTWzpY2NjZHWGBcbGttYvLmJT3xwNPbyDyArF879WrzLEhE5rEgCvQbofcP1KKD2MG3ncoThFne/392r3L2qpKQk8irj4PEl28hIMz4+uhlW/QHO+hLkFsW7LBGRw4ok0JcAlWY2zsyyCEJ7Xt9GZjYJGAq8Ft0ST7zO7hBPvVXDZVNKKVpybzBnyzm3xLssEZEjGjDQ3b0buAV4HlgDPO7uq8zsbjO7qlfT64HH3P1wwzEJY8Haena2dfKFCW2wZh6c/RUYPCzeZYmIHNGAty0CuPt8YH6fdd/ps/zd6JUVX48t2cbwghyqNv8nZBfCOV+Nd0kiIgPSk6J9NLS0s/DdRv5mUgu2bj6cewsMGhLvskREBqRA72Pe27WEHK5t+Q3kDIGzvhzvkkREIqJA7+Ppt7bzsbId5G55Ac67DXIK4l2SiEhEFOi9rKtrZfWOFm7L/AMMLoKZX4p3SSIiEVOg9/L0shpy0noYtXsJTL8esvPiXZKISMQU6GE9IeeZZbXMHdOG9XRA+RnxLklE5Kgo0MPe2LiLupZ2PlrWEKwYmRQTRopIClGghz29bDt52RlMtY2QUwhDx8W7JBGRo6JAB9o6uvnvFTuYPW04GXXLg9659TcnmYjIyUuBDjyzfDt7O3v45JmlUL9awy0ikpBSPtDdnd++vpUpIwqYkVULoS4FuogkpJQP9OXb9rBmRwufOms0tmNZsFKBLiIJKOUD/XdvbCU3K52rTy+H2mXBA0WFFQP/oIjISSalA715XxfPvl3LnNPLycvOgFpdEBWRxJXSgf7UWzV0dIf45MzR0LkPGtZouEVEElbKBrq788jircyoGMK08kKoXwneo0AXkYSVsoG+tWkf1Q1tXHNGebCiVhdERSSxpWygv7GpCYCzx4df/Fy7DPLKIH9EHKsSETl2KRvoizc1MXRwJhNKwjMq1i7TBVERSWgpHegfHDuMtDSDjjZoXAcjZsS7LBGRY5aSgb6jeT9bm/Yxc9ywYEXdO4BrylwRSWgRBbqZzTKzdWZWbWZ3HqbNx81stZmtMrNHoltmdC0Oj5+fNa7X+Dmohy4iCS1joAZmlg7cB3wIqAGWmNk8d1/dq00lcBdwnrvvNrPSWBUcDYs3NZGXncGUEfnBitplUFAO+WXxLUxE5DhE0kOfCVS7+0Z37wQeA+b0afNF4D533w3g7g3RLTO6Fm9q4swxQ8lIDx/+gQuiIiIJLJJALwe29VquCa/rbSIw0cxeNbPXzWxWtAqMtl1tHaxvaHtv/Ly9GXZVw0gNt4hIYhtwyAXo7z4+72c7lcDFwChgkZlNc/c9h2zI7GbgZoDRo0cfdbHRsGTzbgDOOhDoO94O/lQPXUQSXCQ99Bqg9/SDo4Dafto84+5d7r4JWEcQ8Idw9/vdvcrdq0pKSo615uOyeFMT2RlpnDaqMFhx8IKoAl1EElskgb4EqDSzcWaWBcwF5vVp81/AJQBmVkwwBLMxmoVGy5LNTZw+egjZGenBitplMGQ05BbFtzARkeM0YKC7ezdwC/A8sAZ43N1XmdndZnZVuNnzwC4zWw28CHzD3XfFquhj1dLexaraZmaOHfbeSl0QFZEkEckYOu4+H5jfZ913en3vwB3hr5PWX6p3EnI4b0JxsGJfE+zeDGfeEM+yRESiIqWeFH353UbysjM4Y8zQYMWO5cGf6qGLSBJImUB3d15e18h5E4rI7H3/OcCI6fErTEQkSlIm0Ksb2qhtbufiSb0eYq1dBsPGw6Ch8StMRCRKUibQX1rXCMCFE8O3S4ZCsF0XREUkeaRMoL/8biOVpXmUDxkUhPkf74CWGjjlsniXJiISFSkR6Ps6u1m8qYmLJpYEYf7sbfDmr+H8O2DGJ+NdnohIVER022Kie33jLjp7Qlw8sQSevRWW/RYu/CZc8vd6Q5GIJI2UCPSX1zUyKDOdDxY0BWF+zi1w6T/EuywRkahKiSGXl95t5JxTishueCdYMf36+BYkIhIDSR/om3fuZcuufcH4ee0yyMiBksnxLktEJOqSPtAXre91u2LtMhj+AUhPiZEmEUkxSR/oC9fvpGLYIMYOzQ7mPtd95yKSpJI60Lt6Qry2YRcXVJZgu6qhay+UnxHvskREYiKpA33Z1j20dXRzYWXxe/O2qIcuIkkqqQN90fpG0tOMc04JB3pWHhRNiHdZIiIxkdSBvnD9TmZUDKFwUCbUvhXMqpiWHu+yRERiImkDfc++Tt6p2cMFlcXQ0wV1KzTcIiJJLWkD/ZXqnbjDBZUl0LgWutsV6CKS1JI20Be9u5OCnAymjyrUBVERSQlJGejuzqL1jZw3oZiM9LQg0LMLYei4eJcmIhIzSRnoGxr3UtvcHgy3QBDoI6dDWlIerogIkKSBvvDd4HH/CyqLobsD6lZquEVEkl5EgW5ms8xsnZlVm9md/Xx+g5k1mtny8NcXol9q5Batb2R8cS4VwwZD9Z8h1AWjPhjPkkREYm7AWarMLB24D/gQUAMsMbN57r66T9Pfu/stMajxqHR09/D6xiY+XjUqeDvRi/cEL4KeODvepYmIxFQkPfSZQLW7b3T3TuAxYE5syzp2b27ezf6unmD8fO1zUL8CLvqWZlgUkaQXSaCXA9t6LdeE1/V1rZm9Y2ZPmllFfxsys5vNbKmZLW1sbDyGcge2cP1OMtKMs8cPhZfugaJKmHZdTPYlInIyiSTQ+3vppvdZfhYY6+4fAP4MPNTfhtz9fnevcveqkpKSo6s0QovWN3LmmKHkVT8HDavh4jvVOxeRlBBJoNcAvXvco4Da3g3cfZe7d4QX/x9wZnTKOwqhHvas/F/K6l7iM0Vrgt55yWSY+tETXoqISDxE0nVdAlSa2ThgOzAX+GTvBmY2wt13hBevAtZEtcpIrP8fhjw5lweygJXhdR9/WJNxiUjKGDDQ3b3bzG4BngfSgQfcfZWZ3Q0sdfd5wG1mdhXQDTQBN8Sw5v41BDfd3Gh388ubLiA9axCUTjnhZYiIxEtEg8vuPh+Y32fdd3p9fxdwV3RLOzq+awO7GEJu5QWkj9JbiUQk9STNk6L7695lQ2h48DJoEZEUlDSBTtNGNoeGc2GlAl1EUlNyBHp7C4M7d7E3fwzDC3PiXY2ISFwkRaC37lgHwNAKXQQVkdSVFIG+fvXbAEycMj3OlYiIxE9SBHrD5uCWxUlTPhDnSkRE4ifhAz0Ucnp2rmd3RgkZOXnxLkdEJG4SPtBX1jYzoqeW7iF6vZyIpLaED/QX1zYy1urIHzkp3qWIiMRVwgf6kjUbGGZt5JRNjHcpIiJxldCBvqutg73hWxYpOiW+xYiIxFlCB/qrG3YxhrpgYZgCXURSW0IH+ramfYxLq8MxGKaLoiKS2hI60Bta2qnMqMeGVEBGdrzLERGJq4QO9LqWdk5Jr9dwi4gICR7o9c3tjArt0AVRERESPNC7WurJ9b3qoYuIkMCBHgo5uXu3BgvqoYuIJG6g79rbSYUfuGVxfHyLERE5CSRsoNe3tFNmTcFCQXl8ixEROQkkbKA3tLZTZrvpziqArMHxLkdEJO4iCnQzm2Vm68ys2szuPEK768zMzawqeiX2r665gzLbg+cNj/WuREQSwoCBbmbpwH3AbOBU4HozO7WfdvnAbcAb0S6yP8GQy27SC0eeiN2JiJz0IumhzwSq3X2ju3cCjwFz+mn3z8APgfYo1ndYDa3tjEjbQ1rBiBOxOxGRk14kgV4ObOu1XBNed5CZnQ5UuPtzUaztiBqa91PMbsjXkIuICEQW6NbPOj/4oVkacC/w9QE3ZHazmS01s6WNjY2RV9mPfXsayKAH8tVDFxGByAK9BqjotTwKqO21nA9MA14ys83A2cC8/i6Muvv97l7l7lOiOuoAAAaZSURBVFUlJSXHXjVgbTvCe1cPXUQEIgv0JUClmY0zsyxgLjDvwIfu3uzuxe4+1t3HAq8DV7n70phUDHT1hMja3xAsqIcuIgJEEOju3g3cAjwPrAEed/dVZna3mV0V6wL709Aa3LIIqIcuIhKWEUkjd58PzO+z7juHaXvx8Zd1ZPUt7ZSxO1jIK4v17kREEkJCPinaEL4HvTtnmF5sISISlpCBXt/SQanplkURkd4SMtDrWtoZbrtJL9BToiIiByRkoNe3BE+JWoF66CIiByRkoO9s3sdQmnXLoohILwkZ6J3NdaQT0hi6iEgvCRno3hZ+U5F66CIiByVcoO/r7Ca/c2ewoB66iMhBCRfoDS0dlFn4oSL10EVEDkq4QK9raafUduMY5JbGuxwRkZNGwgX6gcf+ewaXQHpEMxeIiKSEhAv0hpYOSm0PpvFzEZFDJFygn19ZzIwh+0kr1Pi5iEhvCRfoU0YUMCzUhOmCqIjIIRIu0Onpgr2NusNFRKSPxAv0tvrgT42hi4gcIvECvVVPiYqI9CcBA10vhxYR6U8CBrp66CIi/Um8QC8YCZOvhMHF8a5EROSkkniPWk7+cPAlIiKHiKiHbmazzGydmVWb2Z39fP5lM1thZsvN7BUzOzX6pYqIyJEMGOhmlg7cB8wGTgWu7yewH3H309x9BvBD4N+jXqmIiBxRJD30mUC1u290907gMWBO7wbu3tJrMRfw6JUoIiKRiGQMvRzY1mu5BjirbyMz+ypwB5AFXBqV6kREJGKR9NCtn3Xv64G7+33ufgrwLeDb/W7I7GYzW2pmSxsbG4+uUhEROaJIAr0GqOi1PAqoPUL7x4Cr+/vA3e939yp3ryopKYm8ShERGVAkgb4EqDSzcWaWBcwF5vVuYGaVvRY/DKyPXokiIhKJAcfQ3b3bzG4BngfSgQfcfZWZ3Q0sdfd5wC1mdjnQBewGPhfLokVE5P3MPT43pJhZI7DlGH+8GNgZxXISRSoedyoeM6TmcafiMcPRH/cYd+93zDpugX48zGypu1fFu44TLRWPOxWPGVLzuFPxmCG6x514c7mIiEi/FOgiIkkiUQP9/ngXECepeNypeMyQmsediscMUTzuhBxDFxGR90vUHrqIiPSRcIE+0FS+ycDMKszsRTNbY2arzOxr4fXDzOx/zWx9+M+h8a412sws3cyWmdlz4eVxZvZG+Jh/H364LamY2RAze9LM1obP+Tkpcq5vD//9Xmlmj5pZTrKdbzN7wMwazGxlr3X9nlsL/DScbe+Y2RlHu7+ECvQIp/JNBt3A1919CnA28NXwcd4JvODulcAL4eVk8zVgTa/lHwD3ho95N3BTXKqKrZ8Af3L3ycB0guNP6nNtZuXAbUCVu08jeGhxLsl3vh8EZvVZd7hzOxuoDH/dDPz8aHeWUIFOBFP5JgN33+Hub4W/byX4B15OcKwPhZs9xGHmzElUZjaKYOqIX4aXjWDmzifDTZLxmAuAC4FfAbh7p7vvIcnPdVgGMMjMMoDBwA6S7Hy7+0Kgqc/qw53bOcDDHngdGGJmR/Xy5EQL9P6m8i2PUy0nhJmNBU4H3gDK3H0HBKEPlMavspj4D+CbQCi8XATscffu8HIynu/xQCPw6/BQ0y/NLJckP9fuvh34EbCVIMibgTdJ/vMNhz+3x51viRboEU3lmyzMLA94CvjbPi8RSTpmdiXQ4O5v9l7dT9NkO98ZwBnAz939dGAvSTa80p/wuPEcYBwwkuDFOLP7aZps5/tIjvvve6IF+tFO5ZuwzCyTIMx/5+5Ph1fXH/gVLPxnQ7zqi4HzgKvMbDPBUNqlBD32IeFfySE5z3cNUOPub4SXnyQI+GQ+1wCXA5vcvdHdu4CngXNJ/vMNhz+3x51viRboA07lmwzCY8e/Ata4e+/3s87jvZksPwc8c6JrixV3v8vdR7n7WILzusDdPwW8CFwXbpZUxwzg7nXANjObFF51GbCaJD7XYVuBs81scPjv+4HjTurzHXa4czsP+Gz4bpezgeYDQzMRc/eE+gKuAN4FNgD/EO96YnSM5xP8qvUOsDz8dQXBmPILBPPNvwAMi3etMTr+i4Hnwt+PBxYD1cATQHa864vB8c4AlobP938BQ1PhXAPfA9YCK4HfANnJdr6BRwmuEXQR9MBvOty5JRhyuS+cbSsI7gA6qv3pSVERkSSRaEMuIiJyGAp0EZEkoUAXEUkSCnQRkSShQBcRSRIKdBGRJKFAFxFJEgp0EZEk8f8Bs4shH3g1zWMAAAAASUVORK5CYII=\n", 238 | "text/plain": [ 239 | "
" 240 | ] 241 | }, 242 | "metadata": { 243 | "needs_background": "light" 244 | }, 245 | "output_type": "display_data" 246 | } 247 | ], 248 | "source": [ 249 | "import matplotlib.pyplot as plt\n", 250 | "plt.plot(train_scores2)\n", 251 | "plt.plot(test_scores2)\n", 252 | "plt.show()" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": {}, 258 | "source": [ 259 | "# Adam效果" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": 13, 265 | "metadata": {}, 266 | "outputs": [ 267 | { 268 | "data": { 269 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU1f3/8dfJbkLYQlgDYTEskU2ICLiL+16Ximvdamu1at1bq7XWX+tSv/3qV0ul1t2CCmpRUbSKQEWUPexLSELCFrIRyEKSmfP7Y0YaQyIBZnIz976fj0cekztzmfu53snbO+eec66x1iIiIpEvyukCREQkNBToIiIuoUAXEXEJBbqIiEso0EVEXCLGqQ136dLF9u3b16nNi4hEpMWLFxdba1Obes2xQO/bty+LFi1yavMiIhHJGJPf3GtqchERcQkFuoiISyjQRURcQoEuIuISCnQREZdQoIuIuIQCXUTEJRzrhy4RzlpY+jqUF+z/mjEw+FzoMSL8dVSVwuJXoK46vNvpPgwyLwjvNrxm6zJYNzPwWfKaQWdBr9Ehf1sFuhyaJa/BB7cHF0yjFy385y9w9hMw+vpAwIfD1qXw9rVQvrmJGkIpGDhHXwPnPAWxR4RxWx5gLSx8ET75NfjrCO+xa6OSuyvQJcSshdpKiG93cP+uYht8+hCkHw8/+QCiGrXcVZXCuzfDh7+Czd/AyfeDiQ5d3QA5n8PHD0BSKtz0BaSF/o9jH78PvvwTzH0Kti2HC/4PjugUvu21VHIPiIlrnW1VlcLe3Yf/PtYHXz4O2W9Bxplw8Qtt47+lSxin7liUlZVlNfTfQdVl8N7PIecLOPtJGH1dy8+kp14FG/8Nt8yHlAFNr+P3BwLwyz+x7ww31PqfApf8A5JSwvP+ja2fBe/+FGp2tc72DqRTP7j89UBzULhYCwsmwWcPgb8+RG9q4JQH4YS79z8ZkAMyxiy21mY1+ZoC3YO2LYe3roGKrdB9aKDpYuRVcM6fIS7xh//t6n8FmjlOewSO/9WBt7VlMRStDUXV35fQHgadA1EhPvM/kF2FkDvX+Xbf+prA/zCry+C8v8DIK0O/jb27YcYvYdV7gf/Wg88Lzft2HQK9RoXmvTxIgd6W7dkJnz4IWTdAn7Hh396S1+GjuyGpC1z2auAPa84TgZ9uw+Dy16Bz//+uvy0bZv8RavcElrevgI594KezIVotdo7asxOmXQ9586BXVujb9svyoGILTPgdHHdH+K6FyEFRoLdl71wPq96FqBg4/Q8w9pbw/OHUVcPMewM9U/qfHGyq6PLf19d/GmhOsDbQrjnobFj6Jnx0F8QnQ0pGYL24RDjjscBZljjPVw/z/gyb5oT+vWPi4Pi7oP9JoX9vOWQK9LZq7UyYegWMvx1KN8HaDyHzwsDX21Cy/kA76PZsOOEeOOU3TTdVlOUHmlO2LYM+42HzfOh3IlzyErRrcvplEWllPxTo+s7slJpdgbPfrkfBqQ9BdCx89Qx8/vtAO3WoxXeAK6YGzryb0ykdbpgFH98HS14NnJ2d+tvWb6cWkUOiM3SnfHBnIDRv+vf3+6NWlsDeMPSiSEoNNJ20VM0uSOgQ+jpE5LDoDL0t2DQHpkwEX21g2V8P427bf3BBUkrrdcP7IQpzkUNireWf327m8Y/XUl3ra3KdRy8cypXH9gn5thXoreU/f4G4djDqmsDyEZ3gmJucrUlEQqq61seD76/g3SVbGD8ghaP7dGxyvSE9DuLb8kFQoLeGorWwaXagPfrEe52uRiTkfH7Lq/Pz2FC0x+lSHLU4v5QNRXu4Y0IGt0/IIDqqdbt6KtBbw7eTITo+MK+JiMuUVtZyx9SlzNtQTJd2cRgP91dPjo/hpeuO4ZRBXR3ZvgI93KrLYfkUGHbZ9/t9i0SQwrIqdu7eu9/zJXtqefhfKymurOWJS4Zx+TGhbxeWllOgh9vSN6CuCo692elKRA7JwrxSJk5egM/fdI+4tE5H8O4t4xnaSxfSnaZADye/L9Dc0md868wNLhJiNXU+7p+eTff2CTx20dD9Zro1wOj0TiQnxDpSn3yfAj2cvvkblOfD6Y86XYnIPm8vKuCl/+RSHzzjTk6I4bfnDmF0euf91n3ui41s2lnJazeM4cSBGi3c1mnuynCoq4b3b4VZv4EjTwvdLHUih6GmzscD07O5b1o2sdFRDOqWzKBuyRRV7OXyFxbw8le5NBxouHprBX+bk8Mlo9IU5hFCZ+ihtmsLTLk8MCvhSfcHfjR0XhxWUFrFLW8uZuWWCm49ZQB3nT5oX5e6XdV13P32Mn7/wWq+2VTKqPRA3+l3l2yhY2IsD52nidgiRYsC3RhzFvAMEA28aK19vNHrfYBXgY7BdR6w1s4Mca2RYe6TULwBrnwHBp7hdDUizF5bxJ1vLcNvLX+/NovTM7t97/UOR8Qy+ZosJs3J4S+freeTVdsBiIuJ4tmJR9MxsZXuiiSH7YCBboyJBp4HTgcKgYXGmBnW2tUNVvst8La1dpIxJhOYCfQNQ71tm98Haz8KTIClMBeH+fyWZ/69nme/2MiQHu3529WjSE9JanLdqCjDraccyU0n9KPeF2h2iY4yJMTq22UkackZ+hhgo7V2E4AxZipwIdAw0C3QPvh7B2BrKIuMGJsXQOVOGKK7w4uzGg72uXR0Go9dNLRF4RwfE028GmIjVksOXS+goMFyIXBso3UeAT41xvwSSAJOa+qNjDE3AzcD9OnjwgEIaz4IjAjN0Nm5OGdZQTm/eGMxxXtq+dPFw5h4TG9Pj970kpYEelOfhMYjDK4AXrHWPm2MGQe8bowZaq31f+8fWTsZmAyB6XMPpeA2y9pAoB85AeLbOV2NeIS1llfm5/HVxpJ9y/M2FJOaHM+0W8YxPK3pyaHEnVoS6IVA7wbLaezfpHIjcBaAtfZrY0wC0AUoCkWREWHLEqgoDEzAJdIKKmrquPvt5Xy2egf9U5NIiAk0qZw9rDu/v+AoXcz0oJYE+kIgwxjTD9gCTAQa32J8MzABeMUYMwRIAHaGstA2b82MwH1BB53ldCXiUjk795ATnM1wb72fpz9dR2FZNQ+fl8n1x/VVs4ocONCttfXGmNuAWQS6JL5krV1ljHkUWGStnQHcDfzdGPMrAs0x11mnboXkBGsDgd7vxMA85yIhZG1gatrHPlqzb3QnQLf28Uy9eSxZffcf4Sne1KLr2cE+5TMbPfdwg99XA8eFtrQIUrQ6cJPn8bc7XYm4TFVtPQ9MX8GM5Vs5bUhXbp+QQVTwTLxvlyTaqUuKNKBPQyisngEYGHyu05WIi+yt93H5CwtYtXUX9545iFtOGkBUK98wQSKLAj0U1syA9PHQzplJ7cWdnp+dw4otu5h01SjOHtbD6XIkAmhyrsNVvDHQ5KLBRBJC67bvZtKXG7loZE+FubSYAv1wrZkReByiGRUlNHx+y33Ts0lOiOXh849yuhyJIAr0w7VmBvQaDR3SnK5EXOLlr3JZXlDO787PpHOS+pJLyynQD0d5AWxdquYWCQlrLVO+3cyTn6zj1MFduWBET6dLkgiji6KHY80Hgcch5ztbh0S8mjofD72/kncWF3JCRheevmyEBgrJQVOgH441H0C3oZAywOlKJILll1Ty8zeWsGZbBbdPyOCOCRn7bj4hcjAU6Idq9w7Y/DWc/IDTlUgE+/fqHfzq7WVEGcPL1x3DKYPV9VUOnQL9UK39ELBqP5dDYq3l6U/X89zsjQzt1Z5JV42md+dEp8uSCKdAPxR11fD185A6GLrqfoty8N5eVMBzszfy46w0Hr2wZTefEDkQBfqhmPMElObANe+DLlzJQSqqqOGxj9Ywpl9nHr94uIbzS8io2+LB2rYcvnoWRl4NA05xuhqJQA//axV76/08fvEwhbmElAL9YPjq4V+3QWIKnPmY09VIBPp4xTY+WbWdX502kP6purOVhJaaXA7G1/8H27Phx69p3nNp1qqtu3h1fh51vv1vCTB3/U6O6tmen57Qz4HKxO0U6C1VkgNfPg6Dz4PMC52uRtqotxcV8ND7K4mLjqJTE8P2u3dI4M+XjSAmWl+OJfQU6C3h98OM2yE6Hs75s9PVSCsrr6pl9daKA643Y/lWpi4s4LgjU3h24tGktItvhepE/kuB3hJLXoX8/8D5z0J7TWXqJQs2lXDbP5dSvGdvi9a/9ZQB3HX6II30FEco0A+kYit89jD0PQFGXet0NdJKrLX8fd4mnvhkHekpiTx16XAS4364r3hKu3iO7KoLneIcBfoPsRY+uht8dXDBs+pz7hEVNXXc9042n6zazjnDuvPEJcNJToh1uiyRA1Kg/5BV78G6mXD6H6Bzf6erkVawdnsFt7yxhM2lVTx4zhBuOqGfZj2UiKFAb05VKcy8F3qMhLG/cLoaCZM563eybHM5AFV19bw6P4/khFim/HQsY/p1drg6kYOjQG/OrN9ATTlc+z5E6z+TG2UXlnP9y9/ib9BdfPyAFP738pF0bZ/gXGEih0hJ1ZSN/4blU+CEu6H7MKerkTCo8/m5b1o2qcnxfHrnSSQnBP4UNBRfIpkCvSmzHoSUDDjxPqcrkTB5YU4Oa7fvZvI1o+mQqAue4g4artZYeQHsXAtZ10Osvna70caiPTz7+UbOHd6DM47q7nQ5IiGjQG8sb17gsd+JztYhYbGrqo6731nOEXHRPHL+UU6XIxJSanJpLHduYDbFrvpjd5uVW3Zxy5uL2b6rhmcnHk1qsobmi7so0BuyNhDofU+AKH15cZN3FhXw4PsrSUmK462fjWNUH82WKe6j1GqodBNUbFFzi8vk7NzDfdOzyUrvxIe/PF5hLq6lQG8od07gsd9JztYhIfXa/Dxio6J4RjMgissp0BvKnQvJPSFlgNOVSIhU1NQxbXEh543ooTZzcT0F+nf8fsidF2hu0dwdrvHOokIqa31cP153CBL3U6B/p2g1VBWr/dxFfH7Lq/PzyErvxLC0Dk6XIxJ2CvTv5M4NPCrQXePLdUVsLq3iuuP6Ol2KSKtQoH8nd25gityOvZ2uRELklfl5dG+fwJkaDSoeoUAH8NVD/leB/ucS8Xx+y1Oz1jJvQzHXjEsnVjdkFo/QwCKAbcthbwX0V3fFSFe8Zy+3T1nK/JwSrhjTm5tO0MVQ8Y4WBbox5izgGSAaeNFa+3gT6/wYeASwwHJr7ZUhrDO8vut/rjP0iJNbXMn907KpqKkDYHtFDdW1Pp66dDiXZan5TLzlgIFujIkGngdOBwqBhcaYGdba1Q3WyQB+DRxnrS0zxnQNV8FhkTsXumZCu8gq2+v8fsu97yxn3Y7djB+QAsDAbsn87KT+HNVTvVrEe1pyhj4G2Git3QRgjJkKXAisbrDOT4HnrbVlANbaolAXGjb1e2HzAhj9E6crkYP0xjf5LMov4+nLRnDJ6DSnyxFxXEuuFvUCChosFwafa2ggMNAY85UxZkGwiWY/xpibjTGLjDGLdu7ceWgVh1rhIqivVnfFCLOlvJonPl7LCRlduHhU44+jiDe15Ay9qWGTttFyDJABnAykAfOMMUOtteXf+0fWTgYmA2RlZTV+D2fkzgUTBenHOV2JHEDl3np8NvCxefC9FVjgjz8ahtHIXhGgZYFeCDS8upQGbG1inQXW2jog1xizjkDALwxJleGUOxd6jIQjOjpdiTSjps7HIzNWMXVhwfeef/i8THp3TnSoKpG2pyWBvhDIMMb0A7YAE4HGPVjeB64AXjHGdCHQBLMplIWGRW0lFC6Ecbc6XYk0o6C0ilveXMzKLRVcMzad9JRAgKcmx3Pe8J4OVyfSthww0K219caY24BZBLotvmStXWWMeRRYZK2dEXztDGPMasAH3GutLQln4SGxeQH469R+7iBrLdMWF/L5mqavo3+9qQRrLf/4SRYThnRr5epEIkuL+qFba2cCMxs993CD3y1wV/AncuTOhahY6DPW6Uo8qbrWx2/fX8n0JYWkdTqCpLj9P44jenfksQuH0idFTSsiB+LtkaK5cyHtGIhLcroSz9lcUsXP3ljM2u0V3DEhg9snZBAdpYubIofDm4Huq4cvHoWtS+DUh5yuxnNq6nz85OVvKa2s5aXrjuGUQRrQJRIK3gv03Ttg2g2Q/x/IuhHG/9Lpijznmc83kFtcyRs3HsvxGV2cLkfENbwV6LWV8PdToaoEfvQCjJjodEWes3LLLibP3cRlo9MU5iIh5q1Az58PFYUw8Z8w+Fynq/Gcep+fB97NplNiHL89N9PpckRcx1uBnjsHouOg/ylOV+JJf/0yh5VbKvjrVaPokBjrdDkiruOxQJ8LaWMgTl3gWlNNnY/ff7CaKd9u5rzhPTh7qO4gJBIO3gn0qlLYlg0n/9rpSjylsKyKX7y5hOzCXfz8pAHcc8ZAzb0iEibeCfT8rwCrUaGtaHdNHZe/sICK6jpeuGa07u0pEmbeCfTcuRCbCL1GO12JZzz5yTq27qpm2s/HMzq9k9PliLied+6emzsX+oyDmDinK/GEhXmlvL4gn+vG91WYi7QSbwT67h2wc62aW1pJTZ2P+6dn06vjEdxzxiCnyxHxDG80ueTNCzwq0FvFs59vYNPOSl67YQxJ8d74iIm0Bd74a8udAwkdoMcIpytxtdp6P3+cuYZX5udx6eg0ThyY6nRJIp7ikUCfC31PgKhopytxFWv/exfBHRV7ufWfS1icX8YNx/Xj1+cMdrAyEW9yf6BvXwFleXDsLU5X4iqfrd7B/dOzKa2s3fdcYlw0z115tO4kJOIQdwe63wcf3AmJKTDsMqercYV6n5//+Ww9f/0yh6G92nPtuHQAoozh3OE9GJDazuEKRbzL3YH+zQuwZRFc/CIkpThdTUTaWl7N3+bkULnXB0DOzj0sKyjnijF9+N35mSTEqhlLpK1wb6CX5cEXf4CMM2HYpU5XE5HmbdjJ7VOWUlXro0u7eADiY6J48tLh/Dirt8PViUhj7gl0vx+2LoX6msDynCfARMN5/wOaO2SfmjofK7bswu+3P7je15tKeObzDQzsmsykq0fRX00pIm2eewJ9w6cw5fLvP3fu09AhzZl62qCNRXv4+RuL2Vi0p0XrXziyJ3+6eBiJTdy8WUTaHvf8pW5ZDCYKrp4eODNP6AA9RzpdVZvxUfY27pu2nITYaJ6ZOJLUYBNKc5LiYxie1kEzI4pEEPcE+o6VkJIBA051upI258V5m3jsozWM6tORv141mu4dEpwuSUTCwD2Bvn0F9B7jdBVtzoYdu3nik7WckdmN564cRVyMN6bvEfEid/x1V5fBrgLoNtTpStoUn99y//Rs2sXH8MeLhynMRVzOHX/hO1YFHrsPc7aONub1r/NYsrmch8/P3NftUETcyx2Bvn1l4FGBvk9hWRVPzlrHyYNSuWhkL6fLEZFW4I5A37ECErtAu25OV9Im5BVXctOrizDAYxcNVU8VEY9wx0XR7Sug+1ANIAI+XbWdu99eTnS0YdLVo0nrlOh0SSLSSiI/0H31ULQWxvzU6UocVe/z8/Rn65n0ZQ7DenXgr1eNondnhbmIl0R+oJdsAN9eT7ef79y9l9unLOXrTSWaNEvEwyI/0D1yQXTp5jK2lFfv93xVrY+nP11HeVUdT106nMs0aZaIZ0V+oO9YAdFx0GWg05WERb3PzxOfrOXv83KbXSc9JZH3fjGGzJ7tW7EyEWlrIj/Qt6+A1EEQHet0JSFXVFHDbVOW8m1uKdeOS+fqsek0ddm3d+dENbGIiBsCfSUceZrTVYREYVkV1/7jW7ZXBKYArq33ExNt+N/LR3LR0epLLiI/LLIDfU8RVBYFuixGOGstv3lvJdsrarhyTB+MgagowyWj0hjYLdnp8kQkAkR2oG9fEXh0wQXR95ZuYe76nTxyfibXHdfP6XJEJAJF9kjR4vWBx9QhztZxmIr37OXRD1czqk9HrhnX1+lyRCRCRfYZelk+xCZBUhenKzkgay0bi/awt96/32vPz95I1V4fT1wynOgojXYVkUPTokA3xpwFPANEAy9aax9vZr1LgXeAY6y1i0JWZXPK86FTepsf8l9RU8c9by/n09U7ml3nrtMHkqG2chE5DAcMdGNMNPA8cDpQCCw0xsyw1q5utF4ycDvwTTgKbVJZHnRMb7XNHYo12yq45Y3FFJZVc88ZA5u8wNkuPoax/VMcqE5E3KQlZ+hjgI3W2k0AxpipwIXA6kbr/QF4ErgnpBU2x9pAk0u/E1tlc429+U0+S/LLefi8TDokxgZLsryxIJ+3FhXgD7as5OzcQ8fEWKbePJasvp0dqVVEvKElgd4LKGiwXAgc23AFY8zRQG9r7YfGmGYD3RhzM3AzQJ8+fQ6+2oaqSqCu0pEz9PU7dvPIjFXU+Szf5pUw6arR9E9N4oHpK5ixfCsj0jrQvWPgvp3DenXgnjMHkZqsG0yISHi1JNCbaqC2+140Jgr4C3Ddgd7IWjsZmAyQlZVlD7D6DyvLDzx2at1A9/kt903LJjkhlqcuHc5v31/JxZPm07NDAptLq7j3zEHcctIAonRxU0RaWUsCvRBoOONTGrC1wXIyMBT4Mngjhe7ADGPMBWG9MFqeF3hs5TP0V+bnsaygnGcmjmTCkG6M7N2RO99axuqtFbx2w7Ecn9H2e9yIiDu1JNAXAhnGmH7AFmAicOV3L1prdwH7UswY8yVwT9h7uThwhl5QWsWfZ63jlEGpXDCiJwAp7eJ57YYx1PstsdGR3a1fRCLbARPIWlsP3AbMAtYAb1trVxljHjXGXBDuAptVng9HdIb41unqV1Baxc2vLybKwGM/Gva927oZYxTmIuK4FvVDt9bOBGY2eu7hZtY9+fDLaoGy/FY7O5+9tog731qG31qeu2oUvToe0SrbFRE5GJE7UrQ8H7oPD/tmJs/N4Y8z15LZoz1/u3o0fVJ0WzcRaZsis53A74PygrCfoS/dXMbjH6/l7KHdefcX4xXmItKmRWag794G/rqw9nCprffzwPQVdE1O4MlLh+sGEiLS5kVmk0sr9HCZ9GUO63bs5sVrs0hOcN/dkETEfSLzDL38u0APz7zhG3bs5rnZGzh/RE9Oy+wWlm2IiIRaZAZ6WT5goENayN/a57fcPz2bpPgYfnd+ZsjfX0QkXCIz0MvzoX1PiAn9/CivfZ3Hks2BSbe6tNP8KyISOSIz0Mvyw3JBtLCsiqdmreOkgan8SDdlFpEIE5mBXh76QUXf3aQZ4P/9aOj3RoKKiESCyAv0+r1QsTXkZ+jf3aT5/rMGk9ZJ/c1FJPJEXqDvKgRsSM/Q/X7LH2euZXR6J64Z27bvgCQi0pzIC/SyvMBjCM/QV2+roHjPXq4Zm655zEUkYkVeoJeHflDR/JxiAMYN0H09RSRyRV6gx7eH3mMhuUfI3nJ+TgkDUpPo1j4hZO8pItLaIi/Qh10KN86CqNDMrVLn8/NtbinjB+hOQyIS2SIv0EMsu7Ccqlof49XcIiIRzvOBPn9jCQBj+yvQRSSyKdBzSsjs0Z5OSXFOlyIiclg8Heg1dT4Wby5Tc4uIuIKnA31Jfhm19X6OO1IXREUk8nk60OfnlBAdZTimX2enSxEROWyeDvSvcooZkdaBdvGReeMmEZGGPBvou2vqyC7cpdGhIuIang30hXml+PyW4zSgSERcwrOBPn9jCXExUYxK7+R0KSIiIeHdQM8pYXSfTiTEhmYKARERp3ky0Msqa1m9rUL9z0XEVTwZ6As2BYb7jz9SgS4i7uHJQJ+fU0JiXDTD0zo6XYqISMh4NNCLGdOvM7HRntx9EXEpzyXajooacnZWqv1cRFzHc4H+dU6w/Vz9z0XEZTwX6PNziulwRCyZPdo7XYqISEh5MNBLGNc/hago43QpIiIh5alAn7N+J4Vl1RyfoeYWEXEfzwR65d56fvPuCgakJnFZVprT5YiIhJxn5o19atY6tu6q5p2fjSM+RsP9RcR9PHGGvji/jFe/zuPaselk9dXNLETEnVoU6MaYs4wx64wxG40xDzTx+l3GmNXGmGxjzOfGmPTQl3poauv93D89mx7tE7j3rMFOlyMiEjYHDHRjTDTwPHA2kAlcYYzJbLTaUiDLWjscmAY8GepCD9VHK7aysWgPj1xwlO5MJCKu1pIz9DHARmvtJmttLTAVuLDhCtba2dbaquDiAqBNXHW01vLyV3kMSE3i9MxuTpcjIhJWLQn0XkBBg+XC4HPNuRH4+HCKCpUlm8vJLtzFdeP7Yoz6nYuIu7WkDaKpJLRNrmjM1UAWcFIzr98M3AzQp0+fFpZ46F6Zn0dyQgwXj2oTXxhERMKqJWfohUDvBstpwNbGKxljTgMeBC6w1u5t6o2stZOttVnW2qzU1NRDqbfFtu+q4eMV27g8qzdJajsXEQ9oSaAvBDKMMf2MMXHARGBGwxWMMUcDLxAI86LQl3nw3vwmH5+1XDuur9OliIi0igMGurW2HrgNmAWsAd621q4yxjxqjLkguNpTQDvgHWPMMmPMjGberlVU1/r45zebmTC4G31SEp0sRUSk1bSoLcJaOxOY2ei5hxv8flqI6zpkBaVV3PLmYkoqa7nphH5OlyMi0mpc1bg8e10Rd05dht9a/n5tFmP76yYWIuIdrgn05QXl3PDKQgZ3b8/frh5FekqS0yWJiLQq1wT6ks1lWAuvXn8MXdsnOF2OiEirc83kXHnFlSTHx5CaHO90KSIijnBNoOeWVJHeJVEjQkXEs1wT6HnFlfRVu7mIeJgrAr223k9hWRX9uijQRcS7XBHohWVV+C06QxcRT3NFoOeVVALQt4tGhYqId7kj0IsDU7HrDF1EvMwdgV5SSXJCDJ2T4pwuRUTEMa4I9NxgDxd1WRQRL3NFoOeVVNJXPVxExOMiPtBr6/1sKaumn6bJFRGPi/hAL/iuy6LO0EXE4yI+0POKA10WNbuiiHhd5Ad6SaDLokaJiojXRX6gF1fSPiGGTomxTpciIuKoyA/0YA8XdVkUEa+L+EDP1SyLIiJAhAf63nofW8ur1cNFRIQID/SC0mr8FvppUi4RkcgO9Fx1WRQR2SdiA93vt7w4bxPJCTEM7JbsdDkiIo6L2ECfurCAb3JL+e25Q2gXH9A2AXcAAASjSURBVON0OSIijovIQN++q4Y/zVzD+AEp/Dirt9PliIi0CREX6NZaHvrXSur8fv508TD1PxcRCYq4QJ+5Yjufrd7BXacP1MVQEZEGIi7Q2yXEcHpmN244rp/TpYiItCkRdzXxpIGpnDQw1ekyRETanIg7QxcRkaYp0EVEXEKBLiLiEgp0ERGXUKCLiLiEAl1ExCUU6CIiLqFAFxFxCWOtdWbDxuwE8g/xn3cBikNYTqTw4n57cZ/Bm/vtxX2Gg9/vdGttk6MrHQv0w2GMWWStzXK6jtbmxf324j6DN/fbi/sMod1vNbmIiLiEAl1ExCUiNdAnO12AQ7y4317cZ/DmfntxnyGE+x2RbegiIrK/SD1DFxGRRhToIiIuEXGBbow5yxizzhiz0RjzgNP1hIMxprcxZrYxZo0xZpUx5o7g852NMZ8ZYzYEHzs5XWuoGWOijTFLjTEfBpf7GWO+Ce7zW8aYOKdrDDVjTEdjzDRjzNrgMR/nkWP9q+Dne6UxZooxJsFtx9sY85IxpsgYs7LBc00eWxPwbDDbso0xow52exEV6MaYaOB54GwgE7jCGJPpbFVhUQ/cba0dAowFbg3u5wPA59baDODz4LLb3AGsabD8BPCX4D6XATc6UlV4PQN8Yq0dDIwgsP+uPtbGmF7A7UCWtXYoEA1MxH3H+xXgrEbPNXdszwYygj83A5MOdmMRFejAGGCjtXaTtbYWmApc6HBNIWet3WatXRL8fTeBP/BeBPb11eBqrwIXOVNheBhj0oBzgReDywY4FZgWXMWN+9weOBH4B4C1ttZaW47Lj3VQDHCEMSYGSAS24bLjba2dC5Q2erq5Y3sh8JoNWAB0NMb0OJjtRVqg9wIKGiwXBp9zLWNMX+Bo4Bugm7V2GwRCH+jqXGVh8b/AfYA/uJwClFtr64PLbjze/YGdwMvBpqYXjTFJuPxYW2u3AH8GNhMI8l3AYtx/vKH5Y3vY+RZpgW6aeM61/S6NMe2A6cCd1toKp+sJJ2PMeUCRtXZxw6ebWNVtxzsGGAVMstYeDVTisuaVpgTbjS8E+gE9gSQCTQ6Nue14/5DD/rxHWqAXAr0bLKcBWx2qJayMMbEEwvxNa+27wad3fPcVLPhY5FR9YXAccIExJo9AU9qpBM7YOwa/koM7j3chUGit/Sa4PI1AwLv5WAOcBuRaa3daa+uAd4HxuP94Q/PH9rDzLdICfSGQEbwSHkfgIsoMh2sKuWDb8T+ANdba/2nw0gzgJ8HffwL8q7VrCxdr7a+ttWnW2r4EjusX1tqrgNnApcHVXLXPANba7UCBMWZQ8KkJwGpcfKyDNgNjjTGJwc/7d/vt6uMd1NyxnQFcG+ztMhbY9V3TTItZayPqBzgHWA/kAA86XU+Y9vF4Al+1soFlwZ9zCLQpfw5sCD52drrWMO3/ycCHwd/7A98CG4F3gHin6wvD/o4EFgWP9/tAJy8ca+D3wFpgJfA6EO+24w1MIXCNoI7AGfiNzR1bAk0uzwezbQWBHkAHtT0N/RcRcYlIa3IREZFmKNBFRFxCgS4i4hIKdBERl1Cgi4i4hAJdRMQlFOgiIi7x/wHZaBTtsp05LgAAAABJRU5ErkJggg==\n", 270 | "text/plain": [ 271 | "
" 272 | ] 273 | }, 274 | "metadata": { 275 | "needs_background": "light" 276 | }, 277 | "output_type": "display_data" 278 | } 279 | ], 280 | "source": [ 281 | "import matplotlib.pyplot as plt\n", 282 | "plt.plot(test_scores1)\n", 283 | "plt.plot(test_scores2)\n", 284 | "plt.show()" 285 | ] 286 | }, 287 | { 288 | "cell_type": "markdown", 289 | "metadata": {}, 290 | "source": [ 291 | "Adam的效果明显要好。只是我看不懂原理。" 292 | ] 293 | }, 294 | { 295 | "cell_type": "code", 296 | "execution_count": null, 297 | "metadata": {}, 298 | "outputs": [], 299 | "source": [] 300 | } 301 | ], 302 | "metadata": { 303 | "kernelspec": { 304 | "display_name": "Python 3", 305 | "language": "python", 306 | "name": "python3" 307 | }, 308 | "language_info": { 309 | "codemirror_mode": { 310 | "name": "ipython", 311 | "version": 3 312 | }, 313 | "file_extension": ".py", 314 | "mimetype": "text/x-python", 315 | "name": "python", 316 | "nbconvert_exporter": "python", 317 | "pygments_lexer": "ipython3", 318 | "version": "3.7.6" 319 | } 320 | }, 321 | "nbformat": 4, 322 | "nbformat_minor": 2 323 | } 324 | -------------------------------------------------------------------------------- /Chapter8/8-5-3.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter8/mnist_loader.py: -------------------------------------------------------------------------------- 1 | #### Libraries 2 | # Standard library 3 | import pickle 4 | import gzip 5 | 6 | # Third-party libraries 7 | import numpy as np 8 | 9 | #### Load the MNIST data 10 | def get_part(data, seed, size): 11 | X, y = data 12 | n = X.shape[0] 13 | if n <= size:return data 14 | if seed: 15 | np.random.seed(seed) 16 | shuffle_indexes = np.random.permutation(n) 17 | part_indexes = shuffle_indexes[:size] 18 | return X[part_indexes], y[part_indexes] 19 | 20 | def vectorized_result(j): 21 | """Return a 10-dimensional unit vector with a 1.0 in the jth 22 | position and zeroes elsewhere. This is used to convert a digit 23 | (0...9) into a corresponding desired output from the neural 24 | network.""" 25 | e = np.zeros(10) 26 | e[j] = 1.0 27 | return e 28 | 29 | def vectorize(data): 30 | vector_y = [vectorized_result(y) for y in data[1]] 31 | return (data[0], np.array(vector_y)) 32 | #return (data[0], data[1]) 33 | 34 | def load_data_shared(filename, seed, train_size, vali_size, test_size): 35 | f = gzip.open(filename, 'rb') 36 | training_data, validation_data, test_data = pickle.load(f,encoding='bytes') 37 | 38 | return get_part(training_data, seed, train_size),\ 39 | get_part(validation_data, seed, vali_size),\ 40 | get_part(test_data, seed, test_size) -------------------------------------------------------------------------------- /Chapter9/mnist_loader.py: -------------------------------------------------------------------------------- 1 | #### Libraries 2 | # Standard library 3 | import pickle 4 | import gzip 5 | 6 | # Third-party libraries 7 | import numpy as np 8 | 9 | #### Load the MNIST data 10 | def get_part(data, seed, size): 11 | X, y = data 12 | n = X.shape[0] 13 | if n <= size:return data 14 | if seed: 15 | np.random.seed(seed) 16 | shuffle_indexes = np.random.permutation(n) 17 | part_indexes = shuffle_indexes[:size] 18 | return X[part_indexes], y[part_indexes] 19 | 20 | def vectorized_result(j): 21 | """Return a 10-dimensional unit vector with a 1.0 in the jth 22 | position and zeroes elsewhere. This is used to convert a digit 23 | (0...9) into a corresponding desired output from the neural 24 | network.""" 25 | e = np.zeros(10) 26 | e[j] = 1.0 27 | return e 28 | 29 | def vectorize(data): 30 | vector_y = [vectorized_result(y) for y in data[1]] 31 | return (data[0], np.array(vector_y)) 32 | #return (data[0], data[1]) 33 | 34 | def load_data_shared(filename, seed, train_size, vali_size, test_size): 35 | f = gzip.open(filename, 'rb') 36 | training_data, validation_data, test_data = pickle.load(f,encoding='bytes') 37 | 38 | return get_part(training_data, seed, train_size),\ 39 | get_part(validation_data, seed, vali_size),\ 40 | get_part(test_data, seed, test_size) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # play-with-handwritten-digits 2 | -------------------------------------------------------------------------------- /SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Table of contents 2 | 3 | * [Introduction](README.md) 4 | * 第6章 深度前馈网络 5 | * 6.2 基于梯度的学习 6 | * [6.2.2.1 用于高斯输出分布的线性神单元](Chapter6/6-2-2-1.md) 7 | * [深度线性网络实验](Chapter6/deep-linear-network.md) 8 | * [6.2.2.2 用于Bernoulli输出分布的sigmoid单元](Chapter6/6-2-2-2.md) 9 | * [6.2.2.3 用于Multinoulli输出分布的softmax单元](Chapter6/6-2-2-3.md) 10 | * 6.3 隐藏单元 11 | * [6.3.1 ReLU及其扩展](Chapter6/6-3-1.md) 12 | * [6.3.2 logistic sigmoid与双曲正切函数](Chapter6/6-3-2.md) 13 | * 第7章 深度学习中的正则化 14 | * 7.1 参数范数惩罚 15 | * [7.1.1 L2参数正则化](Chapter7/7-1-1.md) 16 | * [7.1.2 L1参数正则化](Chapter7/7-1-2.md) 17 | * [7.8 提前终止](Chapter7/7-8.md) 18 | * [7.11 Bagging 和其他集成方法](Chapter7/7-11.md) 19 | * [7.12 Dropout](Chapter7/7-12.md) 20 | * [第8章 深度模型中的优化](Chapter8/0Optimization.md) 21 | * 8.3 基本算法 22 | * [8.3.1 随机梯度下降](Chapter8/8-3-1.md) 23 | * [8.3.2 动量](Chapter8/8-3-2.md) 24 | * 8.5 自适应学习率算法 25 | * [8.5.1 AdaGrad](Chapter8/8-5-1.md) 26 | * [8.5.2 RMSProp](Chapter8/8-5-2.md) 27 | * 第11章 实践方法论 28 | * [11.2 默认的基准模型](Chapter11/11-2.md) 29 | * 第12章 应用 30 | * 12.1 大规模深度学习 31 | * [12.1.2 GPU 实现](Chapter12/12-1-2.md) 32 | -------------------------------------------------------------------------------- /book.json: -------------------------------------------------------------------------------- 1 | { 2 | "gitbook": "3.2.3", 3 | "author": "windmissing", 4 | "plugins": [ 5 | "katex", 6 | "alerts", 7 | "mygitalk", 8 | "github", 9 | "github-buttons", 10 | "splitter", 11 | "-sharing", "sharing-plus", 12 | "donate", 13 | "copy-code-button", 14 | "anchor-navigation-ex", 15 | "sectionx", 16 | "click-reveal" 17 | ], 18 | "pluginsConfig": { 19 | "mygitalk": { 20 | "clientID": "43dbd838dc1b0ba757f1", 21 | "clientSecret": "7029e5dabc2cb6bf3d538efcc461c6219239cecc", 22 | "repo": "DeepLearningPractise", 23 | "owner": "windmissing", 24 | "admin": ["windmissing"], 25 | "distractionFreeMode": false 26 | }, 27 | "github": { 28 | "url": "https://github.com/windmissing/DeepLearningPractise" 29 | }, 30 | "github-buttons": { 31 | "repo": "windmissing/DeepLearningPractise", 32 | "types": [ 33 | "star", 34 | "watch", 35 | "fork" 36 | ], 37 | "size": "small" 38 | }, 39 | "donate": { 40 | "wechat": "http://windmissing.github.io/images/2020/weixin.jpg", 41 | "alipay": "http://windmissing.github.io/images/2020/alipay.jpg", 42 | "title": "", 43 | "button": "赏", 44 | "alipayText": "支付宝打赏", 45 | "wechatText": "微信打赏" 46 | }, 47 | "anchor-navigation-ex": { 48 | "isShowTocTitleIcon": true, 49 | "tocLevel1Icon": "fa fa-hand-o-right", 50 | "tocLevel2Icon": "fa fa-hand-o-right", 51 | "tocLevel3Icon": "fa fa-hand-o-right" 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /mnist.pkl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/windmissing/DeepLearningPractise/0f324b5caa82a52bb44ba6749cd68572e0b36f6d/mnist.pkl.gz -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "requires": true, 3 | "lockfileVersion": 1, 4 | "dependencies": { 5 | "array-union": { 6 | "version": "1.0.2", 7 | "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", 8 | "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", 9 | "requires": { 10 | "array-uniq": "^1.0.1" 11 | } 12 | }, 13 | "array-uniq": { 14 | "version": "1.0.3", 15 | "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", 16 | "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=" 17 | }, 18 | "async": { 19 | "version": "2.6.3", 20 | "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", 21 | "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", 22 | "requires": { 23 | "lodash": "^4.17.14" 24 | }, 25 | "dependencies": { 26 | "lodash": { 27 | "version": "4.17.15", 28 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", 29 | "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" 30 | } 31 | } 32 | }, 33 | "balanced-match": { 34 | "version": "1.0.0", 35 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 36 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" 37 | }, 38 | "boolbase": { 39 | "version": "1.0.0", 40 | "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", 41 | "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" 42 | }, 43 | "brace-expansion": { 44 | "version": "1.1.11", 45 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 46 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 47 | "requires": { 48 | "balanced-match": "^1.0.0", 49 | "concat-map": "0.0.1" 50 | } 51 | }, 52 | "commander": { 53 | "version": "2.20.3", 54 | "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", 55 | "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" 56 | }, 57 | "concat-map": { 58 | "version": "0.0.1", 59 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 60 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" 61 | }, 62 | "debug": { 63 | "version": "3.2.6", 64 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", 65 | "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", 66 | "requires": { 67 | "ms": "^2.1.1" 68 | } 69 | }, 70 | "domelementtype": { 71 | "version": "1.3.1", 72 | "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", 73 | "integrity": "sha1-0EjESzew0Qp/Kj1f7j9DM9eQSB8=" 74 | }, 75 | "email-addresses": { 76 | "version": "3.1.0", 77 | "resolved": "https://registry.npmjs.org/email-addresses/-/email-addresses-3.1.0.tgz", 78 | "integrity": "sha512-k0/r7GrWVL32kZlGwfPNgB2Y/mMXVTq/decgLczm/j34whdaspNrZO8CnXPf1laaHxI6ptUlsnAxN+UAPw+fzg==" 79 | }, 80 | "entities": { 81 | "version": "1.1.2", 82 | "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", 83 | "integrity": "sha1-vfpzUplmTfr9NFKe1PhSKidf6lY=" 84 | }, 85 | "es5-ext": { 86 | "version": "0.10.53", 87 | "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", 88 | "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", 89 | "dev": true, 90 | "requires": { 91 | "es6-iterator": "~2.0.3", 92 | "es6-symbol": "~3.1.3", 93 | "next-tick": "~1.0.0" 94 | } 95 | }, 96 | "es6-iterator": { 97 | "version": "2.0.3", 98 | "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", 99 | "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", 100 | "dev": true, 101 | "requires": { 102 | "d": "1", 103 | "es5-ext": "^0.10.35", 104 | "es6-symbol": "^3.1.1" 105 | }, 106 | "dependencies": { 107 | "d": { 108 | "version": "1.0.1", 109 | "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", 110 | "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", 111 | "dev": true, 112 | "requires": { 113 | "es5-ext": "^0.10.50", 114 | "type": "^1.0.1" 115 | } 116 | } 117 | } 118 | }, 119 | "es6-symbol": { 120 | "version": "3.1.3", 121 | "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", 122 | "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", 123 | "dev": true, 124 | "requires": { 125 | "d": "^1.0.1", 126 | "ext": "^1.1.2" 127 | }, 128 | "dependencies": { 129 | "d": { 130 | "version": "1.0.1", 131 | "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", 132 | "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", 133 | "dev": true, 134 | "requires": { 135 | "es5-ext": "^0.10.50", 136 | "type": "^1.0.1" 137 | } 138 | } 139 | } 140 | }, 141 | "escape-string-regexp": { 142 | "version": "1.0.5", 143 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", 144 | "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" 145 | }, 146 | "ext": { 147 | "version": "1.4.0", 148 | "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", 149 | "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", 150 | "dev": true, 151 | "requires": { 152 | "type": "^2.0.0" 153 | }, 154 | "dependencies": { 155 | "type": { 156 | "version": "2.0.0", 157 | "resolved": "https://registry.npmjs.org/type/-/type-2.0.0.tgz", 158 | "integrity": "sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow==", 159 | "dev": true 160 | } 161 | } 162 | }, 163 | "filename-reserved-regex": { 164 | "version": "1.0.0", 165 | "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-1.0.0.tgz", 166 | "integrity": "sha1-5hz4BfDeHJhFZ9A4bcXfUO5a9+Q=" 167 | }, 168 | "filenamify": { 169 | "version": "1.2.1", 170 | "resolved": "https://registry.npmjs.org/filenamify/-/filenamify-1.2.1.tgz", 171 | "integrity": "sha1-qfL/0RxQO+0wABUCknI3jx8TZaU=", 172 | "requires": { 173 | "filename-reserved-regex": "^1.0.0", 174 | "strip-outer": "^1.0.0", 175 | "trim-repeated": "^1.0.0" 176 | } 177 | }, 178 | "filenamify-url": { 179 | "version": "1.0.0", 180 | "resolved": "https://registry.npmjs.org/filenamify-url/-/filenamify-url-1.0.0.tgz", 181 | "integrity": "sha1-syvYExnvWGO3MHi+1Q9GpPeXX1A=", 182 | "requires": { 183 | "filenamify": "^1.0.0", 184 | "humanize-url": "^1.0.0" 185 | } 186 | }, 187 | "fs-extra": { 188 | "version": "8.1.0", 189 | "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", 190 | "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", 191 | "requires": { 192 | "graceful-fs": "^4.2.0", 193 | "jsonfile": "^4.0.0", 194 | "universalify": "^0.1.0" 195 | } 196 | }, 197 | "fs.realpath": { 198 | "version": "1.0.0", 199 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 200 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" 201 | }, 202 | "g": { 203 | "version": "2.0.1", 204 | "resolved": "https://registry.npmjs.org/g/-/g-2.0.1.tgz", 205 | "integrity": "sha1-C1lj69DKcOO8jGdmk0oCGCHIuFc=" 206 | }, 207 | "gh-pages": { 208 | "version": "2.2.0", 209 | "resolved": "https://registry.npmjs.org/gh-pages/-/gh-pages-2.2.0.tgz", 210 | "integrity": "sha512-c+yPkNOPMFGNisYg9r4qvsMIjVYikJv7ImFOhPIVPt0+AcRUamZ7zkGRLHz7FKB0xrlZ+ddSOJsZv9XAFVXLmA==", 211 | "requires": { 212 | "async": "^2.6.1", 213 | "commander": "^2.18.0", 214 | "email-addresses": "^3.0.1", 215 | "filenamify-url": "^1.0.0", 216 | "fs-extra": "^8.1.0", 217 | "globby": "^6.1.0" 218 | } 219 | }, 220 | "gitbook-plugin-alerts": { 221 | "version": "0.2.0", 222 | "resolved": "https://registry.npmjs.org/gitbook-plugin-alerts/-/gitbook-plugin-alerts-0.2.0.tgz", 223 | "integrity": "sha1-yMd4tBeY+AiwO+uQiCLpKKT74Tc=" 224 | }, 225 | "glob": { 226 | "version": "7.1.6", 227 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", 228 | "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", 229 | "requires": { 230 | "fs.realpath": "^1.0.0", 231 | "inflight": "^1.0.4", 232 | "inherits": "2", 233 | "minimatch": "^3.0.4", 234 | "once": "^1.3.0", 235 | "path-is-absolute": "^1.0.0" 236 | } 237 | }, 238 | "globby": { 239 | "version": "6.1.0", 240 | "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", 241 | "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=", 242 | "requires": { 243 | "array-union": "^1.0.1", 244 | "glob": "^7.0.3", 245 | "object-assign": "^4.0.1", 246 | "pify": "^2.0.0", 247 | "pinkie-promise": "^2.0.0" 248 | } 249 | }, 250 | "graceful-fs": { 251 | "version": "4.2.3", 252 | "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", 253 | "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" 254 | }, 255 | "humanize-url": { 256 | "version": "1.0.1", 257 | "resolved": "https://registry.npmjs.org/humanize-url/-/humanize-url-1.0.1.tgz", 258 | "integrity": "sha1-9KuZ4NKIF0yk4eUEB8VfuuRk7/8=", 259 | "requires": { 260 | "normalize-url": "^1.0.0", 261 | "strip-url-auth": "^1.0.0" 262 | } 263 | }, 264 | "inflight": { 265 | "version": "1.0.6", 266 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 267 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 268 | "requires": { 269 | "once": "^1.3.0", 270 | "wrappy": "1" 271 | } 272 | }, 273 | "inherits": { 274 | "version": "2.0.4", 275 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 276 | "integrity": "sha1-D6LGT5MpF8NDOg3tVTY6rjdBa3w=" 277 | }, 278 | "is-plain-obj": { 279 | "version": "1.1.0", 280 | "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", 281 | "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=" 282 | }, 283 | "jsonfile": { 284 | "version": "4.0.0", 285 | "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", 286 | "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", 287 | "requires": { 288 | "graceful-fs": "^4.1.6" 289 | } 290 | }, 291 | "lodash": { 292 | "version": "3.10.1", 293 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", 294 | "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=" 295 | }, 296 | "minimatch": { 297 | "version": "3.0.4", 298 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 299 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 300 | "requires": { 301 | "brace-expansion": "^1.1.7" 302 | } 303 | }, 304 | "ms": { 305 | "version": "2.1.2", 306 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", 307 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" 308 | }, 309 | "next-tick": { 310 | "version": "1.0.0", 311 | "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", 312 | "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=", 313 | "dev": true 314 | }, 315 | "normalize-url": { 316 | "version": "1.9.1", 317 | "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-1.9.1.tgz", 318 | "integrity": "sha1-LMDWazHqIwNkWENuNiDYWVTGbDw=", 319 | "requires": { 320 | "object-assign": "^4.0.1", 321 | "prepend-http": "^1.0.0", 322 | "query-string": "^4.1.0", 323 | "sort-keys": "^1.0.0" 324 | } 325 | }, 326 | "nth-check": { 327 | "version": "1.0.2", 328 | "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", 329 | "integrity": "sha1-sr0pXDfj3VijvwcAN2Zjuk2c8Fw=", 330 | "requires": { 331 | "boolbase": "~1.0.0" 332 | } 333 | }, 334 | "object-assign": { 335 | "version": "4.1.1", 336 | "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", 337 | "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" 338 | }, 339 | "once": { 340 | "version": "1.4.0", 341 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 342 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 343 | "requires": { 344 | "wrappy": "1" 345 | } 346 | }, 347 | "path-is-absolute": { 348 | "version": "1.0.1", 349 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 350 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" 351 | }, 352 | "pify": { 353 | "version": "2.3.0", 354 | "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", 355 | "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" 356 | }, 357 | "pinkie": { 358 | "version": "2.0.4", 359 | "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", 360 | "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" 361 | }, 362 | "pinkie-promise": { 363 | "version": "2.0.1", 364 | "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", 365 | "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", 366 | "requires": { 367 | "pinkie": "^2.0.0" 368 | } 369 | }, 370 | "prepend-http": { 371 | "version": "1.0.4", 372 | "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", 373 | "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=" 374 | }, 375 | "query-string": { 376 | "version": "4.3.4", 377 | "resolved": "https://registry.npmjs.org/query-string/-/query-string-4.3.4.tgz", 378 | "integrity": "sha1-u7aTucqRXCMlFbIosaArYJBD2+s=", 379 | "requires": { 380 | "object-assign": "^4.1.0", 381 | "strict-uri-encode": "^1.0.0" 382 | } 383 | }, 384 | "safe-buffer": { 385 | "version": "5.2.0", 386 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", 387 | "integrity": "sha1-t02uxJsRSPiMZLaNSbHoFcHy9Rk=" 388 | }, 389 | "sort-keys": { 390 | "version": "1.1.2", 391 | "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", 392 | "integrity": "sha1-RBttTTRnmPG05J6JIK37oOVD+a0=", 393 | "requires": { 394 | "is-plain-obj": "^1.0.0" 395 | } 396 | }, 397 | "strict-uri-encode": { 398 | "version": "1.1.0", 399 | "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", 400 | "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=" 401 | }, 402 | "strip-outer": { 403 | "version": "1.0.1", 404 | "resolved": "https://registry.npmjs.org/strip-outer/-/strip-outer-1.0.1.tgz", 405 | "integrity": "sha512-k55yxKHwaXnpYGsOzg4Vl8+tDrWylxDEpknGjhTiZB8dFRU5rTo9CAzeycivxV3s+zlTKwrs6WxMxR95n26kwg==", 406 | "requires": { 407 | "escape-string-regexp": "^1.0.2" 408 | } 409 | }, 410 | "strip-url-auth": { 411 | "version": "1.0.1", 412 | "resolved": "https://registry.npmjs.org/strip-url-auth/-/strip-url-auth-1.0.1.tgz", 413 | "integrity": "sha1-IrD6OkE4WzO+PzMVUbu4N/oM164=" 414 | }, 415 | "trim-repeated": { 416 | "version": "1.0.0", 417 | "resolved": "https://registry.npmjs.org/trim-repeated/-/trim-repeated-1.0.0.tgz", 418 | "integrity": "sha1-42RqLqTokTEr9+rObPsFOAvAHCE=", 419 | "requires": { 420 | "escape-string-regexp": "^1.0.2" 421 | } 422 | }, 423 | "type": { 424 | "version": "1.2.0", 425 | "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", 426 | "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==", 427 | "dev": true 428 | }, 429 | "universalify": { 430 | "version": "0.1.2", 431 | "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", 432 | "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" 433 | }, 434 | "util-deprecate": { 435 | "version": "1.0.2", 436 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 437 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" 438 | }, 439 | "wrappy": { 440 | "version": "1.0.2", 441 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 442 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" 443 | } 444 | } 445 | } 446 | --------------------------------------------------------------------------------