├── forward_mapping_functions.py ├── README.md ├── snip_prunner.py └── testing_mnist.ipynb /forward_mapping_functions.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | def snip_forward_conv2d(self, x): 6 | return F.conv2d(x, self.weight * self.weight_mask, self.bias, 7 | self.stride, self.padding, self.dilation, self.groups) 8 | 9 | def snip_forward_conv1d(self, x): 10 | return F.conv1d(x, self.weight * self.weight_mask, self.bias) 11 | 12 | def snip_forward_linear(self, x): 13 | return F.linear(x, self.weight * self.weight_mask, self.bias) 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PySNIP 2 | Unofficial implementation of SNIP (ICLR 19) in PyTorch. 3 | SNIP is a single shot neural network prunning technique which prunes the network before training based on sensitivity of connections of the randomly initialized weights. 4 | 5 | ## Usage 6 | ```python 7 | from snip_prunner import Prunner 8 | from model import my_model 9 | from loss_func import my_loss 10 | 11 | prunner = Prunner(my_model, my_loss, train_dataloader) 12 | prunned_model, masks = prunner.prun(compression_factor=0.9, num_batch_sampling=1) 13 | 14 | """ 15 | Now continue training prunned_model 16 | as you would do in normal setup 17 | """ 18 | ``` 19 | Refer test_mnist.ipynb for experiments on MNIST 20 | 21 | ## MNIST Results 22 | | Parameters / Batches | 1 | 10 | 23 | |----------------------|---------|----------| 24 | | 90% | 97.74 | 97.70 | 25 | | 75% | 97.79 | 97.79 | 26 | | 50% | 97.74 | 97.67 | 27 | | 10% | 96.69 | 96.69 | 28 | | 2% | 93.01 | 93.69 | 29 | 30 | ### ToDo 31 | Run experiments using ResNet Model on CIFAR 10 32 | 33 | ## Paper 34 | [SNIP](https://openreview.net/pdf?id=B1VZqjAcYX) 35 | -------------------------------------------------------------------------------- /snip_prunner.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import types 5 | import copy 6 | from forward_mapping_functions import * 7 | 8 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 9 | 10 | forward_mapping_dict = { 11 | 'Linear': snip_forward_linear, 12 | 'Conv2d': snip_forward_conv2d, 13 | 'Conv1d': snip_forward_conv1d 14 | } 15 | 16 | class Prunner: 17 | 18 | def __init__(self, model, criterion, dataloader): 19 | self.model = copy.deepcopy(model).to(device) 20 | self.prun_model = copy.deepcopy(model).to(device) 21 | self.criterion = criterion.to(device) 22 | self.dataloader = dataloader 23 | self.variance_scaling_init() 24 | self.update_forward_pass() 25 | 26 | def apply_hook(self, masks): 27 | layers = filter(lambda l: type(l).__name__ in forward_mapping_dict, self.prun_model.modules()) 28 | def apply_masking(mask): 29 | def hook(weight): 30 | return weight * mask 31 | return hook 32 | for layer, mask in zip(layers, masks): 33 | assert layer.weight.shape == mask.shape 34 | layer.weight.data = layer.weight.data * mask 35 | layer.weight.register_hook(apply_masking(mask)) 36 | 37 | def prun(self, compression_factor=0.5, num_batch_sampling=1): 38 | grads, grads_list = self.compute_grads(num_batch_sampling) 39 | keep_params = int((1 - compression_factor) * len(grads)) 40 | values, idxs = torch.topk(grads / grads.sum(), keep_params, sorted=True) 41 | threshold = values[-1] 42 | masks = [(grad / grads.sum() > threshold).float() for grad in grads_list] 43 | self.apply_hook(masks) 44 | return self.prun_model, masks 45 | 46 | def compute_grads(self, num_batch_sampling=1): 47 | moving_average_grads = 0 48 | for i, (data, labels) in enumerate(self.dataloader): 49 | if i == num_batch_sampling: 50 | break 51 | data, labels = data.to(device), labels.to(device) 52 | out = self.model(data) 53 | loss = self.criterion(out, labels) 54 | self.model.zero_grad() 55 | loss.backward() 56 | grads_list = [] 57 | for layer in self.model.modules(): 58 | if type(layer).__name__ in forward_mapping_dict: 59 | grads_list.append(torch.abs(layer.weight_mask.grad)) 60 | grads = torch.cat([torch.flatten(grad) for grad in grads_list]) 61 | if i == 0: 62 | moving_average_grads = grads 63 | moving_average_grad_list = grads_list 64 | else: 65 | moving_average_grads = ((moving_average_grads * i) + grads) / (i + 1) 66 | moving_average_grad_list = [((mv_avg_grad * i) + grad) / (i + 1) 67 | for mv_avg_grad, grad in zip(moving_average_grad_list, grads_list)] 68 | return moving_average_grads, moving_average_grad_list 69 | 70 | def variance_scaling_init(self): 71 | for layer in self.model.modules(): 72 | if type(layer).__name__ in forward_mapping_dict: 73 | layer.weight_mask = nn.Parameter(torch.ones_like(layer.weight).to(device)) 74 | nn.init.xavier_normal_(layer.weight) 75 | layer.weight.requires_grad = False 76 | 77 | def update_forward_pass(self): 78 | for layer in self.model.modules(): 79 | if type(layer).__name__ in forward_mapping_dict: 80 | layer.forward = types.MethodType(forward_mapping_dict[type(layer).__name__], layer) 81 | 82 | -------------------------------------------------------------------------------- /testing_mnist.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "kernelspec": { 6 | "name": "python3", 7 | "display_name": "Python 3" 8 | }, 9 | "language_info": { 10 | "codemirror_mode": { 11 | "name": "ipython", 12 | "version": 3 13 | }, 14 | "file_extension": ".py", 15 | "mimetype": "text/x-python", 16 | "name": "python", 17 | "nbconvert_exporter": "python", 18 | "pygments_lexer": "ipython3", 19 | "version": "3.6.7" 20 | }, 21 | "colab": { 22 | "name": "Untitled.ipynb", 23 | "version": "0.3.2", 24 | "provenance": [] 25 | }, 26 | "accelerator": "GPU" 27 | }, 28 | "cells": [ 29 | { 30 | "cell_type": "code", 31 | "metadata": { 32 | "id": "k4LyfUd2TtuQ", 33 | "colab_type": "code", 34 | "colab": {} 35 | }, 36 | "source": [ 37 | "import torch\n", 38 | "import torch.nn as nn\n", 39 | "import numpy as np\n", 40 | "import copy\n", 41 | "from torchvision.datasets import MNIST, CIFAR10\n", 42 | "from torch.utils.data import DataLoader\n", 43 | "from torchvision import transforms\n", 44 | "import torch.nn.functional as F\n", 45 | "from snip_prunner import Prunner" 46 | ], 47 | "execution_count": 0, 48 | "outputs": [] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "metadata": { 53 | "id": "oq7mllZHTtuX", 54 | "colab_type": "code", 55 | "colab": {} 56 | }, 57 | "source": [ 58 | "class LinearLayer(nn.Module):\n", 59 | " def __init__(self, input_dim, output_dim, act='relu', use_bn=False):\n", 60 | " super(LinearLayer, self).__init__()\n", 61 | " self.use_bn = use_bn\n", 62 | " self.lin = nn.Linear(input_dim, output_dim)\n", 63 | " self.act = nn.ReLU() if act == 'relu' else act\n", 64 | " if use_bn:\n", 65 | " self.bn = nn.BatchNorm1d(output_dim)\n", 66 | " def forward(self, x):\n", 67 | " if self.use_bn:\n", 68 | " return self.bn(self.act(self.lin(x)))\n", 69 | " return self.act(self.lin(x))" 70 | ], 71 | "execution_count": 0, 72 | "outputs": [] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "metadata": { 77 | "id": "DuPsr2jjTtue", 78 | "colab_type": "code", 79 | "colab": {} 80 | }, 81 | "source": [ 82 | "class Net(nn.Module):\n", 83 | " def __init__(self, in_dims, hid_dims, out_dims):\n", 84 | " super(Net, self).__init__()\n", 85 | " self.layers = nn.Sequential(LinearLayer(in_dims, hid_dims, use_bn=True),\n", 86 | " LinearLayer(hid_dims, hid_dims, use_bn=True),\n", 87 | " nn.Linear(hid_dims, out_dims))\n", 88 | " def forward(self, x):\n", 89 | " return self.layers(x.view(x.shape[0], -1))" 90 | ], 91 | "execution_count": 0, 92 | "outputs": [] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "metadata": { 97 | "id": "FCUF85yCTtui", 98 | "colab_type": "code", 99 | "colab": {} 100 | }, 101 | "source": [ 102 | "torch.manual_seed(2019)\n", 103 | "model = Net(784, 300, 10)\n", 104 | "train_ds = MNIST('../data/', train=True, transform=transforms.ToTensor(), download=True)\n", 105 | "val_ds = MNIST('../data/', train=False, transform=transforms.ToTensor(), download=True)\n", 106 | "train_dl = DataLoader(train_ds, batch_size=100, shuffle=True)\n", 107 | "val_dl = DataLoader(val_ds, batch_size=100, shuffle=True)\n", 108 | "criterion = nn.CrossEntropyLoss()\n", 109 | "prunner = Prunner(model, criterion, train_dl)" 110 | ], 111 | "execution_count": 0, 112 | "outputs": [] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "metadata": { 117 | "id": "ZnddfRNfTtum", 118 | "colab_type": "code", 119 | "colab": {} 120 | }, 121 | "source": [ 122 | "model, masks = prunner.prun(compression_factor=0.98, num_batch_sampling=1)" 123 | ], 124 | "execution_count": 0, 125 | "outputs": [] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "metadata": { 130 | "id": "uGNx-jNOTtuq", 131 | "colab_type": "code", 132 | "colab": { 133 | "base_uri": "https://localhost:8080/", 134 | "height": 191 135 | }, 136 | "outputId": "a321a4f5-b277-4153-e1a4-ad0adcb4345f" 137 | }, 138 | "source": [ 139 | "for n, w in model.named_parameters():\n", 140 | " print(n, (w != 0).sum())" 141 | ], 142 | "execution_count": 36, 143 | "outputs": [ 144 | { 145 | "output_type": "stream", 146 | "text": [ 147 | "layers.0.lin.weight tensor(3840, device='cuda:0')\n", 148 | "layers.0.lin.bias tensor(300, device='cuda:0')\n", 149 | "layers.0.bn.weight tensor(300, device='cuda:0')\n", 150 | "layers.0.bn.bias tensor(0, device='cuda:0')\n", 151 | "layers.1.lin.weight tensor(1899, device='cuda:0')\n", 152 | "layers.1.lin.bias tensor(300, device='cuda:0')\n", 153 | "layers.1.bn.weight tensor(300, device='cuda:0')\n", 154 | "layers.1.bn.bias tensor(0, device='cuda:0')\n", 155 | "layers.2.weight tensor(824, device='cuda:0')\n", 156 | "layers.2.bias tensor(10, device='cuda:0')\n" 157 | ], 158 | "name": "stdout" 159 | } 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "metadata": { 165 | "id": "rfErx_gYTtuw", 166 | "colab_type": "code", 167 | "colab": { 168 | "base_uri": "https://localhost:8080/", 169 | "height": 69 170 | }, 171 | "outputId": "180eb56a-022c-4197-b13f-620ee52f8fa3" 172 | }, 173 | "source": [ 174 | "for mask in masks:\n", 175 | " print(mask.sum())" 176 | ], 177 | "execution_count": 37, 178 | "outputs": [ 179 | { 180 | "output_type": "stream", 181 | "text": [ 182 | "tensor(3840., device='cuda:0')\n", 183 | "tensor(1899., device='cuda:0')\n", 184 | "tensor(824., device='cuda:0')\n" 185 | ], 186 | "name": "stdout" 187 | } 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "metadata": { 193 | "id": "uFlZHWH7Ttuz", 194 | "colab_type": "code", 195 | "colab": {} 196 | }, 197 | "source": [ 198 | "from torched.trainer_utils import Train" 199 | ], 200 | "execution_count": 0, 201 | "outputs": [] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "metadata": { 206 | "id": "iRoGaJC-Ttu3", 207 | "colab_type": "code", 208 | "colab": {} 209 | }, 210 | "source": [ 211 | "trainer = Train(model, [train_dl, val_dl], cuda=True)" 212 | ], 213 | "execution_count": 0, 214 | "outputs": [] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "metadata": { 219 | "id": "MuH5uLUHTtu7", 220 | "colab_type": "code", 221 | "colab": { 222 | "base_uri": "https://localhost:8080/", 223 | "height": 35 224 | }, 225 | "outputId": "08334a4a-ac06-4f80-c489-9690803ff193" 226 | }, 227 | "source": [ 228 | "trainer.lr_find(crit=criterion, opt='adamW')" 229 | ], 230 | "execution_count": 40, 231 | "outputs": [ 232 | { 233 | "output_type": "stream", 234 | "text": [ 235 | "Train Loss 0.744214: 66%|██████▌ | 394/600 [00:06<00:03, 62.06it/s]\n" 236 | ], 237 | "name": "stderr" 238 | } 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "metadata": { 244 | "id": "823Z-KndTtvA", 245 | "colab_type": "code", 246 | "colab": { 247 | "base_uri": "https://localhost:8080/", 248 | "height": 273 249 | }, 250 | "outputId": "d32991a2-1432-4bee-8fad-d8343dabbba7" 251 | }, 252 | "source": [ 253 | "trainer.plot_lrs()" 254 | ], 255 | "execution_count": 41, 256 | "outputs": [ 257 | { 258 | "output_type": "display_data", 259 | "data": { 260 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEACAYAAAC08h1NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8XHW5+PHPk8lkX5utS9qmdG+h\nLRBAaIGWpRRQ8OdVARGEKyIKLtflinoFxXuV671XEEURsQouILIoSFkKspUWSgql+76naZt9T2b7\n/v6Yc07OTCZN2k4yWZ7365VXM2eZ+eY0ec73PN9NjDEopZQaOZISXQCllFIDSwO/UkqNMBr4lVJq\nhNHAr5RSI4wGfqWUGmE08Cul1AijgV8ppUYYDfxKKTXC9Br4RWS8iLwqIptEZKOIfCXGMdeKyDoR\nWS8iK0VkrmvfHmv7WhGpiPcPoJRS6tgk9+GYAPB1Y8x7IpINrBGR5caYTa5jdgPnG2PqReRS4EHg\nLNf+RcaYmvgVWyml1PHqNfAbY6qAKuv7ZhHZDIwDNrmOWek65W2g9EQKVVhYaMrKyk7kLZRSakRZ\ns2ZNjTGmqC/H9qXG7xCRMuBU4J2jHPZZ4HnXawO8JCIG+LUx5sHePqesrIyKCs0KKaVUX4nI3r4e\n2+fALyJZwJPAV40xTT0cs4hw4F/g2rzAGFMpIsXAchHZYox5I8a5NwM3A0yYMKGvxVJKKXWM+tSr\nR0S8hIP+n4wxT/VwzBzgIeBKY0ytvd0YU2n9ewR4Gjgz1vnGmAeNMeXGmPKioj49rSillDoOfenV\nI8Bvgc3GmJ/2cMwE4CngOmPMNtf2TKtBGBHJBBYDG+JRcKWUUsenL6me+cB1wHoRWWtt+w4wAcAY\n8wBwB1AA/DJ8nyBgjCkHSoCnrW3JwJ+NMS/E9SdQSil1TPrSq2cFIL0ccxNwU4ztu4C53c9QSimV\nKDpyVymlRphhF/g3VDYSDOlykkop1ZNhFfjf31fPh3++gifW7E90UZRSatAaVoH/6fcrAXh+wyGC\nIUNIa/5KKdXNsAn8/mCI59ZVkSTw2tZqZnzveW7545pEF0sppQadY5qyYTALGcPXFk8jOUn4yQtb\nqW318dKmwwSCIZI9w+b+ppRSJ2zYRMTUZA/XnjWRq86YwD+/vpAvXzgVgC2HmhNcMqWUGlyGTeB3\ny83wctUZ4wFYs7c+waVRSqnBZVgGfoCxuWmMy0vnze26DIBSSrkN28AvIlw8q4Q3t1fT5gskujhK\nKTVoDNvAD7B4dgmdgRBvbKtOdFGUUmrQGNaB/4yyUWSmeDTdo5RSLsM68Hs9SZx1UgErd9b2frBS\nSo0QwzrwA5wzuYDdNa0s+O9/UrGnjoMN7YkuklJKJdSwD/wfO62U0vx0DtS38/EHVvGJB1YlukhK\nKZVQwz7wj8pM4Y+fPct5Xak1fqXUCNeXpRfHi8irIrJJRDaKyFdiHCMicp+I7BCRdSJymmvfZ0Rk\nu/X1mXj/AH0xsSAj4rVO3qaUGsn6UuMPAF83xswCPgTcKiKzoo65FJhqfd0M/ApAREYBdwJnEV5k\n/U4RyY9T2ftMRDhvWtcC7gcbtdavlBq5eg38xpgqY8x71vfNwGZgXNRhVwKPmLC3gTwRGQNcAiw3\nxtQZY+qB5cCSuP4EffTwjWfw2M0fAuCXr+3kht+t1gVblFIj0jHNzikiZcCpwDtRu8YB7tVPDljb\neto+4ESEmWNy8HqEP7+zD4DNVU2cPC43EcVRSqmE6XPjrohkAU8CXzXGNMW7ICJys4hUiEhFdXX/\njLTNTfdy/dllzuu3d2n/fqXUyNOnwC8iXsJB/0/GmKdiHFIJjHe9LrW29bS9G2PMg8aYcmNMeVFR\nUaxD4uL2S2ew9IZyygoyWKUDu5RSI1BfevUI8FtgszHmpz0c9gxwvdW750NAozGmCngRWCwi+Vaj\n7mJrW8J4PUlcMKOE0yeOYl1lYyKLopRSCdGXHP984DpgvYistbZ9B5gAYIx5AFgGXAbsANqAG619\ndSLyQ+Bd67y7jDF18Sv+8ZsxOpsn3ztAXauPUZkpiS6OUkoNmF4DvzFmBSC9HGOAW3vYtxRYelyl\n60fTR2cDsPVQM2dPLkhwaZRSauAM+5G7PZlhBf6NBxt5ceMhHdSllBoxhs1i68eqKDuVUZkp/Odz\nmwF44NOnc/60ItJTPAkumVJK9a8RW+MXEb64cLLz+pY/rmH2nS+wcofO3a+UGt5GbOAH+OyCSdx3\nzalkp4YffEIGvvKXtTR1+BNcMqWU6j8jOvCLCFfMHUtGaji9843F06hu7uQvq/f3cqZSSg1dIzrw\n2/77X+bwoZNG8bnzTmL8qHTW7m9IdJGUUqrfjNjGXbeF04tZOL0YgDnj8lhXqYFfKTV8aY0/yiml\nueyva6e+1ZfooiilVL/QGn+UOaXh2Trf2V3HS5sOcfLYXP51waQEl0oppeJHA3+U0yfmk5ni4ZY/\nrgHgqfcqNfArpYYVTfVESU32ONM52GpaOhNUGqWUij8N/DF89aJpnD4xn9/dcAYAK3X6ZqXUMKKp\nnhjOm1bEedOK8AVCjM1N41tPrGNaSRYzRuckumhKKXXCtMZ/FCnJSTz1xfl4koQHXtuZ6OIopVRc\naODvxejcNK46Yzz/WFelA7uUUsNCX1bgWioiR0RkQw/7vykia62vDSISFJFR1r49IrLe2lcR78IP\nlFsXTWF0bhrf/OsHAFQ1tvM/L24hqFM5K6WGoL7U+H8PLOlppzHmf4wx84wx84BvA69HrbK1yNpf\nfmJFTZxRmSlcOW8su2paCQRDfP3xD7j/1Z1sPKhLNyqlhp5eA78x5g2gr8slXgM8ekIlGqTG5KYT\nDBlu+N27Ti+fpvZAgkullFLHLm45fhHJIPxk8KRrswFeEpE1InJzvD4rEcblpQOwwjVf/6GmjkQV\nRymljls8u3N+BHgrKs2zwBhTKSLFwHIR2WI9QXRj3RhuBpgwYUIcixUfY/LSum07rIFfKTUExbNX\nz9VEpXmMMZXWv0eAp4EzezrZGPOgMabcGFNeVFQUx2LFx5jc9G7bDjVq4FdKDT1xCfwikgucD/zd\ntS1TRLLt74HFQMyeQUNBTlr3h6M/r97Hm9urE1AapZQ6fn3pzvkosAqYLiIHROSzInKLiNziOuz/\nAS8ZY1pd20qAFSLyAbAaeM4Y80I8Cz+QRMT5/pPlpRRnpxIMGW5+ZA3769po8wU4UN+WwBIqpVTf\niDGDry96eXm5qagYfN3+1+5vICPFw7SSbD7+q5VU7K0nM8VDekoyC6YU8NbOWt797kWJLqZSagQS\nkTV97Tavc/Ucg3nj85zv7716Hntr20hNTuLjD6zib2sPAtDhD5Lm9SSqiEop1SsN/MepND+D0vwM\nfIFQxPbaVp/T9VMppQYjnavnBKUkJ1GQmeK8rtW5+5VSg5wG/jgozunq41/bomv1KqUGNw38cVCS\nk+p8X9PSyb7aNj73SAWtnTqlg1Jq8NHAHwcl2a4af6uPf245zPJNh/lAp3FWSg1CGvjjwF3jr23p\nZHdNeDjDzuqWRBVJKaV6pL164iA6x19tNfDurG7t6RSllEoYDfxxsHh2Cfvr2nhrZw27alqpbg4H\n/l01GviVUoOPpnrioDg7jW9fNpOPzhvH2v0NVDa0A7BLUz1KqUFIA38cfXbBJL50wRQAzijLp7Kh\nneYOPyFdolEpNYho4I8jEeHri6ez478u5aozJmAMnPL9l7jx9+8mumhKKeXQwN8Pkj1JjM/vmrbh\n9W3VrNnb19UrlVKqf2njbj8pHZUR8XrZ+kMcqG+nqd3PdWeXJaZQSimFBv5+M9rVxXNSYSYrd9by\n2xW7ATh7cgFlBZkke/SBSyk18PqyEMtSETkiIjFXzxKRhSLSKCJrra87XPuWiMhWEdkhIrfHs+CD\nnSepa+GWj506js1VTc7ri376Bt9/dmMiiqWUUn3K8f8eWNLLMW8aY+ZZX3cBiIgHuB+4FJgFXCMi\ns06ksENNdmr4geqCmcXd9j1ecWCgi6OUUkAfUj3GmDdEpOw43vtMYIcxZheAiDwGXAlsOo73GpJW\n3H4BxhjyMlK67fMFQhhjIpZ0VEqpgRCvJPPZIvKBiDwvIrOtbeOA/a5jDljbRozcdK8T9B+87nTn\nCcC29XBzIoqllBrh4hH43wMmGmPmAj8H/nY8byIiN4tIhYhUVFdXx6FYg8vi2aP5+23znddej7B0\nxW4CwdBRzlJKqfg74cBvjGkyxrRY3y8DvCJSCFQC412HllrbenqfB40x5caY8qKiohMt1qDknszt\n8lPG8HjFAT7923cSWCKl1Eh0woFfREaLlagWkTOt96wF3gWmisgkEUkBrgaeOdHPG8qyXKmeb182\nk5KcVFbvrqPDH0xgqZRSI02vjbsi8iiwECgUkQPAnYAXwBjzAPBx4AsiEgDagauNMQYIiMhtwIuA\nB1hqjBnxfRj/fcl0Zo7JoSQnjR9ccTK3/HENm6uaOHVCfqKLppQaIfrSq+eaXvb/AvhFD/uWAcuO\nr2jD0xcXTnG+n1OaC8D6ykYN/EqpAaNDRxNoTG4ahVmpvL9Pl2hUSg0cDfwJJCKcM7mAN7fXOFM3\nP7p6H69uPZLgkimlhjMN/Al23rQialo62XwoPKXDPcu3sdSa00cppfqDBv4EO29aIV6P8KU/v8+R\npg6qWzrZX9eW6GIppYYxDfwJVpydxs+vOY1dNa089u5+jIED9e06sEsp1W808A8CF80sJs2bxIsb\nDwEQCBmqGjsSXCql1HClgX8QSPYkMXtsLhsPdk3dvL+ujR8v28w5P34lgSVTSg1HGvgHiVPG5Ua8\n3lvXxq/f2MXBxg6qGtsTVCql1HCkgX+QuOyUMc73uele3ttb77z+xAOr2HGkJRHFUkoNQxr4B4kz\nJ43ih1fO5tZFkzlvWhGvbq0m3esBwo29v12xK8ElVEoNFxr4B5Hrzi7jm5fMYNH0cN/+dn+QhdPD\nM5VWNmhjr1IqPnSx9UHo/GlFiIAxcFX5eHLSvLy3r773E5VSqg+0xj8IFWSlMrc0D4CxeemcVJRJ\nZUN7j9M3/2jZZlbuqBnIIiqlhjAN/IPUxbNKSBIozU/npKIsjIHdNa3djjPG8NCbu5wxAEop1RtN\n9QxSN507iflTCinISmXWmBwAVu6sZab1va3dHyRkoLkzkIhiKqWGoF5r/CKyVESOiMiGHvZfKyLr\nRGS9iKwUkbmufXus7WtFpCKeBR/uUpM9zBsfTvdMKc7itAl5/GHVHmcWT1uLFfBbOjTwK6X6pi+p\nnt8DS46yfzdwvjHmFOCHwINR+xcZY+YZY8qPr4gK4LqzJ7Knto13dtfhD4a49+VtHKhvo7UznPdv\n1sCvlOqjXgO/MeYNoO4o+1caY+wuJ28TXlRdxdmS2WPITPHw9PsHWHegkXtf3s6C/36VmpZOoKvm\nr5RSvYl34+5ngeddrw3wkoisEZGb4/xZI0p6iodLTh7NCxsOUdnQNYXDqp21gAZ+pVTfxS3wi8gi\nwoH/W67NC4wxpwGXAreKyHlHOf9mEakQkYrq6up4FWtYWTClkKaOAK9v7bo+7+4JP4w1d/gTVSyl\n1BATl8AvInOAh4ArjTG19nZjTKX17xHgaeDMnt7DGPOgMabcGFNeVFQUj2INO2dOGgXAMx9UkpHi\noSAzhdW77cCvNX6lVN+ccOAXkQnAU8B1xphtru2ZIpJtfw8sBmL2DFJ9U5qfwdjcNPxBw+jcNGaN\nzaEzEF6wpTMQwhfQxVuUUr3rS3fOR4FVwHQROSAinxWRW0TkFuuQO4AC4JdR3TZLgBUi8gGwGnjO\nGPNCP/wMI8q5U8NPQ4VZqUwryY7Yt/FgI8vWVyWiWEqpIaTXAVzGmGt62X8TcFOM7buAud3PUCfi\nolkl/KViP/vr2rjcNZUzwE9e2Mo7u2vZdNcS0qyZPZVSKppO2TDELJhSCMCSk0dTmp8ese+d3bWE\nDDp3v1LqqDTwDzHpKR4+uGMx371sJuNHZUTsswf1bjvczHPrqmjzdTX4VjW2c+PvVtPYrr1/lBrp\nNPAPQbkZXpI9SYzLS4+5f/mmw9z65/f4xwdd+f7Vu+t4dWs1GysbB6qYSqlBSgP/EJaZ2r2JpiAz\nhRXbw1M0V1ujegHqW30A1Fr/KqVGLp2dc5io+I+L6AyEuOvZjby48TAANS2dPLnmAK2+AHV24Hfd\nDJRSI5MG/iHu9W8uJEmEwqxUAEpy0px9da0+/vTOXurb/MyfUuBsa+kM8KEfvcK9V83jolklCSm3\nUipxNNUzxE0syIxo5HUH/toWH4caO6hqbHdq/DWtPtbua6ClM8CvXt854OVVSiWe1viHmdGuwF/d\n3Mnh5k6CIcOu6vDqXbUtnew40gzAxKheQUqpkUED/zDjrvFvPdzsfL/lUPj7ulYfGw82AZCRqoO8\nlBqJNPAPM6NzU4+6v7bFR4u1eIuu2qXUyKSBf5hx1/hjqWnpJGCN9NIZPZUambRxd5jJitG335aX\n4aWpI0CbL1zj33KomT++vRdjTI/nKKWGHw38w4yI8I8vLeDlr51HSU4qY3LTuHBGMbPG5HDtWRMi\njq1saOc//raBd3b3uLKmUmoY0lTPMHTyuFwAVt1+IYGQISU5fH9fs7ee+18Nd+EszU/nQH14CceH\nV+7hQycVJKawSqkBpzX+YSwpSZygDzCxoKv75tTiLOd7rfErNbJo4B9BCjJTnO+nuhZxqWv10dCm\nc/goNVL0KfCLyFIROSIiMZdOlLD7RGSHiKwTkdNc+z4jItutr8/Eq+Dq2ImI8/0o6yaQl+EFYHdN\nK4FgiIMN7Qkpm1Jq4PS1xv97YMlR9l8KTLW+bgZ+BSAio4A7gbMIL7R+p4jkH29h1Yk7fWI+eRle\nkpPCN4FTrPaAXdWtPPneARb972s6Z79Sw1yfAr8x5g3gaIngK4FHTNjbQJ6IjAEuAZYbY+qMMfXA\nco5+A1H97K+fP5s1/3Ex7VaXzhmjs/EkCbtrWtlb20ZnIMSemtYEl1Ip1Z/ileMfB+x3vT5gbetp\nu0qQpCTBkySkesP/9ePy0inNT2dvXZszkdveurZEFlEp1c8GTXdOEbmZcJqICRMm9HK0OlGfOaeM\nJBGu/dBE/rGuiprmTjrSwr8Oe7XGr9SwFq8afyUw3vW61NrW0/ZujDEPGmPKjTHlRUVFcSqW6klq\nsoebzj0JryeJwqxUals7I2r8++vaKLv9Odbub0hwSZVS8RavwP8McL3Vu+dDQKMxpgp4EVgsIvlW\no+5ia5saRAqyUqht8XUF/tpW3t5VC8Dv3tqdyKIppfpBn1I9IvIosBAoFJEDhHvqeAGMMQ8Ay4DL\ngB1AG3Cjta9ORH4IvGu91V3GGB0tNMgUZKVS1+ajMxACYG9tGxkp4V+N/Vbuf5RrDIBSamjrU+A3\nxlzTy34D3NrDvqXA0mMvmhooRVkpGAMtnQHSvR6ONHdypLkDgPf2NXDaD5ez/vuLyU7zJrikSql4\n0JG7ioKsrjn855SG+/VvqGyKOOa3KzTlo9RwoYFfRUzlcNrE8Pi69ZWRjbqvbjlCS2eAf6w76Gzb\neqiZax58m/pWne5BqaFEA7+KqPEvmFIIwI4jLQB8+cKp3HBOGVsPN/OtJ9dx25/fZ5u1pOMTa/az\nalctv3lz18AXWil13DTwK4qyw4E/Jy2ZcyYXkJ/hJWTCUzd/7eJpzB6bQ4c/xHPrqgA43BTO/+dl\nhJ8UXt58uNt7tnQGmPm9F3hhw6EB+imUUn2lgV+Rm+5l6Q3lvPHvixARJhRkApBjNebOHJMTcbw9\nj789o+e2wy00dfijjmmj3R/kW0+u6+/iK6WOkQZ+BcAFM0qcGnyZNW9/tjWSd0pxljOpG0ClFfjd\nk7lttBqDOwNBalo6abUWdNcJ35QafDTwq24mjgoH/pz0cI0/zevhX04rdfb/4tUd3LN8Gw1tfgqt\n9oENlY0APPTmbi655w2aXAHf7hqqlBocNPCrbiZGpXoA7rxiFl+9aCo51lPAz17ZTkO7n5OKMhmb\nm8Z6K/DvqWmlttXHwcauef0PNmjgV2ow0cCvupkYleoByEhJ5qsXTSPf1fVzx5EW8tK9zBqbw5ZD\n4VRPTUsnALuruyZ66/AHB6LYSqk+0sCvuumq8Xcf2P2b68u5cX4ZEF6yMS/Dy5TibHbXtOIPhqhp\nCTf47qnVwK/UYKWBX3VTmJXCNy+ZzhXzxnbbN60km28tmYHXE27szU33Mq0kC3/QsLe2jVq7xl/j\nDvyhgSm4UqpPBs18/GrwEBFuXTSlx/1pXg+zx+aydn8DeRkpTC0OL9y+5N43CIQMAHtquxZz6Qxo\njV+pwURr/Oq42HP6BEOGycXh1JAd9O3t6V4PoKkepQYbDfzquCyeNRoIj/rNSEnm8+ed1O2Y4pxw\nV88Of4jfrtjN6t06I7dSg4GmetRxWTC1kGVfPpfpo8Npnm9fNpPlmw6zy5XbL85OZW9tG4+s2sPO\n6lamFmex/GvnJ6jESilbn2r8IrJERLaKyA4RuT3G/ntEZK31tU1EGlz7gq59z8Sz8CqxZo3NweMa\n0Xv+9MglM+05gHZaXTvbXSmfS+55g+8/s3EASqmUitZrjV9EPMD9wMXAAeBdEXnGGLPJPsYY82+u\n478EnOp6i3ZjzLz4FVkNVt+5bCbnTi3kmbUH+dvag4zOScfrEfzBcO6/sqGdNl+AjJRkth5uZuvh\nZr5/xewEl1qpkacvqZ4zgR3GmF0AIvIYcCWwqYfjryG8NKMaYbyeJC6YUcK5U4u49kMTmVqcxV8r\n9uMPBkgSCBnYfriFk8flJrqoSo1ofUn1jAP2u14fsLZ1IyITgUnAP12b00SkQkTeFpGPHndJ1ZDh\n9SRxRtko8jJSSLV69pRPHAXA5qomnbhNqQSLd6+eq4EnjDHu/nsTjTHlwKeAe0VkcqwTReRm6wZR\nUV1dHediqURJ84Z/xWaNzaEoO5UVO2qo0xW7lEqovgT+SmC863WptS2Wq4FH3RuMMZXWv7uA14jM\n/7uPe9AYU26MKS8qKop1iBqCUpPDv2I5acksnFbEG9uqnfl8lFKJ0ZfA/y4wVUQmiUgK4eDerXeO\niMwA8oFVrm35IpJqfV8IzKfntgE1DBlrTFdWWjLnTSuiqSPAyp21rv2mhzOVUv2l18BvjAkAtwEv\nApuBx40xG0XkLhG5wnXo1cBjJvIveSZQISIfAK8Cd7t7A6nhzx8Kz9OTlep1Zv3cXNXk7G/vYVTv\nqp21zhz/Sqn46tMALmPMMmBZ1LY7ol5/P8Z5K4FTTqB8aogLWl05s9KSnUVbtluLtQM0d4S7d0a7\n5jdvA7Dn7ssHoJRKjSw6ZYPqV35r/p7s1GQKssJz+bsncGtq99PU4XcWcFdK9T8N/KpfBYJWqict\nmdRkT7c5/ps6Alzwv69x1o9eSUTxlBqRNPCrfmWP2s1KDQf8QmsaB9v7++qdxVvs/v3BkDb4KtWf\nNPCrfuW3a/x24Lfy/KdNyAPgqfe6egY/unofrZ0BWjoDzjbt9aNU/GngV/3KnqPfXr/Xrs2fPbkA\nCK/ba7v7+S18/fEPaO7oGtnbU68fpdTx08Cv+lWJldrJtGr8diPugilFiIAvGGJyUaZz/AsbD3HT\nwxXOa53eQQ1XVY3tCeuyrPPxq371l8+fTcXeOryecB3jghnFPLJqL3NKcynJTuNQUwfF2Wl8+9KZ\nZKcl89R7lfylomtqqMZ2P2Ny0xNVfKX6zfk/eQ1fMJSQLsta41f9avyoDP7fqaXO6+99eBbvfOdC\nMlOTKc0PB/TinFQumlXCWScV8OG5YyLOb2zTGr8annxW+xfAsvVVTPvu87S62rf6kwZ+NaC8niRK\nctIAGGcHfldPn+jafWO7n9W763h45Z4BK6NSA6nDH+TbT63HFwxxpHlg5rHSVI9KmNx0LwCjMt2B\nPy3imIZ2P8+tr+LFjYe4YEYxhVmppKd4BrScSvWnpna/05bldz0F9Cet8auESbHy/oauLpt2I7Dt\n359Yx8aDTXT4Q5z7k1f50qPvD2gZlepvDa4ODO2+genFpoFfJcw8qy//1OLsmPvHWrV/d5fPlzcf\n7v+CKTWANh7s6tnTMUDdlzXwq4T58JyxvPRv53HxrJKY+1/75iJca7kDkGmleb72+Fq++dcP+ruI\nKkH+8PZenn7/QKKLMSC2H+6q2AzUuBXN8auEmlbSvbb/r/MnsXJnDSnJSWSneSP68mdYqaCKPfU0\nd/gxxiAi3d5DDW3f+9sGgIgeYcPV4aauBl2t8asR646PzOKFr54HdC3d6BYKGQ41dlDf5ueZDw5y\n6l0vcURn91RDiHsqkiPNXb+7A1Xj71PgF5ElIrJVRHaIyO0x9t8gItUistb6usm17zMist36+kw8\nC6+GvzRvZA+e6uZO7n5hi9MH+iuPraW+zc/Lm48konhKHZfOQFfvHfeU5O2+QdKrR0Q8wP3ApcAs\n4BoRmRXj0L8YY+ZZXw9Z544C7gTOAs4E7hSR/LiVXg17EwvC0zmcOiGPFGv93gff2NXtuC2Hmrpt\nU+p4+IMh1h/o36kU3L13Bmuq50xghzFmlzHGBzwGXNnH978EWG6MqTPG1APLgSXHV1Q1Et3zybnc\n+ZFZPPWFc7j3qnkR+7JdXT9X7azV6ZxVXNz9/BY+8osV7Kpu6f3g49Tq6xqh627DGkypnnHAftfr\nA9a2aP8iIutE5AkRGX+M5yoVU0FWKjfOn4SIdOvjf8HMYuf77Uda+A+rQbAnzR1+Lvjf13hp4yEe\neH0n39BeQSoGu7bfn6Noe+qvP5hq/H3xLFBmjJlDuFb/8LG+gYjcLCIVIlJRXV0dp2Kp4aR8Yj6X\nzO7q+rlwehEA31g8jU+Wl/L0+wecP6hfvbaTe5Zvizj/ta3V7Kpp5Q9v72X17jpW7awduMKrISPZ\nE+4l1p9PkG09BP6BGsDVl+6clcB41+tSa5vDGOP+C3oI+Inr3IVR574W60OMMQ8CDwKUl5frM7vq\nJjM1mV9fV86+2jZ21rQwrSSblOQk5o7P49QJ+TxecYD7X93B71fucRZz+beLpznnL98UHvz11o4a\nJhZkRsz7rwaPRC++k2yNKO/JV9jAAAAeI0lEQVTP6RPsVE+aN4kOf/hz8jK8dAQGT+B/F5gqIpMI\nB/KrgU+5DxCRMcaYKuvlFcBm6/sXgR+5GnQXA98+4VKrEW1CQQYTCjIAWHvHxWSkJOMPhshJS+YX\nr+6IODYYMnisUWCrdtUyLi+dyoZ2dte0kiToOIBBKOCqaYdChqToUXz9LNn6PHvZ0P5g1+wLs1I5\nUN+O1yNkpiQPnl49xpgAcBvhIL4ZeNwYs1FE7hKRK6zDviwiG0XkA+DLwA3WuXXADwnfPN4F7rK2\nKRUXGSnhuovXk8T8KYUA5Gd4ufyU8PTOc77/Im/tqKG5w091cydnlHV1KgsZXeErUTr8QXyB2EHO\nvb2zh2PiKXqxH7ui0ObrPkXyhspG6lp9x/U5Hf6g8yTa6gr8AKnJHtJTPIMrx2+MWWaMmWaMmWyM\n+S9r2x3GmGes779tjJltjJlrjFlkjNniOnepMWaK9fW7/vkxlIJzrOUcp4/O5hPl4RGfrb4gyzcd\nZk9NGwCnTojsTdzSEXv+83tf3sZH73+LS3/2Jg+9uYu/Vuzn9ifX9akc2w43D9gsi0PV9UtX86Nl\nm2Puiwz8xxcIgyFDoA//B2v21jP3By85aUAAr8cO/OHPbvcFeWnjIY40d/Dhn6/g3584vk4BV/7i\nLU6+80XrPcO/d3bgT/Mmke71DKpePUoNCXaN//I5Y51FXgDWVzayu7YVCI8HcGvuYeGLLVXNrK9s\nZHNVE//53Ga++cQ6Hnt3f8xj3V7ceIjF97zBXysG9zwzD6/cQ2VDe6/HPbeuim8/tT7un7+/ro39\ndW0x97kXKLHz39E++/t3ue637wBQ09LJ27siG+o/+etVzLaC7NF8sL8BCLf72JKTwmHRXhTlgdd3\ncvMf1nDzI2us7cHjavjderjZ+d5+yijOcdX4vR6dnVOpY3VSURYV/3ERnz5rAmPzugL/xoONzgyf\n0TOB9lTjr2/zxfzj7u0P/q5nNwHQ1EPDcbsvSNntz/HIqj1HfZ/+VNvSyZ3PbOSGpat7PfaNbdX8\n44ODcS9Da2cgZs+W+lZfRI2/p9THK1uO8Ob2cLB+eOUerv/takKu/5s1e+vpDIR6bSi20zru/9fk\nqBp/yHqPtdZN4mBjO7PueIHqE+juubumlfwML6OtRYmSPUKqN2nAGnc18KthpTArFRFxcv8QrjW+\nvOkwY3PTui3icqipIyJg2Bp6WPKxJeoJYUNlI/VWztcY49Sie7pB1LeFj70/qhF6INllq2npPXC1\n+YNxTz8YY2j1dX/fNXvrOfWHy3nGdaNxB8LWzgDXL13NzqiBVXWtPnzBUMynt4Y2f7ccvpvdbhxy\n3SDs62MH/qyo8SN7a9voDIQiplqwHW7q6PY7Es0Yw87qVk4qyiLD+n0MhozW+JWKh7duv4C/3Pwh\nADZVNVFWGJ7+wd2J5/N/WMO9L3f1969saMcfDDkBGuDWRZOd76P/qK996B1++Vo4iLsDT0811YDV\nU6Q/e4z0xm4w7Uu6ot0XIBAycW2z6AyECIZMtyD3/r56IPyUYXOnejZXNfHGtmrW7KmPOK/Zempr\nihHgH161h7k/eIl398TuU2JfA3fgt5847MZd95NJmdWbDGJ3DLjmN2/z05e2ddvu1hkIsau6lclF\nmU4FJRQyg69xV6mhaFxeOrPG5jiv7cDvieq+ufStPUC4IXH+3f/k3/6yNmJVpCWzx/DLa08Dwqmh\nFzZUUXb7c+yva6Ox3c/+unAtv7al62bR5gvy97WVPLxyDxsqGym7/Tn21rY6wcI/AL1VemI3mMaK\n+9XNnWxz5aKdBs44BiT7Pdv8kTdRX4ybS6frc+2nqeg0mn0zjlWz/+Pbe4GuXH40u3dNyPXR9o3x\nkVV7ufflbU4wfuqL51Cc07U0aJsvSGO7nznff5FXrAWCqho6OBij7cSdcjrc1EFNSycnFWWRmWrV\n+I3Rxl2l4iU7zUthVgoAk6wJ36L7hduBsL41HDj+sa4qIs+cm+51HvcvufcNbvnjewBOg+Jha1rd\nutau1Em7P8ijq/fx8Mo9PLEm3NC7fNNh5w+7Mxji/17aymf6kGePN7sWHavGf98r27nxd+86r+0g\n3WH9+9eK/Vx/gmW2G02j+6z7A93L0+H6fzhQbwV+V4A3xjgD8ZpirFtbY92MvdagrEAwxKOr9xEI\nhli7v8GZnsGdUnL/39/78nbafEHyM7ycNiGfdNdsse2+IBsPNtLUEeD7z27EHwzR7g/GvAG5u6Vu\nOhieUHBSYabzfsEQLJxezKfPmtjt3P6gC7GoYW9iQSY1LT4m9VDj9wcNxhhqW2PnvHPTvWSldf9T\nsWvGhxvDgd9d42/3Balt8VHb6iPJ+ryQMU7t0R8MsfFgE+sr+3cWyFi6avwx2jba/U67R1KSOOkY\n+4a1Zm89bx/HVBez7niBS2aP5p6r5rm6SUbW+O3g6w/Gbty1a9KHXLl1XzDkpHre2V3HtNGxl/G0\nb/Z/XXOAbz+1nuYOPz9a5vQ6j7iZRI8vaPMFnQDtXh+iwx90fmdqW3xOORpiBH73zeCg9fsyKjPF\n+ayQMSw5eTRLTh4ds/zxpjV+NexNtPKyTqonxkjQjQebeG9vfbftAFlpyREzgdo2VIZrbkeaOwmF\njDOwJys1ORz4W300tvudAOseMGZMOBg0tPmchWUeenNXj71Qthxq4qaHK+KSA7Zr/LECf7sv3FXR\nTqfY6Rj7nMZ2P75g6Ji6MxpjaPMFefr98EwvdmqmzR+M+HkbrHYVd8N6R4xUT1VjV+Bv6ww6Afdn\nr2zn/17aGrPG/eAbO7nr2U3O00ZlfWQ6pqkjQDBkCIUMnVEpp3Z/wOkU4K7xN3X4nd+BNl+Q3TXh\nLsPRbQ3L1lext7ar66rdqJ6Zkuy870DPLKs1fjXsnTw2l1c2H2H8qHAXz6vPGM9DK3ZHHPPhn6+I\neW52WjKeJIlZ499gLZIdCBlqW8O1ewi3LbR0BpwGYjsQ2U8BtvBNIdw4eePv32VzVROXzxlDVmoy\ntz+5ntsvncH4UeGb1qcfWk1NSzj/Pqc0cizCsep0ataGTQebItpB7EBb2+ojLyOlW43f/lk6/MFu\ns6X2JDoQ242mxoRTIPZiO/aNs9Y1MrYzEGLpit0888FBJ2i7c+itvkDEnEvbDrc4n+eeB2d/XTvP\nb6jitgumWGWIvIE2d/i5+ZEKSnLTutX4mzsCTiOse2GgO/6+MeK4DdbTm/vnrW/18cU/vRdxnN0N\nNCs12amExOpZ1p+0xq+GvevPnshr31hIanL4j/Y7l81kww8uAWBqcRaXzxnT47m56V4gskvf/CkF\njM1Nc2qaEG6wq2v1kZHiIT/Ty8GGduzKrB3IfvbK9oipoO0AUdfmc9JGnf4Q7+1r4Ln1VXzukQog\nXBu0a4nuG8fxcveUuey+NyOmILADvL3NPXrVXea+PHkYY3hvX31EDR3CA6Bsr2+rdmr99me6A2en\nP8j7+xtYu7+B/fXhWrP7/Vo7gxE9rXbXtDrnzxjddUOD8JOE02MnqvxN7QF2VLew43ALvqi+9Acb\n2p2aefSKcNDVS2yfNSCtpTPgjBqONZ7DDvwZqZ6u7pwDPDGdBn417CV7ksjPTHFeJyUJWanJbPjB\nJTz7pQX83yfmcsM5Zc7+K+aOBSAlOckJ/JmucQH3fHJet1q3HfhHZaaQ7vU4DZEAVT2MkG20Uhru\nwWItnV012C2Hmmn3Bdl+pKuXzeGmcK+Rstuf4/n1Vd3ftA+ip0Fw15jtAF/b4sMY49wIOqJq/H3p\nffKLf+7gY79cybNRA8Dcc+B8/g9r+LW1olqsOXA6/F395e0blru2XtPSGdE7qa7Vxz4rrTJzTGS+\nv90fdG6g0QP3mjr8NHcEaOrwd+tdVNnQ7srxdw/8v7muHCAindNkvX9zjAGC7hp/RoJSPRr41YiV\nlZpMmtdDmtfDZad01fp/+sm5LPvyuZTmp5OTFg787p5AozJTnNlBbYebOqlt9VGQmUJGSnJE8NjX\ny9QEDa4xA62dgYhafVVjO3Wu14eaOth6KHwjePQoU0g0d/h7bC/ojJoGoam9+/iDulYfHf6Q89TS\nPdXTe3fU37wZDugVVr/7bCtd1hqVZnnc+jnqYwya6/AHOeJqzE3xRIasWIOo7BG200u6N/TaN+To\n6SrafEHq28JtMp3+EPPG53HNmeOtMoScAJ0eFfivKh/PWSeNAmBfXauz3b5OMQN/SydJAqnJSV39\n+LXGr9TAs/P/EH5CmDU2hyvnjuOyU7r3skj2JLFwWpHzWiQckI80dVCUndatVtjbDJPuhsY2X5Ba\n14ja+jZ/REA83NTp5IV7GgtwpLmDU77/Ure1iYMhQ2Obv1uaJtbSf/Vtvoiaud3oawey3lI9lQ3t\nTq23Ym948JR9E22LGgS3q6aVndUtEYPmbE0d/oiVsCYXZ0XsPxQj8K+xGumnR6V6AGd+oD01rd32\nGRNumPUFQ8wpzeUzrqfArlRPZMjMSU8mKzWZ5CSJuMHbN3P309RUq+zVzZ1kpiYjIqR5k1g8q4Sl\nN5zRrTz9SQO/UkBJdlq3bV+5aCrXnV0W8/gzJ41yvi/MSuVIUwfVzZ0U56Q6tcO+em9f1+Cils4A\nNa6UR0Obj4Z2n/U5KRxp6nACcqwBT9B1I1lmpYI+/dA73P/qDn72ynbm3vVStyUFYwX+2hZfREql\n3R+MCGIf/vkKHnh9J396Z2/Mhsl1rgFT9m67m2ZrjCkNvvTn98lMSeYs13WdWJBBxd76iHLMiOqu\neaQp8mexA/CY3DRKrAnQ3Owaf6CH1EqrL0ibL0iKJ4lRGV3pQafGH/V/m53mRUTIz0yJeAqyr6nd\n/nDfNafy99vmO/vt1KGI8OD15Zw7tYiBpIFfKbpSOdGphG7HWRmfZE8S//uJudx71TxG56RxoL6d\n2lYfRVmpTnDweoQCV9tCT+xujhDOf9e1+EhNDpejvs3vdG+cWpzN4eYOpya9Zm898+/+Z7eUjp02\nsBeYWbGjhv95cSv/3HLYee0WEfh9dqqnMyKP3xFjYNLdz2/hu09v4OXNh4m2rrIRr0eY7eoxZL9f\ndKoHwlNqXDizmJljuo4/dXwe77tuimneJCaMikyxHbIaev/7X07huS8vcPaXFWQyqTCTu66cTaYr\nWPdlHV1fIERKchJ5rsDv5PiTowN/OIDnZ3gjtkeneuZPLiDd63F+f+wRu4nSp8AvIktEZKuI7BCR\n22Ps/5qIbLIWW39FRCa69gVFZK319Uw8C69UPL3+zYWsuH1Rj/s/uGMxa+9c7Lz++OmlfPTUcZTk\npLLR6tpZnJPqBInS/AzGuaaHPppzp4anlN54sImNVY1MsdICDVbeOc2bxMSCDKoaOiLyxpUN7d3a\nEFqsXjNJErmG6/j8cFBcdyBy0Jjd8yQUMk5aqrY1qsbviz0iFSJ72fiDIf6+tpJfvbaTaSXZTtor\nxZNES2eAf6w72OP75LlGSAPMGx/ZgD42L50cq7HdvsYvbDxEcpJw3rQiZo/NdQbplRVmIiJcf3ZZ\nRACPVpSdyn9+9ORu21OSk0hJTnJGfafb3Tlj1PgB5zNS7Bt2q4+XNx3moRW7nOPckwf2tStsf+k1\n8IuIB7gfuBSYBVwjIrOiDnsfKLcWW3+CrjV3AdqNMfOsrytQapCaWJBJcYyUjy03w+vkqd1KctKc\nPHxxdpqTFijJSWVc3tED/0mFmYzNTePOj4T/pB5ZtZf9de2UFWSSJOEuiA1tPvLSUzh5XC61rT5n\nMjPb2qh5aOyUjCdJImbg7Cng1rf5uPIXKyJmxaxrjcrx9zAVgX2+7f9e2sZXHlsLwFmTCpxFTS6f\nMwZj4LY/v88Taw50ayQFyM1IiRgvcYqr59Qp43KZV5rn/Gz2QjsAv/jUaYzJDV9nO/BPKux6MohO\nibk/+76rT+XiWSXdymIH8DJrmo+eGneja/wTRmWQ7vWwv76dHz+/mf117aRaNxH3+7h7iSVCXz79\nTGCHMWYXgIg8BlwJbLIPMMa86jr+beDT8SykUoNZiWviruLsVGeAUUlOGsXZ3fPMADlpyTR1BPjr\nLWdT4Cy/l+SqcXeSl5FCfZuP+jY/eRleFs0oBsJzCbl95bG1JCclOeMR7CcCEaHaFfh7moZ5b00b\nHxxoZOXOrhRQXasv4mnhaIHfzpsbY3j2g4OcUZbPz685jcKsFKoaO3hzew1tvkBESuuW8ydzz8uR\ns1jmpnsj0jKnTcjjfz4+hyQRPnbaOCDcELzuQCNfu3gaj6wKT8B2yeyuwH1SUfhJyQ7YAKX56RFz\n55fmp7PdWp8hPcVDfownAnvMR1lhJhV76525fqIbd+3Ab++fPTYHjwivb6tmZ3W4AdkO+mDV9K3G\n3UTqS6pnHODuN3bA2taTzwLPu16niUiFiLwtIh89jjIqNaiNdgf+nFSnQa84u+ca/9O3zufLF0xh\nlKsNINnVZfTUCfnkZXj50zv7WL7pMHkZXsblpcfsogjw+5XhkciPrt7HT5eHA2qSEBHwanoY/LXH\nWp3MTtnkZ3i7pXpi5fht+2rbeOaDg9z3yg4qG9r5ZPl4RuemkexJYvyoDD511oSInk4fP73UGTeR\n46rh56V7ufTkrm61IsInysfzL6eXIiKICJOLslh6wxnkZaTw8dNL+cnH5zhtGQALphRy+sR8Tp/Y\ntcTmg9eVc/+nTnOWORzvaifISPFEBGZbV40/fOyhxvDNLbrGbz8B2jfsmxacxMSCDGfhH4js0ml/\n9pDI8feViHwaKAf+x7V5ojGmHPgUcK+ITO7h3JutG0RFdXV1rEOUGpQWWPl5CPfwsWvWJTlpjLEC\n/0lFmXzt4mnOcZOLsvja4ukRQctu9PzmJdP52sXTIuYHsgeSzYgalHTXlbO5Yu5YtlQ1EwoZHlu9\nzwn2IRNZy69r9VEU9QQi0jXwyG4oHZefji8QYsuh8Dw0qcnhqQ96Cvyr99Tx5Uff556XtzF7bA6X\nntJ9JLQ7YN79sVPIzfDywytn8+yXFkT8jLkZXh749GnctmhKzM9y+99PzOWT5eMjtk0oyODJL5zj\nPEVBOI9/+ZwxTC4KPwW4bzaxUk4AqVYN3u5tY48Cju6qawf+Oz8yi59dPY9TSnOduaFidRSwB5X1\n1omgv/Xl0ysB99UttbZFEJGLgO8CVxhjnN82Y0yl9e8u4DXg1FgfYox50BhTbowpLyoa2K5NSp2I\nsXnp/O3W+dz5kVl4PUncfN5JXDSzhE+Uj3dq9Dlp3j4FMwinC7yeJHZVd/U1t2vjE10pDIALZhSz\ncHoRzZ0Bth9piRgx3OYLUNMcWct3d4fcc/flnDu1yOlt4wR+62Z1/6s7mT+lgLKCTKdxNznGBHdJ\nEj7nyS+cw1NfPKfbilVARBfXZCvoXXd2GRMLMp3jc608+ZKTx/CNS6Yf9Rodj29fNhMgoudQdPdM\nm13jnzs+j3e+c6GTaooO/HaqpzQ/gyvnhY+ZYP0fXTFvbLf3nWV9dqzxBwOpL4H/XWCqiEwSkRTg\naiCid46InAr8mnDQP+Lani8iqdb3hcB8XG0DSg0X88bnceP8SQCMyU3noc+Uk5vu5eSxuUwuyuRb\nS2Z0WwegJ3abgXs0px04J1qpgokFGey5+3JK8zM4oyzc9/1vaysjJjhr7QxS3RIZYGaNiRzUlJfe\n1VhtryA2Lq8rFXLboqmkp3ho9QVoavd36yHz4TljeOc7F/HMbfM5fWK+kxuPFt0bxs2ugbvL0h/m\njc/jrdsv4F8XTHK29VTjd/9fleSkOU9mZQUZfO7crvNjTd43tzQXr0e4cX5Zt33TrFRdT6O5B0qv\nLQzGmICI3Aa8CHiApcaYjSJyF1BhjHmGcGonC/irdYH2WT14ZgK/FpEQ4ZvM3cYYDfxqxEhP8fDK\n1xdGbItOt0SzG4Qfu/lsthxqojA71QnYZVZvFfc0DuNHZXDBjGJ+9drOiPdp6Qx0WxB89rjciNdj\nY7RBuLugzh2fy+icNLYfaSY7LZnc9OSI9FFehrfXnwd6DrAAOeleDjZ2OOms/hTd5mKX69KTR/P8\nhkM8c9t8vvXkemaNid2WkuxJ4ruXz6IwK5X7XtnuNOq6zSnNY8MPLiE12UNRdioXWo3yAFOKszhz\n0ii+euHUOP5Ux65PTcvGmGXAsqhtd7i+v6iH81YCp5xIAZUaTt66/QKyeunKZ/cyOaU0l1NKIwP1\nhFHhNEL0wuJfWDiZf245ErGttTPA3to2irNTnYFLC6dHplHLouYcgsjgmJGSzISCDP659QglOWnd\ngnNeeu8D1KCXwG/lyXMGIPBHs2v2v7z2NKdW//xXzu31vM+fP5nPnx+zuRLo6hX07ncjQ6PXk8Tj\nnz/7eIsbNzpyV6kBNC4v3cllR5tjBfmjpYTsAUX2BGK2clcvFlubL8iumtaIgVDR4xDsxWncSqMG\nnY0flYEvEGLb4Zbugb+HnyWanUuP1aiZk55Mmjcp5syXA0Wkb2m44UIXYlFqkHj882f3OvmZiLDl\nh0u6BVAR4dZFk3lk5d6IpwFfIMTZkwuoauzgqxd1pRfG5obbESbFCPz5mSlce9YEZ3pqu12hpqWz\nW+AvzOo9zQNdNf5Ub/fAX5iV2qd0kYofDfxKDRL2FNF9OS6WbyyezjcWT+e6366murmTrdbiLtNL\nsiO6Ta7+zoVOY2usAWaFWSn81//rytC658dxB/7f3XgG50wu6LW8AMnWCN5YZf+3i6dxfQ+T4fWX\n579ybsJ71iSSpnqUGibsQU5/vOksvrioK/88qSiyVl+ck+akfEQkIv9+8ricbj1zxualO6tMuQP/\nounFPfbiiTYqM4VRmSn84IrZ3faV5KRFLP84EGaOyWHR9OLeDxymtMav1DBk99aZVpIVc8ppt40/\nuITmzgD//sQHfO/D0dNwhfu0zxqTw8aDTeSke3nnOxcec3lSkz28972Lj/k81T808Cs1DJVPzOe1\nbyxkwqiMXscPJCUJuelefm0tIRjLgimFbDzYRGO7P2JuIjU0aeBXahgSkZg9do7X5847ibX7G/jo\nqUebpksNFRr4lVK9KsxK5S+DoP+5ig9t3FVKqRFGA79SSo0wGviVUmqE0cCvlFIjjAZ+pZQaYTTw\nK6XUCKOBXymlRhgN/EopNcKIcS3vNliISDWw13qZCzS6dhcCNf1chOjP7K9zezv2aPtj7TvebXpN\nj2179LZEXM9Yn9sf5/blOL2mx3ZuvP/u7e15xpi+LVhujBnUX8CDUa8rBvoz++vc3o492v5Y+453\nm17TY9se4/oN+PUcqGval+P0msb3mh7r3/3xlHsopHqeHWKfeSzn9nbs0fbH2nci2/rbYL+mx7I9\nelsirueJfm5fz+3LcXpNj+3ceP/dH8tnA4M01XM0IlJhjOl5GkF1zPSaxpdez/jTaxpfQ6HGH+3B\nRBdgGNJrGl96PeNPr2kcDbkav1JKqRMzFGv8SimlToAGfqWUGmE08Cul1AgzbAK/iCwUkTdF5AER\nWZjo8gwXIpIpIhUi8uFEl2U4EJGZ1u/oEyLyhUSXZzgQkY+KyG9E5C8isjjR5RkKBkXgF5GlInJE\nRDZEbV8iIltFZIeI3N7L2xigBUgDDvRXWYeKOF1TgG8Bj/dPKYeWeFxTY8xmY8wtwCeB+f1Z3qEg\nTtf0b8aYzwG3AFf1Z3mHi0HRq0dEziMctB8xxpxsbfMA24CLCQfyd4FrAA/w46i3+FegxhgTEpES\n4KfGmGsHqvyDUZyu6VyggPDNtMYY84+BKf3gFI9raow5IiJXAF8A/mCM+fNAlX8witc1tc77P+BP\nxpj3Bqj4Q9agWGzdGPOGiJRFbT4T2GGM2QUgIo8BVxpjfgwcLe1QD6T2RzmHknhcUytllgnMAtpF\nZJkxJtSf5R7M4vV7aox5BnhGRJ4DRnTgj9PvqQB3A89r0O+bQRH4ezAO2O96fQA4q6eDReRjwCVA\nHvCL/i3akHVM19QY810AEbkB64mqX0s3NB3r7+lC4GOEKyfL+rVkQ9cxXVPgS8BFQK6ITDHGPNCf\nhRsOBnPgPybGmKeApxJdjuHIGPP7RJdhuDDGvAa8luBiDCvGmPuA+xJdjqFkUDTu9qASGO96XWpt\nU8dPr2n86TWNP72m/WwwB/53gakiMklEUoCrgWcSXKahTq9p/Ok1jT+9pv1sUAR+EXkUWAVMF5ED\nIvJZY0wAuA14EdgMPG6M2ZjIcg4lek3jT69p/Ok1TYxB0Z1TKaXUwBkUNX6llFIDRwO/UkqNMBr4\nlVJqhNHAr5RSI4wGfqWUGmE08Cul1AijgV8ppUYYDfxKKTXCaOBXSqkR5v8DEkvE1otb/EQAAAAA\nSUVORK5CYII=\n", 261 | "text/plain": [ 262 | "
" 263 | ] 264 | }, 265 | "metadata": { 266 | "tags": [] 267 | } 268 | } 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "metadata": { 274 | "id": "jsYXyR_BTtvE", 275 | "colab_type": "code", 276 | "colab": { 277 | "base_uri": "https://localhost:8080/", 278 | "height": 260 279 | }, 280 | "outputId": "0e23969b-8a5d-45a7-8fd9-8013286ea155" 281 | }, 282 | "source": [ 283 | "trainer.train(1e-4, 3, 2, criterion, opt='adamW')" 284 | ], 285 | "execution_count": 42, 286 | "outputs": [ 287 | { 288 | "output_type": "display_data", 289 | "data": { 290 | "application/vnd.jupyter.widget-view+json": { 291 | "model_id": "91f627ceece24456a32355e62f3208dd", 292 | "version_minor": 0, 293 | "version_major": 2 294 | }, 295 | "text/plain": [ 296 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 297 | ] 298 | }, 299 | "metadata": { 300 | "tags": [] 301 | } 302 | }, 303 | { 304 | "output_type": "stream", 305 | "text": [ 306 | "Train Loss 1.502261: 100%|██████████| 600/600 [00:10<00:00, 58.65it/s]\n", 307 | "Valid Loss 1.200479: 100%|██████████| 100/100 [00:01<00:00, 85.08it/s]\n", 308 | "Train Loss 0.813754: 100%|██████████| 600/600 [00:10<00:00, 50.12it/s]\n", 309 | "Valid Loss 0.559981: 100%|██████████| 100/100 [00:01<00:00, 84.20it/s]\n", 310 | "Train Loss 0.545776: 100%|██████████| 600/600 [00:10<00:00, 58.79it/s]\n", 311 | "Valid Loss 0.498388: 100%|██████████| 100/100 [00:01<00:00, 85.93it/s]\n", 312 | "Train Loss 0.427372: 100%|██████████| 600/600 [00:10<00:00, 59.48it/s]\n", 313 | "Valid Loss 0.330076: 100%|██████████| 100/100 [00:01<00:00, 83.81it/s]\n", 314 | "Train Loss 0.320786: 100%|██████████| 600/600 [00:09<00:00, 60.25it/s]\n", 315 | "Valid Loss 0.272724: 100%|██████████| 100/100 [00:01<00:00, 86.45it/s]\n", 316 | "Train Loss 0.281855: 100%|██████████| 600/600 [00:10<00:00, 59.19it/s]\n", 317 | "Valid Loss 0.254479: 100%|██████████| 100/100 [00:01<00:00, 85.24it/s]\n", 318 | "Train Loss 0.269442: 100%|██████████| 600/600 [00:10<00:00, 59.53it/s]\n", 319 | "Valid Loss 0.250065: 100%|██████████| 100/100 [00:01<00:00, 85.42it/s]\n" 320 | ], 321 | "name": "stderr" 322 | } 323 | ] 324 | }, 325 | { 326 | "cell_type": "code", 327 | "metadata": { 328 | "id": "XxM1SZJjTtvI", 329 | "colab_type": "code", 330 | "colab": { 331 | "base_uri": "https://localhost:8080/", 332 | "height": 191 333 | }, 334 | "outputId": "7147be3c-2ba2-48b4-b2dd-d616a5f13a98" 335 | }, 336 | "source": [ 337 | "for n, w in trainer.model.named_parameters():\n", 338 | " print(n, (w != 0).sum())" 339 | ], 340 | "execution_count": 43, 341 | "outputs": [ 342 | { 343 | "output_type": "stream", 344 | "text": [ 345 | "layers.0.lin.weight tensor(3840, device='cuda:0')\n", 346 | "layers.0.lin.bias tensor(300, device='cuda:0')\n", 347 | "layers.0.bn.weight tensor(300, device='cuda:0')\n", 348 | "layers.0.bn.bias tensor(216, device='cuda:0')\n", 349 | "layers.1.lin.weight tensor(1899, device='cuda:0')\n", 350 | "layers.1.lin.bias tensor(300, device='cuda:0')\n", 351 | "layers.1.bn.weight tensor(300, device='cuda:0')\n", 352 | "layers.1.bn.bias tensor(238, device='cuda:0')\n", 353 | "layers.2.weight tensor(824, device='cuda:0')\n", 354 | "layers.2.bias tensor(10, device='cuda:0')\n" 355 | ], 356 | "name": "stdout" 357 | } 358 | ] 359 | }, 360 | { 361 | "cell_type": "code", 362 | "metadata": { 363 | "id": "mr9Tc3ZrTtvQ", 364 | "colab_type": "code", 365 | "colab": {} 366 | }, 367 | "source": [ 368 | "def epoch(loader, model, opt=None):\n", 369 | " \"\"\"Standard training/evaluation epoch over the dataset\"\"\"\n", 370 | " total_loss, total_err = 0.,0.\n", 371 | " for X,y in loader:\n", 372 | " X,y = X.cuda(), y.cuda()\n", 373 | " yp = model(X)\n", 374 | " loss = nn.CrossEntropyLoss()(yp,y)\n", 375 | " if opt:\n", 376 | " opt.zero_grad()\n", 377 | " loss.backward()\n", 378 | " opt.step()\n", 379 | " \n", 380 | " total_err += (yp.max(dim=1)[1] != y).sum().item()\n", 381 | " total_loss += loss.item() * X.shape[0]\n", 382 | " return total_err / len(loader.dataset), total_loss / len(loader.dataset)" 383 | ], 384 | "execution_count": 0, 385 | "outputs": [] 386 | }, 387 | { 388 | "cell_type": "code", 389 | "metadata": { 390 | "id": "PfEGetJMTtvU", 391 | "colab_type": "code", 392 | "colab": {} 393 | }, 394 | "source": [ 395 | "err, loss = epoch(val_dl, trainer.model)" 396 | ], 397 | "execution_count": 0, 398 | "outputs": [] 399 | }, 400 | { 401 | "cell_type": "code", 402 | "metadata": { 403 | "id": "7vNg1pmQTtvX", 404 | "colab_type": "code", 405 | "colab": { 406 | "base_uri": "https://localhost:8080/", 407 | "height": 35 408 | }, 409 | "outputId": "c171798d-83a7-4e44-b54a-90951e5219d5" 410 | }, 411 | "source": [ 412 | "1 - err" 413 | ], 414 | "execution_count": 46, 415 | "outputs": [ 416 | { 417 | "output_type": "execute_result", 418 | "data": { 419 | "text/plain": [ 420 | "0.9329" 421 | ] 422 | }, 423 | "metadata": { 424 | "tags": [] 425 | }, 426 | "execution_count": 46 427 | } 428 | ] 429 | }, 430 | { 431 | "cell_type": "code", 432 | "metadata": { 433 | "id": "cLNBTTLVTtvd", 434 | "colab_type": "code", 435 | "colab": { 436 | "base_uri": "https://localhost:8080/", 437 | "height": 1000 438 | }, 439 | "outputId": "22b8c895-2082-430f-ba3f-43c20f422663" 440 | }, 441 | "source": [ 442 | "COMPRESSION_FACTORS = [0.1, 0.25, 0.50, 0.90, 0.98]\n", 443 | "BATCH_SAMPLING = [1, 10]\n", 444 | "ACCURACIES = []\n", 445 | "PARAMETERS = []\n", 446 | "for cf in COMPRESSION_FACTORS:\n", 447 | " for bs in BATCH_SAMPLING:\n", 448 | " net = Net(784, 300, 10)\n", 449 | " prunner = Prunner(net, criterion, train_dl)\n", 450 | " model, masks = prunner.prun(compression_factor=cf, num_batch_sampling=bs)\n", 451 | " trainer = Train(model, [train_dl, val_dl], cuda=True)\n", 452 | " trainer.train(1e-4, 3, 2, criterion, opt='adamW')\n", 453 | " err, _ = epoch(val_dl, trainer.model)\n", 454 | " ACCURACIES.append(1 - err)\n", 455 | " PARAMETERS.append(sum([(w != 0).sum() for _, w in trainer.model.named_parameters()]))" 456 | ], 457 | "execution_count": 47, 458 | "outputs": [ 459 | { 460 | "output_type": "display_data", 461 | "data": { 462 | "application/vnd.jupyter.widget-view+json": { 463 | "model_id": "99ddf2ed22cf4e9e9e1f69dd10384bf7", 464 | "version_minor": 0, 465 | "version_major": 2 466 | }, 467 | "text/plain": [ 468 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 469 | ] 470 | }, 471 | "metadata": { 472 | "tags": [] 473 | } 474 | }, 475 | { 476 | "output_type": "stream", 477 | "text": [ 478 | "Train Loss 0.579240: 100%|██████████| 600/600 [00:10<00:00, 47.50it/s]\n", 479 | "Valid Loss 0.324463: 100%|██████████| 100/100 [00:01<00:00, 84.16it/s]\n", 480 | "Train Loss 0.225078: 100%|██████████| 600/600 [00:10<00:00, 57.50it/s]\n", 481 | "Valid Loss 0.152903: 100%|██████████| 100/100 [00:01<00:00, 84.49it/s]\n", 482 | "Train Loss 0.141308: 100%|██████████| 600/600 [00:10<00:00, 59.99it/s]\n", 483 | "Valid Loss 0.136242: 100%|██████████| 100/100 [00:01<00:00, 84.35it/s]\n", 484 | "Train Loss 0.116216: 100%|██████████| 600/600 [00:10<00:00, 58.96it/s]\n", 485 | "Valid Loss 0.097312: 100%|██████████| 100/100 [00:01<00:00, 82.90it/s]\n", 486 | "Train Loss 0.076728: 100%|██████████| 600/600 [00:10<00:00, 58.72it/s]\n", 487 | "Valid Loss 0.083073: 100%|██████████| 100/100 [00:01<00:00, 87.87it/s]\n", 488 | "Train Loss 0.056323: 100%|██████████| 600/600 [00:10<00:00, 49.96it/s]\n", 489 | "Valid Loss 0.077468: 100%|██████████| 100/100 [00:01<00:00, 85.16it/s]\n", 490 | "Train Loss 0.047678: 100%|██████████| 600/600 [00:10<00:00, 59.13it/s]\n", 491 | "Valid Loss 0.076484: 100%|██████████| 100/100 [00:01<00:00, 84.88it/s]\n" 492 | ], 493 | "name": "stderr" 494 | }, 495 | { 496 | "output_type": "display_data", 497 | "data": { 498 | "application/vnd.jupyter.widget-view+json": { 499 | "model_id": "b6552500c233442d8347f3fd7080b8bb", 500 | "version_minor": 0, 501 | "version_major": 2 502 | }, 503 | "text/plain": [ 504 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 505 | ] 506 | }, 507 | "metadata": { 508 | "tags": [] 509 | } 510 | }, 511 | { 512 | "output_type": "stream", 513 | "text": [ 514 | "Train Loss 0.551629: 100%|██████████| 600/600 [00:09<00:00, 50.76it/s]\n", 515 | "Valid Loss 0.311137: 100%|██████████| 100/100 [00:01<00:00, 87.80it/s]\n", 516 | "Train Loss 0.215195: 100%|██████████| 600/600 [00:10<00:00, 59.35it/s]\n", 517 | "Valid Loss 0.144761: 100%|██████████| 100/100 [00:01<00:00, 82.99it/s]\n", 518 | "Train Loss 0.135641: 100%|██████████| 600/600 [00:10<00:00, 50.01it/s]\n", 519 | "Valid Loss 0.130807: 100%|██████████| 100/100 [00:01<00:00, 84.64it/s]\n", 520 | "Train Loss 0.111491: 100%|██████████| 600/600 [00:10<00:00, 59.62it/s]\n", 521 | "Valid Loss 0.093668: 100%|██████████| 100/100 [00:01<00:00, 83.81it/s]\n", 522 | "Train Loss 0.072818: 100%|██████████| 600/600 [00:10<00:00, 50.36it/s]\n", 523 | "Valid Loss 0.079237: 100%|██████████| 100/100 [00:01<00:00, 83.51it/s]\n", 524 | "Train Loss 0.052689: 100%|██████████| 600/600 [00:10<00:00, 50.75it/s]\n", 525 | "Valid Loss 0.073805: 100%|██████████| 100/100 [00:01<00:00, 85.06it/s]\n", 526 | "Train Loss 0.044621: 100%|██████████| 600/600 [00:10<00:00, 50.15it/s]\n", 527 | "Valid Loss 0.073268: 100%|██████████| 100/100 [00:01<00:00, 81.44it/s]\n" 528 | ], 529 | "name": "stderr" 530 | }, 531 | { 532 | "output_type": "display_data", 533 | "data": { 534 | "application/vnd.jupyter.widget-view+json": { 535 | "model_id": "59edb14f87e3494d9090b380c01f6732", 536 | "version_minor": 0, 537 | "version_major": 2 538 | }, 539 | "text/plain": [ 540 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 541 | ] 542 | }, 543 | "metadata": { 544 | "tags": [] 545 | } 546 | }, 547 | { 548 | "output_type": "stream", 549 | "text": [ 550 | "Train Loss 0.568829: 100%|██████████| 600/600 [00:10<00:00, 58.90it/s]\n", 551 | "Valid Loss 0.322708: 100%|██████████| 100/100 [00:01<00:00, 81.98it/s]\n", 552 | "Train Loss 0.226664: 100%|██████████| 600/600 [00:10<00:00, 59.45it/s]\n", 553 | "Valid Loss 0.148147: 100%|██████████| 100/100 [00:01<00:00, 84.03it/s]\n", 554 | "Train Loss 0.140796: 100%|██████████| 600/600 [00:10<00:00, 58.85it/s]\n", 555 | "Valid Loss 0.131555: 100%|██████████| 100/100 [00:01<00:00, 84.33it/s]\n", 556 | "Train Loss 0.113898: 100%|██████████| 600/600 [00:10<00:00, 58.74it/s]\n", 557 | "Valid Loss 0.094995: 100%|██████████| 100/100 [00:01<00:00, 84.93it/s]\n", 558 | "Train Loss 0.074536: 100%|██████████| 600/600 [00:10<00:00, 59.33it/s]\n", 559 | "Valid Loss 0.078419: 100%|██████████| 100/100 [00:01<00:00, 84.14it/s]\n", 560 | "Train Loss 0.055034: 100%|██████████| 600/600 [00:10<00:00, 59.12it/s]\n", 561 | "Valid Loss 0.072654: 100%|██████████| 100/100 [00:01<00:00, 82.20it/s]\n", 562 | "Train Loss 0.046104: 100%|██████████| 600/600 [00:10<00:00, 59.08it/s]\n", 563 | "Valid Loss 0.072474: 100%|██████████| 100/100 [00:01<00:00, 86.85it/s]\n" 564 | ], 565 | "name": "stderr" 566 | }, 567 | { 568 | "output_type": "display_data", 569 | "data": { 570 | "application/vnd.jupyter.widget-view+json": { 571 | "model_id": "57430cfa958a47999982076f2f83a439", 572 | "version_minor": 0, 573 | "version_major": 2 574 | }, 575 | "text/plain": [ 576 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 577 | ] 578 | }, 579 | "metadata": { 580 | "tags": [] 581 | } 582 | }, 583 | { 584 | "output_type": "stream", 585 | "text": [ 586 | "Train Loss 0.564984: 100%|██████████| 600/600 [00:10<00:00, 59.11it/s]\n", 587 | "Valid Loss 0.319184: 100%|██████████| 100/100 [00:01<00:00, 81.79it/s]\n", 588 | "Train Loss 0.217199: 100%|██████████| 600/600 [00:10<00:00, 58.73it/s]\n", 589 | "Valid Loss 0.147742: 100%|██████████| 100/100 [00:01<00:00, 86.21it/s]\n", 590 | "Train Loss 0.135781: 100%|██████████| 600/600 [00:10<00:00, 59.24it/s]\n", 591 | "Valid Loss 0.131766: 100%|██████████| 100/100 [00:01<00:00, 84.77it/s]\n", 592 | "Train Loss 0.111096: 100%|██████████| 600/600 [00:10<00:00, 58.65it/s]\n", 593 | "Valid Loss 0.097672: 100%|██████████| 100/100 [00:01<00:00, 83.94it/s]\n", 594 | "Train Loss 0.071949: 100%|██████████| 600/600 [00:10<00:00, 50.36it/s]\n", 595 | "Valid Loss 0.080551: 100%|██████████| 100/100 [00:01<00:00, 87.24it/s]\n", 596 | "Train Loss 0.052584: 100%|██████████| 600/600 [00:10<00:00, 59.27it/s]\n", 597 | "Valid Loss 0.076655: 100%|██████████| 100/100 [00:01<00:00, 82.53it/s]\n", 598 | "Train Loss 0.043895: 100%|██████████| 600/600 [00:10<00:00, 58.92it/s]\n", 599 | "Valid Loss 0.075099: 100%|██████████| 100/100 [00:01<00:00, 81.62it/s]\n" 600 | ], 601 | "name": "stderr" 602 | }, 603 | { 604 | "output_type": "display_data", 605 | "data": { 606 | "application/vnd.jupyter.widget-view+json": { 607 | "model_id": "d0ed001349694ab39fce6ba5b7f97a7c", 608 | "version_minor": 0, 609 | "version_major": 2 610 | }, 611 | "text/plain": [ 612 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 613 | ] 614 | }, 615 | "metadata": { 616 | "tags": [] 617 | } 618 | }, 619 | { 620 | "output_type": "stream", 621 | "text": [ 622 | "Train Loss 0.597221: 100%|██████████| 600/600 [00:10<00:00, 50.61it/s]\n", 623 | "Valid Loss 0.335977: 100%|██████████| 100/100 [00:01<00:00, 87.54it/s]\n", 624 | "Train Loss 0.235150: 100%|██████████| 600/600 [00:10<00:00, 58.40it/s]\n", 625 | "Valid Loss 0.154408: 100%|██████████| 100/100 [00:01<00:00, 80.99it/s]\n", 626 | "Train Loss 0.150037: 100%|██████████| 600/600 [00:10<00:00, 57.80it/s]\n", 627 | "Valid Loss 0.137783: 100%|██████████| 100/100 [00:01<00:00, 84.33it/s]\n", 628 | "Train Loss 0.121238: 100%|██████████| 600/600 [00:10<00:00, 59.21it/s]\n", 629 | "Valid Loss 0.099244: 100%|██████████| 100/100 [00:01<00:00, 84.28it/s]\n", 630 | "Train Loss 0.081774: 100%|██████████| 600/600 [00:10<00:00, 58.15it/s]\n", 631 | "Valid Loss 0.083902: 100%|██████████| 100/100 [00:01<00:00, 82.72it/s]\n", 632 | "Train Loss 0.061101: 100%|██████████| 600/600 [00:10<00:00, 58.86it/s]\n", 633 | "Valid Loss 0.078120: 100%|██████████| 100/100 [00:01<00:00, 86.19it/s]\n", 634 | "Train Loss 0.052224: 100%|██████████| 600/600 [00:10<00:00, 59.08it/s]\n", 635 | "Valid Loss 0.077118: 100%|██████████| 100/100 [00:01<00:00, 84.25it/s]\n" 636 | ], 637 | "name": "stderr" 638 | }, 639 | { 640 | "output_type": "display_data", 641 | "data": { 642 | "application/vnd.jupyter.widget-view+json": { 643 | "model_id": "051da34606ef4c2b9258f5004ffd6857", 644 | "version_minor": 0, 645 | "version_major": 2 646 | }, 647 | "text/plain": [ 648 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 649 | ] 650 | }, 651 | "metadata": { 652 | "tags": [] 653 | } 654 | }, 655 | { 656 | "output_type": "stream", 657 | "text": [ 658 | "Train Loss 0.600702: 100%|██████████| 600/600 [00:10<00:00, 58.74it/s]\n", 659 | "Valid Loss 0.340123: 100%|██████████| 100/100 [00:01<00:00, 81.91it/s]\n", 660 | "Train Loss 0.235824: 100%|██████████| 600/600 [00:10<00:00, 59.31it/s]\n", 661 | "Valid Loss 0.155959: 100%|██████████| 100/100 [00:01<00:00, 81.26it/s]\n", 662 | "Train Loss 0.148497: 100%|██████████| 600/600 [00:10<00:00, 59.35it/s]\n", 663 | "Valid Loss 0.139545: 100%|██████████| 100/100 [00:01<00:00, 84.07it/s]\n", 664 | "Train Loss 0.120669: 100%|██████████| 600/600 [00:10<00:00, 48.96it/s]\n", 665 | "Valid Loss 0.098499: 100%|██████████| 100/100 [00:01<00:00, 86.76it/s]\n", 666 | "Train Loss 0.080130: 100%|██████████| 600/600 [00:10<00:00, 58.92it/s]\n", 667 | "Valid Loss 0.085373: 100%|██████████| 100/100 [00:01<00:00, 82.55it/s]\n", 668 | "Train Loss 0.061044: 100%|██████████| 600/600 [00:10<00:00, 58.95it/s]\n", 669 | "Valid Loss 0.078647: 100%|██████████| 100/100 [00:01<00:00, 86.74it/s]\n", 670 | "Train Loss 0.052978: 100%|██████████| 600/600 [00:10<00:00, 59.53it/s]\n", 671 | "Valid Loss 0.077972: 100%|██████████| 100/100 [00:01<00:00, 87.02it/s]\n" 672 | ], 673 | "name": "stderr" 674 | }, 675 | { 676 | "output_type": "display_data", 677 | "data": { 678 | "application/vnd.jupyter.widget-view+json": { 679 | "model_id": "659b693530bb45889e88a6db200abacb", 680 | "version_minor": 0, 681 | "version_major": 2 682 | }, 683 | "text/plain": [ 684 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 685 | ] 686 | }, 687 | "metadata": { 688 | "tags": [] 689 | } 690 | }, 691 | { 692 | "output_type": "stream", 693 | "text": [ 694 | "Train Loss 0.908646: 100%|██████████| 600/600 [00:10<00:00, 49.77it/s]\n", 695 | "Valid Loss 0.576039: 100%|██████████| 100/100 [00:01<00:00, 84.51it/s]\n", 696 | "Train Loss 0.384706: 100%|██████████| 600/600 [00:10<00:00, 49.91it/s]\n", 697 | "Valid Loss 0.252410: 100%|██████████| 100/100 [00:01<00:00, 84.50it/s]\n", 698 | "Train Loss 0.254255: 100%|██████████| 600/600 [00:10<00:00, 59.24it/s]\n", 699 | "Valid Loss 0.226386: 100%|██████████| 100/100 [00:01<00:00, 84.37it/s]\n", 700 | "Train Loss 0.201884: 100%|██████████| 600/600 [00:10<00:00, 58.96it/s]\n", 701 | "Valid Loss 0.152886: 100%|██████████| 100/100 [00:01<00:00, 82.25it/s]\n", 702 | "Train Loss 0.145404: 100%|██████████| 600/600 [00:10<00:00, 58.61it/s]\n", 703 | "Valid Loss 0.126134: 100%|██████████| 100/100 [00:01<00:00, 85.37it/s]\n", 704 | "Train Loss 0.122407: 100%|██████████| 600/600 [00:10<00:00, 59.62it/s]\n", 705 | "Valid Loss 0.117613: 100%|██████████| 100/100 [00:01<00:00, 85.71it/s]\n", 706 | "Train Loss 0.112847: 100%|██████████| 600/600 [00:10<00:00, 58.93it/s]\n", 707 | "Valid Loss 0.116246: 100%|██████████| 100/100 [00:01<00:00, 85.11it/s]\n" 708 | ], 709 | "name": "stderr" 710 | }, 711 | { 712 | "output_type": "display_data", 713 | "data": { 714 | "application/vnd.jupyter.widget-view+json": { 715 | "model_id": "e6c0b267d2e34b6fa37dc67ecbeb82b9", 716 | "version_minor": 0, 717 | "version_major": 2 718 | }, 719 | "text/plain": [ 720 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 721 | ] 722 | }, 723 | "metadata": { 724 | "tags": [] 725 | } 726 | }, 727 | { 728 | "output_type": "stream", 729 | "text": [ 730 | "Train Loss 0.899953: 100%|██████████| 600/600 [00:10<00:00, 59.58it/s]\n", 731 | "Valid Loss 0.563455: 100%|██████████| 100/100 [00:01<00:00, 84.70it/s]\n", 732 | "Train Loss 0.380311: 100%|██████████| 600/600 [00:10<00:00, 58.46it/s]\n", 733 | "Valid Loss 0.256279: 100%|██████████| 100/100 [00:01<00:00, 82.61it/s]\n", 734 | "Train Loss 0.255473: 100%|██████████| 600/600 [00:10<00:00, 50.41it/s]\n", 735 | "Valid Loss 0.229022: 100%|██████████| 100/100 [00:01<00:00, 87.09it/s]\n", 736 | "Train Loss 0.205060: 100%|██████████| 600/600 [00:10<00:00, 59.53it/s]\n", 737 | "Valid Loss 0.156413: 100%|██████████| 100/100 [00:01<00:00, 83.44it/s]\n", 738 | "Train Loss 0.147704: 100%|██████████| 600/600 [00:10<00:00, 58.61it/s]\n", 739 | "Valid Loss 0.129519: 100%|██████████| 100/100 [00:01<00:00, 83.77it/s]\n", 740 | "Train Loss 0.124054: 100%|██████████| 600/600 [00:10<00:00, 59.48it/s]\n", 741 | "Valid Loss 0.120353: 100%|██████████| 100/100 [00:01<00:00, 82.59it/s]\n", 742 | "Train Loss 0.115274: 100%|██████████| 600/600 [00:10<00:00, 57.92it/s]\n", 743 | "Valid Loss 0.119134: 100%|██████████| 100/100 [00:01<00:00, 84.70it/s]\n" 744 | ], 745 | "name": "stderr" 746 | }, 747 | { 748 | "output_type": "display_data", 749 | "data": { 750 | "application/vnd.jupyter.widget-view+json": { 751 | "model_id": "2c70880424db44a084c8f07610953258", 752 | "version_minor": 0, 753 | "version_major": 2 754 | }, 755 | "text/plain": [ 756 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 757 | ] 758 | }, 759 | "metadata": { 760 | "tags": [] 761 | } 762 | }, 763 | { 764 | "output_type": "stream", 765 | "text": [ 766 | "Train Loss 1.478377: 100%|██████████| 600/600 [00:10<00:00, 58.22it/s]\n", 767 | "Valid Loss 1.187216: 100%|██████████| 100/100 [00:01<00:00, 82.43it/s]\n", 768 | "Train Loss 0.798353: 100%|██████████| 600/600 [00:10<00:00, 59.08it/s]\n", 769 | "Valid Loss 0.552721: 100%|██████████| 100/100 [00:01<00:00, 84.52it/s]\n", 770 | "Train Loss 0.541047: 100%|██████████| 600/600 [00:10<00:00, 58.99it/s]\n", 771 | "Valid Loss 0.492757: 100%|██████████| 100/100 [00:01<00:00, 83.19it/s]\n", 772 | "Train Loss 0.429721: 100%|██████████| 600/600 [00:10<00:00, 59.01it/s]\n", 773 | "Valid Loss 0.332853: 100%|██████████| 100/100 [00:01<00:00, 81.86it/s]\n", 774 | "Train Loss 0.332341: 100%|██████████| 600/600 [00:10<00:00, 59.09it/s]\n", 775 | "Valid Loss 0.280423: 100%|██████████| 100/100 [00:01<00:00, 85.15it/s]\n", 776 | "Train Loss 0.295493: 100%|██████████| 600/600 [00:10<00:00, 50.33it/s]\n", 777 | "Valid Loss 0.262048: 100%|██████████| 100/100 [00:01<00:00, 84.51it/s]\n", 778 | "Train Loss 0.283006: 100%|██████████| 600/600 [00:10<00:00, 59.61it/s]\n", 779 | "Valid Loss 0.260017: 100%|██████████| 100/100 [00:01<00:00, 81.85it/s]\n" 780 | ], 781 | "name": "stderr" 782 | }, 783 | { 784 | "output_type": "display_data", 785 | "data": { 786 | "application/vnd.jupyter.widget-view+json": { 787 | "model_id": "d77dc6a293834d1c897632cf112f2c01", 788 | "version_minor": 0, 789 | "version_major": 2 790 | }, 791 | "text/plain": [ 792 | "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))" 793 | ] 794 | }, 795 | "metadata": { 796 | "tags": [] 797 | } 798 | }, 799 | { 800 | "output_type": "stream", 801 | "text": [ 802 | "Train Loss 1.351790: 100%|██████████| 600/600 [00:10<00:00, 58.85it/s]\n", 803 | "Valid Loss 1.023331: 100%|██████████| 100/100 [00:01<00:00, 82.19it/s]\n", 804 | "Train Loss 0.699126: 100%|██████████| 600/600 [00:10<00:00, 58.77it/s]\n", 805 | "Valid Loss 0.486799: 100%|██████████| 100/100 [00:01<00:00, 83.76it/s]\n", 806 | "Train Loss 0.478460: 100%|██████████| 600/600 [00:10<00:00, 59.15it/s]\n", 807 | "Valid Loss 0.436088: 100%|██████████| 100/100 [00:01<00:00, 86.20it/s]\n", 808 | "Train Loss 0.384783: 100%|██████████| 600/600 [00:10<00:00, 49.11it/s]\n", 809 | "Valid Loss 0.300667: 100%|██████████| 100/100 [00:01<00:00, 82.35it/s]\n", 810 | "Train Loss 0.297129: 100%|██████████| 600/600 [00:10<00:00, 58.54it/s]\n", 811 | "Valid Loss 0.257974: 100%|██████████| 100/100 [00:01<00:00, 88.33it/s]\n", 812 | "Train Loss 0.265162: 100%|██████████| 600/600 [00:10<00:00, 59.55it/s]\n", 813 | "Valid Loss 0.241354: 100%|██████████| 100/100 [00:01<00:00, 86.30it/s]\n", 814 | "Train Loss 0.254860: 100%|██████████| 600/600 [00:10<00:00, 59.37it/s]\n", 815 | "Valid Loss 0.239391: 100%|██████████| 100/100 [00:01<00:00, 85.39it/s]\n" 816 | ], 817 | "name": "stderr" 818 | } 819 | ] 820 | }, 821 | { 822 | "cell_type": "code", 823 | "metadata": { 824 | "id": "cxuP8q2qYEUg", 825 | "colab_type": "code", 826 | "colab": { 827 | "base_uri": "https://localhost:8080/", 828 | "height": 35 829 | }, 830 | "outputId": "fd11d7c7-b2ae-47a5-aecb-6ffc0268842f" 831 | }, 832 | "source": [ 833 | "ACCURACIES" 834 | ], 835 | "execution_count": 48, 836 | "outputs": [ 837 | { 838 | "output_type": "execute_result", 839 | "data": { 840 | "text/plain": [ 841 | "[0.9774, 0.977, 0.9779, 0.9779, 0.9774, 0.9767, 0.9669, 0.9669, 0.9301, 0.9369]" 842 | ] 843 | }, 844 | "metadata": { 845 | "tags": [] 846 | }, 847 | "execution_count": 48 848 | } 849 | ] 850 | }, 851 | { 852 | "cell_type": "code", 853 | "metadata": { 854 | "id": "SHCpbZwsbWQg", 855 | "colab_type": "code", 856 | "colab": { 857 | "base_uri": "https://localhost:8080/", 858 | "height": 191 859 | }, 860 | "outputId": "4846ac3e-47eb-47c8-c955-d3b84e957033" 861 | }, 862 | "source": [ 863 | "PARAMETERS" 864 | ], 865 | "execution_count": 49, 866 | "outputs": [ 867 | { 868 | "output_type": "execute_result", 869 | "data": { 870 | "text/plain": [ 871 | "[tensor(230229, device='cuda:0'),\n", 872 | " tensor(264825, device='cuda:0'),\n", 873 | " tensor(228164, device='cuda:0'),\n", 874 | " tensor(247959, device='cuda:0'),\n", 875 | " tensor(165905, device='cuda:0'),\n", 876 | " tensor(165896, device='cuda:0'),\n", 877 | " tensor(34582, device='cuda:0'),\n", 878 | " tensor(34570, device='cuda:0'),\n", 879 | " tensor(8230, device='cuda:0'),\n", 880 | " tensor(8213, device='cuda:0')]" 881 | ] 882 | }, 883 | "metadata": { 884 | "tags": [] 885 | }, 886 | "execution_count": 49 887 | } 888 | ] 889 | }, 890 | { 891 | "cell_type": "code", 892 | "metadata": { 893 | "id": "wVEr3DIEbaBU", 894 | "colab_type": "code", 895 | "colab": {} 896 | }, 897 | "source": [ 898 | "" 899 | ], 900 | "execution_count": 0, 901 | "outputs": [] 902 | } 903 | ] 904 | } --------------------------------------------------------------------------------