├── .gitignore ├── LICENSE ├── README.md ├── data.ipynb ├── deploy.py ├── digits.ipynb ├── digitz.py ├── fence.jpg ├── graph.ipynb ├── kernels.ipynb ├── kids.jpg ├── models.py ├── powerpoint ├── 9squares.png └── DeepLearningwithPyTorch.pptx ├── pytorchmnist.yml ├── score.py ├── simple.ipynb ├── squares.ipynb ├── superfile.onnx ├── tensors.ipynb ├── triple.ipynb ├── utils ├── __init__.py ├── draw.py ├── helpers.py ├── square.py ├── tree.json └── viz.py └── wedding.jpg /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # data 107 | .vscode/ 108 | data/ 109 | aml_config/ 110 | outputs/ 111 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Seth Juarez 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Azure Notebooks](https://notebooks.azure.com/launch.svg)](https://notebooks.azure.com/import/gh/sethjuarez/DeepLearningWithPyTorch) 2 | 3 | # Deep Learning with PyTorch 4 | This repo is a resource for my Deep Learning with PyTorch talk. It 5 | contains all of the code that was demonstrated as well as the deck. 6 | 7 | ## Inspiration 8 | This talk is inspired by a PyTorch tutorial available [online](https://pytorch.org/tutorials/beginner/pytorch_with_examples.html). 9 | It is also inspired by a silly nine squares image problem. 10 | 11 | ![9 Squares](powerpoint/9squares.png "9 Squares") 12 | 13 | The purpose of this silly problem is to provide the basic intution 14 | behind models, cost, accuracy, and optimization principles behind 15 | machine learning. 16 | 17 | ## Overview 18 | 19 | ### Deep Learning Principles 20 | 21 | - input 22 | - model function 23 | - cost function 24 | - optimization method 25 | 26 | ### PyTorch 27 | 28 | - Tensors 29 | - DataSets 30 | - DataLoaders 31 | - Models 32 | - Loss (or Cost) 33 | - Optimization 34 | 35 | ### Tools 36 | 37 | - ONNX 38 | - Azure Machine Learning service 39 | -------------------------------------------------------------------------------- /data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch\n", 10 | "from torch.utils.data import DataLoader\n", 11 | "from torch.utils.data.dataset import Dataset\n", 12 | "from torchvision import datasets, transforms" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": {}, 18 | "source": [ 19 | "# Dataset" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 2, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "class SquareDataset(Dataset):\n", 29 | " def __init__(self, size):\n", 30 | " self.size = size\n", 31 | " self.X = torch.randint(255, (size, 9), dtype=torch.float)\n", 32 | "\n", 33 | " real_w = torch.tensor([[1,1,1,0,0,0,0,0,0],\n", 34 | " [0,0,0,1,1,1,0,0,0],\n", 35 | " [0,0,0,0,0,0,1,1,1]], \n", 36 | " dtype=torch.float)\n", 37 | "\n", 38 | " y = torch.argmax(self.X.mm(real_w.t()), 1)\n", 39 | " \n", 40 | " self.Y = torch.zeros(size, 3, dtype=torch.float) \\\n", 41 | " .scatter_(1, y.view(-1, 1), 1)\n", 42 | "\n", 43 | " def __getitem__(self, index):\n", 44 | " return (self.X[index], self.Y[index])\n", 45 | "\n", 46 | " def __len__(self):\n", 47 | " return self.size" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 3, 53 | "metadata": {}, 54 | "outputs": [ 55 | { 56 | "name": "stdout", 57 | "output_type": "stream", 58 | "text": [ 59 | "(tensor([ 54., 182., 47., 142., 200., 197., 220., 215., 33.]), tensor([0., 1., 0.]))\n", 60 | "(tensor([198., 171., 26., 140., 28., 9., 205., 48., 113.]), tensor([1., 0., 0.]))\n", 61 | "(tensor([ 64., 7., 167., 4., 9., 160., 169., 113., 214.]), tensor([0., 0., 1.]))\n" 62 | ] 63 | } 64 | ], 65 | "source": [ 66 | "squares = SquareDataset(256)\n", 67 | "print(squares[34])\n", 68 | "print(squares[254])\n", 69 | "print(squares[25])" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 4, 75 | "metadata": {}, 76 | "outputs": [ 77 | { 78 | "name": "stdout", 79 | "output_type": "stream", 80 | "text": [ 81 | "tensor([[152., 127., 155., 219., 81., 140., 112., 77., 102.],\n", 82 | " [ 77., 58., 228., 164., 229., 155., 111., 223., 141.],\n", 83 | " [106., 250., 87., 62., 105., 254., 0., 210., 136.],\n", 84 | " [190., 108., 134., 204., 145., 251., 146., 171., 99.],\n", 85 | " [ 88., 36., 190., 108., 122., 4., 231., 22., 70.]]) \n", 86 | "\n", 87 | " tensor([[0., 1., 0.],\n", 88 | " [0., 1., 0.],\n", 89 | " [1., 0., 0.],\n", 90 | " [0., 1., 0.],\n", 91 | " [0., 0., 1.]])\n" 92 | ] 93 | } 94 | ], 95 | "source": [ 96 | "dataloader = DataLoader(squares, batch_size=5)\n", 97 | "\n", 98 | "for batch, (X, Y) in enumerate(dataloader):\n", 99 | " print(X, '\\n\\n', Y)\n", 100 | " break" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "metadata": {}, 106 | "source": [ 107 | "# Digits\n", 108 | "Transforms!" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 5, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "digits = datasets.MNIST('data', train=True, download=True,\n", 118 | " transform=transforms.Compose([\n", 119 | " transforms.ToTensor(),\n", 120 | " transforms.Lambda(lambda x: x.view(28*28))\n", 121 | " ]),\n", 122 | " target_transform=transforms.Compose([\n", 123 | " transforms.Lambda(lambda y: \n", 124 | " torch.zeros(10, dtype=torch.float).scatter_(0, torch.tensor(y), value=1))\n", 125 | " ])\n", 126 | " )" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 6, 132 | "metadata": {}, 133 | "outputs": [ 134 | { 135 | "name": "stdout", 136 | "output_type": "stream", 137 | "text": [ 138 | "tensor([[0., 0., 0., ..., 0., 0., 0.],\n", 139 | " [0., 0., 0., ..., 0., 0., 0.],\n", 140 | " [0., 0., 0., ..., 0., 0., 0.],\n", 141 | " ...,\n", 142 | " [0., 0., 0., ..., 0., 0., 0.],\n", 143 | " [0., 0., 0., ..., 0., 0., 0.],\n", 144 | " [0., 0., 0., ..., 0., 0., 0.]]) \n", 145 | "\n", 146 | " tensor([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", 147 | " [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", 148 | " [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n", 149 | " [0., 0., 0., 0., 0., 0., 0., 1., 0., 0.],\n", 150 | " [0., 0., 0., 0., 0., 0., 0., 0., 1., 0.],\n", 151 | " [0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],\n", 152 | " [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n", 153 | " [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.],\n", 154 | " [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", 155 | " [0., 0., 0., 0., 0., 0., 0., 1., 0., 0.]])\n" 156 | ] 157 | } 158 | ], 159 | "source": [ 160 | "dataloader = DataLoader(digits, batch_size=10, shuffle=True)\n", 161 | "\n", 162 | "for batch, (X, Y) in enumerate(dataloader):\n", 163 | " print(X, '\\n\\n', Y)\n", 164 | " break" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [] 173 | } 174 | ], 175 | "metadata": { 176 | "file_extension": ".py", 177 | "kernelspec": { 178 | "display_name": "Python 3", 179 | "language": "python", 180 | "name": "python3" 181 | }, 182 | "language_info": { 183 | "codemirror_mode": { 184 | "name": "ipython", 185 | "version": 3 186 | }, 187 | "file_extension": ".py", 188 | "mimetype": "text/x-python", 189 | "name": "python", 190 | "nbconvert_exporter": "python", 191 | "pygments_lexer": "ipython3", 192 | "version": "3.6.10" 193 | }, 194 | "mimetype": "text/x-python", 195 | "name": "python", 196 | "npconvert_exporter": "python", 197 | "pygments_lexer": "ipython3", 198 | "version": 3 199 | }, 200 | "nbformat": 4, 201 | "nbformat_minor": 2 202 | } 203 | -------------------------------------------------------------------------------- /deploy.py: -------------------------------------------------------------------------------- 1 | import json 2 | import azureml 3 | from azureml.core.model import Model 4 | from azureml.core import Workspace, Run 5 | from azureml.core.image import ContainerImage, Image 6 | from azureml.core.conda_dependencies import CondaDependencies 7 | from azureml.core.webservice import Webservice, AciWebservice 8 | 9 | def load_workspace(): 10 | # use this code to set up config file 11 | #subscription_id ='' 12 | #resource_group ='' 13 | #workspace_name = '' 14 | 15 | #try: 16 | # ws = Workspace(subscription_id = subscription_id, resource_group = resource_group, workspace_name = workspace_name) 17 | # ws.write_config() 18 | # print('Workspace configuration succeeded. You are all set!') 19 | # return ws 20 | #except: 21 | # print('Workspace not found. TOO MANY ISSUES!!!') 22 | 23 | ws = Workspace.from_config() 24 | return ws 25 | 26 | def main(): 27 | # get workspace 28 | ws = load_workspace() 29 | model = Model.register(ws, model_name='pytorch_mnist', model_path='model.pth') 30 | 31 | # create dep file 32 | myenv = CondaDependencies() 33 | myenv.add_pip_package('numpy') 34 | myenv.add_pip_package('torch') 35 | with open('pytorchmnist.yml','w') as f: 36 | print('Writing out {}'.format('pytorchmnist.yml')) 37 | f.write(myenv.serialize_to_string()) 38 | print('Done!') 39 | 40 | # create image 41 | image_config = ContainerImage.image_configuration(execution_script="score.py", 42 | runtime="python", 43 | conda_file="pytorchmnist.yml", 44 | dependencies=['./models.py']) 45 | 46 | image = Image.create(ws, 'pytorchmnist', [model], image_config) 47 | image.wait_for_creation(show_output=True) 48 | 49 | # create service 50 | aciconfig = AciWebservice.deploy_configuration(cpu_cores=1, 51 | memory_gb=1, 52 | description='simple MNIST digit detection') 53 | service = Webservice.deploy_from_image(workspace=ws, 54 | image=image, 55 | name='pytorchmnist-svc', 56 | deployment_config=aciconfig) 57 | service.wait_for_deployment(show_output=True) 58 | 59 | def debug_deploy(): 60 | # get workspace 61 | ws = load_workspace() 62 | # get service 63 | service = ws.webservices['pytorchmnist-svc'] 64 | # write log 65 | with open('deploy.log','w') as f: 66 | f.write(service.get_logs()) 67 | 68 | 69 | if __name__ == '__main__': 70 | # check core SDK version number 71 | print("Using Azure ML SDK Version: ", azureml.core.VERSION) 72 | main() 73 | -------------------------------------------------------------------------------- /digitz.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import argparse 4 | import torch.nn as nn 5 | from pathlib import Path 6 | import torch.onnx as onnx 7 | import torch.optim as optim 8 | import torch.nn.functional as F 9 | from torch.utils.data import DataLoader 10 | from torchvision import datasets, transforms 11 | from torch.optim.lr_scheduler import StepLR 12 | 13 | ################################################################### 14 | # Helpers # 15 | ################################################################### 16 | def info(msg, char = "#", width = 75): 17 | print("") 18 | print(char * width) 19 | print(char + " %0*s" % ((-1*width)+5, msg) + char) 20 | print(char * width) 21 | 22 | def check_dir(path, check=False): 23 | if check: 24 | assert os.path.exists(path), '{} does not exist!'.format(path) 25 | else: 26 | if not os.path.exists(path): 27 | os.makedirs(path) 28 | return Path(path).resolve() 29 | 30 | ################################################################### 31 | # Data Loader # 32 | ################################################################### 33 | def get_dataloader(train=True, batch_size=64, data_dir='data'): 34 | digits = datasets.MNIST(data_dir, train=train, download=True, 35 | transform=transforms.Compose([ 36 | transforms.ToTensor(), 37 | transforms.Lambda(lambda x: x.reshape(28*28)) 38 | ]), 39 | target_transform=transforms.Compose([ 40 | transforms.Lambda(lambda y: 41 | torch.zeros(10, dtype=torch.float).scatter_(0, torch.tensor(y), value=1)) 42 | ]) 43 | ) 44 | 45 | return DataLoader(digits, batch_size=batch_size, shuffle=True, num_workers=0, pin_memory=True) 46 | 47 | ################################################################### 48 | # Saving # 49 | ################################################################### 50 | def save_model(model, device, path, name): 51 | base = Path(path) 52 | onnx_file = base.joinpath('{}.onnx'.format(name)).resolve() 53 | pth_file = base.joinpath('{}.pth'.format(name)).resolve() 54 | 55 | # create dummy variable to traverse graph 56 | x = torch.randint(255, (1, 28*28), dtype=torch.float).to(device) / 255 57 | onnx.export(model, x, onnx_file) 58 | print('Saved onnx model to {}'.format(onnx_file)) 59 | 60 | # saving PyTorch Model Dictionary 61 | torch.save(model.state_dict(), pth_file) 62 | print('Saved PyTorch Model to {}'.format(pth_file)) 63 | 64 | ################################################################### 65 | # Models # 66 | ################################################################### 67 | class Logistic(nn.Module): 68 | def __init__(self): 69 | super(Logistic, self).__init__() 70 | self.layer1 = nn.Linear(28*28, 10) 71 | 72 | def forward(self, x): 73 | x = self.layer1(x) 74 | return F.softmax(x, dim=1) 75 | 76 | class NeuralNework(nn.Module): 77 | def __init__(self): 78 | super(NeuralNework, self).__init__() 79 | self.layer1 = nn.Linear(28*28, 512) 80 | self.layer2 = nn.Linear(512, 512) 81 | self.output = nn.Linear(512, 10) 82 | 83 | def forward(self, x): 84 | x = F.relu(self.layer1(x)) 85 | x = F.relu(self.layer2(x)) 86 | x = self.output(x) 87 | return F.softmax(x, dim=1) 88 | 89 | class CNN(nn.Module): 90 | def __init__(self): 91 | super(CNN, self).__init__() 92 | self.conv1 = nn.Conv2d(1, 10, kernel_size=5) 93 | self.conv2 = nn.Conv2d(10, 20, kernel_size=5) 94 | self.conv2_drop = nn.Dropout2d() 95 | self.fc1 = nn.Linear(320, 50) 96 | self.fc2 = nn.Linear(50, 10) 97 | 98 | def forward(self, x): 99 | x = x.view(-1, 1, 28, 28) 100 | x = F.relu(F.max_pool2d(self.conv1(x), 2)) 101 | x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) 102 | x = x.view(-1, 320) 103 | x = F.relu(self.fc1(x)) 104 | x = F.dropout(x, training=self.training) 105 | x = self.fc2(x) 106 | return F.softmax(x, dim=1) 107 | 108 | ################################################################### 109 | # Train/Test # 110 | ################################################################### 111 | def train(model, device, dataloader, cost, optimizer, epoch): 112 | model.train() 113 | for batch, (X, Y) in enumerate(dataloader): 114 | X, Y = X.to(device), Y.to(device) 115 | optimizer.zero_grad() 116 | pred = model(X) 117 | loss = cost(pred, Y) 118 | loss.backward() 119 | optimizer.step() 120 | 121 | if batch % 100 == 0: 122 | print('loss: {:>10f} [{:>5d}/{:>5d}]'.format(loss.item(), batch * len(X), len(dataloader.dataset))) 123 | 124 | def test(model, device, dataloader, cost): 125 | model.eval() 126 | test_loss = 0 127 | correct = 0 128 | with torch.no_grad(): 129 | for batch, (X, Y) in enumerate(dataloader): 130 | X, Y = X.to(device), Y.to(device) 131 | pred = model(X) 132 | 133 | test_loss += cost(pred, Y).item() 134 | correct += (pred.argmax(1) == Y.argmax(1)).type(torch.float).sum().item() 135 | 136 | test_loss /= len(dataloader.dataset) 137 | correct /= len(dataloader.dataset) 138 | print('\nTest Error:') 139 | print('acc: {:>0.1f}%, avg loss: {:>8f}'.format(100*correct, test_loss)) 140 | 141 | ################################################################### 142 | # Main Loop # 143 | ################################################################### 144 | def main(data_dir, output_dir, log_dir, epochs, batch, lr, model_kind): 145 | # use GPU 146 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 147 | 148 | # get data loaders 149 | training = get_dataloader(train=True, batch_size=batch, data_dir=data_dir) 150 | testing = get_dataloader(train=False, batch_size=batch, data_dir=data_dir) 151 | 152 | # model 153 | if model_kind == 'linear': 154 | model = Logistic().to(device) 155 | elif model_kind == 'nn': 156 | model = NeuralNework().to(device) 157 | else: 158 | model = CNN().to(device) 159 | 160 | info('Model') 161 | print(model) 162 | 163 | # cost function 164 | cost = torch.nn.BCELoss() 165 | 166 | # optimizers 167 | optimizer = optim.Adam(model.parameters(), lr=lr) 168 | scheduler = StepLR(optimizer, 5) 169 | 170 | for epoch in range(1, epochs + 1): 171 | info('Epoch {}'.format(epoch)) 172 | scheduler.step() 173 | print('Current learning rate: {}'.format(scheduler.get_lr())) 174 | train(model, device, training, cost, optimizer, epoch) 175 | test(model, device, testing, cost) 176 | 177 | # save model 178 | info('Saving Model') 179 | save_model(model, device, output_dir, 'model') 180 | 181 | 182 | if __name__ == '__main__': 183 | parser = argparse.ArgumentParser(description='CNN Training for Image Recognition.') 184 | parser.add_argument('-d', '--data', help='directory to training and test data', default='data') 185 | parser.add_argument('-o', '--output', help='output directory', default='outputs') 186 | parser.add_argument('-g', '--logs', help='log directory', default='logs') 187 | 188 | parser.add_argument('-e', '--epochs', help='number of epochs', default=15, type=int) 189 | parser.add_argument('-b', '--batch', help='batch size', default=100, type=int) 190 | parser.add_argument('-l', '--lr', help='learning rate', default=0.001, type=float) 191 | 192 | parser.add_argument('-m', '--model', help='model type', default='cnn', choices=['linear', 'nn', 'cnn']) 193 | 194 | args = parser.parse_args() 195 | 196 | # enforce folder locatations 197 | args.data = check_dir(args.data).resolve() 198 | args.outputs = check_dir(args.output).resolve() 199 | args.logs = check_dir(args.logs).resolve() 200 | 201 | main(data_dir=args.data, output_dir=args.output, log_dir=args.logs, 202 | epochs=args.epochs, batch=args.batch, lr=args.lr, model_kind=args.model) -------------------------------------------------------------------------------- /fence.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sethjuarez/DeepLearningWithPyTorch/81b8f58a27494ea41178fd667dcc7811de33d75d/fence.jpg -------------------------------------------------------------------------------- /graph.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import re\n", 10 | "import json\n", 11 | "import torch\n", 12 | "import torch.nn as nn\n", 13 | "import torch.nn.functional as F\n", 14 | "import utils.viz as torchviz" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "x = torch.tensor([[255., 232, 132, 12, 343, 21, 0, 32, 12]]) / 255\n", 24 | "y = torch.tensor([1., 0, 0])" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 3, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "w = torch.rand(9, 3, requires_grad=True)\n", 34 | "b = torch.rand(1, 3, requires_grad=True)" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 4, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "h = x.mm(w) + b\n", 44 | "loss = ((h - y)**2).mean()\n", 45 | "\n", 46 | "# here\n" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 5, 52 | "metadata": {}, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "image/svg+xml": [ 57 | "\r\n", 58 | "\r\n", 60 | "\r\n", 62 | "\r\n", 63 | "\r\n", 65 | "\r\n", 66 | "g\r\n", 67 | "\r\n", 68 | "\r\n", 69 | "0\r\n", 70 | "\r\n", 71 | "Mean\r\n", 72 | "\r\n", 73 | "\r\n", 74 | "1\r\n", 75 | "\r\n", 76 | "Pow\r\n", 77 | "\r\n", 78 | "\r\n", 79 | "0->1\r\n", 80 | "\r\n", 81 | "\r\n", 82 | "\r\n", 83 | "\r\n", 84 | "2\r\n", 85 | "\r\n", 86 | "Sub\r\n", 87 | "\r\n", 88 | "\r\n", 89 | "1->2\r\n", 90 | "\r\n", 91 | "\r\n", 92 | "\r\n", 93 | "\r\n", 94 | "3\r\n", 95 | "\r\n", 96 | "Add\r\n", 97 | "\r\n", 98 | "\r\n", 99 | "2->3\r\n", 100 | "\r\n", 101 | "\r\n", 102 | "\r\n", 103 | "\r\n", 104 | "8\r\n", 105 | "\r\n", 106 | "Const\r\n", 107 | "\r\n", 108 | "\r\n", 109 | "2->8\r\n", 110 | "\r\n", 111 | "\r\n", 112 | "\r\n", 113 | "\r\n", 114 | "4\r\n", 115 | "\r\n", 116 | "Mm\r\n", 117 | "\r\n", 118 | "\r\n", 119 | "3->4\r\n", 120 | "\r\n", 121 | "\r\n", 122 | "\r\n", 123 | "\r\n", 124 | "7\r\n", 125 | "\r\n", 126 | "Var\r\n", 127 | "\r\n", 128 | "\r\n", 129 | "3->7\r\n", 130 | "\r\n", 131 | "\r\n", 132 | "\r\n", 133 | "\r\n", 134 | "5\r\n", 135 | "\r\n", 136 | "Const\r\n", 137 | "\r\n", 138 | "\r\n", 139 | "4->5\r\n", 140 | "\r\n", 141 | "\r\n", 142 | "\r\n", 143 | "\r\n", 144 | "6\r\n", 145 | "\r\n", 146 | "Var\r\n", 147 | "\r\n", 148 | "\r\n", 149 | "4->6\r\n", 150 | "\r\n", 151 | "\r\n", 152 | "\r\n", 153 | "\r\n", 154 | "\r\n" 155 | ], 156 | "text/plain": [ 157 | "" 158 | ] 159 | }, 160 | "execution_count": 5, 161 | "metadata": {}, 162 | "output_type": "execute_result" 163 | } 164 | ], 165 | "source": [ 166 | "torchviz.draw(loss)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 6, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "loss.backward()" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 7, 181 | "metadata": {}, 182 | "outputs": [ 183 | { 184 | "name": "stdout", 185 | "output_type": "stream", 186 | "text": [ 187 | "tensor([[1.3103, 0.9271, 1.6677],\n", 188 | " [1.1921, 0.8435, 1.5172],\n", 189 | " [0.6783, 0.4799, 0.8633],\n", 190 | " [0.0617, 0.0436, 0.0785],\n", 191 | " [1.7625, 1.2471, 2.2432],\n", 192 | " [0.1079, 0.0764, 0.1373],\n", 193 | " [0.0000, 0.0000, 0.0000],\n", 194 | " [0.1644, 0.1163, 0.2093],\n", 195 | " [0.0617, 0.0436, 0.0785]]) \n", 196 | " tensor([[1.3103, 0.9271, 1.6677]])\n" 197 | ] 198 | } 199 | ], 200 | "source": [ 201 | "print(w.grad, '\\n', b.grad)" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "metadata": {}, 207 | "source": [ 208 | "# Linear" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 8, 214 | "metadata": {}, 215 | "outputs": [], 216 | "source": [ 217 | "class LinearModel(nn.Module):\n", 218 | " def __init__(self, x, y):\n", 219 | " super(LinearModel, self).__init__()\n", 220 | " self.layer1 = nn.Linear(x, y)\n", 221 | "\n", 222 | " def forward(self, x):\n", 223 | " x = self.layer1(x)\n", 224 | " return F.softmax(x, dim=1)" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 9, 230 | "metadata": {}, 231 | "outputs": [ 232 | { 233 | "data": { 234 | "image/svg+xml": [ 235 | "\r\n", 236 | "\r\n", 238 | "\r\n", 240 | "\r\n", 241 | "\r\n", 243 | "\r\n", 244 | "g\r\n", 245 | "\r\n", 246 | "\r\n", 247 | "0\r\n", 248 | "\r\n", 249 | "Mse\r\n", 250 | "\r\n", 251 | "\r\n", 252 | "1\r\n", 253 | "\r\n", 254 | "Softmax\r\n", 255 | "\r\n", 256 | "\r\n", 257 | "0->1\r\n", 258 | "\r\n", 259 | "\r\n", 260 | "\r\n", 261 | "\r\n", 262 | "2\r\n", 263 | "\r\n", 264 | "Addmm\r\n", 265 | "\r\n", 266 | "\r\n", 267 | "1->2\r\n", 268 | "\r\n", 269 | "\r\n", 270 | "\r\n", 271 | "\r\n", 272 | "3\r\n", 273 | "\r\n", 274 | "Var\r\n", 275 | "\r\n", 276 | "\r\n", 277 | "2->3\r\n", 278 | "\r\n", 279 | "\r\n", 280 | "\r\n", 281 | "\r\n", 282 | "4\r\n", 283 | "\r\n", 284 | "Const\r\n", 285 | "\r\n", 286 | "\r\n", 287 | "2->4\r\n", 288 | "\r\n", 289 | "\r\n", 290 | "\r\n", 291 | "\r\n", 292 | "5\r\n", 293 | "\r\n", 294 | "T\r\n", 295 | "\r\n", 296 | "\r\n", 297 | "2->5\r\n", 298 | "\r\n", 299 | "\r\n", 300 | "\r\n", 301 | "\r\n", 302 | "6\r\n", 303 | "\r\n", 304 | "Var\r\n", 305 | "\r\n", 306 | "\r\n", 307 | "5->6\r\n", 308 | "\r\n", 309 | "\r\n", 310 | "\r\n", 311 | "\r\n", 312 | "\r\n" 313 | ], 314 | "text/plain": [ 315 | "" 316 | ] 317 | }, 318 | "execution_count": 9, 319 | "metadata": {}, 320 | "output_type": "execute_result" 321 | } 322 | ], 323 | "source": [ 324 | "model = LinearModel(9, 3)\n", 325 | "cost = nn.MSELoss()\n", 326 | "pred = model(x)\n", 327 | "loss = cost(pred, y.view(1, -1))\n", 328 | "torchviz.draw(loss)" 329 | ] 330 | }, 331 | { 332 | "cell_type": "markdown", 333 | "metadata": {}, 334 | "source": [ 335 | "# Neural Networks" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": 10, 341 | "metadata": {}, 342 | "outputs": [], 343 | "source": [ 344 | "class NeuralNework(nn.Module):\n", 345 | " def __init__(self, x, y):\n", 346 | " super(NeuralNework, self).__init__()\n", 347 | " hidden = int(x/2)\n", 348 | " self.layer1 = nn.Linear(x, hidden)\n", 349 | " self.layer2 = nn.Linear(hidden, hidden)\n", 350 | " self.output = nn.Linear(hidden, y)\n", 351 | "\n", 352 | " def forward(self, x):\n", 353 | " x = F.relu(self.layer1(x))\n", 354 | " x = F.relu(self.layer2(x))\n", 355 | " x = self.output(x)\n", 356 | " return F.softmax(x, dim=1)" 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": 11, 362 | "metadata": {}, 363 | "outputs": [ 364 | { 365 | "data": { 366 | "image/svg+xml": [ 367 | "\r\n", 368 | "\r\n", 370 | "\r\n", 372 | "\r\n", 373 | "\r\n", 375 | "\r\n", 376 | "g\r\n", 377 | "\r\n", 378 | "\r\n", 379 | "0\r\n", 380 | "\r\n", 381 | "Mse\r\n", 382 | "\r\n", 383 | "\r\n", 384 | "1\r\n", 385 | "\r\n", 386 | "Softmax\r\n", 387 | "\r\n", 388 | "\r\n", 389 | "0->1\r\n", 390 | "\r\n", 391 | "\r\n", 392 | "\r\n", 393 | "\r\n", 394 | "2\r\n", 395 | "\r\n", 396 | "Addmm\r\n", 397 | "\r\n", 398 | "\r\n", 399 | "1->2\r\n", 400 | "\r\n", 401 | "\r\n", 402 | "\r\n", 403 | "\r\n", 404 | "3\r\n", 405 | "\r\n", 406 | "Var\r\n", 407 | "\r\n", 408 | "\r\n", 409 | "2->3\r\n", 410 | "\r\n", 411 | "\r\n", 412 | "\r\n", 413 | "\r\n", 414 | "4\r\n", 415 | "\r\n", 416 | "Relu\r\n", 417 | "\r\n", 418 | "\r\n", 419 | "2->4\r\n", 420 | "\r\n", 421 | "\r\n", 422 | "\r\n", 423 | "\r\n", 424 | "15\r\n", 425 | "\r\n", 426 | "T\r\n", 427 | "\r\n", 428 | "\r\n", 429 | "2->15\r\n", 430 | "\r\n", 431 | "\r\n", 432 | "\r\n", 433 | "\r\n", 434 | "5\r\n", 435 | "\r\n", 436 | "Addmm\r\n", 437 | "\r\n", 438 | "\r\n", 439 | "4->5\r\n", 440 | "\r\n", 441 | "\r\n", 442 | "\r\n", 443 | "\r\n", 444 | "6\r\n", 445 | "\r\n", 446 | "Var\r\n", 447 | "\r\n", 448 | "\r\n", 449 | "5->6\r\n", 450 | "\r\n", 451 | "\r\n", 452 | "\r\n", 453 | "\r\n", 454 | "7\r\n", 455 | "\r\n", 456 | "Relu\r\n", 457 | "\r\n", 458 | "\r\n", 459 | "5->7\r\n", 460 | "\r\n", 461 | "\r\n", 462 | "\r\n", 463 | "\r\n", 464 | "13\r\n", 465 | "\r\n", 466 | "T\r\n", 467 | "\r\n", 468 | "\r\n", 469 | "5->13\r\n", 470 | "\r\n", 471 | "\r\n", 472 | "\r\n", 473 | "\r\n", 474 | "8\r\n", 475 | "\r\n", 476 | "Addmm\r\n", 477 | "\r\n", 478 | "\r\n", 479 | "7->8\r\n", 480 | "\r\n", 481 | "\r\n", 482 | "\r\n", 483 | "\r\n", 484 | "9\r\n", 485 | "\r\n", 486 | "Var\r\n", 487 | "\r\n", 488 | "\r\n", 489 | "8->9\r\n", 490 | "\r\n", 491 | "\r\n", 492 | "\r\n", 493 | "\r\n", 494 | "10\r\n", 495 | "\r\n", 496 | "Const\r\n", 497 | "\r\n", 498 | "\r\n", 499 | "8->10\r\n", 500 | "\r\n", 501 | "\r\n", 502 | "\r\n", 503 | "\r\n", 504 | "11\r\n", 505 | "\r\n", 506 | "T\r\n", 507 | "\r\n", 508 | "\r\n", 509 | "8->11\r\n", 510 | "\r\n", 511 | "\r\n", 512 | "\r\n", 513 | "\r\n", 514 | "12\r\n", 515 | "\r\n", 516 | "Var\r\n", 517 | "\r\n", 518 | "\r\n", 519 | "11->12\r\n", 520 | "\r\n", 521 | "\r\n", 522 | "\r\n", 523 | "\r\n", 524 | "14\r\n", 525 | "\r\n", 526 | "Var\r\n", 527 | "\r\n", 528 | "\r\n", 529 | "13->14\r\n", 530 | "\r\n", 531 | "\r\n", 532 | "\r\n", 533 | "\r\n", 534 | "16\r\n", 535 | "\r\n", 536 | "Var\r\n", 537 | "\r\n", 538 | "\r\n", 539 | "15->16\r\n", 540 | "\r\n", 541 | "\r\n", 542 | "\r\n", 543 | "\r\n", 544 | "\r\n" 545 | ], 546 | "text/plain": [ 547 | "" 548 | ] 549 | }, 550 | "execution_count": 11, 551 | "metadata": {}, 552 | "output_type": "execute_result" 553 | } 554 | ], 555 | "source": [ 556 | "model = NeuralNework(9, 3)\n", 557 | "cost = nn.MSELoss()\n", 558 | "pred = model(x)\n", 559 | "loss = cost(pred, y.view(1, -1))\n", 560 | "torchviz.draw(loss)" 561 | ] 562 | }, 563 | { 564 | "cell_type": "markdown", 565 | "metadata": {}, 566 | "source": [ 567 | "# Convolutional Neural Network" 568 | ] 569 | }, 570 | { 571 | "cell_type": "code", 572 | "execution_count": 12, 573 | "metadata": {}, 574 | "outputs": [], 575 | "source": [ 576 | "from torch.utils.data import DataLoader\n", 577 | "from torch.utils.data.dataset import Dataset\n", 578 | "from torchvision import datasets, transforms\n", 579 | "\n", 580 | "digits = datasets.MNIST('data', train=True, download=True,\n", 581 | " transform=transforms.Compose([\n", 582 | " transforms.ToTensor(),\n", 583 | " transforms.Lambda(lambda x: x.view(28*28))\n", 584 | " ]),\n", 585 | " target_transform=transforms.Compose([\n", 586 | " transforms.Lambda(lambda y: \n", 587 | " torch.zeros(10, dtype=torch.float).scatter_(0, torch.tensor(y), value=1))\n", 588 | " ])\n", 589 | " )" 590 | ] 591 | }, 592 | { 593 | "cell_type": "code", 594 | "execution_count": 13, 595 | "metadata": {}, 596 | "outputs": [], 597 | "source": [ 598 | "class CNN(nn.Module):\n", 599 | " def __init__(self):\n", 600 | " super(CNN, self).__init__()\n", 601 | " self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n", 602 | " self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n", 603 | " self.conv2_drop = nn.Dropout2d()\n", 604 | " self.fc1 = nn.Linear(320, 50)\n", 605 | " self.fc2 = nn.Linear(50, 10)\n", 606 | "\n", 607 | " def forward(self, x):\n", 608 | " x = x.view(-1, 1, 28, 28)\n", 609 | " x = F.relu(F.max_pool2d(self.conv1(x), 2))\n", 610 | " x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n", 611 | " x = x.view(-1, 320)\n", 612 | " x = F.relu(self.fc1(x))\n", 613 | " x = F.dropout(x, training=self.training)\n", 614 | " x = self.fc2(x)\n", 615 | " return F.softmax(x, dim=1)" 616 | ] 617 | }, 618 | { 619 | "cell_type": "code", 620 | "execution_count": 14, 621 | "metadata": {}, 622 | "outputs": [ 623 | { 624 | "data": { 625 | "image/svg+xml": [ 626 | "\r\n", 627 | "\r\n", 629 | "\r\n", 631 | "\r\n", 632 | "\r\n", 634 | "\r\n", 635 | "g\r\n", 636 | "\r\n", 637 | "\r\n", 638 | "0\r\n", 639 | "\r\n", 640 | "Binary\r\n", 641 | "\r\n", 642 | "\r\n", 643 | "1\r\n", 644 | "\r\n", 645 | "Softmax\r\n", 646 | "\r\n", 647 | "\r\n", 648 | "0->1\r\n", 649 | "\r\n", 650 | "\r\n", 651 | "\r\n", 652 | "\r\n", 653 | "2\r\n", 654 | "\r\n", 655 | "Addmm\r\n", 656 | "\r\n", 657 | "\r\n", 658 | "1->2\r\n", 659 | "\r\n", 660 | "\r\n", 661 | "\r\n", 662 | "\r\n", 663 | "3\r\n", 664 | "\r\n", 665 | "Var\r\n", 666 | "\r\n", 667 | "\r\n", 668 | "2->3\r\n", 669 | "\r\n", 670 | "\r\n", 671 | "\r\n", 672 | "\r\n", 673 | "4\r\n", 674 | "\r\n", 675 | "Mul\r\n", 676 | "\r\n", 677 | "\r\n", 678 | "2->4\r\n", 679 | "\r\n", 680 | "\r\n", 681 | "\r\n", 682 | "\r\n", 683 | "25\r\n", 684 | "\r\n", 685 | "T\r\n", 686 | "\r\n", 687 | "\r\n", 688 | "2->25\r\n", 689 | "\r\n", 690 | "\r\n", 691 | "\r\n", 692 | "\r\n", 693 | "5\r\n", 694 | "\r\n", 695 | "Relu\r\n", 696 | "\r\n", 697 | "\r\n", 698 | "4->5\r\n", 699 | "\r\n", 700 | "\r\n", 701 | "\r\n", 702 | "\r\n", 703 | "24\r\n", 704 | "\r\n", 705 | "Const\r\n", 706 | "\r\n", 707 | "\r\n", 708 | "4->24\r\n", 709 | "\r\n", 710 | "\r\n", 711 | "\r\n", 712 | "\r\n", 713 | "6\r\n", 714 | "\r\n", 715 | "Addmm\r\n", 716 | "\r\n", 717 | "\r\n", 718 | "5->6\r\n", 719 | "\r\n", 720 | "\r\n", 721 | "\r\n", 722 | "\r\n", 723 | "7\r\n", 724 | "\r\n", 725 | "Var\r\n", 726 | "\r\n", 727 | "\r\n", 728 | "6->7\r\n", 729 | "\r\n", 730 | "\r\n", 731 | "\r\n", 732 | "\r\n", 733 | "8\r\n", 734 | "\r\n", 735 | "View\r\n", 736 | "\r\n", 737 | "\r\n", 738 | "6->8\r\n", 739 | "\r\n", 740 | "\r\n", 741 | "\r\n", 742 | "\r\n", 743 | "22\r\n", 744 | "\r\n", 745 | "T\r\n", 746 | "\r\n", 747 | "\r\n", 748 | "6->22\r\n", 749 | "\r\n", 750 | "\r\n", 751 | "\r\n", 752 | "\r\n", 753 | "9\r\n", 754 | "\r\n", 755 | "Relu\r\n", 756 | "\r\n", 757 | "\r\n", 758 | "8->9\r\n", 759 | "\r\n", 760 | "\r\n", 761 | "\r\n", 762 | "\r\n", 763 | "10\r\n", 764 | "\r\n", 765 | "Max\r\n", 766 | "\r\n", 767 | "\r\n", 768 | "9->10\r\n", 769 | "\r\n", 770 | "\r\n", 771 | "\r\n", 772 | "\r\n", 773 | "11\r\n", 774 | "\r\n", 775 | "Mul\r\n", 776 | "\r\n", 777 | "\r\n", 778 | "10->11\r\n", 779 | "\r\n", 780 | "\r\n", 781 | "\r\n", 782 | "\r\n", 783 | "12\r\n", 784 | "\r\n", 785 | "Mkldnn\r\n", 786 | "\r\n", 787 | "\r\n", 788 | "11->12\r\n", 789 | "\r\n", 790 | "\r\n", 791 | "\r\n", 792 | "\r\n", 793 | "21\r\n", 794 | "\r\n", 795 | "Const\r\n", 796 | "\r\n", 797 | "\r\n", 798 | "11->21\r\n", 799 | "\r\n", 800 | "\r\n", 801 | "\r\n", 802 | "\r\n", 803 | "13\r\n", 804 | "\r\n", 805 | "Relu\r\n", 806 | "\r\n", 807 | "\r\n", 808 | "12->13\r\n", 809 | "\r\n", 810 | "\r\n", 811 | "\r\n", 812 | "\r\n", 813 | "19\r\n", 814 | "\r\n", 815 | "Var\r\n", 816 | "\r\n", 817 | "\r\n", 818 | "12->19\r\n", 819 | "\r\n", 820 | "\r\n", 821 | "\r\n", 822 | "\r\n", 823 | "20\r\n", 824 | "\r\n", 825 | "Var\r\n", 826 | "\r\n", 827 | "\r\n", 828 | "12->20\r\n", 829 | "\r\n", 830 | "\r\n", 831 | "\r\n", 832 | "\r\n", 833 | "14\r\n", 834 | "\r\n", 835 | "Max\r\n", 836 | "\r\n", 837 | "\r\n", 838 | "13->14\r\n", 839 | "\r\n", 840 | "\r\n", 841 | "\r\n", 842 | "\r\n", 843 | "15\r\n", 844 | "\r\n", 845 | "Mkldnn\r\n", 846 | "\r\n", 847 | "\r\n", 848 | "14->15\r\n", 849 | "\r\n", 850 | "\r\n", 851 | "\r\n", 852 | "\r\n", 853 | "16\r\n", 854 | "\r\n", 855 | "Const\r\n", 856 | "\r\n", 857 | "\r\n", 858 | "15->16\r\n", 859 | "\r\n", 860 | "\r\n", 861 | "\r\n", 862 | "\r\n", 863 | "17\r\n", 864 | "\r\n", 865 | "Var\r\n", 866 | "\r\n", 867 | "\r\n", 868 | "15->17\r\n", 869 | "\r\n", 870 | "\r\n", 871 | "\r\n", 872 | "\r\n", 873 | "18\r\n", 874 | "\r\n", 875 | "Var\r\n", 876 | "\r\n", 877 | "\r\n", 878 | "15->18\r\n", 879 | "\r\n", 880 | "\r\n", 881 | "\r\n", 882 | "\r\n", 883 | "23\r\n", 884 | "\r\n", 885 | "Var\r\n", 886 | "\r\n", 887 | "\r\n", 888 | "22->23\r\n", 889 | "\r\n", 890 | "\r\n", 891 | "\r\n", 892 | "\r\n", 893 | "26\r\n", 894 | "\r\n", 895 | "Var\r\n", 896 | "\r\n", 897 | "\r\n", 898 | "25->26\r\n", 899 | "\r\n", 900 | "\r\n", 901 | "\r\n", 902 | "\r\n", 903 | "\r\n" 904 | ], 905 | "text/plain": [ 906 | "" 907 | ] 908 | }, 909 | "execution_count": 14, 910 | "metadata": {}, 911 | "output_type": "execute_result" 912 | } 913 | ], 914 | "source": [ 915 | "model = CNN()\n", 916 | "cost = torch.nn.BCELoss()\n", 917 | "pred = model(digits[0][0])\n", 918 | "loss = cost(pred, digits[0][1].view(1, -1))\n", 919 | "torchviz.draw(loss)" 920 | ] 921 | }, 922 | { 923 | "cell_type": "code", 924 | "execution_count": null, 925 | "metadata": {}, 926 | "outputs": [], 927 | "source": [] 928 | }, 929 | { 930 | "cell_type": "code", 931 | "execution_count": null, 932 | "metadata": {}, 933 | "outputs": [], 934 | "source": [] 935 | } 936 | ], 937 | "metadata": { 938 | "file_extension": ".py", 939 | "kernelspec": { 940 | "display_name": "Python 3", 941 | "language": "python", 942 | "name": "python3" 943 | }, 944 | "language_info": { 945 | "codemirror_mode": { 946 | "name": "ipython", 947 | "version": 3 948 | }, 949 | "file_extension": ".py", 950 | "mimetype": "text/x-python", 951 | "name": "python", 952 | "nbconvert_exporter": "python", 953 | "pygments_lexer": "ipython3", 954 | "version": "3.6.10" 955 | }, 956 | "mimetype": "text/x-python", 957 | "name": "python", 958 | "npconvert_exporter": "python", 959 | "pygments_lexer": "ipython3", 960 | "version": 3 961 | }, 962 | "nbformat": 4, 963 | "nbformat_minor": 2 964 | } 965 | -------------------------------------------------------------------------------- /kernels.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import math\n", 11 | "import shutil\n", 12 | "import numpy as np\n", 13 | "from PIL import Image\n", 14 | "import matplotlib.pyplot as plt" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "# Utility\n", 22 | "Resize and crop image to be square" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "def open_and_resize(file, resize=(500,500)):\n", 32 | " image = Image.open(file).convert('RGB')\n", 33 | " w, h = image.size\n", 34 | " left = int((w - h) / 2 if w > h else 0)\n", 35 | " upper = int((h - w) / 2 if h > w else 0)\n", 36 | " right = int(w - ((w - h) / 2) if w > h else w)\n", 37 | " lower = int(h - ((h - w) / 2) if h > w else h)\n", 38 | " image = image.crop((left, upper, right, lower))\n", 39 | " image.thumbnail(resize)\n", 40 | " return image" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "# Convolution\n", 48 | "Implements filtering on a single image with selected pad and stride" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "def conv2d(X, W, pad=1, stride=1):\n", 58 | " # filter\\kernel size\n", 59 | " f, f, _ = W.shape\n", 60 | " n_C = 1\n", 61 | " \n", 62 | " # new output volume\n", 63 | " n_H = int(np.floor(X.shape[0] - f + 2 * pad) / (stride * 1.)) + 1\n", 64 | " n_W = int(np.floor(X.shape[1] - f + 2 * pad) / (stride * 1.)) + 1\n", 65 | " \n", 66 | " Z = np.zeros((n_H, n_W, n_C))\n", 67 | " \n", 68 | " # padding\n", 69 | " x = np.pad(X, ((pad, pad), (pad, pad), (0, 0)), 'constant', constant_values=(0, 0))\n", 70 | " for h in range(n_H):\n", 71 | " for w in range(n_W):\n", 72 | " for c in range(n_C):\n", 73 | " vert_start = h * stride\n", 74 | " vert_end = vert_start + f\n", 75 | " horiz_start = w * stride\n", 76 | " horiz_end = horiz_start + f\n", 77 | "\n", 78 | " Z[h, w, c] = np.sum(W[:,:,c] * x[vert_start:vert_end,horiz_start:horiz_end,:])\n", 79 | " return Z" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "# Max Pooling\n", 87 | "Implements max pooling with the given pool size and stride." 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "def max_pooling2d(X, pool_size=2, stride=2):\n", 97 | " # new output volume\n", 98 | " n_H = int(np.floor(X.shape[0] - pool_size) / (stride * 1.)) + 1\n", 99 | " n_W = int(np.floor(X.shape[1] - pool_size) / (stride * 1.)) + 1\n", 100 | " n_C = X.shape[2]\n", 101 | " \n", 102 | " Z = np.zeros((n_H, n_W, n_C))\n", 103 | " for h in range(n_H):\n", 104 | " for w in range(n_W):\n", 105 | " for c in range(n_C):\n", 106 | " vert_start = h * stride\n", 107 | " vert_end = vert_start + pool_size\n", 108 | " horiz_start = w * stride\n", 109 | " horiz_end = horiz_start + pool_size\n", 110 | "\n", 111 | " Z[h, w, c] = np.amax(X[vert_start:vert_end,horiz_start:horiz_end,c])\n", 112 | " \n", 113 | " return Z" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "# Filter\\Kernel" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "w = np.zeros((3, 3, 3))\n", 130 | "t = np.array([[1, 1, 1], [0, 0, 0], [-1, -1, -1]]) / 3\n", 131 | "#t = np.array([[0, 0, 0], [0, 1., 0], [0, 0, 0]]) / 3\n", 132 | "w[:,:,0] = t\n", 133 | "w[:,:,1] = t\n", 134 | "w[:,:,2] = t" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "# Example Filter Application\n", 142 | "Run filter on selected image" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "image = np.array(open_and_resize(\"wedding.jpg\", resize=(400,400)))\n", 152 | "plt.imshow(image)" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "n = conv2d(image, w)\n", 162 | "plt.imshow(Image.fromarray(n[:,:,0]).convert(\"L\"), cmap=\"gray\")" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": null, 168 | "metadata": {}, 169 | "outputs": [], 170 | "source": [ 171 | "x = max_pooling2d(n, pool_size=5)\n", 172 | "plt.imshow(Image.fromarray(x[:,:,0]).convert(\"L\"), cmap='gray')" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [] 181 | } 182 | ], 183 | "metadata": { 184 | "kernelspec": { 185 | "display_name": "Python 3", 186 | "language": "python", 187 | "name": "python3" 188 | }, 189 | "language_info": { 190 | "codemirror_mode": { 191 | "name": "ipython", 192 | "version": 3 193 | }, 194 | "file_extension": ".py", 195 | "mimetype": "text/x-python", 196 | "name": "python", 197 | "nbconvert_exporter": "python", 198 | "pygments_lexer": "ipython3", 199 | "version": "3.6.10" 200 | }, 201 | "varInspector": { 202 | "cols": { 203 | "lenName": 16, 204 | "lenType": 16, 205 | "lenVar": 40 206 | }, 207 | "kernels_config": { 208 | "python": { 209 | "delete_cmd_postfix": "", 210 | "delete_cmd_prefix": "del ", 211 | "library": "var_list.py", 212 | "varRefreshCmd": "print(var_dic_list())" 213 | }, 214 | "r": { 215 | "delete_cmd_postfix": ") ", 216 | "delete_cmd_prefix": "rm(", 217 | "library": "var_list.r", 218 | "varRefreshCmd": "cat(var_dic_list()) " 219 | } 220 | }, 221 | "types_to_exclude": [ 222 | "module", 223 | "function", 224 | "builtin_function_or_method", 225 | "instance", 226 | "_Feature" 227 | ], 228 | "window_display": false 229 | } 230 | }, 231 | "nbformat": 4, 232 | "nbformat_minor": 2 233 | } 234 | -------------------------------------------------------------------------------- /kids.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sethjuarez/DeepLearningWithPyTorch/81b8f58a27494ea41178fd667dcc7811de33d75d/kids.jpg -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class Logistic(nn.Module): 6 | def __init__(self): 7 | super(Logistic, self).__init__() 8 | self.layer1 = nn.Linear(28*28, 10) 9 | 10 | def forward(self, x): 11 | x = self.layer1(x) 12 | return F.softmax(x, dim=1) 13 | 14 | class NeuralNework(nn.Module): 15 | def __init__(self): 16 | super(NeuralNework, self).__init__() 17 | self.layer1 = nn.Linear(28*28, 512) 18 | self.layer2 = nn.Linear(512, 512) 19 | self.output = nn.Linear(512, 10) 20 | 21 | def forward(self, x): 22 | x = F.relu(self.layer1(x)) 23 | x = F.relu(self.layer2(x)) 24 | x = self.output(x) 25 | return F.softmax(x, dim=1) 26 | 27 | class CNN(nn.Module): 28 | def __init__(self): 29 | super(CNN, self).__init__() 30 | self.conv1 = nn.Conv2d(1, 10, kernel_size=5) 31 | self.conv2 = nn.Conv2d(10, 20, kernel_size=5) 32 | self.conv2_drop = nn.Dropout2d() 33 | self.fc1 = nn.Linear(320, 50) 34 | self.fc2 = nn.Linear(50, 10) 35 | 36 | def forward(self, x): 37 | x = x.view(-1, 1, 28, 28) 38 | x = F.relu(F.max_pool2d(self.conv1(x), 2)) 39 | x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) 40 | x = x.view(-1, 320) 41 | x = F.relu(self.fc1(x)) 42 | x = F.dropout(x, training=self.training) 43 | x = self.fc2(x) 44 | return F.softmax(x, dim=1) 45 | 46 | if __name__ == '__main__': 47 | a = Logistic() 48 | b = NeuralNework() 49 | c = CNN() -------------------------------------------------------------------------------- /powerpoint/9squares.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sethjuarez/DeepLearningWithPyTorch/81b8f58a27494ea41178fd667dcc7811de33d75d/powerpoint/9squares.png -------------------------------------------------------------------------------- /powerpoint/DeepLearningwithPyTorch.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sethjuarez/DeepLearningWithPyTorch/81b8f58a27494ea41178fd667dcc7811de33d75d/powerpoint/DeepLearningwithPyTorch.pptx -------------------------------------------------------------------------------- /pytorchmnist.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | 13 | - pip: 14 | # Required packages for AzureML execution, history, and data preparation. 15 | - azureml-defaults 16 | - numpy 17 | - torch 18 | -------------------------------------------------------------------------------- /score.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | import torch 4 | import datetime 5 | import numpy as np 6 | from models import CNN 7 | from io import StringIO 8 | 9 | from azureml.core.model import Model 10 | 11 | def init(): 12 | global model, device 13 | 14 | try: 15 | model_path = Model.get_model_path('pytorch_mnist') 16 | except: 17 | model_path = 'model.pth' 18 | 19 | device = torch.device('cpu') 20 | 21 | model = CNN() 22 | model.load_state_dict(torch.load(model_path, map_location=device)) 23 | model.to(device) 24 | model.eval() 25 | 26 | print('Initialized model "{}" at {}'.format(model_path, datetime.datetime.now())) 27 | 28 | def run(raw_data): 29 | prev_time = time.time() 30 | 31 | post = json.loads(raw_data) 32 | 33 | # load and normalize image 34 | image = np.loadtxt(StringIO(post['image']), delimiter=',') / 255. 35 | 36 | # run model 37 | with torch.no_grad(): 38 | x = torch.from_numpy(image).float().to(device) 39 | pred = model(x).detach().numpy()[0] 40 | 41 | # get timing 42 | current_time = time.time() 43 | inference_time = datetime.timedelta(seconds=current_time - prev_time) 44 | 45 | payload = { 46 | 'time': inference_time.total_seconds(), 47 | 'prediction': int(np.argmax(pred)), 48 | 'scores': pred.tolist() 49 | } 50 | 51 | return payload 52 | 53 | if __name__ == "__main__": 54 | img = '0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,232,39,0,0,0,0,0,0,0,0,0,62,81,0,0,0,0,0,0,0,0,0,0,0,0,0,0,120,180,39,0,0,0,0,0,0,0,0,0,126,163,0,0,0,0,0,0,0,0,0,0,0,0,0,2,153,210,40,0,0,0,0,0,0,0,0,0,220,163,0,0,0,0,0,0,0,0,0,0,0,0,0,27,254,162,0,0,0,0,0,0,0,0,0,0,222,163,0,0,0,0,0,0,0,0,0,0,0,0,0,183,254,125,0,0,0,0,0,0,0,0,0,46,245,163,0,0,0,0,0,0,0,0,0,0,0,0,0,198,254,56,0,0,0,0,0,0,0,0,0,120,254,163,0,0,0,0,0,0,0,0,0,0,0,0,23,231,254,29,0,0,0,0,0,0,0,0,0,159,254,120,0,0,0,0,0,0,0,0,0,0,0,0,163,254,216,16,0,0,0,0,0,0,0,0,0,159,254,67,0,0,0,0,0,0,0,0,0,14,86,178,248,254,91,0,0,0,0,0,0,0,0,0,0,159,254,85,0,0,0,47,49,116,144,150,241,243,234,179,241,252,40,0,0,0,0,0,0,0,0,0,0,150,253,237,207,207,207,253,254,250,240,198,143,91,28,5,233,250,0,0,0,0,0,0,0,0,0,0,0,0,119,177,177,177,177,177,98,56,0,0,0,0,0,102,254,220,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,169,254,137,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,169,254,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,169,254,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,169,255,94,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,169,254,96,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,169,254,153,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,169,255,153,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,96,254,153,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0' 55 | data = { 56 | 'image': img 57 | } 58 | 59 | init() 60 | out = run(json.dumps(data)) 61 | print(out) -------------------------------------------------------------------------------- /simple.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import math\n", 11 | "import shutil\n", 12 | "import numpy as np\n", 13 | "from PIL import Image\n", 14 | "import matplotlib.pyplot as plt\n", 15 | "from utils.draw import draw_single" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 2, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "def generate(count):\n", 25 | " X = np.random.randint(0, high=255, size=(count, 9))\n", 26 | " Y = X.dot(np.array([1, 1, 1, 0, 0, 0, -1, -1, -1]))\n", 27 | " Y[Y > 0] = 1\n", 28 | " Y[Y < 0] = -1\n", 29 | " return X, Y" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 3, 35 | "metadata": {}, 36 | "outputs": [ 37 | { 38 | "data": { 39 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/4AAAGRCAYAAADcoWhrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZjVdf3//8eL2ZmN2VgHGBBlXHBBWtTcutzST34slzJzz8w0XMr0k6JCftKstEjN1LBM0szUbHHLzF2TUQlIR1BBHGQZUGAQGJb37w8539+cc6Cc52vgnHl+7rfr4ro6p3m83s/nvF7vc86TQQhJkggAAAAAAPjUJ9cFAAAAAACArYfBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcCzng38IYZcQwsMhhPYQQpLreqxCCOeEEKaFENaGEH6Z63qsvPSR4uh8uehD8tGLl/vESx8pHs6W5KcPyU8vXvqQfPTi5bXLSx+Su156/T0i+dkTL32k5PJ85Xzwl7RO0t2STs91IZEWSLpS0pRcFxLJSx8pXs6Xlz4kH714uU+89JHi4WxJfvqQ/PTipQ/JRy9eXru89CH56sXDPSL52RMvfaTk7HwVbusLZkqSpFVSawhhVK5riZEkyb2SFEIYJ6kxx+WYeekjxdH5ctGH5KMXL/eJlz5SPJwtyU8fkp9evPQh+ejFy2uXlz4kd730+ntE8rMnXvpIyeX5yoef+AMAAAAAgK2EwR8AAAAAAMe2+eAfQjghhNCx6deD2/r68M3L+fLSh+SrF+QXL2fLSx+Sn1689CH56gXYGrhHsDXl0/na5v+Nf5IkUyVN3dbXxf8NXs6Xlz4kX70gv3g5W176kPz04qUPyVcvwNbAPYKtKZ/OV87/cr8QQpBUIql40+NSSUmSJGtzWlg3hRAK9eH3s0BSwaY+1idJsj63lXWPlz5SHJ0vF31IPnrxcp946SPFw9mS/PQh+enFSx+Sj168vHZ56UNy10uvv0ckP3vipY+UXJ6vfPhv/IdLWi1p1qbHqyW15q4cs0v1Ye0XS/rypv99aU4rsvHSR4qX8+WlD8lHL17uEy99pHg4W5KfPiQ/vXjpQ/LRi5fXLi99SL568XCPSH72xEsfKTk7XyFJkm1xHQAAAAAAkAP58BN/AAAAAACwlTD4AwAAAADgGIM/AAAAAACOMfgDAAAAAOBYt/45v8LCwqSoqMh0oZ133tmUS9m4caM5+/LLL7cnSdKQehxCMP+NhjU1NeY6JKmxsdGcnTFjRlofUlwvxcXF5lok+55s2LBBGzduDF2fq6+vT4YNG2Zab9asWf/5i/6NMWPGmLMtLS1pe1JdXZ0MGDDAtFbMGZekqqoqczbzHpGksrKypLq6epvXIkmrVq0yZxcsWJDWS11dXTJ06FDTWvPnzzfXIUnLli2LiWftSW1tbTJkyBDTYjNnzoypRRUVFeZsR0dHWi/FxcVJWVmZaa0PPvjAXIck7brrrubsSy+9lLUndXV15teuV155xVyLJDU1NZmzc+fOTeulvr4+sa63evVqcx2S9Pbbb5uzmWdLkvr165cMGjTItF55ebm5Fkn65z//ac6uW7eux/Zk3bp15jok6b333jNn58+fn7UnxcXFSWlpqWm92Ht+xIgRptzixYu1fPnytM8qZWVlSWVlpWm9/v37m3IpkZ910vaksrIyqa+vNy304b9EZvfWW2/FxLPOVn19fTJ8+HDTYp2dnTG1aM6cOebsmjVr0nopKChICgtt/8p6bB/W92NJWr16dY/OJtbXiZSY78XGjRuz9sQ69+64447mOjZd25zNnE266tYJKyoq0qhRo0xFTJs2zZRL6ejoMGcrKyvnRV28i0MPPTQqf/XVV5uzTU1NPdaHJA0cODAqv2bNGlNuc8PQsGHD9PTTT5vWi/1NpZizGUJI25MBAwbo+uuvN60VM+xK0sEHH2zObu4eqa6u1kknnbTNa5Gk559/3py97LLL0noZOnSoHn30UdNa559/vrkOSZo6dWpMPGtPhgwZogceeMC02MiRI2Nq0bhx48zZv//972m9lJWVaZ999jGt9fLLL5vrkOLOVnFxcdaeDBs2TI8//rhpvdjfSL7iiivM2VNOOSWtl6amJvNr4YwZM8x1SNI555xjzj755JNZezJo0CBNmTLFtN5ee+1lrkWK+839tra2HtuTBQsWmOuQpN///vfm7Pjx47P2pLS0VJ/85CdN6/3jH/8w1yJJ1113nSm3udf/yspKHXPMMab1zjvvPFMuZfTo0THxtD2pr6/XxIkTTQvF/tDo+OOPj4lnna3hw4frhRdeMC0W85uOknTEEUeYs6+99lpaL4WFhRo8eLBprblz55rrkOLO1iuvvNKjs4l1zkyZN89ezsqVK9PCRUVFsv7g6IknnjDXIcX9AC1zNumKP+oPAAAAAIBjDP4AAAAAADjG4A8AAAAAgGMM/gAAAAAAOMbgDwAAAACAYwz+AAAAAAA4xuAPAAAAAIBjDP4AAAAAADjG4A8AAAAAgGMM/gAAAAAAOMbgDwAAAACAYwz+AAAAAAA4xuAPAAAAAIBjhd354u23315/+ctfTBcKIZhyKRdffHFUvqsRI0Zo0qRJpuxBBx0Ude1bb701Kp9pu+22049+9CNTdocddoi69pgxY0y5DRs2ZD03Y8YMNTU1mdbbc889TbmU2LPZVWlpqXbaaSdTdrfddou69ptvvhmVz7Rhwwa9//77puzBBx8cde1DDjkkKt9VYWGhGhoaTNlHH3006to/+clPzNlzzz0367lZs2apubnZtN6sWbPMtUjS4MGDzdmampq0xytXrtRjjz1mWuvuu+821yFJH//4x6PymdavX6+lS5eastb305QDDzzQnD3llFPSHre1tel//ud/TGs9+eST5jokmV8zt3Tt1157TXvvvbdpvfLycnMtknTXXXeZs5/97GfTHi9cuFBXX321aS3rXqbsvvvuUflMI0eONH9v9tprr6hrf+Mb3zDlFixYkPVce3u7brvtNtN6xxxzjCmXcuqpp5qzmTUvX77c/PrT0tJirkOSkiQxZ3vy85okffDBB1H5zs7OHqrkw89cK1euNGWfffbZqGtv7qx/VJs7142NjbrgggtM602ZMsVciyStWLHCnM08X/X19frKV75iWuvmm2821yFJ++yzT1R+S/iJPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOMfgDAAAAAOAYgz8AAAAAAI4x+AMAAAAA4BiDPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOMfgDAAAAAOAYgz8AAAAAAI4x+AMAAAAA4BiDPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOFXbni+fOnauTTjrJdKHm5mZTLmXGjBlR+a5WrVqlf/zjH6bsl7/85ahrP/roo1H5TPPnz9f48eNN2Xnz5kVde8899zTlZs6cmfVcdXW1jjjiCNN6q1atMuVSli5das7W1dWlPS4uLlZjY6NprY0bN5rrkKR+/fpF5TOtW7dO7777ril71VVXRV27sLBbL01p1q9fn/Z49uzZOvTQQ01rPf/88+Y6JOmOO+6IymcaO3aspk2bZsr++Mc/jrr2O++8E5XvqqSkRKNGjTJljzrqqKhrJ0lizoYQsp57/fXXddBBB5nWW7JkibkWSero6IjKd9XZ2Wne49bW1qhrP/vss1H5TDU1NTr44INN2c3tcXfMnz8/Kt9Vv379dOSRR5qyn//856Ouvccee0TlM02fPj3r/fKjirlnJfvnxy9+8YtZz/Xr1898v3/605825VK22247c/a2227Les76meMLX/iCuQ5Juv7666PyPWnMmDFR+Z58P9lxxx113333mdaKnbEyPzfFam9v180332zKXnnllVHXnjx5clS+qxCCioqKTNnYee/CCy+Mym8JP/EHAAAAAMAxBn8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcY/AHAAAAAMAxBn8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcY/AHAAAAAMAxBn8AAAAAABwLSZJ89C8OYYmkeVuvnK1meJIkDakHXvqQem0vXvqQOFv5iD3JP+xJ/mFP8g97kl+89CFxtvIRe5J/3O5JSrcGfwAAAAAA0LvwR/0BAAAAAHCMwR8AAAAAAMcY/AEAAAAAcCyng38I4ZwQwrQQwtoQwi9zWUssL73QR34KIewSQng4hNAeQujVfzGHl17oI794uue97InkoxdPZ0tiT/KRhz2R6CMfeeiF+73n5Pon/gskXSlpSo7r6AleeqGP/LRO0t2STs91IT3ASy/0kV883fNe9kTy0YunsyWxJ/nIw55I9JGPPPTC/d5DCrf1BbtKkuReSQohjJPUmMtaYnnphT7yU5IkrZJaQwijcl1LLC+90Ed+8XTPe9kTyUcvns6WxJ7kIw97ItFHPvLQC/d7z8n1T/wBAAAAAMBWxOAPAAAAAIBjDP5AHgohnBBC6Nj068Fc1xPDSy/0ga3F05546sUL9iT/eNkT+sg/nnrxIp/2JKf/jT+AzUuSZKqkqbmuoyd46YU+sLV42hNPvXjBnuQfL3tCH/nHUy9e5NOe5HTwDyEUbqqhQFJBCKFU0vokSdbnsi4LL73QR34KIQRJJZKKNz0ulZQkSbI2p4UZeOmFPvKLp3vey55IPnrxdLYk9iQfedgTiT7ykYdeuN97Tq7/qP+lklZLuljSlzf970tzWpGdl17oIz8N14c9zNr0eLWk1tyVE8VLL/SRXzzd8172RPLRi6ezJbEn+cjDnkj0kY889ML93kNCkiTb4joAAAAAACAHcv0TfwAAAAAAsBUx+AMAAAAA4BiDPwAAAAAAjjH4AwAAAADgWLf+Ob8QgvlvAqypqbFGo7333nvtSZI0pB7H9BFr8ODB5uyCBQvS+pCkkpKSpG/fvqb1NmzYYK5FkoYPH27KtbW1admyZaHrc0VFRUlpaalpPWsuZenSpeZskiRpexKzH9ttt525Dkl6/fXXzdmVK1dmna26urpk6NChpvUWL15srmVTPeZsR0dHWi8VFRVJbW2taa3+/fub69hUiznb2tqatSd9+/ZNqqurTesNHDjQXMumeszZ1atXp/VSUFCQFBUVmdbaZZddzHVI0ttvv23OLlmyJGtPysvLE+v7W1tbm7kWSRoyZIg529bWltZLfX190tTUZFprxowZ5jokqbOzMybeo/fJsmXLYmrRoEGDzNl58+al9VJbW5s0Njaa1po5c6a5DinutW/RokVZe1JWVpZUVVWZ1ov53CRJs2fPNuXWrl2rdevWpX1W6du3r7mPRYsWmXIpu+22mzk7ffr0rM8q5eXlprWsrxMpffrYf+bY0tKSdbZiXrtWrVplrkWS5s+fb86uWrUqrZfS0tKkoqLCtFbsPTJnzhxzNvP9XfrwM31JSYlpvebmZnMtktTS0hITT+ulqqoqGTBggGkh6/2Vsnr1anP29ddfz9qTlG4N/jEOOuigbXWpLL/73e/mZT5XWGhrff36uH8y8swzzzRnL7/88qw++vbtqwMOOMC0XsxQIkk33XSTKXfUUUdlPVdaWqrdd9/dtN5OO+1kyqXcdttt5uy6devS9qRv37468MADTWvde++95jqkuHvsscceyzpbQ4cO1d/+9jfTetddd525Fkl66qmnzNknnngirZfa2lpdeOGFprXOOecccx2S9Mwzz5iz++67b9aeVFdX69RTTzWtd9FFF5lrkaT999/fnJ0+fXpaL0VFRebfOJw2bZq5DiluT2+44YasPampqdH48eNN68XuyTe+8Q1z9uKLL07rpampyfy9jR0E5s3L+rZ2K575RHV1tU455RTTYr/5zW9iatGECRPM2a985StpvTQ2NuqBBx4wrbXDDjuY65CkE0880Zz94Q9/mLUnVVVVOv74403rTZw40VyLJB122GGm3OZ+Q6uqqkonn3yyab1rrrnGlEv561//as42NDSk7Ul5ebkOPvhg01q33nqruQ5JqqysNGdDCFlnK+a168UXXzTXIknnnXeeOfvss8+m9VJRUaEjjzzStFbsPWK9riS98sorWXtSUlKiMWPGmNZ77rnnzLVIUgjhP3/RlqX1MmDAAF177bWmhfbZZ5+YOqJ+Q/2AAw7Y4psqf9QfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHCrvzxcOHD9fll19uulBnZ6cpl3LxxRdH5bsKIaiwsFut/z+NjY1R1/70pz9tzm7ue19bW6svfvGLpvWsuZRjjjnGlHvjjTeynhs9erSeeuop03oPPPCAKZdy8803R+W76ujo0HPPPWfKhhCirm090/9uvdraWlP29ddfj7p2W1tbVL6rvn37arfddjNl77nnnqhrH3vssVH5TAUFBaqsrDRlv/CFL0Rd+5VXXjFnM892WVmZdt55Z9Najz76qLkOSbrhhhui8pmSJDG/v/3whz+MuvY3v/lNczbzPfWNN94wv6ZfffXV5jokqby83Jw98sgjs54rKChQVVWVab0xY8aYa5Gk008/3Zz9yle+kvZ44cKF+v73v29aK/Yz06RJk8zZzZ3rdevWmV/Xd9ppJ3MtkrRgwYKofFeLFi3SNddcY8quXLky6toVFRVR+a7ef/99/fGPfzRlTz755Khr33fffVH5TGvWrNGrr75qyh522GFR1/7f//1fc/bZZ59Ne9zU1KQpU6aY1lq8eLG5Dkk6+uijzdnNfTaoqKjQfvvtZ1ov9nPXEUccYc7++c9/Tnu8Zs0azZkzx7TWHnvsYa5Dkg444ICo/JbwE38AAAAAABxj8AcAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcY/AHAAAAAMAxBn8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcY/AHAAAAAMAxBn8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHCvszhcvX75cf/zjH00Xuvfee025lF//+tfm7DPPPJP2uLm5WXfccYdprbFjx5rrkKSVK1dG5TMVFxdr2LBhpmySJFHXvuyyy0y5RYsWZT03e/ZsHX744VH1WB1yyCHm7COPPJL2uLGxUZdffrlprfHjx5vrkKSdd97ZnH3++eeznps5c6aam5tN69XU1JhrkaQ5c+ZE5btqbW3V/vvvb8p+73vfi7r2bbfdFpXPtHTpUvNr17/+9a+oay9YsCAq31X//v119tlnm7LLly+PuvanPvUpc/bpp5/Oem7FihV69NFHTeutWLHCXIsk3X///VH5rt5//33z+3Rra2vUtZ966qmofKa2tjZ95zvfMWXnzp0bde2f/exnUfmu3nvvPfMeW18nUkIIUflMgwYN0qWXXmrKnnLKKVHXbmtrM+XGjRuX9dzuu++uxx9/3LReQ0ODKZcyevToqHxXY8aM0UMPPWTK/vSnP426dlVVlTm7udfMWbNmaaeddjKt99e//tVcixT/ftRVS0uL+b675ZZboq49Y8aMqHymiooKfeITnzBl77rrrqhrx7wGZ85TbW1tuuiii0xrnX/++eY6JGnJkiXm7L97reEn/gAAAAAAOMbgDwAAAACAYwz+AAAAAAA4xuAPAAAAAIBjDP4AAAAAADjG4A8AAAAAgGMM/gAAAAAAOMbgDwAAAACAYwz+AAAAAAA4xuAPAAAAAIBjDP4AAAAAADjG4A8AAAAAgGMM/gAAAAAAOMbgDwAAAACAYyFJko/+xSEskTRv65Wz1QxPkqQh9cBLH1Kv7cVLHxJnKx+xJ/mHPck/7En+YU/yi5c+JM5WPmJP8o/bPUnp1uAPAAAAAAB6F/6oPwAAAAAAjjH4AwAAAADgWM4H/xDCLiGEh0MI7SGEXv3fHXjoJYRwTghhWghhbQjhl7mux8pLH5KfXrz0kcL9nn/Yk/zDnuQXL7146SPFw30i+ejD09ny0ouXPlJyeZ/kfPCXtE7S3ZJOz3UhPcBDLwskXSlpSq4LieSlD8lPL176SOF+zz/sSf5hT/KLl1689JHi4T6RfPTh6Wx56cVLHyk5u08Kt/UFMyVJ0iqpNYQwKte1xPLQS5Ik90pSCGGcpMYcl2PmpQ/JTy9e+kjhfs8/7En+YU/yi5devPSR4uE+kXz04elseenFSx8pubxP8uEn/gAAAAAAYCth8AcAAAAAwLFtPviHEE4IIXRs+vXgtr5+T/LUC4B/j/s9/7An+Yc9Af4zL/eJlz6ArSmf7pNt/t/4J0kyVdLUbX3drcFTLwD+Pe73/MOe5B/2BPjPvNwnXvoAtqZ8uk9y/pf7hRCCpBJJxZsel0pKkiRZm9PCDDz0EkIo1IfnokBSwaYe1idJsj63lXWPlz4kP7146SOF+z3/sCf5hz3JL1568dJHiof7RPLRh6ez5aUXL32k5PQ+SZIkp78kNUlKMn7NzXVd/1d7kXTFZnq4Itd1/V/tw1MvXvro0g/3e579Yk/y7xd7kl+/vPTipY8u/fT6+8RLH57OlpdevPTRpZ+c3SdhUwEAAAAAAMAh/lZ/AAAAAAAcY/AHAAAAAMAxBn8AAAAAABxj8AcAAAAAwLFu/XN+/fr1SwYPHmy6UN++fU25lA8++MCcffXVV9uTJGlIPS4tLU3Ky8tNa9XV1ZnrkKS2tjZz9oMPPkjrQ/pwTwYOHGhab+nSpeZaJKm9vd2cTZIkdH0cQkj69LH9PtTGjRvNdUjS6NGjzdnW1ta0PSkpKTGfrYaGhv/8Rf/Gm2++ac6uX78+62yFEMx/82dhYdy/FBrzvXj33XfTeqmvr0+amppMa7W0tJjrkKSxY8easy+99NJm9+TDfwVm24v8i2DTeunTp09iPSPWvUypqqoyZ1taWrL2pKysLKmsrDStt2TJEnMtUtz3Yu7cuWm91NTUJEOGDImqx2rWrFkx8aw9qa2tNfcS81lDkt555x1ztrOzM62XmNfgXXbZxVyHFPf5IPM1WJIKCwuT4uJi03rV1dXmWiT7Z4QVK1Zo9erVaS+45eXlSU1NjWm9999/35Trcm1zdvHixWl7UlFRkVg/z65cudJchyRZz4EkLVq0KOts1dfXJ8OGDTOt9/LLL5trkaRRo0aZs3PmzOmxzyqxZ+uNN96Iiffo58eCgoKYWqLyma/Bufz8OGjQIHN2c6/BKd365DV48GDdcccdpiJiPgBL0rRp08zZj33sY/O6Pi4vL9fhhx9uWuuEE04w1yFJEyZMMGenTZs2L/O5gQMHasqUKab1fvnLX5prkaRbbrklKt9Vnz59VFpaasrGflC7+eabzdn9998/62wddNBBprXOOusscx2SdNxxx5mz7e3tWWcrhvWDUcoZZ5xhzk6aNCmtl6amJvPrR+yQ/dxzz5mzJSUlWXsSQjDfJxs2bDDXIkmdnZ0x8bReCgsLNWDAANNCP/3pT2Pq0KGHHmrOhhCy9qSyslLHHHOMab2bbrrJXIskXXHFFebsKaecktbLkCFDdM8995jWir1PmpubY+JZezJkyBD94Q9/MC0W81lDkr797W+bs/Pmzeux1+H7778/Kv+rX/3KnP3ud7+b1UdxcbG2335703rWz2spq1evNuXuvPPOrOdqamo0fvx403r33nuvKZfyiU98wpydPHly2p7U1dXpkksuMa316KOPmuuQpJEjR5qz11xzTdbZGjZsmJ588knTetbftE259tprzdkjjzyyxz6rWF/vUo466qiYeI9+fqyoqIjK9+vXz5zNfA3O5efHM88805y94oortrgn/FF/AAAAAAAcY/AHAAAAAMAxBn8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcY/AHAAAAAMAxBn8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcK+zOFy9dulR33HGH6UIvvPCCKZfy9a9/PSrf1eDBg3XZZZeZstb+Ux588EFztqGhYbPPb9y40bTeX/7yF3MtknTggQeactOmTct6rrm5WXfeeadpvaOPPtqUS3nyySej8l2tWrVKL774oik7b968qGt3dHRE5TcnhGDKLV68OOq6b775pjk7adKktMezZ8/WYYcdZlrr2GOPNdchSWvXro3KZ6qvr9dxxx1nypaXl0dd+7e//a05m3m2161bp3feece01n333WeuQ7K/Xm7JsGHDdOONN5qy9fX1Udc+5ZRTovJdlZaWqrm52ZQdNWpU1LUvv/xyc3bixIlZzy1atEjXXnutab1LLrnEXIu05ffpjyLzPikvL9fuu+9uWuvdd9811yFJ7e3tUflM69ev13vvvWfKXnXVVVHX3nnnnU25FStWZD03cOBAffvb3zat949//MOUS5k8eXJUvquGhgZ99atfNWV/8YtfRF37uuuuM2evueaarOemT5+ugQMHmtY766yzzLVI0vz586PyXa1cuVJPPPGEKfvKK69EXTvmft/c+9iee+5pnvsKCgrMtUjSHnvsYc5mvgavWbNGr776qmmt8847z1yHJI0YMSIqvyX8xB8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxwq788WLFy/WddddZ7pQc3OzKZfS2tpqzo4ePTrt8ezZs3X44Yeb1pozZ465DkmaOHFiVD5TRUWFPvWpT5myF154YdS1v/a1r5lye++9d9ZzZWVl2nXXXU3rvf/++6ZcymuvvRaV76qzs1Nz5841Zd96662oa8+YMcOc3dz3vri4WIMHDzatd+6555prkaT6+vqofFd1dSzjHKYAACAASURBVHU6+eSTTdkTTjgh6tqTJk2KymcaNmyYrr/+elN2jz32iLr2008/bc4OHTo07XGfPn3Ut29f01o33XSTuQ5JuvPOO6PymaZPn67+/fubsk888UTUtR977DFz9tlnn017vHLlSj3++OOmtW688UZzHVL8e1GmJUuW6IYbbjBlzzjjjKhrv/jii+ZsCCHtcXNzs/m++9KXvmSuQ5LGjx9vzv7sZz/Lem7dunWaP3++ab3nn3/eXIsk7b///qbckiVLsp6bOXOm+fPsiBEjTLmUmD2ZPHly2uOWlpas8/ZRjRs3zlyHJJ122mlR+c1JksSUi33t+tOf/hSV76q4uFiNjY2m7I477hh1beu9uTUUFnZrNM1y1113mbPHHnts2uP29nb94he/MK11xx13mOuQpAkTJkTlt4Sf+AMAAAAA4BiDPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOMfgDAAAAAOAYgz8AAAAAAI4x+AMAAAAA4BiDPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOMfgDAAAAAOAYgz8AAAAAAI4x+AMAAAAA4BiDPwAAAAAAjoUkST76F4ewRNK8rVfOVjM8SZKG1AMvfUi9thcvfUicrXzEnuQf9iT/sCf5hz3JL176kDhb+Yg9yT9u9ySlW4M/AAAAAADoXfij/gAAAAAAOMbgDwAAAACAYzkd/EMI54QQpoUQ1oYQfpnLWmI562WXEMLDIYT2EEKv/G9BPO2HxJ7kI/YkvzjrpdefLcnPnnjpI8XR+er1fXg6W1568dKH5K6XXn+/p+Syl1z/xH+BpCslTclxHT3BUy/rJN0t6fRcFxLB035I7Ek+Yk/yi6dePJwtyc+eeOkjxcv58tCHp7PlpRcvfUi+evFwv6fkrJfCbX3BrpIkuVeSQgjjJDXmspZYznppldQaQhiV61qsPO2HxJ7kI/YkvzjrpdefLcnPnnjpI8XR+er1fXg6W1568dKH5K6XXn+/p+Syl1z/xB8AAAAAAGxFDP4AAAAAADjG4A+FEE4IIXRs+vVgrusBe5KP2BNsLZwtbE1ezpeXPgD8Z57u93zqJaf/jT/yQ5IkUyVNzXUd+P+xJ/mHPcHWwtnC1uTlfHnpA8B/5ul+z6decjr4hxAKN9VQIKkghFAqaX2SJOtzWZeFs16CpBJJxZsel0pKkiRZm9PCusHTfkjsST5iT/KLs156/dmS/OyJlz5SHJ2vXt+Hp7PlpRcvfUjueun193tKTntJkiRnvyRdISnJ+HVFLmuil0SSmjbTy9xc1/V/dT/Yk/z8xZ7k1y9nvfT6s+VpT7z00aUfL+er1/fh6Wx56cVLHw576fX3ez70EjYVAAAAAAAAHOIv9wMAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcY/AHAAAAAMCxwu58cXl5edKvXz/ThRYsWGDKpRQWdqvUNOvXr29PkqQh9bi+vj5pamoyrdXS0mKuY9O1zdn29va0PiSptLQ0qaioMK03fPhwcy2StHjxYlNu2bJlWrVqVej6XGlpaVJeXm5ar6SkxJRLGTx4sDnb0tKStidVVVVJ//79TWutXLnSXIckDR061JzN7EOSiouLk7KysqiarKxnWpIWLFiQ1ktlZWXS0NDw7yJbtH593D91W1VVZc7OmjUra08KCwuT4uJi03qNjY3mWiRp0aJF5uyKFSvSegkhmP85GWv/KaNHjzZnZ8yYkbUnMecr5n1Nijtfmfd8zPv7smXLzHVIce+L77zzTtaexJyvyspKcy2StHr1anM287NKTU1NYn1/6tMn7uc6q1atMmffeuutrD2J+dz11ltvmWuR7J913n77bbW3t6d9Vol5X6yrqzPlUtatW2fOZt4ndXV1ifUzw+zZs811SNKGDRvM2bVr12adrerq6mTAgAGm9dra2sy1bKrHnN2wYUNaLwUFBUlRUZFprebmZnMdkvSvf/3LnF23bl3WnhQVFSXWz+axn7uGDBlizr755ps9Ni/G3K+bajFnOzo6svYkpVufOvr166ezzjrLVMSECRNMuZSYF8xFixbN6/q4qalJ06ZNM60V+2b6+c9/3py9+eab52U+V1FRoSOOOMK03s9//nNzLZJ04403mnLXXXdd1nPl5eX6zGc+Y1pvxIgRplzKd7/7XXM2hJC2J/3799cPfvAD01pPPvmkuQ5p89/XjyqzD0kqKyvTPvvsE1WT1d57723OTpgwIa2XhoYGfe973zOtFTPsStKhhx5qzu64445Ze1JcXGweXL///e+ba5HiztdDDz2U1YvVoEGDovIPPPCAOTtixIisPhoaGjRp0iTTetYPqykHH3ywOZt5z/fr109nnnmmaa3f/e535jok6ZRTTjFnv/Wtb/XY2ZKkcePGReVnzpxpzi5ZsiStl8GDB+u3v/2taa3Y37R94YUXzNkTTjgha09iPnedeOKJ5lok6Wc/+5kpt99++2U9V1ZWZn5/iu0j5v3oggsuSNuToUOH6pFHHjGt9dnPftZchxT3G4Vz5szJOlsDBgwwfx79n//5H3MtUtxvgixfvjytl6KiIvNv0Fv3MmXs2LHmbFtbW9aelJSUaMyYMab12tvbzbVI0tVXX23OHnPMMT02L8b+wPtLX/qSOfvEE09s8X2RP+oPAAAAAIBjDP4AAAAAADjG4A8AAAAAgGMM/gAAAAAAOMbgDwAAAACAYwz+AAAAAAA4xuAPAAAAAIBjDP4AAAAAADjG4A8AAAAAgGMM/gAAAAAAOMbgDwAAAACAYwz+AAAAAAA4xuAPAAAAAIBjhd354pKSEm233XamC915552mXMqrr75qzk6aNCntcZIk6uzsNK3V0tJirkOSxo4dG5XP1NTUpF/96lem7BtvvBF17aKiIlMuhJD1XHV1tQ477DDTehdffLEpl3LllVdG5bt644039PnPf96U/cMf/hB17RUrVkTlM61evVrTp083Zdva2qKuvbkzYlVbW6svfOELOamjJ/uQpFGjRun+++83ZZuamnq0lhh77rmnpk2bZsqedtppUde+5557ovKZKioq9KlPfcqUHTFiRNS1r7rqqqh8V++++66+973vmbLr16+PuvawYcOi8pmKioo0cOBAU/Zvf/tb1LVnzJhhzu66665pj9esWWP+7GN9P0350pe+ZM6ecMIJWc/985//NO9zQUGBuRZJamhoMOXWrl2b9dyKFSv00EMPmdZ78MEHTbmU2bNnm7MXXHBB2uMVK1bor3/9q2mtV155xVyHtPnv60e1uffUefPm6YwzzjCtN3fuXHMt0ofn2irzfq+qqtIhhxxiWqumpsZchyRde+215uzmPl81NzfrueeeM603fPhwcy3Sh9/HnrJ48WL95Cc/MWWte5ly6623mrPbb7/9Fv8/fuIPAAAAAIBjDP4AAAAAADjG4A8AAAAAgGMM/gAAAAAAOMbgDwAAAACAYwz+AAAAAAA4xuAPAAAAAIBjDP4AAAAAADjG4A8AAAAAgGMM/gAAAAAAOMbgDwAAAACAYwz+AAAAAAA4xuAPAAAAAIBjDP4AAAAAADjG4A8AAAAAgGOF3fni2tpaHX/88aYLrV692pRLmThxYlS+q9dee0377LOPKVtRURF17ZUrV5qzlZWVWc+1tLQohGBar6WlxVyLJF1yySWm3KpVq7KeW7Zsme666y7Tes8//7wpl7K57+tH1a9fv7THY8eO1TPPPGNaq6qqylyHJB1yyCFR+UzbbbedbrvtNlP2oIMOirr2U089Zc7uu+++aY9XrlypJ5980rTW9OnTzXVIUp8+9t9b3bhxY9Zza9as0euvv25aL0kScy2S1NzcbM62tramPZ49e7YOP/xw01pPP/20uQ5JWrJkiTl74YUXZj23ePFi3Xjjjab1Yl+D33zzzah8V2PHjtW0adNM2eOOOy7q2rH5TLvuuqtefPFFU3aPPfaIunbs+1FXIQQVFBSYsrHvJzvssENUPlNZWZl22mknU/bHP/5x1LV33HHHqHxXO+ywg2666SZT9tlnn4269h//+MeofFfl5eX62Mc+lpM6rJ9Zt2TMmDHm1y5rLiXm82OmYcOGmd9LYr+nCxcujMpniplNdt111x6tJUZDQ4POPvtsU7aoqCjq2gceeGBUfkv4iT8AAAAAAI4x+AMAAAAA4BiDPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOMfgDAAAAAOAYgz8AAAAAAI4x+AMAAAAA4BiDPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOMfgDAAAAAOAYgz8AAAAAAI4x+AMAAAAA4FhIkuSjf3EISyTN23rlbDXDkyRpSD3w0ofUa3vx0ofE2cpH7En+YU/yD3uSf9iT/OKlD4mzlY/Yk/zjdk9SujX4AwAAAACA3oU/6g8AAAAAgGMM/gAAAAAAOJbzwT+EsEsI4eEQQnsIodf+dwchhHNCCNNCCGtDCL/MdT0xvOyJ5KMXT2dLYk/yjYf9SPHQi6ezJfnYE8lPH5KfXjz0wf2en7z0Ifnohfuk5+R88Je0TtLdkk7PdSGRFki6UtKUXBfSA7zsieSjF09nS2JP8o2H/Ujx0IunsyX52BPJTx+Sn1489MH9np+89CH56IX7pIfkzV/uF0IYJWl2kiQh17XECCFcKakxSZJTcl1LLC97IvnoxdPZktiTfONhP1I89OLpbEk+9kTy04fkpxcPfXC/5ycvfUg+euE+iZcPP/EHAAAAAABbCYM/AAAAAACObfPBP4RwQgihY9OvB7f19ZHN05546sUL9iS/eNoPT7144WVPvPQh+enFSx+eeNkTL31IvnrxIp/2pHBbXzBJkqmSpm7r62LLPO2Jp168YE/yi6f98NSLF172xEsfkp9evPThiZc98dKH5KsXL/JpT7b54J8phBAklUgq3vS4VFKSJMnanBbWTSGEQn34/SyQVLCpj/VJkqzPbWXd52VPJB+9eDpbEnuSbzzsR4qHXjydLcnHnkh++pD89OKhD+73/OSlD8lHL9wnPScf/hv/4ZJWS5q16fFqSa25K8fsUn1Y+8WSvrzpf1+a04rsvOyJ5KMXT2dLYk/yjYf9SPHQi6ezJfnYE8lPH5KfXjz0wf2en7z0Ifnohfukh+TNP+cHAAAAAAB6Xj78xB8AAAAAAGwlDP4AAAAAADjG4A8AAAAAgGMM/gAAAAAAONatf86vtrY2GTJkiOlCixYtMuVSioqKzNkFCxa0J0nSkHocQjD/jYYFBQXmOiRpt912M2dfeumltD4kqa6uLhk6dKhpvTVr1phrkaSKigpTbu7cuWpvbw9dn6upqTGfrfnz55tyKQ0NDf/5i7bgjTfeSNuT0tLSxPp9se5jyptvvmnOrlixIutsVVZWJnV1dab1SktLzbXEam1tTeulqqoqse7x3Llzo2rZuHFjTDxrT8rLy5OamhrTYoWFcf96a319vTnb0tKS1kt9fX3S1NRkWuvtt9821yFJjY2N5uzLL7+ctScx7ye1tbXmWiRp7Vr7v/yzatWqrPukf//+prWKi4vNdUhS3759zdnMsyXFna85c+aYa5GkkSNHmrOZ56usrCyprKw0rTVs2DBzHZI0b948c7a9vT2v9sT63trW1qZly5alfVYpLCxMrJ9HYz9zjR492pzNfF/s169fMnDgQNNasff70qVLzdnMz/NS3NlqaWkx1yJJ1u+hJC1cuDCtl9ra2sT6/hR7tmbPnh0T79HZZPr06TG1aMSIEebsW2+9lXWfDB482LRWzPuaFLens2bNytqTlG59EhwyZIjuvfdeUxGTJ0825VIGDBhgzk6YMMH+Dpahuro6Kv/cc8+ZsyUlJVl9DB06VI888ohpvdbWuH85Yt999zXlxo0bl/XckCFD9Pvf/9603rnnnmvKpXzta18zZz/3uc+l7UlFRYU++9nPmtb6yU9+Yq5Dko499lhz9pFHHsk6W3V1dbr0Utu/ltLc3GyuJda+++6b1ktDQ4Ouvvpq01qnnXZaVC0dHR0x8aw9qampMZ9362/ipMR8L0IIab00NTVp2rRpprW+/vWvm+uQpB/96EfmbN++fXvsvUSSPvOZz0TlYwaiF154Ia2X/v3765prrjGtFTPsStLuu+9uzmaeLSnufB155JHmWiRp6tSp5mxVVVVaL5WVlTruuONMa11//fXmOqS498Wf//znPbon//Vf/2WuRbK/tx511FFZzxUVFWnUqFGm9WbOnGnKpdx6663mbOb74sCBA3XLLbeY1ooZrCTptttuM2cvu+yyHj1bH/5z6nannnqqOXvVVVel9dLY2Kg//elPprVeffVVcx2SdNhhh8XENzub/O1vfzMtFvtZZdKkSebsiSeemNbL4MGDdfvtt5vW2tys0x2vvfaaObvjjjtu8bMKf9QfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHCrvzxW+99ZZOOukk04Vef/11Uy5l6dKl5uyECRPSHjc2Nuq8884zrXXEEUeY65CknXbaKSqfadGiRbruuutM2auvvjrq2g899JApt2LFiqznZs2apebmZtN6RUVFplzKqaeeGpXvqrGxUT/84Q9N2dtvvz3q2h//+MfN2UceeSTruY0bN2rNmjWm9S688EJzLZJUUFAQle+qra1N3/nOd0zZb37zm1HXjnm92Nx+vvfee7rrrrtM6y1btsxcixT/2tfV/Pnzde6555qyP/7xj6OuXVJSEpXPNHbsWD3//POm7K677hp17RkzZpizma+by5Yt0913321aK3ZPQghR+Uwvv/yyKisrTdmOjo6oa8e+ZnS1ZMkS3XDDDabsRRddFHXtr371q+bsz3/+86zn3nnnHX372982rRf7Hr9o0SJTbt26dVnPdXZ2au7cuab1ysrKTLmUDz74ICrfVWtrq/bbbz9TNkmSqGvPnj07Kr85GzduNOVqa2ujrjto0KCofFfFxcUaNmyYKWudaVKmTJlizp522mlZz3V2durtt982rTd48GBzLZJ0wAEHROW7mjdvns4++2xTduLEiVHXnjx5clR+S/iJPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOMfgDAAAAAOAYgz8AAAAAAI4x+AMAAAAA4BiDPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOMfgDAAAAAOAYgz8AAAAAAI4x+AMAAAAA4BiDPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOFXbni0eOHKk777zTdKG99trLlEsJIUTle8qUKVOi8nPmzDFnN/c9aGho0Nlnn21ar7m52VyLJB199NGm3OrVq7OeGzJkiM4991zTehdeeKEplzJ8+PCofFcdHR166qmnTNlzzjkn6to9fY/Mnz9f3/jGN0zZE088Merat99+uzmb+X0oLCzUgAEDTGtdccUV5jok6frrr4/KZyopKdHIkSNN2Zdeeinq2tXV1VH5zLWOOOIIU7apqSnq2pdffrk5O3HixKzn2tvbdcstt5jWe/XVV821SNLhhx8ele8pO++8c1R+6tSp5uwJJ5yQ9Vxtba2OOuoo03oDBw401yJJkydPjsp3VVhYqLq6OlP2kksuibr2hAkTovKZkiRRZ2enKVtVVRV17X322Scq39Xuu++u5557zpQtLi6OuvbSpUuj8l2VlZVp++23N2UXLlwYde1DDjnEnP31r3+d9dy8efP01a9+1bSe9bU75fjjj4/KdzVjxgzz59G333476tpPP/10VD7TmjVrNHPmTFO2ra0t6tpnnHFGVL6rmpoafe5znzNlP/OZz0RdOza/JfzEHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwLCRJ8tG/OIQlkuZtvXK2muFJkjSkHnjpQ+q1vXjpQ+Js5SP2JP+wJ/mHPck/7El+8dKHxNnKR+xJ/nG7JyndGvwBAAAAAEDvwh/1BwAAAADAMQZ/AAAAAAAcy/ngH0LYJYTwcAihPYTQa/+7gxDCOSGEaSGEtSGEX+a6Hiv6yE9e7hPJRy9ezpeXPlI4W/nFWS+9/myleOnFQx/O7hFPvfT6syWxJ/ko13uS88Ff0jpJd0s6PdeFRFog6UpJU3JdSCT6yE9e7hPJRy9ezpeXPlI4W/nFUy8ezlaKl1489OHpHvHUi4ezJbEn+Sine1KYi4t2lSRJq6TWEMKoXNcSI0mSeyUphDBOUmOOyzGjj/zk5T6RfPTi5Xx56SOFs5VfnPXS689WipdePPTh7B7x1EuvP1sSe5KPcr0n+fATfwAAAAAAsJUw+AMAAAAA4Ng2H/xDCCeEEDo2/XpwW18f6A083SeeekF+4Wxha/F0trz04qUP5B/OVv5hT7aObf7f+CdJMlXS1G19XaA38XSfeOoF+YWzha3F09ny0ouXPpB/OFv5hz3ZOnL+l/uFEIKkEknFmx6XSkqSJFmb08K6KYRQqA+/nwWSCjb1sT5JkvW5rax76CM/eblPJB+9eDlfXvpI4WzlF2e99PqzleKlFw99OLtHPPXS68+WxJ7ko5zvSZIkOf0lqUlSkvFrbq7rMvRxxWb6uCLXddFH7+6jSz8u7hMvvXg5X1766NIPZyuPfjnrpdefLW+9eOjD2T3iqZdef7bYk/z8les9CZuKAAAAAAAADvG3+gMAAAAA4BiDPwAAAAAAjjH4AwAAAADgGIM/AAAAAACOdeuf86uoqEhqa2tNF/rggw9MuZSmpiZztqWlpT1JkobU4z59+iQFBQWmtdavj/vXFj781yhskiRJ60OSSkpKkvLyctN6FRUV5lokafny5abc6tWr1dnZmfaNKCsrSyorK03rrVu3zpRLef/992PiaXtSX1+fWM/q4sWLY+ow74ckrVixIutslZWVJdXV1ab1GhsbzbVI0nvvvWfOvvnmm2m9hBDMf4NpSUmJuQ5JGjlypDn76quvZu1JdXV1MnDgQNN6sa/BCxcuNGfXr1+f1ktlZWXS0NDw7yJbZH0PSuno6DBnW1tbs/Yk5p5/6623zLVI0rJly2LiPXafFBcXx9ShESNGmLM9vSctLS3mWiTJeq4lacmSJT22J2PGjDHXIcXtaeZnLimul6FDh5prkaT+/fubcnPnzlV7e3vaZ5Xi4uKkrKzMtN6KFStMuZQ+few/q9u4cWPantTV1SXDhw83rRX7ulVaWmrOLly4sEfP1p577mmuRZLmz59vzi5evDitl5j395j3Zyn6bPboa/Dbb78dU4s6OzvN2eXLl6f1UlBQkBQWdmtU/n8GDx5srkOKu98zPwd31a1uamtrdeGFF5qKePnll025lClTppizIYR5XR8XFBSorq7OtNaiRYvMdUhxb6Zr166dl/lceXm5DjnkENN6e+21l7kWSXr44YdNuWeeeSbrucrKSh1zzDGm9d59911TLuX++++PiaftSVNTk6ZNm2Za6Prrr4+pQ3/5y1/M2QcffDDrbFVXV+ukk04yrXfNNdeYa5Gk3/3ud+bscccdl9WLVexvYPzmN78xZ/fYY4+sPgYOHKibb77ZtN5LL71krkWK29OFCxem9dLQ0KBJkyaZ1vryl79srkOSnn32WXN2n332ydqTmHv+xBNPNNciSVOnTjVnkyTJm/vk1ltvNWf33XffHt2TmN+cl6TjjjvOnL3hhht6bE/+/Oc/R+Vjhu3Mz1yxvvWtb0Xlx48fb8qNGzcu67mysjLzZyfrZ6aUvn37mrMdHR1pezJ8+HA98cQTprVOPvlkcx2SNHr0aHP26quv7tGzZX2dSDn33HPN2cmTJ6f1MnDgQN10002mtX7wgx+Y65CkBx98MCbeo6/BX//612NqifqNgz//+c9pvRQWFpoH+IkTJ5rrkOLu92OPPXaL9wl/1B8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcKu/PF8+fP1/jx400XOuCAA0y5lIMPPjgq31Vpaam23377HluvO0aMGGHOPv/881nP1dbW6vjjjzet99///d/mWiTpvPPOi8p3tWHDBq1YscKUHTJkSNS1J0+ebM5m3g/Tp0/XgAEDTGv17dvXXIckNTc3R+UzLVq0SD/4wQ9M2cMOOyzq2i+++GJUvqva2lodeuihpuwNN9wQde2ampqofKbXX3/d/FpaUFAQde1ly5aZs9XV1WmPOzo6Nvt69lH8/e9/N9chSb/4xS+i8pna2tr0ne98x5Q955xzoq59xx13ROW7qq+v11FHHWXK3nLLLVHXPvPMM6Pymd577z3dfffdpuxuu+0Wde0lS5ZE5bsqKSlRY2OjKXvRRRdFXbu0tDQqn6mkpERDhw41ZWfOnBl17d/85jem3OZe88rLy7XXXnuZ1rvkkktMuZTbb7/dnL311lvTHs+fP1/nn3++aa377rvPXIck3XTTTVH5TA0NDTr66KNN2U9+8pNR1957772j8l2tXLlSTzzxhClr3cuUs846y5w98sgjs55rb2/XbbfdZlpvw4YN5lok6eGHH47Kd9XZ2am5c+easrF7snTp0qj8lvATfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcY/AHAAAAAMAxBn8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/4+YSoQAAAw5JREFUAAAAAAAcY/AHAAAAAMAxBn8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcK+zOFxcVFamhocF0oSFDhphyKWvWrInKd7Vq1Sq9+OKLOanj8ccfN2c//elPZz23evVqzZo1y7TejjvuaK5Fko455hhT7rHHHst6bsSIEbrjjjtM6w0aNMiUS7n//vuj8l0NHz5c1157rSnb3t4ede3TTz89Kp+ppKREjY2Npuzee+8dde3LL788Kt9VZ2en5s+fb8pOnTo16tq77LJLVD5TY2OjLrjgAlP2/PPPj7r2r371q6h8V0uXLtXtt99uylZVVUVd2/q6JUn33HNP1nPLli3TnXfeaVrvnHPOMdciSS+99JI5O3bs2LTH1dXVOuKII0xrfetb3zLXIUm//e1vo/KZKioqtN9++5myF110UdS1V65cGZXvauTIkeb3xc19XuiOfffdNyqfKYSg4uJiUzb2vfFPf/qTKbd8+fKs5wYPHmx+f2pubjblUqyfvzenT58+qqysNGU39zrYHUcffbQ5+7WvfS3rueLiYg0bNsy03qJFi8y1SNLGjRuj8l0tWbJEN910kyn7wQcfRF37mmuuicpnevfddzVp0iRT9u9//3vUtW+++eaofFejR4/WrbfeasrGvm716bN1fjbPT/wBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMcY/AEAAAAAcIzBHwAAAAAAxxj8AQAAAABwjMEfAAAAAADHGPwBAAAAAHCMwR8AAAAAAMdCkiQf/YtDWCJp3tYrZ6sZniRJQ+qBlz6kXtuLlz4kzlY+Yk/yD3uSf9iT/MOe5BcvfUicrXzEnuQft3uS0q3BHwAAAAAA9C78UX8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHGPwBwAAAADAMQZ/AAAAAAAcY/AHAAAAAMAxBn8AAAAAABxj8AcAAAAAwDEGfwAAAAAAHPv/ABk9rPJ4NoSrAAAAAElFTkSuQmCC\n", 40 | "text/plain": [ 41 | "
" 42 | ] 43 | }, 44 | "metadata": {}, 45 | "output_type": "display_data" 46 | } 47 | ], 48 | "source": [ 49 | "draw_single(*generate(120))" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 4, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "def train(n=512, epochs=31000, lr = 1e-3):\n", 59 | " # weight vector\n", 60 | " W = np.random.randn(9, 1)\n", 61 | "\n", 62 | " # training loop\n", 63 | " for t in range(epochs):\n", 64 | " # get new training data\n", 65 | " X, y = generate(n)\n", 66 | " X = X / 255\n", 67 | " y = y.reshape(n, 1)\n", 68 | "\n", 69 | " # model function\n", 70 | " h = X.dot(W)\n", 71 | "\n", 72 | " # compute loss\n", 73 | " loss = np.square(h - y).mean()\n", 74 | "\n", 75 | " # compute accuracy\n", 76 | " acc = (np.sign(h) == y).mean()\n", 77 | "\n", 78 | " if t % 5000 == 0:\n", 79 | " print('l: {:>8f}, a {:>.4f} (e {})'.format(loss, acc, t))\n", 80 | "\n", 81 | " # grad + update\n", 82 | " grad_w = 2 * X.T.dot(h - y) / n\n", 83 | " W -= lr * grad_w\n", 84 | "\n", 85 | " return W" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": 5, 91 | "metadata": {}, 92 | "outputs": [ 93 | { 94 | "name": "stdout", 95 | "output_type": "stream", 96 | "text": [ 97 | "l: 2.826098, a 0.5215 (e 0)\n", 98 | "l: 0.565322, a 0.8223 (e 5000)\n", 99 | "l: 0.412433, a 0.9297 (e 10000)\n", 100 | "l: 0.365760, a 0.9746 (e 15000)\n", 101 | "l: 0.347903, a 0.9922 (e 20000)\n", 102 | "l: 0.338252, a 1.0000 (e 25000)\n", 103 | "l: 0.373862, a 1.0000 (e 30000)\n", 104 | "\n", 105 | "Final W = \n", 106 | "\n", 107 | "[[ 1.13179384]\n", 108 | " [ 1.13089455]\n", 109 | " [ 1.13347194]\n", 110 | " [-0.00384232]\n", 111 | " [-0.0078364 ]\n", 112 | " [ 0.00724967]\n", 113 | " [-1.13251403]\n", 114 | " [-1.13009286]\n", 115 | " [-1.12999758]]\n" 116 | ] 117 | } 118 | ], 119 | "source": [ 120 | "W = train()\n", 121 | "print('\\nFinal W = \\n\\n{}'.format(W))" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [] 130 | } 131 | ], 132 | "metadata": { 133 | "file_extension": ".py", 134 | "kernelspec": { 135 | "display_name": "Python 3", 136 | "language": "python", 137 | "name": "python3" 138 | }, 139 | "language_info": { 140 | "codemirror_mode": { 141 | "name": "ipython", 142 | "version": 3 143 | }, 144 | "file_extension": ".py", 145 | "mimetype": "text/x-python", 146 | "name": "python", 147 | "nbconvert_exporter": "python", 148 | "pygments_lexer": "ipython3", 149 | "version": "3.6.10" 150 | }, 151 | "mimetype": "text/x-python", 152 | "name": "python", 153 | "npconvert_exporter": "python", 154 | "pygments_lexer": "ipython3", 155 | "version": 3 156 | }, 157 | "nbformat": 4, 158 | "nbformat_minor": 2 159 | } 160 | -------------------------------------------------------------------------------- /squares.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch\n", 10 | "import torch.nn as nn\n", 11 | "import torch.nn.functional as F\n", 12 | "from utils.draw import draw_squares\n", 13 | "from utils.square import SquareDataset\n", 14 | "from torch.utils.data import DataLoader\n", 15 | "import utils.viz as torchviz" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 2, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "class LinearModel(nn.Module):\n", 25 | " def __init__(self, x, y):\n", 26 | " super(LinearModel, self).__init__()\n", 27 | " self.layer1 = nn.Linear(x, y)\n", 28 | "\n", 29 | " def forward(self, x):\n", 30 | " x = self.layer1(x)\n", 31 | " return x" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 3, 37 | "metadata": {}, 38 | "outputs": [ 39 | { 40 | "name": "stdout", 41 | "output_type": "stream", 42 | "text": [ 43 | "(tensor([115., 81., 97., 143., 117., 54., 245., 72., 50.]), tensor([0., 0., 1.]))\n", 44 | "(tensor([ 4., 223., 70., 56., 105., 95., 75., 86., 41.]), tensor([1., 0., 0.]))\n", 45 | "(tensor([ 1., 13., 248., 161., 210., 24., 8., 124., 42.]), tensor([0., 1., 0.]))\n", 46 | "(tensor([ 18., 83., 36., 143., 218., 163., 45., 205., 59.]), tensor([0., 1., 0.]))\n", 47 | "(tensor([130., 200., 214., 67., 155., 46., 246., 52., 107.]), tensor([1., 0., 0.]))\n" 48 | ] 49 | } 50 | ], 51 | "source": [ 52 | "squares = SquareDataset(60000)\n", 53 | "for i in range(5):\n", 54 | " print(squares[i])" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "# Using the not-so-smart Model" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 4, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "name": "stdout", 71 | "output_type": "stream", 72 | "text": [ 73 | "tensor([[0.5098, 0.7843, 0.8392, 0.2627, 0.6078, 0.1804, 0.9647, 0.2039, 0.4196]])\n", 74 | "tensor([[-0.2050, -0.0188, 0.0037]], grad_fn=)\n" 75 | ] 76 | } 77 | ], 78 | "source": [ 79 | "model = LinearModel(9, 3)\n", 80 | "X = squares[4][0].reshape(-1, 9)/255\n", 81 | "print(X)\n", 82 | "print(model(X))" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 5, 88 | "metadata": {}, 89 | "outputs": [ 90 | { 91 | "data": { 92 | "text/plain": [ 93 | "(Parameter containing:\n", 94 | " tensor([[ 0.1059, -0.2351, 0.0255, -0.1972, 0.2625, 0.2541, -0.0858, -0.0655,\n", 95 | " 0.1103],\n", 96 | " [ 0.2487, -0.0637, 0.2842, 0.2228, -0.2701, -0.0693, 0.0798, -0.2672,\n", 97 | " 0.0266],\n", 98 | " [ 0.1608, -0.1205, -0.1233, 0.1524, -0.1639, 0.2395, -0.0632, -0.2412,\n", 99 | " 0.3286]], requires_grad=True),\n", 100 | " Parameter containing:\n", 101 | " tensor([-0.1997, -0.2496, 0.1082], requires_grad=True))" 102 | ] 103 | }, 104 | "execution_count": 5, 105 | "metadata": {}, 106 | "output_type": "execute_result" 107 | } 108 | ], 109 | "source": [ 110 | "model.layer1.weight, model.layer1.bias" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | "# Looking at the gradients!" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 6, 123 | "metadata": {}, 124 | "outputs": [ 125 | { 126 | "name": "stdout", 127 | "output_type": "stream", 128 | "text": [ 129 | "tensor(0.4841, grad_fn=)\n" 130 | ] 131 | } 132 | ], 133 | "source": [ 134 | "cost = torch.nn.MSELoss()\n", 135 | "Y = squares[4][1].reshape(-1, 3)\n", 136 | "loss = cost(model(X), Y)\n", 137 | "print(loss)" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 7, 143 | "metadata": {}, 144 | "outputs": [ 145 | { 146 | "data": { 147 | "image/svg+xml": [ 148 | "\r\n", 149 | "\r\n", 151 | "\r\n", 153 | "\r\n", 154 | "\r\n", 156 | "\r\n", 157 | "g\r\n", 158 | "\r\n", 159 | "\r\n", 160 | "0\r\n", 161 | "\r\n", 162 | "Mse\r\n", 163 | "\r\n", 164 | "\r\n", 165 | "1\r\n", 166 | "\r\n", 167 | "Addmm\r\n", 168 | "\r\n", 169 | "\r\n", 170 | "0->1\r\n", 171 | "\r\n", 172 | "\r\n", 173 | "\r\n", 174 | "\r\n", 175 | "2\r\n", 176 | "\r\n", 177 | "Var\r\n", 178 | "\r\n", 179 | "\r\n", 180 | "1->2\r\n", 181 | "\r\n", 182 | "\r\n", 183 | "\r\n", 184 | "\r\n", 185 | "3\r\n", 186 | "\r\n", 187 | "Const\r\n", 188 | "\r\n", 189 | "\r\n", 190 | "1->3\r\n", 191 | "\r\n", 192 | "\r\n", 193 | "\r\n", 194 | "\r\n", 195 | "4\r\n", 196 | "\r\n", 197 | "T\r\n", 198 | "\r\n", 199 | "\r\n", 200 | "1->4\r\n", 201 | "\r\n", 202 | "\r\n", 203 | "\r\n", 204 | "\r\n", 205 | "5\r\n", 206 | "\r\n", 207 | "Var\r\n", 208 | "\r\n", 209 | "\r\n", 210 | "4->5\r\n", 211 | "\r\n", 212 | "\r\n", 213 | "\r\n", 214 | "\r\n", 215 | "\r\n" 216 | ], 217 | "text/plain": [ 218 | "" 219 | ] 220 | }, 221 | "execution_count": 7, 222 | "metadata": {}, 223 | "output_type": "execute_result" 224 | } 225 | ], 226 | "source": [ 227 | "torchviz.draw(loss)" 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": {}, 233 | "source": [ 234 | "# Optimizing All Teh Things!" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": 8, 240 | "metadata": {}, 241 | "outputs": [], 242 | "source": [ 243 | "# Use the nn package to define our model and loss function.\n", 244 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", 245 | "model = LinearModel(9, 3)\n", 246 | "model = model.to(device)\n", 247 | "\n", 248 | "cost = torch.nn.MSELoss()\n", 249 | "\n", 250 | "# optimizer which Tensors it should update.\n", 251 | "learning_rate = 1e-2\n", 252 | "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", 253 | "\n", 254 | "# dataset!\n", 255 | "dataloader = DataLoader(squares, batch_size=128)\n", 256 | "\n", 257 | "epochs = 20" 258 | ] 259 | }, 260 | { 261 | "cell_type": "markdown", 262 | "metadata": {}, 263 | "source": [ 264 | "# The Optimization Loop" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 9, 270 | "metadata": {}, 271 | "outputs": [ 272 | { 273 | "name": "stdout", 274 | "output_type": "stream", 275 | "text": [ 276 | "l: 0.175322, (e 0)\n", 277 | "l: 0.144956, (e 1)\n", 278 | "l: 0.127325, (e 2)\n", 279 | "l: 0.117154, (e 3)\n", 280 | "l: 0.111338, (e 4)\n", 281 | "l: 0.108053, (e 5)\n", 282 | "l: 0.106228, (e 6)\n", 283 | "l: 0.105239, (e 7)\n", 284 | "l: 0.104723, (e 8)\n", 285 | "l: 0.104470, (e 9)\n", 286 | "l: 0.104359, (e 10)\n", 287 | "l: 0.104324, (e 11)\n", 288 | "l: 0.104325, (e 12)\n", 289 | "l: 0.104342, (e 13)\n", 290 | "l: 0.104366, (e 14)\n", 291 | "l: 0.104389, (e 15)\n", 292 | "l: 0.104410, (e 16)\n", 293 | "l: 0.104427, (e 17)\n", 294 | "l: 0.104442, (e 18)\n", 295 | "l: 0.104454, (e 19)\n" 296 | ] 297 | } 298 | ], 299 | "source": [ 300 | "for t in range(epochs):\n", 301 | " for batch, (X, Y) in enumerate(dataloader):\n", 302 | " X, Y = X.to(device) / 255, Y.to(device)\n", 303 | " optimizer.zero_grad()\n", 304 | " pred = model(X)\n", 305 | " loss = cost(pred, Y)\n", 306 | " loss.backward()\n", 307 | " optimizer.step()\n", 308 | "\n", 309 | " print('l: {:>8f}, (e {:>3})'.format(loss.item(), t))" 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": 10, 315 | "metadata": {}, 316 | "outputs": [ 317 | { 318 | "name": "stdout", 319 | "output_type": "stream", 320 | "text": [ 321 | "W's and b's:\n", 322 | "Parameter containing:\n", 323 | "tensor([[ 0.5627, 0.5829, 0.5750, -0.2806, -0.2749, -0.2752, -0.2772, -0.2672,\n", 324 | " -0.2802],\n", 325 | " [-0.2565, -0.2729, -0.2635, 0.5925, 0.5858, 0.5760, -0.2636, -0.2734,\n", 326 | " -0.2683],\n", 327 | " [-0.2728, -0.2780, -0.2808, -0.2776, -0.2767, -0.2666, 0.5733, 0.5725,\n", 328 | " 0.5784]], device='cuda:0', requires_grad=True)\n", 329 | "Parameter containing:\n", 330 | "tensor([0.2969, 0.2559, 0.2975], device='cuda:0', requires_grad=True)\n" 331 | ] 332 | } 333 | ], 334 | "source": [ 335 | "print(\"W's and b's:\")\n", 336 | "for p in model.parameters():\n", 337 | " print(p)" 338 | ] 339 | }, 340 | { 341 | "cell_type": "markdown", 342 | "metadata": {}, 343 | "source": [ 344 | "# Trying it out (inference)" 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 26, 350 | "metadata": {}, 351 | "outputs": [], 352 | "source": [] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": null, 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [] 360 | }, 361 | { 362 | "cell_type": "code", 363 | "execution_count": null, 364 | "metadata": {}, 365 | "outputs": [], 366 | "source": [] 367 | } 368 | ], 369 | "metadata": { 370 | "file_extension": ".py", 371 | "kernelspec": { 372 | "display_name": "Python 3", 373 | "language": "python", 374 | "name": "python3" 375 | }, 376 | "language_info": { 377 | "codemirror_mode": { 378 | "name": "ipython", 379 | "version": 3 380 | }, 381 | "file_extension": ".py", 382 | "mimetype": "text/x-python", 383 | "name": "python", 384 | "nbconvert_exporter": "python", 385 | "pygments_lexer": "ipython3", 386 | "version": "3.6.10" 387 | }, 388 | "mimetype": "text/x-python", 389 | "name": "python", 390 | "npconvert_exporter": "python", 391 | "pygments_lexer": "ipython3", 392 | "version": 3 393 | }, 394 | "nbformat": 4, 395 | "nbformat_minor": 2 396 | } 397 | -------------------------------------------------------------------------------- /superfile.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sethjuarez/DeepLearningWithPyTorch/81b8f58a27494ea41178fd667dcc7811de33d75d/superfile.onnx -------------------------------------------------------------------------------- /tensors.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Tensors\n", 8 | "Where the magic begins!" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "metadata": {}, 15 | "outputs": [ 16 | { 17 | "data": { 18 | "text/plain": [ 19 | "'PyTorch v1.5.0, GPU: True'" 20 | ] 21 | }, 22 | "execution_count": 1, 23 | "metadata": {}, 24 | "output_type": "execute_result" 25 | } 26 | ], 27 | "source": [ 28 | "import torch\n", 29 | "f'PyTorch v{torch.__version__}, GPU: {torch.cuda.is_available()}'" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 2, 35 | "metadata": {}, 36 | "outputs": [ 37 | { 38 | "name": "stdout", 39 | "output_type": "stream", 40 | "text": [ 41 | "tensor([[191., 89., 173.],\n", 42 | " [ 20., 204., 232.],\n", 43 | " [122., 18., 173.]]) \n", 44 | " tensor([[0.7113, 0.6239, 0.0638],\n", 45 | " [0.5682, 0.4671, 0.5599],\n", 46 | " [0.0373, 0.0279, 0.1606]])\n" 47 | ] 48 | } 49 | ], 50 | "source": [ 51 | "x = torch.randint(255, (3, 3), dtype=torch.float)\n", 52 | "y = torch.rand(3, 3)\n", 53 | "print(x, '\\n', y)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 3, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "text/plain": [ 64 | "tensor([[191.7113, 89.6239, 173.0638],\n", 65 | " [ 20.5682, 204.4671, 232.5599],\n", 66 | " [122.0373, 18.0279, 173.1606]])" 67 | ] 68 | }, 69 | "execution_count": 3, 70 | "metadata": {}, 71 | "output_type": "execute_result" 72 | } 73 | ], 74 | "source": [ 75 | "x.add_(y)\n", 76 | "x" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "# Make our square nines" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 4, 89 | "metadata": {}, 90 | "outputs": [ 91 | { 92 | "data": { 93 | "text/plain": [ 94 | "tensor([[[172., 251., 234.],\n", 95 | " [ 53., 68., 191.],\n", 96 | " [135., 204., 142.]],\n", 97 | "\n", 98 | " [[250., 84., 8.],\n", 99 | " [ 16., 120., 153.],\n", 100 | " [215., 237., 50.]],\n", 101 | "\n", 102 | " [[ 13., 131., 57.],\n", 103 | " [ 89., 160., 70.],\n", 104 | " [128., 71., 173.]],\n", 105 | "\n", 106 | " ...,\n", 107 | "\n", 108 | " [[210., 16., 177.],\n", 109 | " [101., 110., 229.],\n", 110 | " [111., 99., 164.]],\n", 111 | "\n", 112 | " [[ 57., 253., 172.],\n", 113 | " [191., 137., 141.],\n", 114 | " [ 61., 84., 65.]],\n", 115 | "\n", 116 | " [[231., 207., 174.],\n", 117 | " [253., 6., 231.],\n", 118 | " [ 74., 87., 254.]]])" 119 | ] 120 | }, 121 | "execution_count": 4, 122 | "metadata": {}, 123 | "output_type": "execute_result" 124 | } 125 | ], 126 | "source": [ 127 | "X = torch.randint(255, (400, 3, 3), dtype=torch.float)\n", 128 | "X" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 5, 134 | "metadata": {}, 135 | "outputs": [ 136 | { 137 | "data": { 138 | "text/plain": [ 139 | "tensor([[172., 251., 234., ..., 135., 204., 142.],\n", 140 | " [250., 84., 8., ..., 215., 237., 50.],\n", 141 | " [ 13., 131., 57., ..., 128., 71., 173.],\n", 142 | " ...,\n", 143 | " [210., 16., 177., ..., 111., 99., 164.],\n", 144 | " [ 57., 253., 172., ..., 61., 84., 65.],\n", 145 | " [231., 207., 174., ..., 74., 87., 254.]])" 146 | ] 147 | }, 148 | "execution_count": 5, 149 | "metadata": {}, 150 | "output_type": "execute_result" 151 | } 152 | ], 153 | "source": [ 154 | "X = X.view(-1, 9)\n", 155 | "X" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": 6, 161 | "metadata": {}, 162 | "outputs": [ 163 | { 164 | "data": { 165 | "text/plain": [ 166 | "tensor([[657., 312., 481.],\n", 167 | " [342., 289., 502.],\n", 168 | " [201., 319., 372.],\n", 169 | " ...,\n", 170 | " [403., 440., 374.],\n", 171 | " [482., 469., 210.],\n", 172 | " [612., 490., 415.]])" 173 | ] 174 | }, 175 | "execution_count": 6, 176 | "metadata": {}, 177 | "output_type": "execute_result" 178 | } 179 | ], 180 | "source": [ 181 | "w = [[1, 1, 1, 0, 0, 0, 0, 0, 0], \n", 182 | " [0, 0, 0, 1, 1, 1, 0, 0, 0], \n", 183 | " [0, 0, 0, 0, 0, 0, 1, 1, 1]]\n", 184 | "\n", 185 | "magic_w = torch.tensor(w, dtype=torch.float)\n", 186 | "X.mm(magic_w.t())" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 7, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "data": { 196 | "text/plain": [ 197 | "tensor([0, 2, 2, 1, 1, 1, 0, 2, 2, 2, 1, 2, 2, 0, 0, 2, 1, 2, 1, 0, 1, 0, 1, 0,\n", 198 | " 0, 1, 0, 1, 2, 0, 1, 0, 2, 2, 2, 1, 2, 0, 2, 2, 2, 0, 0, 2, 2, 0, 1, 2,\n", 199 | " 1, 2, 2, 0, 2, 0, 2, 2, 1, 0, 2, 0, 2, 1, 0, 1, 2, 1, 2, 0, 2, 0, 1, 2,\n", 200 | " 1, 1, 1, 0, 1, 2, 2, 1, 2, 1, 1, 1, 1, 2, 1, 0, 1, 2, 0, 0, 0, 1, 0, 0,\n", 201 | " 0, 2, 1, 1, 2, 2, 2, 2, 0, 2, 1, 2, 1, 1, 1, 2, 0, 2, 1, 1, 0, 2, 1, 2,\n", 202 | " 2, 2, 1, 1, 2, 0, 0, 2, 2, 2, 2, 2, 1, 0, 1, 2, 2, 2, 1, 2, 2, 0, 0, 0,\n", 203 | " 2, 0, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 0, 0, 2, 2, 1, 2, 0, 2, 2, 1, 0, 1,\n", 204 | " 2, 0, 2, 1, 2, 2, 1, 2, 2, 1, 1, 1, 0, 2, 1, 2, 0, 0, 2, 2, 1, 0, 0, 1,\n", 205 | " 0, 2, 2, 1, 1, 1, 2, 0, 1, 1, 0, 2, 1, 0, 2, 2, 1, 0, 2, 1, 0, 0, 2, 1,\n", 206 | " 2, 0, 0, 2, 1, 1, 0, 0, 2, 1, 0, 2, 0, 2, 0, 1, 2, 0, 1, 0, 0, 0, 0, 1,\n", 207 | " 1, 0, 1, 0, 1, 2, 0, 0, 2, 2, 0, 0, 0, 0, 1, 2, 1, 1, 0, 1, 2, 2, 1, 0,\n", 208 | " 1, 2, 1, 2, 2, 2, 1, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 2, 1, 2, 2,\n", 209 | " 2, 1, 1, 2, 1, 1, 0, 1, 0, 2, 2, 1, 0, 2, 1, 2, 2, 1, 1, 0, 1, 1, 0, 2,\n", 210 | " 1, 1, 2, 2, 0, 0, 2, 2, 1, 2, 2, 0, 1, 1, 0, 2, 0, 1, 1, 0, 1, 2, 0, 2,\n", 211 | " 2, 0, 2, 1, 0, 0, 0, 1, 2, 0, 0, 1, 0, 2, 1, 2, 1, 0, 2, 1, 0, 2, 0, 2,\n", 212 | " 1, 2, 0, 2, 1, 2, 2, 1, 2, 2, 0, 0, 0, 0, 0, 2, 1, 2, 0, 2, 0, 2, 0, 2,\n", 213 | " 0, 0, 1, 2, 2, 1, 2, 1, 0, 2, 2, 0, 1, 1, 0, 0])" 214 | ] 215 | }, 216 | "execution_count": 7, 217 | "metadata": {}, 218 | "output_type": "execute_result" 219 | } 220 | ], 221 | "source": [ 222 | "y = torch.argmax(X.mm(magic_w.t()), 1)\n", 223 | "y" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": 8, 229 | "metadata": {}, 230 | "outputs": [ 231 | { 232 | "data": { 233 | "text/plain": [ 234 | "tensor([[1., 0., 0.],\n", 235 | " [0., 0., 1.],\n", 236 | " [0., 0., 1.],\n", 237 | " ...,\n", 238 | " [0., 1., 0.],\n", 239 | " [1., 0., 0.],\n", 240 | " [1., 0., 0.]])" 241 | ] 242 | }, 243 | "execution_count": 8, 244 | "metadata": {}, 245 | "output_type": "execute_result" 246 | } 247 | ], 248 | "source": [ 249 | "y = torch.zeros(400, 3, dtype=torch.float).scatter_(1, y.view(-1, 1), value=1)\n", 250 | "y" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 9, 256 | "metadata": {}, 257 | "outputs": [ 258 | { 259 | "data": { 260 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/4AAAGRCAYAAADcoWhrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeXTV1bn/8c+GQAIkATIwDxGUIohelbZWW60VRWtrawfbYm29DnW2ztVe/V28teq11LYOWLmKOFScWm1tVVy34IDWXoOIMqNMMso8BQLC9/dHzlk9QxJynn0Sv9l5v9ZirZxv9rP3fs6z9/ecncSji6JIAAAAAAAgTO0+7QkAAAAAAIDmw8EfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGB5Pfg75y51zlU752qdc5Oa0H6Mc26Zc26Hc+4551xZI22rnHPTnHM1zrn5zrlR+Zx7PeOVOeeeTcxtmXNuzH7aX+mcW+Oc2+Kcm+icK2yk7QTn3ALn3D7n3Nl5n3z6WEHkkRivybk45w5xzk1xzq13zkVN6LvF1lcoeSTGCyIX7l3x2/PkEa88EuOFst/bZE0S7WOZS2D34CByCWVtJcbj3kVNmkVr2+/5/o3/Kkm3SJq4v4bOueGS7pd0lqSekmokjW8kZLKkmZLKJf2HpGecc5W+E27EvZJ2J+Z2pqT7EnPO4pwbLel6SSdIqpI0SNLNjfQ9S9LFkt7J43wbEkoeUg65SNoj6SlJ5zax75ZcX6HkIYWTC/eu+O158ohXHlI4+71N1iTmuYR0Dw4ll1DWlsS9q0rUhPePkhRFUd7/JZ6ASftpc6ukx1MeD1bdAiipp+0QSbWp35P0uqQLm2n+XRJzGZJy7VFJtzfQ/nFJt6Y8PkHSmiaMM13S2c2RQ0h5WHJJaXNg3TJvtE2Lra9Q8ggtl5QxuHfFYM+TR7zysOSS0iZW+70t1yTOuaSM1arvwaHkEtLa4t5FTVpiz7eW/f5p/jf+w1X30yVJUhRFHyqxCBpouziKom0p12YlrjeHIZL2RlG0sInjpeWS+Lqnc668mebXVKHkIeWeSy5acn2FkocUVi654N7V/MjjX23jkIcUzn5vyzWJcy65iPM9OFdxzSWktcW9619tqcmn61Pf75/mwb9Y0paMa1sklXi2zYdcx8tsn/y6uebXVKHkITXvGmjJ9RVKHs09XkvnkgvuXc2PPP7VVo20b0mh7Pe2XJM455KLON+DcxXXXEJaW9y7/tVWjbRvSaHUJFef+n7/NA/+2yWVZlwrlbTNs20+5DpeZvvk1801v6YKJQ+peddAS66vUPJo7vFaOpdccO9qfuTxr7ZqpH1LCmW/t+WaxDmXXMT5HpyruOYS0tri3vWvtmqkfUsKpSa5+tT3+6d58J8j6bDkA+fcIEmFkhY20HaQcy71pxyHJa43h4WSCpxzBzVxvLRcEl+vjaJoQzPNr6lCyUPKPZdctOT6CiUPKaxccsG9q/mRx7/axiEPKZz93pZrEudcchHne3Cu4ppLSGuLe9e/2lKTT9env9/z/MEGBZKKJN2mug9pKJJU0EDb4ZK2SvqS6j7k4TFJTzTS91uSxiX6PF3SZkmVzfghDU+o7tMUu0g6RnV/XjG8gbYnS1ojaZik7pKmqpEPqJDUMZHHG5LOT3zdjjzymotLzGeYpCjxdWEc1lcoeYSUC/eu+O158ohXHoZc4rzf22pNYpuLwroHB5FLKGvLkAv3LmqSSx6tar/nO/mxiYKk/hvbSPsxkpZL2iHpz5LKGmlbJekVSTslLZA0qrkWY2K8MknPJea2XNKY/bS/StLaREEf2s+CfKWe5+nL5JG/XBLrJXNuS+OwvkLJI6RcuHfFb8+TR7zyyDWXmO/3NlmTOOeisO7BQeQSytrKNRfuXdQkxzxa1X53iY4BAAAAAECAPs3/xh8AAAAAADQzDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMAKcmlcVFQUlZSU7L9hPdavX2+KSyooyGmqaT755JP1URRVJh8758yfaDh48GDzPCSpW7du5tgZM2ak5SFJFRUVUVVVlam/hQvr+99GNl2vXr1McWvXrtWWLVtc6rVOnTpFpaWlpv4+/vhjU1zS4Ycfbo6dOXNmWk0qKiqigQMHmvratm2beR6StHv3bnPssmXLstZWt27dot69e5v6W7NmjXkuklRZWbn/Rg1YtGhRWi5FRUVRcXGxqa/a2lrzPCRp+/btPuFZNenYsWPUqVMnU2fW/ZpUVFRkjn3vvffScikrK4v69u1r6uuTTz4xz0OS5s+f7xNe7z3Yuud37NjhMxft3bvXHPvBBx+k5dK9e3dzTXbu3GmeR2Jsc2x9r4vt27ePrO8ZfO49kt8+y8zF597lu7Z69uxpjq3v9aS4uDgqLy839ef7/tG6PhOfgp32XqVdu3bmtbVnzx5TXNKAAQPMscuXL896r2J977hixQrzPKS694AestZWYWGheZ/4vK5JknVcSVq4cGFaLgUFBVGHDh1MffXv3988D0nq3LmzOXbWrFl5vQdbn4Mkn322e/fuvO2TGTNmmOchSR07djTHZuaRKqeqlJSU6Fvf+pZpEhMmTDDFJfm8MVi3bt0yr8FTjBs3ziv+m9/8pjnWOZeVR1VVlaqrq039jRo1yjwXSbruuutMcZdeemnWtdLSUn3/+9839XfXXXeZ4pJee+01c2xJSUlaTQYOHKi33nrL1Nerr75qnockLVmyxBz7k5/8JGtt9e7dW5MmTTL1d/vtt5vnIkmXXHKJOfbEE09My6W4uFhf//rXTX198MEH5nlI0vTp033Cs2rSqVMnHXPMMabOrrnmGp+5aNiwYebY3r17p+XSt29f/elPfzL15XsIOProo33Cs2ris+f/+c9/+sxFW7duNceeeuqpeavJu+++a56HJJ1xxhnm2PpeFwsKCtSnTx9TfxdccIF5LpJ0/fXXm2Mzc/G5d1nXZNK1115rjj333HOzalJeXq6f//znpv4eeOAB81wkafbs2aa4+n7wW1BQoIqKClN/q1evNsUl3XDDDebYiy66KK0mPu8dfdaG5P0+OmttFRcXa/To0abOhg4d6jMX8+uxJI0aNSotlw4dOph/ueh7Nhk5cqQ5trKyst57cL9+/Uz9WX/hlLRy5Upz7NKlS7P2ifV12ucX1pLMr2FSdh6p+FN/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACVpBL44EDB+r+++83DXTrrbea4pL+/d//3Rz7/PPPpz0+4ogj9MYbb5j6uuuuu8zzkKQLLrjAKz7TmjVrdNttt5liZ82a5TX2xx9/bIr75JNP6u3r7rvvNvUXRZEpLqm4uNgrPtWmTZv05JNPmmLPOussr7Hbt2/vFZ+pXbt2KioqMsVa45Juuukmr/hUe/bs0erVq02x1jWZ9G//9m/mWOdc1rXS0lKddNJJpv7+3//7f+a5SNL06dO94lPNmTNHhx56qCl2586dXmP/8Ic/NMc+9thjWdfWrFmj//7v/zb1d+ONN5rnItW/Rqzmzp1rXq++Ndm1a5dXfKaSkhIdf/zxptgbbrjBa+zDDjvMKz7Vhg0bNGnSJFOs7+vi7NmzveIzLV++XJdeeqkp1nr/TqqoqDDFjRw5MuvaAQccoPHjx5v683kfK0l/+9vfvOJTrVixQtddd50p9rjjjvMa+7zzzjPHDh06NOvaxo0bNXnyZFN/vq/xX//6173iUw0dOlTTpk0zxZ5++uleY1vvlw3ZvXu3Fi9ebIpduXKl19g//vGPzbETJkxIezx79mwNGzbM1Ne+ffvM85Ckf/7zn+bYL3zhCw1+j9/4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAABK8il8bp16zR+/HjTQJMnTzbFJX3zm980xz7//PNpj+fMmaMRI0aY+lq7dq15HpL05ptvmmMnTJiQdW3btm167bXXTP395je/Mc9Fks466yyv+FSlpaU65phjTLF//OMfvcaeOnWqOfbzn/982uOdO3dq7ty5pr6iKDLPQ5Kefvppc+wZZ5yRda1Tp0467LDDTP098cQT5rlI0iWXXGKOfeutt9Ie9+rVS9dee62pr7KyMvM8msPGjRv1hz/8wRRbXV3tNfbMmTO94lOVlZXptNNOM8X269fPa+y9e/d6xWeqqanRO++8Y4q97LLLvMY+8MADzbEffPBB2uODDz5YTz75pKmvOXPmmOchSRUVFV7xmWpra7Vo0SJTbO/evb3Gfvnll73iUxUVFamqqsoUe+KJJ3qNnbk+fPXv31/XXXedKbaystJr7MWLF5viamtrs65t3LjR/Fq7fPlyU1xSPt9z9ezZU1dccYUp9pRTTvEae8eOHV7xmTp06KCePXuaYi+99FKvsceOHWuO3blzZ9rjdevW1fs+vymmTZtmnockOee84jMNGTJE9913nyn2K1/5itfYF198sVd8qkMOOcT83sm3JkcccYRXfEP4jT8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMBcFEVNb+zcOknLmm86zWZgFEWVyQeh5CG12lxCyUNibcURNYkfahI/1CR+qEm8hJKHxNqKI2oSP8HWJCmngz8AAAAAAGhd+FN/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgeT34O+fKnHPPOud2OOeWOefGNNL2EOfcFOfceufcfj9owDlX5Zyb5pyrcc7Nd86Nyufc6xnvUudctXOu1jk3qQntxyRy3uGce845V9ZI2xbLhZrEsiZB5JEYL4j1RU1iWZMm55Fof6Vzbo1zbotzbqJzrrCRthOccwucc/ucc2fnffLpY7G24re22mRNEu3juk+CyCMxXhC5hLLfE+OFUhPuXfGrSavKI9+/8b9X0m5JPSWdKek+59zwBtrukfSUpHOb2PdkSTMllUv6D0nPOOfq/cTCPFkl6RZJE/fXMJHj/ZLOUl3uNZLGNxLSkrlQk/jVJJQ8pHDWFzVpmljeu5xzoyVdL+kESVWSBkm6uZG+Z0m6WNI7eZxvQ1hbTcN+twlln4SShxROLqHsdymcmnDvil9NWlceURTl5Z+kLonEh6Rce1TS7fuJO7BuGo22GSKpVlJJyrXXJV2Yr/k3MvYtkibtp82tkh5PeTw48VyU1NO2xXKhJvGrSUh5hLi+qEk8apJrHpIel3RryuMTJK1pwjjTJZ3dnGuKtRWvtdWWaxLXfRJKHiHlEtJ+D6UmGWNx74pBTVpjHvn8jf8QSXujKFqYcm2WpIZ+OpiL4ZIWR1G0rRn6zofhqpuPJCmKog+VWAgNtG2pXKhJQoxqkqs459FW1xc1yX/fmXLNI60mia97OufKm2FuzYm1lf++fYVUk7juk1DykMLJJaT9HkpNcsW9q/m1ujzyefAvlrQl49oWSSUx7zsfcplfS+ZCTdLFoSa5inMebXV9UZP89+07Vmb75NdxWC+5YG3lv29fIdUkrvsklDykcHIJab+HUpNcce9qfq0uj3we/LdLKs24VippWz1t49R3PuQyv5bMhZqki0NNchXnPNrq+qIm+e/bd6zM9smv47BecsHayn/fvkKqSVz3SSh5SOHkEtJ+D6UmueLe1fxaXR75PPgvlFTgnDso5dphkubkoe85kgY551J/IpKvvvNhjurmI0lyzg2SVKi656S+ti2VCzVJiFFNchXnPNrq+qIm+e87U655pNUk8fXaKIo2NMPcmhNrK/99+wqpJnHdJ6HkIYWTS0j7PZSa5Ip7V/NrfXnk+UMOnlDdp0J2kXSM6v6EYXgDbZ2kIknDJEWJrwsb6fstSeMS7U6XtFlSZTN+YENBYqzbVPdBDUWSChpoO1zSVklfSuT+mKQn4pALNYllTYLII6T1RU1iWZNc8jhZ0ppEHt0lTVUjH0IlqWMihzcknZ/4uh1rq82srbZakzjvkyDyCCmXUPZ7YDXh3hW/mrSqPPKdfJmk5yTtkLRc0phG2lYlbg6p/5bup/0rknZKWiBpVHMtxsR4Y+uZ39hG2o9J5LxD0p8llcUhF2oSy5oEkUdI64uaxLImTc4j0f4qSWtV90bnITX+pvOVevL+MmurzaytNlmTRPu47pMg8ggpl1D2e2A14d4Vv5q0qjxcomMAAAAAABCgfP43/gAAAAAAIGY4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAASvIpXFFRUVUVVVlGujDDz80xSXt3r3bHFtTU7M+iqLK5OPy8vKof//+pr4KCnJ6yrLMmDHDJzwtD0nq2LFj1LlzZ1NnO3bs8JmLevbsaYrbtGmTduzY4VKvderUKerataupv7Vr15rikoYOHWqOnT9/flpNnHPmT8s88sgjzfOQpAULFphjt2/fnrW2CgsLo+LiYlN/7dr5/UyxU6dO5tiPPvooLZeKiopowIABpr5mzZplnock7du3zyc8qyZdu3aNevToYerMur+SfHKZOXNm1j3YWpN58+aZ5yFJn/nMZ8yx7733XlZNOnfuHHXr1s3U3+rVq81zkfzuGTNmzMjbvevQQw81z0OSOnToYI7NzEPyy+Wggw7af6NG+LxHmDdvXlouhYWF5td33/cqAwcONMfWVxOfXEpLS81zkaTKysr9N6rH0qVLtX79+rT3KsXFxVF5ebmpv61bt5riknzu4cuWLcva79bXad/93r59e3NsfWurpKQksta4trbWPBdJsq5pSfrggw/ScunevXvUp08fU19z5841z0Py2++Za0vyOy96npM0YsQIc+z777+flkvnzp3NZ5OdO3ea5yFJgwYNMsdmvudKldMrQ1VVlaqrq02T+M53vmOKS1q2bJk5trq6Oi24f//++vvf/27qq6yszDwPyftAlPUkdO7cWccee6yps7fffttnLvrpT39qivvd736Xda1r16760Y9+ZOrvV7/6lSku6eGHHzbHfv7zn7cvzAzWvZX0pS99yRw7ffr0rDyKi4s1evRoU3++b9QOPvhgc+wVV1yRlsuAAQM0ffp0U1+9evUyz0OStm3b5hOeVZMePXrUu3+a4qtf/arPXLxexDp37pxVk6lTp5r6Ouqoo8zzkKQXX3zRHNu3b9+smnTr1k3nnnuuqb9bbrnFPBfJ757hnMvbveull17yiu/du7c5Np95SNL48eO94isqKsyxhx9+eFounTt31vHHH2/qy/e9ygMPPGCOra8mPrlYX4eSLrjgAlPcyJEjs66Vl5fr5z//uak/333icw//yU9+klaTdu3ayfqD/VdeecU8D8nvBxj1ra3KykrzvdT3l5L1rZGm+upXv5qWS58+ffT444+3+Dwk6cYbbzTHnn/++Vk18Tkv+v7i6IUXXjDH9u/fPy2Xrl276uyzzzb15ftLiscee8wcW1JS0uDrIn/qDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwApyabxhwwY9+uijpoHmzJljikvq3bu3V3yq+fPn66ijjjLFvv32215jr1y50hzbt2/frGtbtmzR888/b+rvgQceMM9Fkm688UZT3Pr167OurVu3Tvfff7+pv71795rikiZOnOgVn2rw4MG68847TbHvvPOO19hnnHGGOXb69OlZ12pra7VkyRJTf2+99ZZ5LpJ0ww03eMWn2rBhgx577DFT7NatW73GnjJlijn25JNPzrr2wQcf6NRTTzX19/DDD5vnIknPPPOMV3yqTZs26Y9//KMpdsGCBV5jW++XDdm5c6dmz55tir3rrru8xv7rX//qFZ9q0KBBuuOOO0yxhx56qNfY//jHP7ziM/Xr109XX321KfbMM8/0Gnvt2rVe8an69OmjsWPHmmI7duzoNfbAgQO94jNFUWR+rfZ9PejSpYspbuPGjVnXtm7dqpdeesnU37nnnmuKSyosLPSKT3XQQQdpwoQJptjTTz/da+xp06Z5xWfau3evtm/fboq96aabvMZetGiRV3yq2tpaLVu2zBT7xhtveI19zjnneMVnevfdd9WtWzdT7PDhw73G9n3fltnX//7v/5pifc+L5eXlXvEN4Tf+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDACnJpvHnzZv35z382DTR//nxTXJJ1XEn6zGc+k/b4kEMOUXV1tamvm2++2TwPSXLOecVnOvLII825XHnllV5jr1mzxis+1eGHH27O48477/Qau6yszCs+VU1Njd5++21T7AUXXOA19n333ecVn2nIkCH6+9//bordu3ev19glJSXm2Ntuuy3tcUVFhX70ox+Z+tq4caN5HpL04IMPesVnqqys1He+8x1T7M9+9jOvsf/zP//THPv888+nPd63b5927Nhh6mvJkiXmeUjSaaed5hWfafv27XrzzTdNsddee63X2IsWLfKKT7Vjxw793//9nyn2/fff9xp74sSJXvGZunXrZq7zwQcf7DX2HXfc4RWfasOGDXrsscdMsb/61a+8xvaJr29dl5aWatSoUab+Mu8fuTrrrLO84lMNHjxYzz77rCm2tLTUa+xt27Z5xaf65JNPzK9vgwYN8hr7s5/9rDm2vv0VRZFqa2tN/a1bt848F6nufVK+fPjhh/rGN75hiv3FL37hNfbs2bPNsfWda/bt22euyYEHHmieiyQNHz7cKz5V//799etf/9oU63veu+aaa8yx48aNa/B7/MYfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgLoqipjd2bp2kZc03nWYzMIqiyuSDUPKQWm0uoeQhsbbiiJrEDzWJH2oSP9QkXkLJQ2JtxRE1iZ9ga5KU08EfAAAAAAC0LvypPwAAAAAAAePgDwAAAABAwPJ68HfOlTnnnnXO7XDOLXPOjdlP+yudc2ucc1uccxOdc4WNtJ3gnFvgnNvnnDs7n/NuYLxLnXPVzrla59ykJrQfk8h5h3PuOedcWSNtq5xz05xzNc65+c65UXmdfPpYQeSRGC+I9RVKHonxglhfudTEOXeIc26Kc269c26//60U+8QmlJqEUo/EeEHkEsp9KzEeNaEmzSKUPBLjhfJ6wj6J2fpqbTXJ92/875W0W1JPSWdKus85N7y+hs650ZKul3SCpCpJgyTd3EjfsyRdLOmdPM63Mask3SJp4v4aJnK8X9JZqsu9RtL4RkImS5opqVzSf0h6xjlX74cw5EEoeUjhrK9Q8pDCWV9NromkPZKeknRuE/tmn9iEUpNQ6iGFk0so9y2JmlCT5hNKHlI4ryfsk/itr9ZVkyiK8vJPUhfVFXBIyrVHJd3eQPvHJd2a8vgESWuaMM50SWfna95NGO8WSZP20+ZWSY+nPB6ceC5K6mk7RFJt6vckvS7pQvIIf32FkkdI6yvXmqS0ObDuFtpoG/ZJG65JKPUILZeUsVrtfYuaUJM43YPjmocll5Q2sXo9yRiXfRKT9dXaapLP3/gPkbQ3iqKFKddmSWroJ2rDE99PbdvTOVeexzm1lLRcoij6UIkF3UDbxVEUbUu51tjz1JLinEco6yuUPCziur5yrUku2Cc2odQklHpIYeWSi7jetyRqIomaNJNQ8pDCeT3JFfskfj71muTz4F8saUvGtS2SSprYPvl1Q+3jLJfcc32eWlKc8whlfYWSh0Vc11dzjsU+sQmlJqHUQworl1zE9b5lGY+aNN42H0KpSSh5SOG8nuSKfRI/n3pN8nnw3y6pNONaqaRt9bStr33y64bax1kuuef6PLWkOOcRyvoKJQ+LuK6v5hyLfWITSk1CqYcUVi65iOt9yzIeNWm8bT6EUpNQ8pDCeT3JFfskfj71muTz4L9QUoFz7qCUa4dJmtNA+zmJ76e2XRtF0YY8zqmlpOXinBskqVB1z0l9bQc551J/YtPY89SS4pxHKOsrlDws4rq+cq1JLtgnNqHUJJR6SGHlkou43rckaiKJmjSTUPKQwnk9yRX7JH4+/Zrk+YMNnlDdJxB2kXSM6v4kYXgDbU+WtEbSMEndJU1VIx+0IamjpCJJb0g6P/F1u2b8kIaCxBi3qe4DJ4okFTTQdrikrZK+lMj9MUlPNNL3W5LGJfo8XdJmSZXk0TbWVyh5hLS+cqyJS8xpmKQo8XVhHPIIaX2FUpNQ6hFSLqHct6gJNWnmmgSRhyGXOL+esE9itr5aW03ynXyZpOck7ZC0XNKY/bS/StLaxJPw0H421iuJDZj678vNuCDH1jPe2Ebaj0nkvEPSnyWVNdK2KpHPTkkLJI0ij7azvkLJI6T1lUtNEvPKzHlpHPIIaX2FUpNQ6hFSLqHct6gJNWnmmgSRR665KN6vJ+yTmK2v1lYTl+gYAAAAAAAEKJ//jT8AAAAAAISTaUwAACAASURBVIgZDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMAKcmlcWloa9ejRwzTQ9u3bTXFJ/fr1M8fOmDFjfRRFlcnHzjnzJxqWlZWZ5yFJ/fv3N8fOmjUrLQ/JL5fy8nLzXCSpqqrKFLd06VKtX7/epV7r3r171LdvX1N/CxYsMMUl9enTxxy7fPnyvK2tESNGmOchSbt27TLHLlq0KGttFRcXR9Y1UlCQ060ly+LFi33C03Lp2LFjVFRUZOqopKRk/40a4VOTjRs3ZtWkoqIisu67999/3zwXSSouLjbHZuZSVlYWWe+FHTp0MM9DkmbMmOETnlUTn/U1aNAgn7lo7ty55tg9e/ak5dKlS5eoe/fupr5WrlxpnockHXnkkebYzNd3SSosLIw6d+5s6m/z5s3mueRB3vbJxo0bvSayYsUKn/CsmnTo0CEqLCw0dea7531qGkVR2nsVn3uw571HpaWl5titW7dm7fdu3bqZ+urdu7d5Hr7q2+8+77sqKyv336gRAwYMMMdm5tKxY8eoU6dOpr58z1j79u3zCc+qSbdu3SLrOlm6dKnPXLzuF9u2bUvLpbi4OLKe+z766CPzPCTJ+r5Cknbt2pVVk6Sc3p336NFDd955p2kSr732mikuady4ceZY59wyr8FTjB492iv+d7/7nTm2R48eectDkk477TSv+IkTJ5riRo4cmXWtb9++evrpp039HX/88aa4pOuvv94ce/HFF+etJi+88IJX/Lx588yxJ510UlYe5eXl5uemoqLCPBdJOuOMM3zC03IpKirS5z73OVNHxx57rM88tHDhQnPsH/7wh6yaVFVVqbq62tTfAQccYJ6LJH3hC18wx06ePDktl/79++vFF1809eXzgzpJat++vTl23759WTXxWV9PPvmkeS6SdOihh5pjV61alZZL9+7ddfnll5v6+tnPfmaehyTzmpbqf33v3LmzTjjhBFN/f/zjH81zkfzW1969e7P2yZQpU0x9PfbYY+Z5SNK1117rE55Vk8LCQvMPt62/FEjyrWkqn3uwc27/jRpx1FFHmWNffvnltJp069ZNF110kamvn//85+Z5SFK7dvY/Ns7n+3lJ+s53vuMVP378eHNsZi6dOnXS0UcfberrzTffNM9DkrZu3eoTnlWT3r17a9KkSabOzjnnHJ+5eP1g6u9//3taLmVlZbr66qtNfV1xxRXmeUj2X65K0vz58xvcJ/ypPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAASvIpfGePXu0atUq00ALFy40xSW1b9/eKz5V7969dd5555liv/CFL3iNXVlZ6RWfqaqqSmPHjjXFLl++3Gvsl19+2RS3devWrGtFRUU6+OCDTf3dddddprikX/7yl17xqY488khVV1ebYq+77jqvsVesWOEVn6myslIXXnihKdY55zV2UVGROXbXrl1pj/ft26cdO3aY+nrggQfM85Ckjz76yCs+0/z58833oPvuu89r7I8//tgcO3ny5LTH7733nvr27Wvqq1+/fuZ5SNJJJ51kjn3ppZeyrg0ZMkRTpkwx9bd48WLzXCRp0KBB5tjM1/KVK1fq+uuvN/W1fv168zwkaeLEiV7xmbZs2aK//OUvptgoirzGXrt2rTm2V69eaY+XL1+uiy++2Gs+Vj7v2YYMGZJ1bejQofrHP/5h6u973/ueeS6SdPrpp5vipk2blnWtpqZG7777rqk/39eDzPtoLjLfr61atUo33XSTqa8zzzzTPA9J+tvf/uYVn2nw4MH61a9+ZYq94447vMb2fd+WqmvXrjrllFNMsb5r6+mnnzbHDhs2LOuaz73L9zX+2WefNcd26dIl7XGPHj102WWXmfo69thjzfOQpJEjR3rFN4Tf+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAASvIpfHq1at1yy23mAZauXKlKS5p8uTJ5tgf/OAHaY/bt2+v0tJSU1/z5883z0OSjjjiCK/4TLt27dKiRYtMsb169fIae8KECaa4devWZV2bMWOGnHOm/mbMmGGKS+rZs6dXfKoVK1boZz/7mSn2a1/7mtfY5557rld8piVLlujMM880xb7//vteY1vXtCT1798/7XFtba0+/PBDU1+nnHKKeR6S9PDDD5tj69sPO3bs0FtvvWXqr0OHDua5SP73vlRFRUWqqqoyxR533HFeYy9fvtwrvj7t27c3xQ0ZMsRr3OOPP94rPlVFRYVOP/10U2ynTp28xv7FL37hFZ+pe/fuGj16tCl269atXmOfd955XvGpampqNHPmTFPsRx995DX2n/70J6/4TDt37tR7771niv3d737nNXbnzp1NcfXda+bNm2d+Hzd37lxTXNJTTz3lFZ+qa9eu+uIXv2iKffDBB73GHjBggFd8ptWrV+vWW281xV577bVeY+/atcsrPlWPHj10+eWXm2KPPfZYr7FfeOEFr/hMNTU1euedd0yxURR5jT1w4ECv+Ezt2tl+R/7973/fa9xvf/vb5tinn366we/xG38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIC5KIqa3ti5dZKWNd90ms3AKIoqkw9CyUNqtbmEkofE2oojahI/1CR+qEn8UJN4CSUPibUVR9QkfoKtSVJOB38AAAAAANC68Kf+AAAAAAAEjIM/AAAAAAABy+vB3zlX5px71jm3wzm3zDk3Zj/tr3TOrXHObXHOTXTOFTbSdoJzboFzbp9z7ux8zruB8Zqci3PuEOfcFOfceufcfv/bCedclXNumnOuxjk33zk3Kr+zTxvrUudctXOu1jk3qQntxyTy3eGce845V9ZI2xbLIzFeKDVhn1CTZhPKng+lJqHUIzEe+z1GaysxXpurSZzzSIwXRC7cu2JZkzZ570q0j2UurW1t5fs3/vdK2i2pp6QzJd3nnBteX0Pn3GhJ10s6QVKVpEGSbm6k71mSLpb0Th7n25gm5yJpj6SnJJ3bxL4nS5opqVzSf0h6xjlX74cw5MEqSbdImri/hon87pd0luryrpE0vpGQlsxDCqcm7JOmoSY2oez5UGoSSj0k9nuV4rW2pDZYE8U7DymcXLh3NU0s90lI966Y59K61lYURXn5J6mL6hIfknLtUUm3N9D+cUm3pjw+QdKaJowzXdLZ+Zp3PnJJaXNg3VPaaJshkmollaRce13Shc2c0y2SJu2nza2SHk95PDjxPJTU07ZF8wilJuwTatKcNckYr9Xu+RBr0prrYalJShv2e/OtqTZZk7jmEVouKWNw74pBTdryvSuuubTGtZXP3/gPkbQ3iqKFKddmSWropx7DE99PbdvTOVeexzlZ5ZpLLoZLWhxF0bZm6NtXWk2iKPpQiQXdQNuWzCOUmrBPmoaatIy47vm2WpO41kNiv6e2jcvaaqs1yQX7pGVw78p/35na8r0rrrm0urWVz4N/saQtGde2SCppYvvk1w21b0m55hKXvn3lMreWziOUmrBPPv2+fceKc01yFdc931ZrEtd6NPd4rC2btlqTuPTd0uPx/tEmlJq05XtXXHNpdWsrnwf/7ZJKM66VStpWT9v62ie/bqh9S8o1l7j07SuXubV0HqHUhH3y6fftO1aca5KruO75tlqTuNajucdjbdm01ZrEpe+WHo/3jzah1KQt37vimkurW1v5PPgvlFTgnDso5dphkuY00H5O4vupbddGUbQhj3OyyjWXXMyRNMg5l/oTm3z17SutJs65QZIKVfd81Ne2JfMIpSbsk6ahJi0jrnu+rdYkrvWQ2O+pbeOyttpqTXLBPmkZ3Lvy33emtnzvimsurW9t5flDDp5Q3ScQdpF0jOr+JGF4A21PlrRG0jBJ3SVNVSMfhiCpo6QiSW9IOj/xdbtm/MCGXHJxifkMkxQlvi5spO+3JI1LtDtd0mZJlc2UR0FinNtU94ETRZIKGmg7XNJWSV9K5P2YpCfikEdgNWGfUJPmrEkQez6UmoRSD0NN2O/cg9tcHiHlwr0rljVpq/eu2ObS2tZWvpMvk/ScpB2Slksas5/2V0lam7hZPLSf5F9JPEmp/77cjAuyybmo7n8tkTm3pftp/4qknZIWSBrVjHmMrWduYxtpPyaR7w5Jf5ZUFoc8AqsJ+4SaNGdNgtjzodQklHrkWhP2O/fgtphHSLmIe1cca9Im711xzqW1rS2X6BgAAAAAAAQon/+NPwAAAAAAiBkO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQsIJcGnfs2DEqKioyDTRkyBBTXFJtba05dvbs2eujKKpMPu7SpUvUvXt3U1+7du0yz0OSSktLzbFLlixJy0Oqy6WsrMzU35YtW8xzkew1+eSTT7R3716Xeq2wsDDq0qWLqb+BAwea4pJ81ta8efPSalJRURFVVVWZ+lq2bJl5HpJkXdOStGjRoqy15ZPL2rVrzXORpM2bN5tjt2/fnreaLF261DwPSerUqZM5dsWKFVk16dy5c2S9h+zdu9c8F0lav369V3i+arJw4UKfeXi9Fs2YMSOrJs458/8ap0ePHua5SH413bBhQ1oun2YePvGZr++S1KFDh6iwsNDUX+/evc1zkaSOHTuaY+fOnZuWS0lJSVReXm7qq6KiwjwPSaqpqTHHZr4uSnX3rm7dupn6833f1bdvX1PcypUrtXHjxrT3KsXFxeaaLF++3BSXVFJSYo7dtm1b1vtg63vH1atXm+chSdZ1IGXftySpoKAg6tChg6k/3/ePPvv9/fffT8ulqKgoKi4uNvVlfT1N8lmb69aty6pJWVlZ1K9fP1N/vu8f27Wz/057zZo1Wa+LzrnGQhrkWxOf959RFGXVJCmng39RUZFGjhxpmsTUqVNNcUkffPCBOfaggw5KO1F1795dl112mamvuXPnmuchSSeeeKI59qyzzso6GZaVlemKK64w9ffiiy+a5yJJS5YsMcWtWLEi61qXLl3Mz83vf/97U1zS4sWLzbEjR45Mq0lVVZXefvttU18XXniheR6S9N3vftcce+KJJ2atraqqKlVXV5v6++1vf2ueiyQ999xz5thXX301qybWPM455xzzPCRpxIgR5tirrroqqyalpaX68Y9/bOpvw4YN5rlI0oMPPugTnreajBo1ymcemjJlijm2oKDA76dzGX7wgx94xW/atMkc+8gjj2TlYn3D5JvH5Zdfbo4dPHhwVh6FhYXmvXfTTTeZ5yJJAwYMMMeOGDEiLZfy8nLdeOONpr7OO+888zwkmfenJH32s5/Nqkm3bt3Mc5o3b555LpJ0xx13mOJOO+20rGvl5eW6/vrrTf1dfPHFprgk6/tvSZo2bVpaTcrKyvTTn/7U1Nftt99unockfe1rXzPHPvzww1lrq0OHDho8eLCpv/Hjx5vnIkmDBg0yx/bv3z8tl+LiYp166qmmvh5++GHzPCTpoosuMsf+/ve/z6pJv3799Ne//tXU37hx48xzkfx+wfrLX/4yLRfnnKw/RL7lllvM85Cks88+2xy7Z8+eBt+r8Kf+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAErCCXxqWlpTrllFNMA7355pumuKTevXt7xadav369/ud//scU+8EHH3iNPXToUK/4TD169NAll1xiir366qu9xj7ggAO84lNt2rRJTz31lCn2hBNO8Br7Bz/4gVd8JuecKW7ChAle4957771e8ZnWr1+vhx56yBR75ZVXeo190kknecWnmjVrlnr06GGKvfPOO73G/spXvmKOveqqq7KurV271jynrVu3muciSe3btzfHZq7t3bt3a+nSpaa+5s6da56HJJ1zzjle8ZmOPPJIVVdXm2Lfffddr7F37txpjn3kkUeyru3bt8/U14gRI8zzkKTp06d7xWcaOnSo/vGPf5hix40b5zX2qaee6hWfavPmzfrzn/9sirXeu5N837Nl6tOnj/7rv/7LFDt+/HivsQcNGuQVn+qjjz6q997cFFVVVV5jV1RUeMWn2r17t1auXGmK/e1vf+s1tvUcIUkPP/xw1rUDDjhAjz76qKm/I444wjwXSbr55pu94lOVlpbq5JNPNsVu27bNa+yuXbt6xWdatmyZzj//fFPslClTvMau77XNKooi7dq1yxT7wx/+0Gvsu+66yxx72WWXNfg9fuMPAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASsIJfGK1eu1HXXXWcaaOvWraa4pFmzZnnFpxo0aJAeeeQRU+yrr77qNfaCBQu84jPt3btX27dvN8U+/fTTXmMvXbrUKz7V4MGDNW7cOFPsEUcc4TV2SUmJV3yqxYsX67vf/a4pdsaMGV5jT5061Su+PlEUmeIuueQSr3E/97nPmWNffvnltMd9+vTR9ddfb+rrrLPOMs9DkoYNG+YVn6m8vFynnnqqKbZTp05eY+dzfc2bN89c43Xr1nmN/d5773nFZ9q0aZOefPJJU+wFF1zgNfaiRYu84lMdfPDBevTRR02xBxxwgNfYZWVl5tgf//jHWdd27typ2bNnm/p79tlnzXORpO7du5tjN23alPZ40KBBmjx5sqkv35ps27bNHFvfa+rOnTvN7+N8712rV682xY0ePTrr2hFHHKHq6mpTf9///vdNcUlLlizxik/Vq1cv8/v5Pn36eI1tfV/RkM6dO+vwww83xZ5yyileY99zzz1e8amWLFmiMWPGmGJXrVrlNfaBBx7oFZ+pe/fu5vXunPMau7a21hyb+XoyYMAA8/vHXbt2mechSZdeeqk59rLLLmvwe/zGHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYC6KoqY3dm6dpGXNN51mMzCKosrkg1DykFptLqHkIbG24oiaxA81iR9qEj/UJF5CyUNibcURNYmfYGuSlNPBHwAAAAAAtC78qT8AAAAAAAHj4A8AAAAAQMDyevB3zpU55551zu1wzi1zzo3ZT/srnXNrnHNbnHMTnXOFjbSd4Jxb4Jzb55w7O5/zbmC8JufinDvEOTfFObfeObff/3bCOVflnJvmnKtxzs13zo3K7+zTxgoij8R4QeQSSh6J8YLY84HV5FLnXLVzrtY5N6kJ7cckct7hnHvOOVfWSFv2SY5C2SOJ8UJZW0HkkRgviFxC2e+J8UKpSRB5JMYLIpfAXk+C2POtLY98/8b/Xkm7JfWUdKak+5xzw+tr6JwbLel6SSdIqpI0SNLNjfQ9S9LFkt7J43wb0+RcJO2R9JSkc5vY92RJMyWVS/oPSc845+r9EIY8CCUPKZxcQslDCmfPh1STVZJukTRxfw0TOd4v6SzV5V4jaXwjIeyT3IWyR6Rw1lYoeUjh5BLKfpfCqUkoeUjh5BLS60koe7515RFFUV7+SeqiusSHpFx7VNLtDbR/XNKtKY9PkLSmCeNMl3R2vuadj1xS2hxY95Q22maIpFpJJSnXXpd0IXmEn0soeVhyieueD6kmGWPfImnSftrcKunxlMeDE89FST1t2SfNnEdc90hIayvEPFp7LqHs95BqEmIerT2XXPeJYvx6Esqeb4155PM3/kMk7Y2iaGHKtVmSGvqpx/DE91Pb9nTOledxTla55pKL4ZIWR1G0rRn6zhRKHlI4uYSShxTOng+pJrlKq0kURR8q8SLWQFv2SW5C2SMWcV1buQolDym+uYSy3y3iWpNchZKHFN9cQno9CWXPt7o88nnwL5a0JePaFkklTWyf/Lqh9i0p11zi0ndLjtWSeTT3eNSkZcaL654PqSa5ymV+7JPmHyuue8QirmsrV6HkIcU3l1D2u0Vca5KrUPKQ4ptLSK8noez5VpdHPg/+2yWVZlwrlbStnrb1tU9+3VD7lpRrLnHpuyXHask8mns8atIy48V1z4dUk1zlMj/2SfOPFdc9YhHXtZWrUPKQ4ptLKPvdIq41yVUoeUjxzSWk15NQ9nyryyOfB/+FkgqccwelXDtM0pwG2s9JfD+17dooijbkcU5WueaSizmSBjnnUn9ik6++M4WShxROLqHkIYWz50OqSa7SauKcGySpUHXPSX1t2Se5CWWPWMR1beUqlDyk+OYSyn63iGtNchVKHlJ8cwnp9SSUPd/68sjzhxw8obpPIOwi6RjV/UnC8AbanixpjaRhkrpLmqpGPgxBUkdJRZLekHR+4ut2+f6gBmMuLjGfYZKixNeFjfT9lqRxiXanS9osqZI82kYuoeRhyCW2ez6wmhQkxrpNdR8yUySpoIG2wyVtlfSlRO6PSXoiDrmEUpNQ9khgayuIPELKJZT9HlhNgsgjpFxy3Cdxfz0JYs+3tjzynXyZpOck7ZC0XNKY/bS/StLaxAZ7aD/Jv5J4klL/fbkZF2STc1Hd/yYjc25L99P+FUk7JS2QNIo82k4uoeSRay6J9rHc84HVZGw98xvbSPsxiZx3SPqzpLI45BJKTULZI4GtrSDyCCmXUPZ7YDUJIo+QcsllnyTax/n1JIg939rycImOAQAAAABAgPL53/gDAAAAAICY4eAPAAAAAEDAOPgDAAAAABAwDv4AAAAAAASsIJfGxcXFUXl5uWmg5cuXm+KShg4dao6dP3/++iiKKpOPy8vLo/79+5v62rNnj3kektSpUydz7IwZM9LykKT27dtHHTp0MPV3yCGHmOciSbt37zbFrVixQhs3bnSp15xz5k+Z7NevnzVUktSzZ09zbGZNunXrFvXq1cvU165du8zzkKSdO3eaYz/++OOstVVcXByVlZWZ+uvRo4d5LpK0atUqc+zq1avTcunYsWNk3XcHHXTQ/hs1YvPmzebYDz/8MKsmPvvkyCOPNM9FkubNm2eOrampyapJ586dTX1Z791Js2fP9gnPqklFRUVUVVVl62z9ep+5yLo/JWnmzJlpuXTt2tV87/LNY9OmTebYKIqyalJQUBAVFhaa+qupqTHPRZL69u1rjl25cmXW60nv3r1NfS1cWN//crzp9u3b5xOe13vX4MGDfeaibt26meKWLl2q9evXp71XadeuXdS+fXtTfwcccIApLumjjz4yx+7atSutJp/me66VK1eaY+vb70VFRVFxcbGpP5/35FLd+1kPebsH++73ESNGmGPff//9rJqUlZVF1nuhz3tZyW+f7N69Oy2XLl26mN8H+5wtJGnOnDnm2Mz9niqng395ebluuOEG0yQuuugiU1zSxIkTzbFHH330stTH/fv319SpU019eW5yHXrooeZY59yyzGsdOnTQwIEDTf1VV1eb5yLZf5jzta99zWvcTD/96U+94q+55hpzbGZNevXqpQkTJpj6WrRokXkekjRr1ixz7N133521tsrKynTttdea+rv44ovNc5GkX/ziF+bYm2++OS2XTp066aijjjL1NWXKFPM8JOm5554zx55++ulZNfHhu98/+9nP+oydlkvnzp31xS9+0dTXb37zG/M8JGnIkCE+4Vk1qaqqMj+3DzzwgM9cNGbMGHNsly5dsu5dv//97019Pfjgg+Z5SNIzzzxjjq2trc2qSWFhoYYNG2bqz3ef+LweXXfddWm59O7dWw899JCprxNPPNE8D0navn27T3he712//vWvveK/8Y1vmOJGjhyZda19+/bmH7jdc889prikq6++2hw7e/bsvNXkiiuu8Iq/6aabzLE7d+7MyqO4uFinnXaaqT/rfSLJ+h4pIW/vH7/85S/7zEPPP/+8ObaqqiqrJn379tWf/vQnU38+B15JuvLKK82xS5cuTculrKzM3J/PPCS/82Jj+50/9QcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAFuTTes2ePVq1aZRooiiJTXNJXv/pVr/hU7733nnr37m2KPeCAA7zGvuCCC7ziMznn1K6d7ec3f/nLX7zG/sY3vuEVn6pbt246/vjjTbGvv/6619j//Oc/veJTLViwQMcdd5wptn///l5jL1++3Bx79913Z13r0aOHLrvsMlN/ffr0Mc9Fkjp06OAVn6qmpkbvvfeeKdY55zX2rbfe6hWfqbKyUt/+9rdNsatXr/Yae8yYMebY6urqtMe7du3S/PnzTX2ddNJJ5nlI0pw5c8yxw4cPBnvnlQAAEhdJREFUz7q2ceNGPf7446b+br/9dvNcJGnq1Kle8amKioo0bNgwU+wJJ5zgNfYf/vAHr/hMnTt31uGHH26KfeSRR7zG/ta3vuUVn2rz5s3m1+mbbrrJa+xf/vKX5titW7dmXevTp48uueQSU38vvfSSeS6SdM8995jiFi5c6DVuptGjR3vF+7yPznwt83nP9bnPfc48D6nuNdmqvtfkDRs2aNKkSab+7r33XvNcpPzWpKamRjNmzDD1NXnyZPM8JGngwIFe8Zlmz56tIUOGmGKt58ykhx56yBybuSd87sHvvPOOeR6SVFCQ0xG9yfiNPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQsIJcGq9Zs0bjxo0zDfTiiy+a4pIOPvhgr/hUQ4cO1eOPP26KPfzww73GnjZtmld8puHDh6u6utoUO3bsWK+xTzvtNFPcq6++mnVt79692rZtm6m/jz76yBSXNGHCBHPsM888k/a4srJS3/72t019WZ/PpAULFnjFZ1q8eLG++93vmmJXr17tNfavf/1rc+zVV1+d9ri4uFhHHXWUqa/a2lrzPCSpd+/eXvGZBgwYoHvuuccUe+6553qNfc0113jFpzrkkEPM960XXnjBa+zf/va3XvGZNm/erOeff94UG0WR19iTJ0/2ik+1ceNG8+vi0Ucf7TW2z/PgnMu6tnHjRj3xxBOm/u6++27zXCRp3rx55tjMXHbt2qX58+eb+iouLjbPQ5K2bNlijq2vJtu3b9frr79u6m/dunXmuUjSpEmTTHHf+973sq6NGDFCr732mqm/4447zhSXdNVVV3nFp9q9e7eWLVtmir3tttu8xraO25DCwkL179//U5nL3LlzveJTrVu3zvx+9LnnnvMa+8orr/SKz1RVVWU+Y/i+b5o5c6ZXfKqePXvq8ssvN8VazwJJ+X5dTOI3/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAXNRFDW9sXPrJC1rvuk0m4FRFFUmH4SSh9RqcwklD4m1FUfUJH6oSfxQk/ihJvESSh4SayuOqEn8BFuTpJwO/gAAAAAAoHXhT/0BAAAAAAgYB38AAAAAAAKW14O/c67MOfesc26Hc26Zc27Mftpf6Zxb45zb4pyb6JwrbKTtBOfcAufcPufc2fmcdwPjBZFLKHkkxgsiF+fcpc65audcrXNuUhPaj0nku8M595xzrqyRtlXOuWnOuRrn3Hzn3Ki8Tj57vFBq0uQ8nHOHOOemOOfWO+f2+99KURObUGoSSh6J8YLIJZQ9khgviNeTwGrS5vZJnPNIjMc+YZ80i9ZWk3z/xv9eSbsl9ZR0pqT7nHPD62vonBst6XpJJ0iqkjRI0s2N9D1L0sWS3snjfBsTSi6h5CGFk8sqSbdImri/hon87pd0luryrpE0vpGQyZJmSiqX9B+SnnHO1fsBH3kSSk2anIekPZKeknRuE/umJjah1CSUPKRwcgllj0jhvJ6EVJM2t08U7zwk9kmV2CfURJKiKMrLP0ldEokPSbn2qKTbG2j/uKRbUx6fIGlNE8aZLunsfM075FxCySO0XFLGukXSpP20uVXS4ymPByeeh5J62g6RVJv6PUmvS7qQmuQvj5Q2B9bdQhttQ03acE1CySOkXELZI/WM12pfT0KqSVvdJ3HNo56x2Sf7H4d9EnBN8vkb/yGS9kZRtDDl2ixJDf30Znji+6ltezrnyvM4J6tQcgklDymsXHKRlkcURR8qcZNpoO3iKIq2pVxr7DnyFUpNcs0jF9TEJpSahJKHFE4uoewRi7i+noRUk7a6T3LR0veuXLFPml8o66vV1SSfB/9iSVsyrm2RVNLE9smvG2rfkkLJJZQ8pLByyUUueef6HPkKpSbN+bxRE5tQahJKHs09XpxrEtc9YhHX15OQatJW90lc+s4H9knzC2V9tbqa5PPgv11Saca1Uknb6mlbX/vk1w21b0mh5BJKHlJYueQil7xzfY58hVKT5nzeqIlNKDUJJY/mHi/ONYnrHrGI6+tJSDVpq/skLn3nA/uk+YWyvlpdTfJ58F8oqcA5d1DKtcMkzWmg/ZzE91Pbro2iaEMe52QVSi6h5CGFlUsu0vJwzg2SVKi656O+toOcc6k/OWzsOfIVSk1yzSMX1MQmlJqEkocUTi6h7BGLuL6ehFSTtrpPctHS965csU+aXyjrq/XVJM8fcvCE6j5JsYukY1T3JwzDG2h7sqQ1koZJ6i5pqhr5UAdJHSUVSXpD0vmJr9vl+4MaQssllDxCykVSQaL/21T3ISBFkgoaaDtc0lZJX0rk/ZikJxrp+y1J4xJ9ni5ps6RKapLXPFxiLsMkRYmvC6kJNQk5j5By+f/t3VtsVNUex/HfDFOmrR3AXsgYgZlYjNFigj4gl4SUeFIN0RAC8aHwACEhjT5gCAkSJRYwmkJ4giBR4vFCgsEAPT74wKUWA9EHLg0ypqRVTosIh1NKyyXHctvnpXMyMxuK/Bfk1OX3k/CwJ/v3X2uxZu/Z/ykXX66RwfG8+DzxbE/+itfJsF3H4HhcJ1wn7EkQPPDGv1xSs6Rrkrol1d/j/OWS/jV4gf39HpvYOrjZub9qH+Ib0ou1+LIOn9YiqfEOYzUOcX794HqvSfqHpPIhzk0PruU/kk5J+tvD2g/P9uQPr2Pw97hwXv9kT9gTn9fh01p8uUYGx2u8w3iNQ5w/LD9PPNuTv9x1MpzXMTge1wnXyf99HcNhTyKDhQEAAAAAgIce5N/xBwAAAAAAwwyNPwAAAAAAHqPxBwAAAADAYzT+AAAAAAB4LHY/J5eUlASjRo0yDTRu3DhTLuuXX34xZ/v6+nqCIKjKHpeUlASJRGKoyF3duHHDPA9JSqfT5mxbW1veOiRpzJgxQTKZNNUrKyszz0WSjh49as4GQRDJPY7H40FJSYmp1hNPPGGehyT99NNP5uzAwEDenlRWVgapVMpU69ixY+Z5SG77efXq1dB7q7S0NBg9erSpnjWXdfv2bXO2o6MjtCfW666vr888D0n6+eefXeKhPYlGo0Esdl+37f8ZMWKEy1yc7uGdnZ0P7B48YcIE8zwk6bfffjNnz507F9oTl2v+1KlT5rlI0lNPPWXOHjt2LG8t8Xg8KC0tNdVy/Xy/cuWKOdvV1RXak3g8Hljvh67/4PGlS5dc4nlrqaioCMaPH28qlMm4/ZfVLr8Pt27dCu1JcXGx+ZofOXKkeS6SVFRUZMpdvHhRV65cyXtWiUajQTRq+5mZy+eaJD3zzDPmbCaTyduTsrKyoKKiwlQrEonc+6QhVFZWmrNHjx4NvbdisVhg3eOamhrzXAbn4xLPW0sikQiqqqqGOv+uXPfE5fnxxIkTd/xctD53dXV1meciSVevXjVnf//997y1FBUVBfF43FTL+lyQ1dvba86eP38+tCdZ9/UEOWrUKC1YsMA0iffff9+Uy6qvrzdn9+zZk/cuSiQSmj9/vqnWuXPnzPOQpE8++cScLS8vD10NyWTSXHP69OnmuUjuN5pcJSUlqq2tNWW/+OILp7Gff/55c7azszNvT1KplH744QdTLdeHG5d1fPfdd6H31ujRo7VkyRJTvbq6OvNcJGlgYMCcraury1tLOp3WkSNHTLWam5vN85CkuXPnusRDexKLxTR27FhTsUcffdRlLvrggw/M2VdffTV0D543b56p1ocffmiehyStWbPGnG1sbAztSSqV0vfff2+qN2vWLPNcJKm1tdWcHTlyZN5aSktL9eKLL5pqNTU1mechSS0tLebs0qVLQ3tSVlaml156yVTv5s2b5rlI0u7du83ZW7du5a1l/Pjx2rt3r6nWpEmTzPMYnIs529vbG9qTRCKhOXPmmOq5ftln/WJq7dq1odei0ajGjBljqnf58mVTLmvnzp3mbE1NTd6eVFRU6O233zbVcv0S2fpcIUmRSCT03ioqKjL/UM36bJAzH5d43lqqqqrMvZL1i4+sGTNmmLOPPfZYaE9cnrsaGhrMc5GkgwcPmrPt7e15a4nH4+Z76bZt28zzkKTt27ebs01NTXf99oQ/6g8AAAAAgMdo/AEAAAAA8BiNPwAAAAAAHqPxBwAAAADAYzT+AAAAAAB4jMYfAAAAAACP0fgDAAAAAOAxGn8AAAAAADxG4w8AAAAAgMdo/AEAAAAA8BiNPwAAAAAAHqPxBwAAAADAYzT+AAAAAAB4LHY/JyeTSa1YscI0UHFxsSmXtWfPHqd8roqKCi1atMiUraurcxq7paXFKV+orKxM06ZNM2WPHz/uNPayZctMuR07doReKy8v12uvvWaq98Ybb5hyWQ0NDeZs4fVw+vRp1dfXm2q57sfkyZPN2UgkEnrt8ccf13vvvWeqN3/+fPNcJCmdTjvlc2UyGT377LOmbE9Pj9PY69atM2dXr14dei0SiWjEiBGmej/++KN5LpLU3t7ulM81MDCg06dPm7Lr1693GruxsdEpXyiTyaimpsaU7ejocBrb9fMo1/Xr13XmzBlTtrq62mnsd955xylfaGBgQN3d3absoUOHnMZ+7rnnzNm2tra8456eHn366aemWhs3bjTPQ5LOnj1rzq5atSr0WiqV0rZt20z1rPe8rF27dplysVj4EXns2LFaunSpqd6aNWtMuaympianfK5EIqGZM2easq+//rrT2O+++65TvtC4ceO0YcMGU9baC2TV1taas62trXnH/f39+vrrr021Fi5caJ6H5P6sU6i3t1fbt283ZQ8cOOA0dmdnp1M+Vzqd1meffWbKvvDCC05j9/X1mbND3Sv4iT8AAAAAAB6j8QcAAAAAwGM0/gAAAAAAeIzGHwAAAAAAj9H4AwAAAADgMRp/AAAAAAA8RuMPAAAAAIDHaPwBAAAAAPAYjT8AAAAAAB6j8QcAAAAAwGM0/gAAAAAAeIzGHwAAAAAAj9H4AwAAAADgMRp/AAAAAAA8RuMPAAAAAIDHYvdzcnt7u6ZPn24a6OTJk6Zc1tatW83ZhoaGvONHHnlEU6ZMMdXq7+83z0OSjhw54pQvdPnyZe3fv9+Uffnll53G/vjjj0255ubm0Gvl5eWqr6831VuwYIEpl3X+/HmnfK5kMqmVK1easq+88orT2C0tLU75Qjdv3tSFCxdM2YMHDzqNvWvXLqd8rmg0quLiYlN2y5YtTmOvXbvWKV/o+vXr6u7uNmWffPJJp7GnTp3qlM9VWVmpxYsXm7Lffvut09iTJk0yZ+/0OZZKpfTRRx+Z6lmvr6x9+/Y55XNVV1dr586dpqzrnnz55ZdO+ULXrl3T4cOHTdmnn37aaexZs2aZs21tbXnHZ8+eNX+ebNq0yTwPSXrrrbfM2VWrVoVeO3PmjJYtW2aq99VXX5nnIklz5851yue6cOGCNm/ebMoGQeA09o0bN8zZzz//PO/40qVL2r17t6mW9dkva+LEiU75QkVFRUomk6bs8uXLncbOZDLmbGtra95xf3+/vvnmG1OtHTt2mOchub83C/3666/me9ebb77pNPbFixfN2aamprzjjo4OzZ4921SrurraPA9JikQiTvm74Sf+AAAAAAB4jMYfAAAAAACP0fgDAAAAAOAxGn8AAAAAADxG4w8AAAAAgMdo/AEAAAAA8BiNPwAAAAAAHqPxBwAAAADAYzT+AAAAAAB4jMYfAAAAAACP0fgDAAAAAOAxGn8AAAAAADxG4w8AAAAAgMdo/AEAAAAA8FgkCII/fnIk8m9JXQ9vOg9NKgiCquyBL+uQ/rRr8WUdEu+t4Yg9GX7Yk+GHPRl+2JPhxZd1SLy3hiP2ZPjxdk+y7qvxBwAAAAAAfy78UX8AAAAAADxG4w8AAAAAgMdo/AEAAAAA8BiNPwAAAAAAHqPxBwAAAADAYzT+AAAAAAB4jMYfAAAAAACP0fgDAAAAAOAxGn8AAAAAADz2X+rXbFHgpQ3cAAAAAElFTkSuQmCC\n", 261 | "text/plain": [ 262 | "
" 263 | ] 264 | }, 265 | "metadata": {}, 266 | "output_type": "display_data" 267 | } 268 | ], 269 | "source": [ 270 | "from utils.draw import draw_xy\n", 271 | "\n", 272 | "draw_xy(X, y)" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": null, 278 | "metadata": {}, 279 | "outputs": [], 280 | "source": [] 281 | } 282 | ], 283 | "metadata": { 284 | "file_extension": ".py", 285 | "kernelspec": { 286 | "display_name": "Python 3", 287 | "language": "python", 288 | "name": "python3" 289 | }, 290 | "language_info": { 291 | "codemirror_mode": { 292 | "name": "ipython", 293 | "version": 3 294 | }, 295 | "file_extension": ".py", 296 | "mimetype": "text/x-python", 297 | "name": "python", 298 | "nbconvert_exporter": "python", 299 | "pygments_lexer": "ipython3", 300 | "version": "3.6.10" 301 | }, 302 | "mimetype": "text/x-python", 303 | "name": "python", 304 | "npconvert_exporter": "python", 305 | "pygments_lexer": "ipython3", 306 | "version": 3 307 | }, 308 | "nbformat": 4, 309 | "nbformat_minor": 2 310 | } 311 | -------------------------------------------------------------------------------- /triple.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import math\n", 11 | "import shutil\n", 12 | "import numpy as np\n", 13 | "from PIL import Image\n", 14 | "from utils.draw import draw_xy\n", 15 | "import matplotlib.pyplot as plt" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 2, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "def generate(count):\n", 25 | " X = np.random.randint(0, high=255, size=(count, 9))\n", 26 | " a = np.array([[1, 1, 1, 0, 0, 0, 0, 0, 0], \n", 27 | " [0, 0, 0, 1, 1, 1, 0, 0, 0], \n", 28 | " [0, 0, 0, 0, 0, 0, 1, 1, 1]])\n", 29 | "\n", 30 | " Y = np.eye(3)[np.argmax(X.dot(a.T), axis=1)]\n", 31 | " return X, Y" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 3, 37 | "metadata": {}, 38 | "outputs": [ 39 | { 40 | "data": { 41 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/4AAAGRCAYAAADcoWhrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeXTV1bn/8c+GhARISEiCzBgQARmqXm5/dnLJLbbUq9brtFpxXrZWadVaV9XWDuC1qK2zVtS2jhTU4tTBqdeham8nBhGDgoAMiswQIIEwfX9/5Jy7zpDpPPsEvtm8X2uxVs6X/ey9n/Ps/T1nJ+HgoigSAAAAAAAIU6cDPQEAAAAAANB+OPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABCyvB3/n3Hecc7Odcw3OuYfb0H6ic26Fc67OOfesc66ihbbVzrnXnHP1zrn3nXPH53PuTYxX4Zx7JjG3Fc65ia20v9I5t8Y5V+uce9A5V9RC2wecc4ucc/uccxfkffLpY4VUkyByCSWPxHhB5JLLfnfOjXbOveSc2+Cci9rQN/cuA9ZWx15bcc4llD2SGC+IXELZ74nxQtknB2VNEu3juk+CWFuJ8YLIpaPlke+f+K+WdIOkB1tr6JwbJel+SedK6i2pXtK9LYTMlDRPUqWk6yTNcs718p1wC34paVdibmdLmpaYcxbn3ARJ10oaL6la0hBJU1roe76kSZLm5nG+zQmpJqHkEkoeUji5tHm/S9ot6UlJF7Wxb+5dNqyt1sV2bSneuYSyR6Rwcgllv0vh7JODsiYx3yehrC0pnFw6Vh5RFOX9jxpvFA+30maqpBkpjw9T4xNX2kTbYZIaUv9O0puSLmmn+XdPzGVYyrXHJN3UTPsZkqamPB4vaU0bxnlL0gXtkUNoNQkxl1Dy6Oi55LrfU9oMbbyFttiGexdr66BcW3HNJaQ9ElIuKWN12P1uqUlKm1jtk4O5JnHdJyGtrVBy6Yh5HMh/4z9Kjd8lkyRFUbRUiSevmbbLoijalnJtfuJ6exgmaW8URYvbOF5aLomvezvnKttpfu0lzjXJVSi5hJKHFN9cct3vueDetX+wtvLbd1NCySWkPRJSLrmI636XwtknuQqpJnHdJyGtrVBy6XB5HMiDf4mk2oxrtZJKPdvmQ67jZbZPft1e82svca5JrkLJJZQ8pPjm0p5jce/aP1hb+e17f48X55rEeY+ElEsu4rrf23u8OL/Gh1STuO6TkNZWKLl0uDwO5MF/u6QeGdd6SNrm2TYfch0vs33y6/aaX3uJc01yFUouoeQhxTeX9hyLe9f+wdrKb9/7e7w41yTOeySkXHIR1/3e3uPF+TU+pJrEdZ+EtLZCyaXD5XEgD/41ko5MPnDODZFUJGlxM22HOOdSv8txZOJ6e1gsqcA5d3gbx0vLJfH12iiKNrbT/NpLnGuSq1ByCSUPKb655Lrfc8G9a/9gbeW376aEkktIeySkXHIR1/0uhbNPchVSTeK6T0JaW6Hk0vHyyPOHHBRIKpZ0oxo/3KBYUkEzbUdJ2irpWDV+OMJ0SY+30PffJd2S6PNUSVsk9cr3BzWkjPe4Gj9Nsbukz6vx1ytGNdP2K5LWSBopqaekV9XCBztI6pLI46+Svpn4ulM75RFSTYLIJZQ8Qsolx/3uEnMaKSlKfF0UhzwMuXDvYm0ddLmEskdCyiWU/W6oSZz3ycFakzjvkyDWVki5dLQ88p385EQiqX8mt9B+oqSVkuokPSepooW21ZJel7RD0iJJx7fXYkyMVyHp2cTcVkqa2Er770lam7jxPdRKIV9v4nka1055hFSTIHIJJY+QcsllvyfmlZnz8jjkkWsuifbcu1hbB1UuoeyRkHIJZb/nWpOY75ODsiaJ9nHdJ0GsrZBy6Wh5uETHAAAAAAAgQAfy3/gDAAAAAIB2xsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAhYQU6NCwqiLl26mAbasWOHKS6pe/fu5ti6uroNURT1Sj72yWP37t3meUjSsGHDzLELFy5My0OSnHPmT2fs3bu3eS6SVFVVZYr7+OOPtXnzZpd6rUuXLlG3bt285mO1detWc2wURWk1qaioiAYMGGDqa8GCBeZ5SNJhhx1mjl26dGnW2iouLo5KSkpM/dXX15vnIkkFBTndmtJs27YtLZeqqqqourra1Ne7775rnockWdeC1HRNSktLo8rKSlN/u3btMs9FkpxzrTdqxurVq7PuwYWFhaa+Ro4caZ6HJM2dO9cnPKsmVVVV0aBBg0ydNTQ0+MxFxcXF5ti5c+em5dKtW7eovLzc1FdpaWnrjVrgs7YWLVqUVROf+/DOnTvNc5GkDz74wCc8LZeePXtG/fv3N3W0Zs0an3l43YPXrl2b1/cqffr0Mc9Fkjp37myK27x5s+rq6tIWp8/riee9R/369TPHfvzxx2k16d69u3m/+74PXr9+vU941toqLy+PrM/N5s2bfeaibdu2mWMzzyY+77k2btxonocklZWVmWNra2uzatK1a9fI+rrQs2dP81wkae3atebYzFx87sHr1q0zz0Pyuwd/8sknWTX5v35z6ahLly7mg+v8+fNNcUljxowxx/79739fkfq4S5cuGj58uKmvjz76yDwPSXriiSfMsWPGjFnRequ2O/fcc73iv/nNb5riTjvttKxr3bp107HHHmvqz/d/pnj11VfNsTt27EiryYABA/SnP/3J1Jf18JB02223mWNPOeWUrLVVUlKik046ydTf22+/bZ6LJFVUVJhjX3vttbRcqqurNXv2bFNfhx9+uHkeknTrrbeaY5uqSWVlpX784x+b+lu1apV5LlLjfdPquuuuS8ulsLBQ1jfP//jHP8zzkPzyiKIoqyaDBg3SW2+9ZepvyZIl5rlI0ogRI8yxRUVFabmUl5fr4osvNvVlvXenzMUce+yxx2bVZMCAAfrjH/9o6m/x4sXmuUjSl770JZ/wtFz69++vp59+2tTR1KlTfeZh/sa+JN166615fa9y0UUXecX36NHDFHfXXXdlXfN5PfFZ55J0xRVXmGOvvvrqrP3+7W9/29SX7/vgadOm+YRnra1+/fpp+vTpps5mzZrlMxe98cYb5ti//vWvabmUlJToxBNPNPX16KOPmuch+d3D//jHP2bVpLS0VKeffrqpvzPPPNM8F0m68847zbG///3vs+7BTz31lKmve+65xzwPye998PXXX9/sPZhf9QcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAFuTQeOXKkZs+ebRpo1qxZprikJ554wis+Vbdu3XT00UebYu+//36vsUePHu0Vn2ns2LHmmkyZMsVr7KefftoUt2XLlqxrgwYN0r333mvq7wtf+IIpLunnP/+5Ofayyy5Le/zJJ5/o+uuvN/X14osvmuchSZ/97Ge94jN169ZNY8eONcU2VeNcPPfcc17xqebMmSPnnCn28ccf9xr7qquu8orPVFVVpYsuusgUu2TJEq+xe/ToYY697rrr0h4fcsghuuKKK0x9FRTk9LKV5YILLjDHPvTQQ1nX5s2bp+7du5v6mzlzpnkukjR58mSv+FSffPKJ+TXhy1/+stfYL730kld8pg8//FBnn322KfbNN9/0Gvv11183x44bNy7t8Y4dO/TOO++Y+vJ9r7J+/Xpz7K233pp1raqqSqeeeqqpv5/+9KfmuUj256JTp+yfja1atUqXX365qb+jjjrKFJf0P//zP17xqRoaGsyvCRUVFV5jn3feeebYRx99NOvae++9Z36vEkWReS6+Mt+XbNy4UY899pipr5/97Gdec/nRj37kFZ9p8+bN5nPfEUcc4TX20qVLveJT1dTUaMSIEaZY37V17rnnesU3h5/4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAABK8il8fr163X//febBnruuedMcUk/+clPzLGzZs1Ke7xjxw4tWLDA1Jc1LumYY47xis+0detWvfTSS6bYefPmeY09cOBAU9yuXbuyri1YsECDBg0y9RdFkSku6YwzzvCKTzVo0CBNmzbNFFtQkNN2zHLeeed5xWfasGGDHnzwQVOs79qaPn26Ofbcc89Ne9y3b19961vfMvXVq1cv8zwk6YorrjDHXnbZZVnX6uvr9fbbb5v6O/roo81zkaQePXp4xadauXKlLr30UlPs8ccf7zW27z7LNHbsWM2ePdsUW1tb6zX2o48+6hWfql+/fvr2t79tir3mmmu8xj7uuOO84vNp1KhRXvEzZ87M00ykZcuW6cwzzzTFlpSUeI29bds2r/hMDQ0NWr58uSm2sLDQa+znn3/eFNfU/ly3bp3uvvtuU3+33HKLKS6pa9eu5tiXX3457XFdXZ3+9re/mfp67733zPOQpNGjR3vFZyovL9d//Md/mGInTJjgNXZRUZFXfKoBAwboyiuvNMVeddVVXmP7vI92zmVd27NnjzZs2GDq7/LLLzfPRZK+//3ve8WnGjVqlJ566ilT7KRJk7zG3rlzp1d8c/iJPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwFwURW1v7Nx6SSvabzrt5tAoinolH4SSh9RhcwklD4m1FUfUJH6oSfxQk/ihJvESSh4SayuOqEn8BFuTpJwO/gAAAAAAoGPhV/0BAAAAAAgYB38AAAAAAALGwR8AAAAAgIDl9eDvnKtwzj3jnKtzzq1wzk1soe1o59xLzrkNzrlWP2jAOVftnHvNOVfvnHvfOXd8PufexHhB5BJKHonxvuOcm+2ca3DOPdyG9hMTOdc55551zlW00HZ/1iSIPBLjBZFLKHkkxgsil1zuXYn2Vzrn1jjnap1zDzrnilpo+4BzbpFzbp9z7oK8Tz59rCDySIwXRC655OHi/7pITVrvm3uwQWA1CSKXUPJIjBdELh3tHpzvn/j/UtIuSb0lnS1pmnNuVDNtd0t6UtJFbex7pqR5kiolXSdplnOuyU8szJNQcgklD0laLekGSQ+21jCR4/2SzlVj7vWS7m0hZH/mEkoeUji5hJKHFE4ubb53OecmSLpW0nhJ1ZKGSJrSQt/zJU2SNDeP821OKHlI4eQS0usiNWkd92CbkGoSSi6h5CGFk0vHugdHUZSXP5K6JxIflnLtMUk3tRI3tHEaLbYZJqlBUmnKtTclXZKv+YeYSyh5NDH2DZIebqXNVEkzUh4flnguSptoe0ByCSWPkHIJJY+Onkuu9y5JMyRNTXk8XtKaNozzlqQL2rEGQeQRUi655pHSJnavi9QkfjXJGPuguQfHuSah5BJKHiHl0hHvwfn8if8wSXujKFqccm2+pOa+e5OLUZKWRVG0rR36bkoouYSSh8UoNc5HkhRF0VIlNmczbeOaSyh5SOHkEkoeUnxzyfXelZZH4uvezrnKdphbLkLJQwonl5BeF6lJ67gH24RUk1ByCSUPKZxcOtw9OJ8H/xJJtRnXaiWVxrzv/T3e/swllDwscplfnHMJJQ8pnFxCyUOKby65jpXZPvn1gX6eQ8lDCieXkF4XqcmB7TsfQrkHx6Xv/T0eNYnfeHGuyQG/B+fz4L9dUo+Maz0kbWuibZz63t/j7c9cQsnDIpf5xTmXUPKQwskllDyk+OaS61iZ7ZNfH+jnOZQ8pHByCel1kZoc2L7zIZR7cFz63t/jUZP4jRfnmhzwe3A+D/6LJRU45w5PuXakpJo89F0jaYhzLvU7Ivnquymh5BJKHhY1apyPJMk5N0RSkRqfk6baxjWXUPKQwskllDyk+OaS670rLY/E12ujKNrYDnPLRSh5SOHkEtLrIjVpHfdgm5BqEkouoeQhhZNLx7sH5/lDDh5X4ycpdpf0eTX+CsOoZto6ScWSRkqKEl8XtdD33yXdkmh3qqQtknrl+4MaQssllDwS4xUkxrpRjR+eUSypoJm2oyRtlXRsIvfpkh6PQy6h5BFSLqHkEVIuOd67viJpTeLe1VPSq2rhQ4IkdUnk8FdJ30x83Yk8Do5ccswj7q+L1CR+NTkY78Fxr0kQuYSSR0i55JjHAb8H5zv5CknPSqqTtFLSxBbaVieKl/pneSvtX5e0Q9IiSce312IMKZdQ8kiMN7mJ+U1uof3ERM51kp6TVBGHXELJI6RcQskjpFyUw70r0f57ktaq8U30Q2r5TcHrTTxH48jj4MgllzwU/9dFahK/mkxuYn6TW2jf4e/BHaAmQeQSSh4h5ZJLHon2B/Qe7BIdAwAAAACAAOXz3/gDAAAAAICY4eAPAAAAAEDAOPgDAAAAABAwDv4AAAAAAASsIJfG5eXlUd++fU0DLVmyxBSXNHz4cHNsTU3NhiiKeiUfd+vWLSorKzP11amT3/dKdu3aZY7dsGFDWh6SVFVVFVVXV5v627x5s3kuklRUVGSKW716tbZs2eJSr5WVlUW9e/c29bd27VpTXNK2bdvMsVEUpdXEOWf+tMyKigrzPCR7PSTpk08+yVpbBQUFUWFhoam/nTt3muciScXFxebYnTt3puXik4fPfpWkyspKc+z69euzalJUVBR169bN1N+WLVvMc5GkAQMGmGM/+uijtFx69OgRHXLIIaa+du/ebZ6HJPXq1av1Rs2YM2dOVk26du0a9ejRw9TfwIEDzXORpKVLl5pjt2zZkpZLz549o379+pn62rBhg3kekrR+/XpzbOY9WJJKS0ujqqoqU3/Lly83z0WSSkpKzLHbt29Py8Xn9b2mxu+/rB4xYoQ59u23386qSffu3aPy8nJTfxs3+v0X19a1sHnzZtXV1aW9V+nUqVPUuXNnU3+jR482xSV9+OGH5tja2tq0mlRUVETWe3p9fb15Hr6WLl2a1/1eWlraeqMWdOnSxRyb+XpSWloaWd8zWPNP+uijj8yxa9euzaqJz3vhPn36mOciSf379zfHZtbE5x7s+17l3XffNcfu27cvqyZJOR38+/btq4cfftg0iZNPPtkUl/S73/3OHDty5MgVqY/Lysp0wQUXmPryvUn43Lh//etfr8i8Vl1drdmzZ5v683lOJWno0KGmuLPPPjvrWu/evXXPPfeY+rv11ltNcUlvvPGGOXbnzp1ZNbE68cQTveIHDx5sjr3++uuz8igsLNRhhx1m6s/3TeeQIUPMsQsXLkzLpbCw0LxWffarJJ1++unm2Pvuuy+rJt26ddP48eNN/T311FPmuUjS9773PZ/YtFwOOeQQ3Xbbbaa+1qxZY56HJF188cXmWOdcVk169OjR5D2tLazPQdIZZ5xhjn3qqafScunXr59mzpxp6us3v/mNeR6SdN9995ljd+3alVWTqqoqXX/99ab+zjvvPPNcJGns2LHm2L/85S9pufi8vo8ZM8Y8D0l6/fXXzbHl5eVZNSkvL9e3vvUtU3+PPfaYeS6SdNFFF5nimnpP0rlzZ1m/gfHqq6+a4pIuvPBCc+xzzz2XVpMBAwbo+eefN/U1d+5c8zx8nXLKKU3u9ylTppj6++IXv+g1H59viGe+nlRWVurHP/6xqS/rGk+6+uqrzbG/+MUv8vY+WJL5fJZ04403mmMza+JzD/Z9r2J9/y1J9fX1zdaEX/UHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgBbk0fv/99/WZz3zGNNCMGTNMcUlHHHGEV3yqzZs3a9asWabY+vp6r7FXr17tFZ9pzpw5cs6ZYq21TPrSl75kituyZUvWtQ8++EATJkww9Td27FhTXNLcuXPNsSNHjkx7XFxcrKFDh5r6evTRR83zkKRdu3aZY6+//vqsa6NGjdLs2bNN/R177LHmuUjSySefbI695ppr0h6Xl5frpJNOMvX1s5/9zDwPSfr444/Nsffdd1/WtX79+mny5Mmm/m6//XbzXCTpmWee8YpPtXLlSn3nO98xxd5///1eY0+fPt0rPlNxcbGGDx9uiv30pz/tNbZ13KZ8+OGHOv/8802x8+bN8xp727Zt5tiHHnoo69rHH3+cdR9oqzPPPNM8F0mqqqoyx/7lL39Je7xgwQINHjzY1Nd7771nnofU9PPqY/Xq1frpT39qij3jjDO8xl63bp0pbvfu3VnX9uzZow0bNpj669mzpykuqUuXLl7xqRYsWKCBAweaYn3eM0nSeeed5xWfaePGjXr44YdNsV/72te8xra+/25KfX29+T3XN77xDa+xN2/ebI79xS9+kXWta9eu5tcn3/fC48aN84pPtXLlSk2aNMkU67vOp02bZo5t6bWcn/gDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAEryKXx2LFj9c9//tM00O9//3tTXNL555/vFZ9q9OjRmj17tin2Jz/5idfYZ511ljl25MiRWdfKy8s1fvx4U39z5swxz0WStm7daorbu3dv1rWxY8eaa/LpT3/aFJe0YcMGr/hM+/btM8Xdd999XuOeffbZXvGZli1bpq9//eum2D/84Q9eY69cudIce80116Q97t+/v2688UZTX7/61a/M85Ckiy++2Cs+U9euXTV69GhT7A033OA1tnXcpuzevVurVq0yxd59991eY3//+9/3is+0b98+1dXVmWIHDx7sNfZvf/tbr/hUO3fuVE1NjSnWOec19qWXXuoVn+lTn/qU+fWktrbWa+xXXnnFHDtt2rS0xz179tQZZ5xh6uuHP/yheR6SdPvtt3vFZ/J5jbfGJeVzn/i8D7733nu9xv7KV75ijv3d736X9rh79+4aM2aMqS/r/S5pwYIF5tim7jVDhw41nzHGjRtnnosk7dq1yxzbpUuXtMe1tbV6+eWXTX1lvu/Jle89PNOAAQN08803m2IvvPBCr7ELCwu94lOtX78+677cVrNmzfIa+5133vGKbw4/8QcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACJiLoqjtjZ1bL2lF+02n3RwaRVGv5INQ8pA6bC6h5CGxtuKImsQPNYkfahI/1CReQslDYm3FETWJn2BrkpTTwR8AAAAAAHQs/Ko/AAAAAAAB4+APAAAAAEDA8nrwd85VOOeecc7VOedWOOcmttB2tHPuJefcBudcq//ewDlX7Zx7zTlX75x73zl3fD7n3sR4bc4l0f5K59wa51ytc+5B51xRC20fcM4tcs7tc85dkPfJp4/1HefcbOdcg3Pu4Ta0n5jIt84596xzrqKFttTEIJQ8EuMFsecP1n0S85oEsU9YWx17bcU5j8R4QayvUPZ7Yrwg1hc1oSbtKaCadKh7cL5/4v9LSbsk9ZZ0tqRpzrlRzbTdLelJSRe1se+ZkuZJqpR0naRZzrkmP7ggT9qci3NugqRrJY2XVC1piKQpLfQ9X9IkSXPzON/mrJZ0g6QHW2uYyO9+SeeqMe96Sfe2EEJNbELJQwpnzx+U+0Txrkko+4S11TaxXFuKdx5SOOsrlP0uhbO+qEnbUBObUGrSse7BURTl5Y+k7mos4LCUa49JuqmVuKGN02ixzTBJDZJKU669KemSfM3fJxdJMyRNTXk8XtKaNozzlqQL2iOHJsa6QdLDrbSZKmlGyuPDEs9DaRNtqclBnIcll5Q2sdvzKWMcVPskrjUJaZ+wtjr22oprHiGtr5D2eyjri5pQE2qSc04d4h6cz5/4D5O0N4qixSnX5ktq7rs3uRglaVkURdvaoe+m5JrLqMTfp7bt7ZyrbKf5tZe0PKIoWqrExmymLTXJXSh5SGHt+VyEtE9ysT9zCWmf5IK1lf++M4WSh0Vc11dI+z2U9UVN2oaa2IRSk1wd8HtwPg/+JZJqM67VSiqNed/5GC+zffLr9ppfe8klb2piE0oeUlh7Phch7ZO49O07Vpz3SS5YW/nve3+OFef7lhTf9RXSfg9lfVGTA9+371jUJP99+zrg9+B8Hvy3S+qRca2HpG1NtI1T3/kYL7N98uv2ml97ySVvamITSh5SWHs+FyHtk7j07TtWnPdJLlhb+e97f44V5/uWFN/1FdJ+D2V9UZMD37fvWNQk/337OuD34Hwe/BdLKnDOHZ5y7UhJNXnou0bSEOdc6nc58tV3U3LNpSbx96lt10ZRtLGd5tde0vJwzg2RVKTG56OpttQkd6HkIYW153MR0j7Jxf7MJaR9kgvWVv77zhRKHhZxXV8h7fdQ1hc1aRtqYhNKTXJ14O/Bef5gg8fV+AmE3SV9Xo2/kjCqmbZOUrGkkZKixNdFLfT9d0m3JNqdKmmLpF7t+CENueTyFUlrErn0lPSqWviACkldEnn8VdI3E193aqc8ChL936jGD84ollTQTNtRkrZKOjaR93RJj1MT8shTLrHd8wfxPolzTYLYJ6ytDr+2YptHSOsrlP0e0vqiJk8PYA8AACAASURBVNSEmrQpjw51D8538hWSnpVUJ2mlpIkttK1OFC/1z/JW2r8uaYekRZKOb6/FmGsuifbfk7Q2UdCHWlmQrzeR+7h2ymNyE2NNbqH9xES+dZKek1RBTcgjH7nEec8frPsk5jUJYp+wtjr22opzHiGtr1D2e0jri5pQE2rSpjwmNzG3yS20P6D3YJfoGAAAAAAABCif/8YfAAAAAADEDAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgBbk07ty5c1RQkFPI/9m1a5cpLunII49svVEz5s+fvyGKol7Jx0VFRVH37t1NfW3ZssU8D0n6t3/7N3PsnDlz0vKQJOec+dMZR48ebZ6LJC1dutQUt2vXLu3Zs8elXquqqoqqq6tN/a1cudIUl7R+/Xqf8LSa9OjRI+rVq1dL7ZtVWlraeqMWzJ8/3yc8a21VVlZGAwcONHVmvU8krVq1yhy7bt26tFx81taSJUvM85Ck2tpan/CsmpSXl0d9+vQxdbZ9+3afuejjjz/2CU/LpWvXrlFZWZmpo549e/rMQ9Z7v5T/e3BlZaV5LpJUWFhojl2zZk3e9klDQ4N5Hom5mGM3bNiQVZOuXbtG1vtpp05+Pw8ZMGCAOTZzffmsLV/dunUzx9bX1+d1nxxxxBHmuUj2XJYvX64NGzakvVfp0qVLVFxcbOpv2LBhprikefPmmWP37duXVpOCgoKoqKjI1Jc1Lmnw4MHm2Llz52atrYqKiqh///6m/hYuXGieiyQddthh5tgPPvgg6x48aNAgU1++9625c+eaY6MoyqpJcXGx+Zzlsz4kacWKFebYzNcTn/uW7z4ZMmSIOfa9997LqklSTu/OCwoK1K9fP9Mkli9fbopLeuWVV8yxVVVVaauge/fuGj9+vKmvP/3pT+Z5SNLs2bPNsc45+2puwjPPPOMVf+aZZ5riFi1alHWturra/NxMmjTJFJc0bdo0n/C0mvTq1Us333yzqaNx48b5zEPWbzgkZK2tgQMH6tVXXzV1VlFR4TMXffe73zXH3nnnnWm5+Kytk046yTwPyft+kVWTPn366IEHHjB19re//c1nLrr22mt9wtNyKSsr0znnnGPqyHrfSTrmmGPMsfm+B5988sle8X379jXH3njjjXnbJ9ZvAifddNNN5thf//rXWTUpLS3V1772NVN/vm/WbrnlFnNsU+vL+oZ+37595nlIfoftOXPm5HWfzJgxwyv+qKOOMsX9+7//e9a14uLiJq+3hfX1NMn6zVJJ2rp1a1pNioqKzDUeOnSoeR6S9Mgjj5hji4uLs9ZW//799eyzz5r6s66NpLvuussce8IJJ6TlMmjQIL3xxhumvkpKSszzkBrXtVVDQ0NWTbp3764TTjjB1N+jjz5qnoskXXrppebYBx54IG/3rkMPPdQr3ue+d/TRRzebB7/qDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwApyaVxYWKjevXubBtq6daspLumyyy7zis+XO++80yv+y1/+cp5m0mjYsGG6//77TbFlZWVeY1trunfv3qxrmzZt0uOPP27q71//+pcpLunGG280x/7gBz9Ie7x69WpNmTLF1NfcuXPN85CkKIrMsc65rGvr1q3THXfcYervqaeeMs9FkhYuXOgVn2rfvn3avn27KfbCCy/0GruwsNAc++yzz2ZdW7ZsmSZOnGjq78knnzTPRZLOOussc+zMmTPTHu/Zs0ebNm0y9XXOOeeY5yFJb775pld8puLiYg0dOtQUO23aNO+xrTLve3V1deZ76f/7f//PPA9JmjBhgld8pm7duulTn/qUKXb69OleY5eUlHjFpxo7dqxmz55tij388MO9xn7rrbfMsV27ds261q9fP11yySWm/j766CPzXKTG12WL2trarGvOOXXp0sXUX1Ovs7kYOXKkOTbzNbW+vl5z5swx9XX88ceb5yFJRUVFXvGZ3n33XfM9+K677vIae/DgwV7xqTZu3Kjf/va3ptgXX3zRa+zvfve75tibb74569rgwYPN91Lre5ykzPcbPjp37qzS0lJTbFPv4XLxuc99ziu+OfzEHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIWEEujUeMGKG///3vpoHWr19vikuaNGmSV3yq/v37a+rUqabYYcOGeY09fPhwr/hMpaWlGjdunCl2yZIlXmN36dLFFNepU/b3mz788EOde+65pv769etniks6/vjjzbE/+MEP0h7v2rVLy5cvN/U1aNAg8zwk6c477/SKz1RfX6+3337bFFtYWOg1dubzmosbb7wx7fHOnTu1aNEiU19XXHGFeR6StGrVKnNsU/uke/fu+vSnP23q75///Kd5LpI0Y8YMc+zMmTPTHm/cuFEPPfSQqa8FCxaY5yFJY8aM8YrPNGTIEPNz4zuXU045xSs+VX19vebNm5e3/nJx0kknmWNfeumlrGudO3dWWVmZqb8+ffqY5yJJdXV1XvGpGhoa9OGHH5pip0+f7jX2nj17vOIzbdiwQb/5zW9MsW+++abX2Ndcc40pbtOmTVnXfPbJCSecYIpL2rdvnzl24cKFaY979uypL33pS6a+XnnlFfM8JOmJJ57wis9UVlamY4891hT7X//1X15jDxw40Cs+VefOnVVaWmqKffbZZ73Gvu2227ziM+3YscP8Wn377bd7jf3OO++YY2tqatIe9+nTR5dffrmprw8++MA8D0navHmzOdY51+zf8RN/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAuSiK2t7YufWSVrTfdNrNoVEU9Uo+CCUPqcPmEkoeEmsrjqhJ/FCT+KEm8UNN4iWUPCTWVhxRk/gJtiZJOR38AQAAAABAx8Kv+gMAAAAAEDAO/gAAAAAABCyvB3/n3Hecc7Odcw3OuYfb0H6ic26Fc67OOfesc66ihbbVzrnXnHP1zrn3nXPH53PuTYxX4Zx7JjG3Fc65ia20v9I5t8Y5V+uce9A5V9RC2wecc4ucc/uccxfkffLpY1ETatJuQskllLWVGC+IXAJaW0HkkRgvlLXV5jycc6Odcy855zY451r9t5Fxrkmccwklj8R4QeQSyn5PjBdELqHkkRgviFw62n7P90/8V0u6QdKDrTV0zo2SdL+kcyX1llQv6d4WQmZKmiepUtJ1kmY555r84II8+aWkXYm5nS1pWmLOWZxzEyRdK2m8pGpJQyRNaaHv+ZImSZqbx/k2h5pQk/asSSi5hLK2pHByCWVthZKHFM7aanMeknZLelLSRW3sO7Y1UbxzCSUPKZxcQtnvUji5hJKHFE4uHWu/R1GU9z9qfJPzcCttpkqakfL4MDU+caVNtB0mqSH17yS9KemSdpp/98RchqVce0zSTc20nyFpasrj8ZLWtGGctyRd0B45UBNq0t41CSWXkNZWSLmEsLZCyiOUtZVrHilthkqKWmkT65rENZdQ8ggpl1D2e0i5hJJHSLl0xP1+IP+N/yg1fkdGkhRF0VIlnrxm2i6LomhbyrX5ievtYZikvVEULW7jeGm5JL7u7ZyrbKf5tRdqEj9xrkmu4ppLSGsrpFxyEde1las45xHK2so1j1zEvSa5iPM9OBfUxCaU/S6Fk0soeUjh5NLh9vuBPPiXSKrNuFYrqdSzbT7kOl5m++TX7TW/9kJN4ifONclVXHMJaW2FlEsu4rq2chXnPEJZW+35vMW9JnHpe3+ORU32z1hx3e9SOLmEkocUTi4dbr8fyIP/dkk9Mq71kLTNs20+5DpeZvvk1+01v/ZCTeInzjXJVVxzCWlthZRLLuK6tnIV5zxCWVvt+bzFvSZx6Xt/jkVN9s9Ycd3vUji5hJKHFE4uHW6/H8iDf42kI5MPnHNDJBVJWtxM2yHOudTvchyZuN4eFksqcM4d3sbx0nJJfL02iqKN7TS/9kJN4ifONclVXHMJaW2FlEsu4rq2chXnPEJZW7nmkYu41yQXcb4H54Ka2ISy36VwcgklDymcXDrefs/zhxwUSCqWdKMaP9ygWFJBM21HSdoq6Vg1fjjCdEmPt9D33yXdkujzVElbJPVqxw9seFyNn6bYXdLn1fjrFaOaafsVSWskjZTUU9KrauGDHSR1SeTxV0nfTHzdqZ3yoCbUpD1rEkQuoaytkHIJaG0FkUdgayuXPFxiLiMlRYmvizpoTWKbSyh5hJRLKPs9pFxCySOkXDrafs938pMTiaT+mdxC+4mSVkqqk/ScpIoW2lZLel3SDkmLJB3fXosxMV6FpGcTc1spaWIr7b8naa0a37Q91EohX2/ieRrXTnlQE2rSnjUJIpdQ1lZIuQS0toLII7C11eY8Es9x5ryWd8SaxDmXUPIIKZdQ9ntIuYSSR0i5dLT97hIdAwAAAACAAB3If+MPAAAAAADaGQd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAhYQS6Ne/bsGfXr18800LJly0xxSbt37zbH7t27d0MURb2SjysqKqKBAwea+vL9XxBWrVpljt2yZUtaHlJjTfr372/qr6amxjwXSSotLTXF7dy5U7t27XKp10pKSqKKigpTf3v27DHFJVnHlaSampq0mpSXl0d9+/Y19fXJJ5+Y5yFJtbW1PuFZa8s5Z17sQ4cO9ZmLdu3aZY5duXJlWi7dunWLysrKTH0VFxeb5yFJlZWV5tg5c+Zk1aRz585R586d9/tcJMl6n5GycyksLIyKiopMfVnjkrZu3WqO3bNnT5M1KSjI6aX0//jmMmDAAHPse++9l5ZLUVFRVFJSYurLZ79K0rBhw8yxc+fOzapJWVlZdMghh5j6q6+vN89Fkqz3fyl7n3Tv3j0qLy839dWnTx/zPCRp7ty5PuF5fa+ybt06n7lo/fr15tgoitLeq/i8xm/evNk8D0my3vslafXq1Vnvg633D8/3Glq7dq05tqGhIWttVVVVRdXV1ab+fM4Wkt/a+uSTT9JyKSsri6z79sMPPzTPQ5JGjBhhjl2wYEFWTXr06GG+By9dutQ8F8nvHpxZE5/zos95T5K2b99ujt29e3dWTZJyerfSr18/zZw50zSJr33ta6a4JJ9DUW1t7YrUxwMHDtSLL75o6sv3Dc53v/tdc+yzzz67IvNa//799fTTT5v6Gz58uHkuknTMMceY4v7xj39kXauoqNBVV11l6m/Dhg2muKSJEyeaY0eOHJlWk759++rRRx819TVlyhTzPCTpT3/6k0941trycdddd3nFL1++3Bw7adKktFzKysp04YUXmvry3SPnn3++OdY5l1WTzp07m9/Qn3POOea5SNLUqVPNsZm5FBUV6cgjjzT1deihh5rnIUmvvPKKOXbdunVZNSkoKJD1G+KDBw82z0WSbrnlFnPs2LFj03IpKSnRhAkTTH2tXr3aPA9Jevnll82xRUVFWTU55JBDzPegefPmmeciST/84Q/NsZn7pLy8XJdccompr2uuucY8D8n7m1JNvld58sknTZ3dfffdPnPRfffd5xWfyuc1/oknnvAa2/oNbEn6yU9+klaTAQMG6Pnnnzf15flew+u+tWTJkqy1VV1drdmzZ5v68713/epXvzLHTp48OS2XPn366IEHHjD1ddZZZ5nnIcm8FiRp4MCBTd6Db7vtNlN/p5xyinkuknTxxRebY6dMmZJ1XnzhhRdMfV155ZXmeUjS3/72N3PsqlWrmn1Pz6/6AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQsIJcGi9btkwTJ040DfT++++b4pKiKDLHOufSHq9du1Z33HGHqa9//etf5nlI0quvvmqOzcxDkhYvXqzx48eb+vN5TiXpv//7v01xCxcuzLr20Ucf6dprrzX1t2PHDlNcUlPPq1Vtba1+//vfm2L/+Mc/eo09ZswYc+y7776bda1nz56aMGGCqb/jjjvOPBdJOuGEE8yxkyZNSntcW1urF154wdTX5z73OfM8JOmtt97yis+0e/durVq1yhR70UUXeY3te79IVV1drQcffNAUO2LECK+x8/laIkm7du3S8uXLTf3ddNNN5rlI0qZNm7ziU0VRZH5ufO9dXbp08YrPtGnTJs2YMcMUW1hY6DV2eXm5V3yqtWvX6s477zTFFhUVeY399NNPm2NPO+20rGuLFi3SuHHjTP1dfvnl5rlI9ntGU/v6/fff12c/+1lTf3v37jXFJfncu37yk594jZ3q6quv9oqvra01xzZ1D96xY0eT72Ha4otf/KJ5LpJ09NFHe8Wnqqur0//+7/+aYpvac7kYMGCAV3ym8vJyffWrXzXFVldXe4399a9/3Rw7ZcqUtMcbN27UI488Yurrn//8p3kekszv9VrDT/wBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIAV5NK4b9++uu6660wD/fCHPzTFJa1bt84rPtX27dv117/+1RRrjUuaNGmSV3ymbt266aijjjLFfvDBB15jf/jhh6a4hoaGJq/v3bvX1N8RRxxhiksaPXq0Ofbdd9/NulZQkNO2+j+vvPKKeR7tobKyUmeffbYp9oILLvAau7q62is+VUNDgxYvXmyKPfnkk73Gfvrpp73iMx122GG67bbbTLETJkzwGts6blNWrlypyy67zBS7adMmr7E/85nPeMVnOuqoo/Tqq6+aYnv27Ok19o9+9COv+FRFRUXmfVdaWuo19hNPPOEVnymKIu3evdsUO336dK+xV61aZY4dOHBg2uOjjjpKs2fPNvV14YUXmuchNb63yKehQ4fq4YcfNsUec8wxXmPfcccdprim1lDfvn31zW9+09Rf//79TXFJhx56qFd8qjVr1mjq1Kmm2K1bt3qNPXfuXK/4TJ988ommTJliir3rrru8xj7rrLO84lN9/PHH5rNSfX2919h9+vTxis/0wQcf6D//8z9NsY888ojX2F/4whe84lOtXr3avLaOO+44r7GtZyxJcs41+3f8xB8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAuiqK2N3ZuvaQV7TeddnNoFEW9kg9CyUPqsLmEkofE2oojahI/1CR+qEn8UJN4CSUPibUVR9QkfoKtSVJOB38AAAAAANCx8Kv+AAAAAAAEjIM/AAAAAAABy+vB3zlX4Zx7xjlX55xb4Zyb2ELb0c65l5xzG5xzrf57A+dctXPuNedcvXPufefc8fmcexPjtTmXRPsrnXNrnHO1zrkHnXNFLbR9wDm3yDm3zzl3Qd4nnz5WSDX5jnNutnOuwTn3cBvaT0zkXOece9Y5V9FC2/2WSyhrKzFeKDUJIo/EeEHkEso+CeweHEQuoeSRGC+UfRLEfSsxXhDr62CtSaJ9XPdJEGsrMV4Q66ujra18/8T/l5J2Seot6WxJ05xzo5ppu1vSk5IuamPfMyXNk1Qp6TpJs5xzTX5wQZ60ORfn3ARJ10oaL6la0hBJU1roe76kSZLm5nG+zQmpJqsl3SDpwdYaJnK8X9K5asy9XtK9LYTsz1xCWVtSODUJJQ8pnFxC2Sch3YNDySWUPKRw9kko9y0pnPV1UNYk5vsklLUlhbO+OtbaiqIoL38kdU8kPizl2mOSbmolbmjjNFpsM0xSg6TSlGtvSrokX/P3yUXSDElTUx6Pl7SmDeO8JemC9sghtJpkjH2DpIdbaTNV0oyUx4clnovSJtrut1xCWVsh1STEPDp6LqHsk5DuwaHkEkoellziuk8yxuqw963Q1tfBWpO47pMQ11ZHX18dcW3l8yf+wyTtjaJoccq1+ZKa+05ULkZJWhZF0bZ26LspueYyKvH3qW17O+cq22l+bRVSTXKVVpMoipYqsTmbabu/cgllbVnEtSa5CiUPKb65hLJPQroHh5JLKHlI4eyTXMX1viWFtb5yEVJN4rpPDta1JcV3fXW4tZXPg3+JpNqMa7WSSmPedz7Gy2yf/Lq95tdWIdUkV7nMb3/mEsrasohrTXIVSh5SfHMJZZ+EdA8OJZdQ8rCMF9d9kqu43rfae7w4v56EVJO47pODdW1J8V1fHW5t5fPgv11Sj4xrPSRta6JtnPrOx3iZ7ZNft9f82iqkmuQql/ntz1xCWVsWca1JrkLJQ4pvLqHsk5DuwaHkEkoelvHiuk9yFdf7VnuPF+fXk5BqEtd9crCuLSm+66vDra18HvwXSypwzh2ecu1ISTV56LtG0hDnXOp3RPLVd1NyzaUm8fepbddGUbSxnebXViHVJFdpNXHODZFUpMbnpKm2+yuXUNaWRVxrkqtQ8pDim0so+ySke3AouYSShxTOPslVXO9bUljrKxch1SSu++RgXVtSfNdXx1tbef6Qg8fV+EmK3SV9Xo2/wjCqmbZOUrGkkZKixNdFLfT9d0m3JNqdKmmLpF75/qAGYy5fkbQmkUtPSa+qhQ/bkNQlkcdfJX0z8XWnGOQR95oUJMa6UY0fnlEsqaCZtqMkbZV0bCL36ZIej0MuoaytwGoSRB4h5RLKPskxj7jfg4PIJZQ8DLnEeZ8Ecd8KaX0dxDWJ8z4JYm2FtL462trKd/IVkp6VVCdppaSJLbStTizE1D/LW2n/uqQdkhZJOr69FmOuuSTaf0/S2sTCfKiVzfV6E7mPO9B5dICaTG5ifpNbaD8xkXOdpOckVcQhl1DWVmA1CSKPkHIJZZ/kkofifw8OIpdQ8sg1l0T7uO6TIO5bIa2vg7UmifZx3SdBrK2Q1ldHW1su0TEAAAAAAAhQPv+NPwAAAAAAiBkO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwApyaVxVVRVVV1ebBlqzZo0pLmn37t3m2HXr1m2IoqhX8nHPnj2j/v37m/qqqfH7byBHjRpljq2pqUnLQ5JKSkqiyspKU3/OOfNcJGnv3r2muE2bNqmuri5t8O7du0c9e/Y09bd582ZTXD7U19en1aRz585RYWGhqa/Bgwd7zcVnjyxdujRrbTnnzJ/8WVVVZZ6LJO3atcscu3Xr1qyadO7c2dTXiBEjzPOQpKVLl5pjM9eW1LhPysvLTf1Z45IWLlzoE56WS2VlZTRgwABTR5s2bfKZh7Zu3eoTm1UTn9fFOXPmmOciSZ062b93v2/fvrRcfPJYvny5eR6SVFFRYY794IMPsmpSWVkZDRo0yNSfz/qQpJKSEnPsO++8k5ZLSUlJZH1ufObh67333suqSadOncz34T59+njNp76+3hRXV1ennTt3pr1X8dkn8+fPN8Ul7dmzxyc8rSY9evSIevXq1VL7Zm3fvt1nHlq3bp1PeNba8nlP7/mcasmSJebYvXv3puXSpUuXqFu3bqa+DjnkEPM8JL/zwOLFi7Nq0qVLl6i4uNjU37Zt28xzkaThw4ebYxctWpSWS1lZWdS7d29TXz7vySX7GUuSVq1alVWTpJwO/tXV1Zo9e7ZpEjfddJMpLsnnRnH77bevSH3cv39/zZo1y9TXEUccYZ6HJD311FPm2BEjRqzIvFZZWalrrrnG1F9RUZF5LpL9DdJtt92Wda1nz566/PLLTf35PKeStG/fPnPs7Nmz02pSWFiogQMHmvp65JFHzPOQpI8++sgce/rpp2etLR+nnHKKV/zq1avNsS+88EJaLp07dza/cfzDH/5gnocknXbaaebYuXPnZtWkvLxcl1xyiam/U0891TwXSRozZoxPeFouAwYM0Msvv2zqaMaMGT7zMI8rSS+++GJWTXxeF32/+epzuNu6dWtaLj55nH/++eZ5SNJZZ51ljj3hhBOyajJo0CC9/vrrpv5efPFF81wkady4cebYPn36pOVSUVGhq6++2tTXZz/7WfM8JL+1OXbs2KyadO7c2fwNniuvvNI8F0maN2+eKe7555/PuuazT3y/gbF27Vqf8LSa9OrVSzfffLOpozfeeMNnHrr77rt9wrPWVv/+/c3vA9evX+8zF331q181x27evDktl27duum4444z9TVp0iTzPCSpoCCn42Ca448/PqsmxcXF+sxnPmPq789//rN5LpL04IMPmmM///nPp+XSu3dv3XPPPaa+fH/g7fNN6Msuu6zZ9/T8qj8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8ApBlYHQAAEuFJREFUAAAAAAEryKXxnDlz5JyzDVSQ01BZTjzxRK/4VGvXrtVtt91mii0rK/Mae/jw4V7xmVatWqWrrrrKFLtjxw6vsa1roSl9+vTR1VdfbYo9/fTTvca+4oorvOJT7dmzR5s3bzbFPvbYY15j33333V7xmUpLS3XMMceYYlevXu019ogRI8yxL7zwQtrj3bt3a9WqVaa+Dj30UPM8JKl3795e8Zlqa2v10ksvmWJ//OMfe409a9Ysc+wZZ5yR9riwsND83EyfPt08D0k655xzzLEvvvhi1rV58+appKTE1N+2bdvMc5Gk/v37e8Wn8nl9P+mkk7zGvu+++7ziM7399tsqLy83xUZR5DX2ggULvOJTlZeX65RTTjHFDhw40Gvsf/3rX17xmaqrq3X77bebYk8++WSvsa2vjX/5y1+yrtXU1GjMmDGm/jp37myKS7LeZyRp+/btaY+XLVums846y9TXt771LfM8JGnhwoXm2JEjR2ZdW7dune68805Tf/fee695LpJ02mmnmWN/85vfpD2OokgNDQ2mviZPnmyehyRdd911XvGZDjnkEF166aWm2EGDBnmN7RufqkuXLqqurjbFfu5zn/Mae8qUKV7xzeEn/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwDj4AwAAAAAQMA7+AAAAAAAEjIM/AAAAAAAB4+APAAAAAEDAOPgDAAAAABAwDv4AAAAAAASMgz8AAAAAAAHj4A8AAAAAQMA4+AMAAAAAEDAO/gAAAAAABIyDPwAAAAAAAePgDwAAAABAwApyaVxVVaVTTz3VNNDYsWNNcUmXXHKJV3yqhoYGLVmyxBR7zDHHeI397rvvesVniqJIDQ0NptipU6d6jT1//nxT3FlnnZV1bc6cOXLOmfrbvHmzKS6pU6f8ff9r3759qq+vN8Xec889XmOfccYZXvGZhg0bpj//+c+m2Ntvv91r7DVr1njFp+rVq5f5uTnuuOO8xv7Rj35kjn3hhReyrnXq1EndunUz9ffII4+Y5yI1ru188dnv//jHP7zGrqmp8YrPNGTIEPN6/+pXv+o19ty5c82xQ4cOTXtcVVWlU045xdTXUUcdZZ6H1PRrQls999xzWdcqKyt14oknmvpbu3ateS6S9Pbbb3vFp1qwYIEGDRpkij300EO9xn7ttde84jOtWLFCl156qSk2iiKvsfv06WOK27hxY5N9XXXVVab++vbta4pL+uijj8yx3/jGN9IeH3bYYfr5z39u6qu8vNw8D0k64ogjvOIzlZWV6aSTTjLFWl+HknzfR6c6/PDD9eKLL5pir7/+eq+xTz75ZK/4TGvXrtWdd95pij366KO9xh4wYIBXfKqamhoNHz7cFOt737rjjju84pvDT/wBAAAAAAgYB38AAAAAAALGwR8AAAAAgIBx8AcAAAAAIGAc/AEAAAAACBgHfwAAAAAAAsbBHwAAAACAgHHwBwAAAAAgYBz8AQAAAAAIGAd/AAAAAAACxsEfAAAAAICAcfAHAAAAACBgHPwBAAAAAAgYB38AAAAAAALmoihqe2Pn1kta0X7TaTeHRlHUK/kglDykDptLKHlIrK04oibxQ03ih5rEDzWJl1DykFhbcURN4ifYmiTldPAHAAAAAAAdC7/qDwAAAABAwDj4AwAAAAAQsLwe/J1zFc65Z5xzdc65Fc65iS20He2ce8k5t8E51+q/N3DOVTvnXnPO1Tvn3nfOHZ/PuTcxXptzSbS/0jm3xjlX65x70DlX1ELbB5xzi5xz+5xzF+R98uljHZQ1iXMuzrnvOOdmO+canHMPt6H9xES+dc65Z51zFS203d81CSKXUPJIjHfQ3bsS7eOaB2uLmrSbUHIJJY/EeKHskyDecyXGCyKXUNZWYrxQatKh7l35/on/LyXtktRb0tmSpjnnRjXTdrekJyVd1Ma+Z0qaJ6lS0nWSZjnnmvzggjxpcy7OuQmSrpU0XlK1pCGSprTQ93xJkyTNzeN8m3NQ1kTxzmW1pBskPdhaw0R+90s6V41510u6t4WQ/V2TUHIJJQ/pILx3xTwP1hY14R7culDykMLZJ6G855LCySWUtSWFU5OOde+KoigvfyR1V2MBh6Vce0zSTa3EDW2cRotthklqkFSacu1NSZfka/4+uUiaIWlqyuPxkta0YZy3JF3QHjkc7DWJcy4pY9wg6eFW2kyVNCPl8WGJ56G0ibYHJI+QcunoeRys96645sHaoibcgw+uPELZJ7nmkdImdu+5QskllLUVUk0yxu0Q9658/sR/mKS9URQtTrk2X1Jz373JxShJy6Io2tYOfTcl11xGJf4+tW1v51xlO82vrQ7mmuRif+eSi7S1FUXRUiVuls20jWseUji5xDmPg/XeFdc8csXaip841yRXoeQS5zxC2SchvecKJZdQ1pYUTk1ydcDvXfk8+JdIqs24ViupNOZ952O8zPbJr9trfm11MNckLn37ymVucc5DCieXOOdxsN674ppHrlhb8RPnmuQqlFzinEco+ySk91yh5BLK2pLCqUmuDvi9K58H/+2SemRc6yFpWxNt49R3PsbLbJ/8ur3m11YHc03i0revXOYW5zykcHKJcx4H670rrnnkirUVP3GuSa5CySXOeYSyT0J6zxVKLqGsLSmcmuTqgN+78nnwXyypwDl3eMq1IyXV5KHvGklDnHOp3+X4/+3cW2iUZx7H8d8444x2TDBxJJpMNJqKVq0HsNKart2ABb0q66XYQrUrWNsKuRG8sFELQt2eoJfC9kKwxUM2aJWWslFw60KTijUWF6HEiLK6SXPwHJO8e+HMModonf8Tt+Pj9wMF8/r+nuf5+7zvO/OfpBmtsUdSaC3nUn+fee7VIAi6H9P6HtXTvCeF+H/XUoisaysUCs2UFNP9f4+Rzi3WOiR/ainmOp7WZ1ex1lEorq3iU8x7UihfainmOny5T3x6z+VLLb5cW5I/e1Ko3//ZNcq/2OBL3f8NhHFJdbr/IwnzHnBuSNI4SXMlBak/xx4y9j8l/SV13p8k9Uqa/Bh/SUMhtayU9O9ULWWS/q6H/IIKSdFUHf+Q9OfUn8cUQR0+7UnR1iIpkppnl+7/MpNxkiIPOHeepH5Jf0jVvVfSl8VQh0+1+FJHar6n8dlVzHVwbbEnPIOfkjpS8/lyn3jxnsunWny5tjzbkyfq2TXaxZdL+pukm5I6Ja15yLk1qc3L/K/jN84/Lum2pH9JWvG4LsZCa0md3yDpampD//obF+TxEWr/4+9dh097Usy1SGocYW2NDzl/Tarem5KaJZUXQx0+1eJLHan5nrpnV5HXwbXFnjzOPfGiFl/qSM3ny33ixXsun2rx5drybE+eqGdXKDUwAAAAAADw0Gj+P/4AAAAAAKDI0PgDAAAAAOAxGn8AAAAAADxG4w8AAAAAgMcihZxcXl4eVFVVmSYaGBgw5dKuXbtmzvb29nYFQTA5/XUkEglisZhprGeeeca8DkmaMmWKOdve3p5Vh+S2J1evXjWvRZKSyaQp19nZqa6urlDmsXg8HpSXl5vGGz9+vCmXVlpaas62tbVl7cm4ceOCkpKSh0Ue6N69e+Z1SFJfX59LPO/aCoVC5t/86XKdS9KYMfbPJK9cuZJVy/jx4wPrHvf09JjXIUmVlZXm7MWLF0e836333dmzZ81rkSTrdS1J169fz6olHA4HkUhBLz//E41GzeuQpNmzZ5uzufe75HafTJ061bwWSerq6jJn7927l1VLSUlJkEgkTGO57onLtTXae+L6ejJnzhxz9vTp01m1JBKJoKamxjTWzZs3zeuQpPPnz7vE8/YkEokEY8eONQ12584dl7WYX0+Gh4cVBEHWe5UxY8YE4XDYNN5zzz1nyqW53Ge590lZWVlgfX36+eefzeuQpPnz55uzI70PjsfjwcSJE03j3bp1y7wWSaqtrTVnc/fE5Rnc399vXockzZgxw5wd6Rk8adKkoLq62jTemTNnzGuRpOnTp5uzue+7XF5LXE2bNs2c7ezszNuTtILeeVVVVam5udm0iI6ODlMu7fPPPzdnm5qaLmZ+HYvFNHfuXNNYixcvNq9DkrZs2WLOPvvssxdzj1VVVampqck03qeffmpeiyTt3r3blKurq8s7Vl5eroaGBtN4zz//vCmXtmLFCnM2FApl7UlJSYlWr15tGuvy5cvmdUjS119/7RLPu7ZcrF+/3ilv/WBOkrZt25ZVS2lpqdasWWMaa//+/eZ1SNL7779vzq5bty5vT5LJpI4ePWoaz/oinLZkyRJztqWlJauWSCRi/uDQmks7ceKEOZt7v7t66623nPJffPGFOXvp0qWsWhKJhBobG01jubxBkaT6+npzdrT3xOWDIUk6efKkORuPx7NqqampUWtrq2msU6dOmdchScuWLXOJ5+3J2LFjZf0Qw/FDCMXjcVNupA9PwuGwrN+kOHz4sCmX5tLQ5N4nlZWV2rdvn2mshQsXmtchydxHSFJtbW3etTVx4kRt3LjRNN6PP/5oXoskHTp0yJzN3ZNEIqEdO3aYxvrmm2/M65CkvXv3mrMjPYOrq6v17bffmsarqKgwr0Ua/fddVtYPCNNc+sVNmzY9sA5+1B8AAAAAAI/R+AMAAAAA4DEafwAAAAAAPEbjDwAAAACAx2j8AQAAAADwGI0/AAAAAAAeo/EHAAAAAMBjNP4AAAAAAHiMxh8AAAAAAI/R+AMAAAAA4DEafwAAAAAAPEbjDwAAAACAx2j8AQAAAADwWKSQkwcGBnTp0iXTRA0NDaZc2pkzZ5zyme7evatffvnFlI3H405z19bWOuVztbe3a9asWabs5s2bnebu7u425QYHB/OORSIRJRIJ03ivvvqqKZf25ptvOuUzDQ0Nqbe315Q9cuSI09yTJ082Z7u6uvKOVVVV6Z133jGN98Ybb5jXIkkff/yxUz7T7du3de7cOVP2hx9+cJp76tSp5uy6devyjkWjUSWTSdN4FRUV5rVI7v8WmSorK7Vt2zZTdvXq1U5z9/X1OeVzhcNhlZaWmrLTpk1zmvvgwYPm7NKlS7O+7unp0YEDB0xjvfbaa+Z1SNKpU6ec8rnKysrMrwvXrl1zmrujo8Mpn2l4eFi3bt0yZa3PvLT6+npztqWlJe9YdXW1PvnkE9N4O3fuNK9FGvk9x6Nob2/POzY0NKTr16+bxjt8+LApl7ZhwwanfKYbN27o+++/N2VXrVrlNPeePXuc8rlisZhqampM2ba2Nqe5XV+PMrk8g1966SWnuZuampzyuc6fP6+6ujpTNhaLOc39wQcfOOUzLV68WCdOnDBlFyxY4DT322+/bc5u2rTpgX/Hd/wBAAAAAPAYjT8AAAAAAB6j8QcAAAAAwGM0/gAAAAAAeIzGHwAAAAAAj9H4AwAAAADgMRp/AAAAAAA8RuMPAAAAAIDHaPwBAAAAAPAYjT8AAAAAAB6j8QcAAAAAwGM0/gAAAAAAeIzGHwAAAAAAj9H4AwAAAADgMRp/AAAAAAA8Fink5Gg0qurqatNEoVDIlEsLgsCczZ17aGhIv/76q2msDz/80LwOSVq7dq1TPtfs2bO1Z88eU/b48eNOcyeTSVMuGo3mHQuHw4rH46bxGhsbTbmHrcdq0qRJev31103ZrVu3Os39wgsvmLPHjh3LOzY4OGi+TyorK81rkaQjR4445TNVVFTovffeM2X7+/ud5nb9d8j1008/mcd88cUXneZetmyZObtly5asr3t6enTo0CHTWE1NTeZ1SFJzc7NTPteiRYvU2tpqys6bN89p7o8++sgpnykIAg0MDJiyS5cudZp74cKFTvlcM2fO1FdffWXKvvLKK05zb9++3Smf6e7du7pw4YIpa312p23YsMGcbWlpyTtWUlKi+vp603grV640r0Wy32fDw8N5xyoqKrR+/XrTeO+++64pl7Zo0SKnfKaenh4dOHDAlP3uu++c5u7r6zNnd+3alXdswoQJWr58uWm8l19+2bwWSZoxY4ZTPlMymRyxvkfh+lqyatUqp3yuwcFBdXd3m7J37txxmvvs2bPm7IIFC7K+7u3tNb9n+Oyzz8zrkKT9+/c75R+E7/gDAAAAAOAxGn8AAAAAADxG4w8AAAAAgMdo/AEAAAAA8BiNPwAAAAAAHqPxBwAAAADAYzT+AAAAAAB4jMYfAAAAAACP0fgDAAAAAOAxGn8AAAAAADxG4w8AAAAAgMdo/AEAAAAA8BiNPwAAAAAAHqPxBwAAAADAY6EgCB795FDoP5IuPr7lPDbTgyCYnP7ClzqkJ7YWX+qQuLaKEXtSfNiT4sOeFB/2pLj4UofEtVWM2JPi4+2epBXU+AMAAAAAgCcLP+oPAAAAAIDHaPwBAAAAAPAYjT8AAAAAAB6j8QcAAAAAwGM0/gAAAAAAeIzGHwAAAAAAj9H4AwAAAADgMRp/AAAAAAA8RuMPAAAAAIDH/guohkVdjzlQagAAAABJRU5ErkJggg==\n", 42 | "text/plain": [ 43 | "
" 44 | ] 45 | }, 46 | "metadata": {}, 47 | "output_type": "display_data" 48 | } 49 | ], 50 | "source": [ 51 | "draw_xy(*generate(120))" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 4, 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "def train(n=512, epochs=56000, lr = 1e-3):\n", 61 | " # weight vector\n", 62 | " W = np.random.randn(9, 3)\n", 63 | " b = np.random.randn(1, 3)\n", 64 | "\n", 65 | " # training loop\n", 66 | " for t in range(epochs):\n", 67 | " # get new training data\n", 68 | " X, y = generate(n)\n", 69 | " X = X / 255\n", 70 | "\n", 71 | " # model function\n", 72 | " h = X.dot(W) + b\n", 73 | "\n", 74 | " # compute loss\n", 75 | " loss = np.square(h - y).mean()\n", 76 | "\n", 77 | " # compute accuracy\n", 78 | " acc = (np.argmax(h, axis=1) == np.argmax(y, axis=1)).mean()\n", 79 | "\n", 80 | " if t % 5000 == 0:\n", 81 | " print('l: {:>8f}, a {:>.4f} (e {})'.format(loss, acc, t))\n", 82 | "\n", 83 | " # grad + update\n", 84 | " grad_w = 2 * X.T.dot(h - y) / n\n", 85 | " W -= lr * grad_w\n", 86 | "\n", 87 | " grad_b = 2 * np.sum(h - y, axis=0) / n\n", 88 | " b -= lr * grad_b\n", 89 | "\n", 90 | " return W, b" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 5, 96 | "metadata": {}, 97 | "outputs": [ 98 | { 99 | "name": "stdout", 100 | "output_type": "stream", 101 | "text": [ 102 | "l: 2.856105, a 0.2734 (e 0)\n", 103 | "l: 0.203259, a 0.5996 (e 5000)\n", 104 | "l: 0.125143, a 0.8164 (e 10000)\n", 105 | "l: 0.109183, a 0.8926 (e 15000)\n", 106 | "l: 0.104883, a 0.9004 (e 20000)\n", 107 | "l: 0.099931, a 0.9375 (e 25000)\n", 108 | "l: 0.104011, a 0.9512 (e 30000)\n", 109 | "l: 0.107277, a 0.9570 (e 35000)\n", 110 | "l: 0.100308, a 0.9668 (e 40000)\n", 111 | "l: 0.104737, a 0.9863 (e 45000)\n", 112 | "l: 0.107053, a 0.9766 (e 50000)\n", 113 | "l: 0.099793, a 0.9961 (e 55000)\n", 114 | "\n", 115 | "Final W = \n", 116 | "\n", 117 | "[[ 0.57775157 -0.27516143 -0.29983488]\n", 118 | " [ 0.57768692 -0.27589319 -0.29935118]\n", 119 | " [ 0.57811469 -0.2764427 -0.29911445]\n", 120 | " [-0.27712936 0.57835142 -0.29856699]\n", 121 | " [-0.27640057 0.5782363 -0.29916655]\n", 122 | " [-0.27706883 0.57939636 -0.29960357]\n", 123 | " [-0.27661276 -0.27371918 0.55306812]\n", 124 | " [-0.27658511 -0.27363858 0.55294286]\n", 125 | " [-0.27656491 -0.2740988 0.55327718]]\n", 126 | "\n", 127 | "Final b = \n", 128 | "\n", 129 | "[[0.29612778 0.28854414 0.40312817]]\n" 130 | ] 131 | } 132 | ], 133 | "source": [ 134 | "W, b = train()\n", 135 | "print('\\nFinal W = \\n\\n{}\\n\\nFinal b = \\n\\n{}'.format(W, b))" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 6, 141 | "metadata": {}, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "top\n" 148 | ] 149 | } 150 | ], 151 | "source": [ 152 | "npix = np.array([[255, 143, 255, 255, 187, 93, 5, 5, 5]])\n", 153 | "a = npix.dot(W) + b\n", 154 | "d = ['top', 'middle', 'bottom']\n", 155 | "print(d[np.argmax(a)])" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [] 164 | } 165 | ], 166 | "metadata": { 167 | "kernelspec": { 168 | "display_name": "Python 3", 169 | "language": "python", 170 | "name": "python3" 171 | }, 172 | "language_info": { 173 | "codemirror_mode": { 174 | "name": "ipython", 175 | "version": 3 176 | }, 177 | "file_extension": ".py", 178 | "mimetype": "text/x-python", 179 | "name": "python", 180 | "nbconvert_exporter": "python", 181 | "pygments_lexer": "ipython3", 182 | "version": "3.6.10" 183 | }, 184 | "varInspector": { 185 | "cols": { 186 | "lenName": 16, 187 | "lenType": 16, 188 | "lenVar": 40 189 | }, 190 | "kernels_config": { 191 | "python": { 192 | "delete_cmd_postfix": "", 193 | "delete_cmd_prefix": "del ", 194 | "library": "var_list.py", 195 | "varRefreshCmd": "print(var_dic_list())" 196 | }, 197 | "r": { 198 | "delete_cmd_postfix": ") ", 199 | "delete_cmd_prefix": "rm(", 200 | "library": "var_list.r", 201 | "varRefreshCmd": "cat(var_dic_list()) " 202 | } 203 | }, 204 | "types_to_exclude": [ 205 | "module", 206 | "function", 207 | "builtin_function_or_method", 208 | "instance", 209 | "_Feature" 210 | ], 211 | "window_display": false 212 | } 213 | }, 214 | "nbformat": 4, 215 | "nbformat_minor": 2 216 | } 217 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sethjuarez/DeepLearningWithPyTorch/81b8f58a27494ea41178fd667dcc7811de33d75d/utils/__init__.py -------------------------------------------------------------------------------- /utils/draw.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | import torch 4 | import shutil 5 | import numpy as np 6 | from PIL import Image 7 | import matplotlib.pyplot as plt 8 | 9 | def draw_squares(squares): 10 | fig, axes = plt.subplots(6, 20, figsize=(18, 7), 11 | subplot_kw={'xticks':[], 'yticks':[]}, 12 | gridspec_kw=dict(hspace=0.1, wspace=0.1)) 13 | for i, ax in enumerate(axes.flat): 14 | X, y = squares[i] 15 | ax.imshow(-1 * (X.reshape(3, 3) - 255), cmap='gray') 16 | ax.set_title('{:.0f} {:.0f} {:.0f}'.format(y[0], y[1], y[2])) 17 | 18 | def draw_digits(digits): 19 | fig, axes = plt.subplots(6, 20, figsize=(18, 7), 20 | subplot_kw={'xticks':[], 'yticks':[]}, 21 | gridspec_kw=dict(hspace=0.1, wspace=0.1)) 22 | for i, ax in enumerate(axes.flat): 23 | X, y = digits[i] 24 | ax.imshow(255 - X.reshape(28,28) * 255, cmap='gray') 25 | ax.set_title('{:.0f}'.format(torch.argmax(y).item())) 26 | 27 | def draw_xy(X, y): 28 | fig, axes = plt.subplots(6, 20, figsize=(18, 7), 29 | subplot_kw={'xticks':[], 'yticks':[]}, 30 | gridspec_kw=dict(hspace=0.1, wspace=0.1)) 31 | for i, ax in enumerate(axes.flat): 32 | ax.imshow(-1 * (X[i].reshape(3, 3) - 255), cmap='gray') 33 | if type(y) is torch.Tensor: 34 | ax.set_title('{:.0f} {:.0f} {:.0f}'.format(y[i][0].item(), y[i][1].item(), y[i][2].item())) 35 | else: 36 | ax.set_title('{:.0f} {:.0f} {:.0f}'.format(y[i][0], y[i][1], y[i][2])) 37 | 38 | def draw_single(X, y): 39 | fig, axes = plt.subplots(6, 20, figsize=(18, 7), 40 | subplot_kw={'xticks':[], 'yticks':[]}, 41 | gridspec_kw=dict(hspace=0.1, wspace=0.1)) 42 | for i, ax in enumerate(axes.flat): 43 | ax.imshow(-1 * (X[i].reshape(3, 3) - 255), cmap='gray') 44 | ax.set_title('{:.0f}'.format(y[i])) 45 | -------------------------------------------------------------------------------- /utils/helpers.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import tensorflow as tf 4 | from functools import wraps 5 | from inspect import getargspec 6 | # pylint: disable-msg=E0611 7 | from tensorflow.python.tools import freeze_graph as freeze 8 | # pylint: enable-msg=E0611 9 | 10 | def info(msg, char = "#", width = 75): 11 | print("") 12 | print(char * width) 13 | print(char + " %0*s" % ((-1*width)+5, msg) + char) 14 | print(char * width) 15 | 16 | def print_info(f): 17 | @wraps(f) 18 | def wrapper(*args, **kwargs): 19 | info('-> {}'.format(f.__name__)) 20 | print('Parameters:') 21 | ps = list(zip(getargspec(f).args, args)) 22 | width = max(len(x[0]) for x in ps) + 1 23 | for t in ps: 24 | items = str(t[1]).split('\n') 25 | print(' {0:<{w}} -> {1}'.format(t[0], items[0], w=width)) 26 | for i in range(len(items) - 1): 27 | print(' {0:<{w}} {1}'.format(' ', items[i+1], w=width)) 28 | ts = time.time() 29 | result = f(*args, **kwargs) 30 | te = time.time() 31 | print('\n -- Elapsed {0:.4f}s\n'.format(te-ts)) 32 | return result 33 | return wrapper 34 | 35 | def print_args(args): 36 | info('Arguments') 37 | ps = args.__dict__.items() 38 | width = max(len(k) for k, _ in ps) + 1 39 | for k, v in ps: 40 | print(' {0:<{w}} -> {1}'.format(k, str(v), w=width)) -------------------------------------------------------------------------------- /utils/square.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.utils.data.dataset import Dataset 3 | 4 | class SquareDataset(Dataset): 5 | def __init__(self, size): 6 | self.size = size 7 | self.X = torch.randint(255, (size, 9), dtype=torch.float) 8 | 9 | real_w = torch.tensor([[1,1,1,0,0,0,0,0,0], 10 | [0,0,0,1,1,1,0,0,0], 11 | [0,0,0,0,0,0,1,1,1]], 12 | dtype=torch.float) 13 | 14 | y = torch.argmax(self.X.mm(real_w.t()), 1) 15 | 16 | self.Y = torch.zeros(size, 3, dtype=torch.float) \ 17 | .scatter_(1, y.view(-1, 1), 1) 18 | 19 | def __getitem__(self, index): 20 | return (self.X[index], self.Y[index]) 21 | 22 | def __len__(self): 23 | return self.size 24 | 25 | if __name__ == "__main__": 26 | data = SquareDataset(10) 27 | for i in range(len(data)): 28 | print(data[i]) 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /utils/tree.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://vega.github.io/schema/vega/v5.json", 3 | "width": 650, 4 | "height": 300, 5 | "padding": 5, 6 | 7 | "signals": [ 8 | { 9 | "name": "labels", "value": true, 10 | "bind": {"input": "checkbox"} 11 | }, 12 | { 13 | "name": "method", "value": "cluster", 14 | "bind": {"input": "select", "options": ["tidy", "cluster"]} 15 | }, 16 | { 17 | "name": "separation", "value": false, 18 | "bind": {"input": "checkbox"} 19 | } 20 | ], 21 | 22 | "data": [ 23 | { 24 | "name": "tree", 25 | "values":[ 26 | {"id": 0, "parentId": null, "name": "Mean", "color": "#286DA8"}, 27 | {"id": 1, "parentId": 0, "name": "Pow", "color": "#286DA8"}, 28 | {"id": 2, "parentId": 1, "name": "Sub", "color": "#286DA8"}, 29 | {"id": 3, "parentId": 2, "name": "Add", "color": "#286DA8"}, 30 | {"id": 4, "parentId": 3, "name": "Mm", "color": "#286DA8"}, 31 | {"id": 5, "parentId": 4, "name": "Const", "color": "#B37D4E"}, 32 | {"id": 6, "parentId": 4, "name": "Var", "color": "#CD5360"}, 33 | {"id": 7, "parentId": 3, "name": "Var", "color": "#CD5360"}, 34 | {"id": 8, "parentId": 2, "name": "Const", "color": "#B37D4E"} 35 | ], 36 | "transform": [ 37 | { 38 | "type": "stratify", 39 | "key": "id", 40 | "parentKey": "parentId" 41 | }, 42 | { 43 | "type": "tree", 44 | "method": {"signal": "method"}, 45 | "separation": {"signal": "separation"}, 46 | "size": [{"signal": "width"}, {"signal": "height"}] 47 | } 48 | ] 49 | }, 50 | { 51 | "name": "links", 52 | "source": "tree", 53 | "transform": [ 54 | { "type": "treelinks" }, 55 | { "type": "linkpath" } 56 | ] 57 | } 58 | ], 59 | "marks": [ 60 | { 61 | "type": "path", 62 | "from": {"data": "links"}, 63 | "encode": { 64 | "enter": { 65 | "stroke": {"value": "#ccc"} 66 | }, 67 | "update": { 68 | "path": {"field": "path"} 69 | } 70 | } 71 | }, 72 | { 73 | "type": "symbol", 74 | "from": {"data": "tree"}, 75 | "encode": { 76 | "enter": { 77 | "text": {"field": "id"}, 78 | "fontSize": {"value": 10}, 79 | "baseline": {"value": "middle"}, 80 | "fill": {"field": "color"}, 81 | "stroke": {"value": "#808080"}, 82 | "size": {"value": 600 } 83 | }, 84 | "update": { 85 | "x": {"field": "x"}, 86 | "y": {"field": "y"} 87 | } 88 | } 89 | }, 90 | { 91 | "type": "text", 92 | "from": {"data": "tree"}, 93 | "encode": { 94 | "enter": { 95 | "text": {"field": "name"}, 96 | "fontSize": {"value": 15}, 97 | "baseline": {"value": "middle"} 98 | }, 99 | "update": { 100 | "x": {"field": "x"}, 101 | "y": {"field": "y"}, 102 | "dx": {"signal": "15"}, 103 | "dy": {"signal": "5"}, 104 | "opacity": {"signal": "labels ? 1 : 0"} 105 | } 106 | } 107 | } 108 | ] 109 | } -------------------------------------------------------------------------------- /utils/viz.py: -------------------------------------------------------------------------------- 1 | import re 2 | import json 3 | import torch 4 | import graphviz 5 | from IPython.display import Javascript, HTML 6 | 7 | def build_graph(g, elements=[], parentId=-1, depth=0): 8 | elm = { 'id': len(elements), 'parentId': None if parentId==-1 else parentId} 9 | if g == None: 10 | elm['name'] = 'Const' 11 | elm['color'] = '#B37D4E' 12 | elif hasattr(g, 'variable'): 13 | elm['name'] = 'Var' 14 | elm['color'] = '#CD5360' 15 | else: 16 | name = g.name() 17 | m = name[:re.search(r'^([^A-Z]*[A-Z]){2}', name).span()[1]-1] 18 | elm['name'] = m 19 | elm['color'] = '#286DA8' 20 | 21 | elements.append(elm) 22 | 23 | if g != None and g.next_functions != None: 24 | depth = depth+1 25 | for subg in g.next_functions: 26 | elements, depth = build_graph(subg[0], elements, elm['id'], depth) 27 | 28 | return elements, depth 29 | 30 | 31 | def draw(g): 32 | graph, depth = build_graph(g.grad_fn, elements=[]) 33 | g = graphviz.Digraph('g') 34 | g.attr('graph', pack='true') 35 | for item in graph: 36 | shape = 'rect' 37 | if item['name'] == 'Const': 38 | shape='rect' 39 | if item['name'] == 'Var': 40 | shape='rect' 41 | g.attr('node', style='filled', fillcolor=item['color'], 42 | color='#303030', fontcolor='white', fontname='Segoe UI', 43 | fontsize='10', fixedsize='false', shape=shape, height='0.2', 44 | width='0.2') 45 | g.node(str(item['id']), label=item['name']) 46 | 47 | for item in graph: 48 | if item['parentId'] != None: 49 | g.attr('edge', arrowsize='0.5', color='#303030') 50 | g.edge(str(item['parentId']), str(item['id'])) 51 | 52 | return g 53 | -------------------------------------------------------------------------------- /wedding.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sethjuarez/DeepLearningWithPyTorch/81b8f58a27494ea41178fd667dcc7811de33d75d/wedding.jpg --------------------------------------------------------------------------------