├── .gitignore ├── README.md └── notebook ├── cnn.ipynb ├── lstm.ipynb ├── mdn-cls.ipynb ├── mdn.ipynb ├── mha.ipynb ├── mln.ipynb ├── mlp.ipynb └── optm.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Yet Another PyTorch Tutorial 2 | #### All notebooks are colab-compatible. 3 | 4 | - MLP [notebook](https://github.com/sjchoi86/yet-another-pytorch-tutorial/blob/main/notebook/mlp.ipynb) 5 | - CNN [notebook](https://github.com/sjchoi86/yet-another-pytorch-tutorial/blob/main/notebook/cnn.ipynb) 6 | - Linear Regression [notebook](https://github.com/sjchoi86/yet-another-pytorch-tutorial/blob/main/notebook/optm.ipynb) 7 | - LSTM [notebook](https://github.com/sjchoi86/yet-another-pytorch-tutorial/blob/main/notebook/lstm.ipynb) 8 | - Multi-Headed Attention [notebook](https://github.com/sjchoi86/yet-another-pytorch-tutorial/blob/main/notebook/mha.ipynb) 9 | - Mixture Density Network [notebook](https://github.com/sjchoi86/yet-another-pytorch-tutorial/blob/main/notebook/mdn.ipynb) 10 | 11 | #### Special Thanks to [Jerry](https://github.com/jjerry-k). 12 | 13 | Contact: Sungjoon (sungjoon-choi@korea.ac.kr) 14 | -------------------------------------------------------------------------------- /notebook/lstm.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"lstm.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyO9gYohIEVn4AOL6FCtGFNe"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"Uyq75dn2Mk6x"},"source":["\n"," \n"," \n","
\n"," Colab\n"," \n"," View Source\n","
"]},{"cell_type":"markdown","metadata":{"id":"PJwtTg9a11du"},"source":["# Classification with LSTM"]},{"cell_type":"code","metadata":{"id":"tepz70nH1wwO","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609935205673,"user_tz":-540,"elapsed":1085,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"9e5e04bb-db53-449b-8711-190a72a97895"},"source":["import numpy as np\n","import matplotlib.pyplot as plt\n","import torch\n","import torch.nn as nn\n","import torch.optim as optim\n","import torch.nn.functional as F\n","%matplotlib inline\n","%config InlineBackend.figure_format='retina'\n","print (\"PyTorch version:[%s].\"%(torch.__version__))\n","device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n","print (\"device:[%s].\"%(device))"],"execution_count":1,"outputs":[{"output_type":"stream","text":["PyTorch version:[1.7.0+cu101].\n","device:[cuda:0].\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"gjSfbrHz2NbN"},"source":["### Dataset and Loader"]},{"cell_type":"code","metadata":{"id":"_apH6GPI2Adq","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609935205674,"user_tz":-540,"elapsed":997,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"35ed37dd-88c9-4fed-cbba-baf61280c6bf"},"source":["from torchvision import datasets,transforms\n","mnist_train = datasets.MNIST(root='./data/',train=True,transform=transforms.ToTensor(),download=True)\n","mnist_test = datasets.MNIST(root='./data/',train=False,transform=transforms.ToTensor(),download=True)\n","BATCH_SIZE = 256\n","train_iter = torch.utils.data.DataLoader(mnist_train,batch_size=BATCH_SIZE,shuffle=True,num_workers=1)\n","test_iter = torch.utils.data.DataLoader(mnist_test,batch_size=BATCH_SIZE,shuffle=True,num_workers=1)\n","print (\"Done.\")"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Done.\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"10evD4Jg2bQ4"},"source":["### Define Model"]},{"cell_type":"code","metadata":{"id":"QoISvH_O2OWO","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609935210005,"user_tz":-540,"elapsed":5312,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"b14ecad1-40f6-4d69-b237-45fe7abde219"},"source":["class RecurrentNeuralNetworkClass(nn.Module):\n"," def __init__(self,name='rnn',xdim=28,hdim=256,ydim=10,n_layer=3):\n"," super(RecurrentNeuralNetworkClass,self).__init__()\n"," self.name = name\n"," self.xdim = xdim\n"," self.hdim = hdim\n"," self.ydim = ydim\n"," self.n_layer = n_layer # K\n","\n"," self.rnn = nn.LSTM(\n"," input_size=self.xdim,hidden_size=self.hdim,num_layers=self.n_layer,batch_first=True)\n"," self.lin = nn.Linear(self.hdim,self.ydim)\n","\n"," def forward(self,x):\n"," # Set initial hidden and cell states \n"," h0 = torch.zeros(self.n_layer,x.size(0),self.hdim).to(device)\n"," c0 = torch.zeros(self.n_layer,x.size(0),self.hdim).to(device)\n"," # RNN\n"," rnn_out,(hn,cn) = self.rnn(x, (h0,c0)) \n"," # x:[N x L x Q] => rnn_out:[N x L x D]\n"," # Linear\n"," out = self.lin(rnn_out[:,-1 :]).view([-1,self.ydim]) \n"," return out \n","\n","R = RecurrentNeuralNetworkClass(\n"," name='rnn',xdim=28,hdim=256,ydim=10,n_layer=2).to(device)\n","loss = nn.CrossEntropyLoss()\n","optm = optim.Adam(R.parameters(),lr=1e-3)\n","print (\"Done.\")"],"execution_count":3,"outputs":[{"output_type":"stream","text":["Done.\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"liD6DC7KANYR"},"source":["### Check How LSTM Works\n","- `N`: number of batches\n","- `L`: sequence lengh\n","- `Q`: input dim\n","- `K`: number of layers\n","- `D`: LSTM feature dimension\n","\n","` Y,(hn,cn) = LSTM(X) `\n","\n","- `X`: [N x L x Q] - `N` input sequnce of length `L` with `Q` dim. \n","- `Y`: [N x L x D] - `N` output sequnce of length `L` with `D` feature dim.\n","- `hn`: [K x N x D] - `K` (per each layer) of `N` final hidden state with `D` feature dim. \n","- `cn`: [K x N x D] - `K` (per each layer) of `N` final hidden state with `D` cell dim. "]},{"cell_type":"code","metadata":{"id":"byX3ViAwARpt","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609935210007,"user_tz":-540,"elapsed":5299,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"a192a6f8-a7b9-417f-9558-8c3d2ae04549"},"source":["np.set_printoptions(precision=3)\n","torch.set_printoptions(precision=3)\n","x_numpy = np.random.rand(2,20,28) # [N x L x Q]\n","x_torch = torch.from_numpy(x_numpy).float().to(device)\n","rnn_out,(hn,cn) = R.rnn(x_torch) # forward path\n","\n","print (\"rnn_out:\",rnn_out.shape) # [N x L x D]\n","print (\"Hidden State hn:\",hn.shape) # [K x N x D]\n","print (\"Cell States cn:\",cn.shape) # [K x N x D]"],"execution_count":4,"outputs":[{"output_type":"stream","text":["rnn_out: torch.Size([2, 20, 256])\n","Hidden State hn: torch.Size([2, 2, 256])\n","Cell States cn: torch.Size([2, 2, 256])\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"XuBUgRKD5vTx"},"source":["### Check parameters"]},{"cell_type":"code","metadata":{"id":"raw5y-vn4rWa","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609935210008,"user_tz":-540,"elapsed":5285,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"21a7f2ab-7095-420b-9e84-e2a4c0051d24"},"source":["np.set_printoptions(precision=3)\n","n_param = 0\n","for p_idx,(param_name,param) in enumerate(R.named_parameters()):\n"," if param.requires_grad:\n"," param_numpy = param.detach().cpu().numpy() # to numpy array \n"," n_param += len(param_numpy.reshape(-1))\n"," print (\"[%d] name:[%s] shape:[%s].\"%(p_idx,param_name,param_numpy.shape))\n"," print (\" val:%s\"%(param_numpy.reshape(-1)[:5]))\n","print (\"Total number of parameters:[%s].\"%(format(n_param,',d')))"],"execution_count":5,"outputs":[{"output_type":"stream","text":["[0] name:[rnn.weight_ih_l0] shape:[(1024, 28)].\n"," val:[-0.018 0.033 0.057 -0.054 -0.017]\n","[1] name:[rnn.weight_hh_l0] shape:[(1024, 256)].\n"," val:[-0.034 -0.043 0.004 0.045 0.001]\n","[2] name:[rnn.bias_ih_l0] shape:[(1024,)].\n"," val:[ 0.005 0.038 -0.008 0.018 -0.02 ]\n","[3] name:[rnn.bias_hh_l0] shape:[(1024,)].\n"," val:[ 0.044 -0.003 -0.032 -0.055 -0.052]\n","[4] name:[rnn.weight_ih_l1] shape:[(1024, 256)].\n"," val:[-0.03 0.042 0.037 -0.037 -0.043]\n","[5] name:[rnn.weight_hh_l1] shape:[(1024, 256)].\n"," val:[-0.042 0.052 -0.012 0.052 -0.012]\n","[6] name:[rnn.bias_ih_l1] shape:[(1024,)].\n"," val:[-0.022 0.015 -0.013 -0.044 0.05 ]\n","[7] name:[rnn.bias_hh_l1] shape:[(1024,)].\n"," val:[-0.017 -0.015 0.047 0.035 -0.031]\n","[8] name:[lin.weight] shape:[(10, 256)].\n"," val:[-0.035 -0.037 0.031 -0.045 0.01 ]\n","[9] name:[lin.bias] shape:[(10,)].\n"," val:[-0.034 0.018 0.044 -0.038 -0.013]\n","Total number of parameters:[821,770].\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"J6rRmikB8dxU"},"source":["### Simple Forward Path "]},{"cell_type":"code","metadata":{"id":"DBdN6qoO8dah","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609935210008,"user_tz":-540,"elapsed":5270,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"24892479-aad1-47f7-9aac-5645c1a0398c"},"source":["np.set_printoptions(precision=3)\n","torch.set_printoptions(precision=3)\n","x_numpy = np.random.rand(3,10,28) # [N x L x Q]\n","x_torch = torch.from_numpy(x_numpy).float().to(device)\n","y_torch = R.forward(x_torch) # [N x 1 x R] where R is the output dim.\n","y_numpy = y_torch.detach().cpu().numpy() # torch tensor to numpy array\n","# print (\"x_torch:\\n\",x_torch)\n","# print (\"y_torch:\\n\",y_torch)\n","print (\"x_numpy %s\"%(x_numpy.shape,))\n","print (\"y_numpy %s\"%(y_numpy.shape,))"],"execution_count":6,"outputs":[{"output_type":"stream","text":["x_numpy (3, 10, 28)\n","y_numpy (3, 10)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"Zi5cIbKG6X3w"},"source":["### Evaluation Function"]},{"cell_type":"code","metadata":{"id":"-STglZMq5xKk","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609935210009,"user_tz":-540,"elapsed":5256,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"e6760929-3d37-4926-e91a-98fd98dd3b16"},"source":["def func_eval(model,data_iter,device):\n"," with torch.no_grad():\n"," n_total,n_correct = 0,0\n"," model.eval() # evaluate (affects DropOut and BN)\n"," for batch_in,batch_out in data_iter:\n"," y_trgt = batch_out.to(device)\n"," model_pred = model.forward(batch_in.view(-1,28,28).to(device))\n"," _,y_pred = torch.max(model_pred,1)\n"," n_correct += (y_pred==y_trgt).sum().item()\n"," n_total += batch_in.size(0)\n"," val_accr = (n_correct/n_total)\n"," model.train() # back to train mode \n"," return val_accr\n","print (\"Done\")"],"execution_count":7,"outputs":[{"output_type":"stream","text":["Done\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"pA-3-qPZ6h5u"},"source":["### Initial Evaluation"]},{"cell_type":"code","metadata":{"id":"qGbdjuhB6Z7U","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609935216009,"user_tz":-540,"elapsed":11241,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"c46c3921-755e-42eb-9f93-37585f0566a1"},"source":["train_accr = func_eval(R,train_iter,device)\n","test_accr = func_eval(R,test_iter,device)\n","print (\"train_accr:[%.3f] test_accr:[%.3f].\"%(train_accr,test_accr))"],"execution_count":8,"outputs":[{"output_type":"stream","text":["train_accr:[0.099] test_accr:[0.096].\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"PWywAU1-Lm0G"},"source":["### Train"]},{"cell_type":"code","metadata":{"id":"sp11_Glg6k7e","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609935272491,"user_tz":-540,"elapsed":67708,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"283902af-add5-47ef-abff-3fcf1cde4791"},"source":["print (\"Start training.\")\n","R.train() # to train mode \n","EPOCHS,print_every = 5,1\n","for epoch in range(EPOCHS):\n"," loss_val_sum = 0\n"," for batch_in,batch_out in train_iter:\n"," # Forward path\n"," y_pred = R.forward(batch_in.view(-1,28,28).to(device))\n"," loss_out = loss(y_pred,batch_out.to(device))\n"," # Update\n"," optm.zero_grad() # reset gradient \n"," loss_out.backward() # backpropagate\n"," optm.step() # optimizer update\n"," loss_val_sum += loss_out\n"," loss_val_avg = loss_val_sum/len(train_iter)\n"," # Print\n"," if ((epoch%print_every)==0) or (epoch==(EPOCHS-1)):\n"," train_accr = func_eval(R,train_iter,device)\n"," test_accr = func_eval(R,test_iter,device)\n"," print (\"epoch:[%d] loss:[%.3f] train_accr:[%.3f] test_accr:[%.3f].\"%\n"," (epoch,loss_val_avg,train_accr,test_accr))\n","print (\"Done\")"],"execution_count":9,"outputs":[{"output_type":"stream","text":["Start training.\n","epoch:[0] loss:[0.642] train_accr:[0.943] test_accr:[0.945].\n","epoch:[1] loss:[0.147] train_accr:[0.965] test_accr:[0.964].\n","epoch:[2] loss:[0.094] train_accr:[0.978] test_accr:[0.975].\n","epoch:[3] loss:[0.065] train_accr:[0.983] test_accr:[0.980].\n","epoch:[4] loss:[0.050] train_accr:[0.988] test_accr:[0.983].\n","Done\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"4JDDHhJtR1aR"},"source":["### Test"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":606},"id":"HrcOUIBrmNf-","executionInfo":{"status":"ok","timestamp":1609935273692,"user_tz":-540,"elapsed":68895,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"e8c3af87-4660-46cb-cdff-51afc0d65522"},"source":["n_sample = 25\n","sample_indices = np.random.choice(len(mnist_test.targets),n_sample,replace=False)\n","test_x = mnist_test.data[sample_indices]\n","test_y = mnist_test.targets[sample_indices]\n","with torch.no_grad():\n"," R.eval() # to evaluation mode \n"," y_pred = R.forward(test_x.view(-1,28,28).type(torch.float).to(device)/255.)\n","y_pred = y_pred.argmax(axis=1)\n","plt.figure(figsize=(10,10))\n","for idx in range(n_sample):\n"," plt.subplot(5, 5, idx+1)\n"," plt.imshow(test_x[idx], cmap='gray')\n"," plt.axis('off')\n"," plt.title(\"Pred:%d, Label:%d\"%(y_pred[idx],test_y[idx]))\n","plt.show()\n","print (\"Done\")"],"execution_count":10,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABHMAAAR7CAYAAAADhugaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd7xcRf3/8fcnJJCEQCgJTWmKEooCIj+RjiAQkSpFekRBUCkaiopo5IsUpYUvKOAXjTQLJSrSEUKRL/ilqZSIiiEgNSEJJQkhyef3x8xyT27OnLt377Zz9/V8PPaxd2fOnDN7Zj93d2fnzJi7CwAAAAAAAOUwoNUVAAAAAAAAQPXozAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoETozMkws0lm5mY2ptV1aVetOkdmNiEed1wzj4v2QGz2rFUxYmbj4nEnNPO4aA/EZs9430QrEJs9IzbRCsRmz4jN6tW9MydzErrf3jCzx83sR2b2/noft5XMbEkzO8rMfmZmj5rZi2Y2Lz7nR83sTDNbtUHHrpzvSY3Yf39gZnuY2QVmdr+ZPWdms+PtH2Z2uZl9rNV1bIZOjM0KM1vDzC40s2fMbI6ZzTCzP5nZl82sIZ3amU6OKY3Yf39hZoPM7OgYnzNj+/wjxmxD/m+2m06Oze7M7PjM85/SoGPwvlkDM9vUzOZn2metVtep0To9Ns1sCTP7opndYWYvm9k7ZvaCmd1lZt8xsyF1Ph6x2QtmtqOZ/dLMpprZXDN7zcweNrNzzewDra5fI3VibGY+V1Zzu7vOxyY2C5jZdr1oG6/nsQfWc2fdvCvp9fi3SRopaaN4+5KZ7ebu9zfw+M20gqSfZB4vkPSGpOUkbRJvR5vZXu5e1+BCVc6WtG7m8UxJwyStE29jzOxkdz+nFZVrgU6KTZnZjpKul7RsTJolaYikLeJtXzP7rLvPbVEVO5aZLSfpJoV2kKR5kmYrxOVxkg41s13c/c8tqmKzdVRsdhc/eP9Xq+uBxZnZEpIulbREq+vSIh0Xm2a2mqQbJVV+8Kp8tl1N0vskbS9pgqQXWlG/TmZmgyRdLumQmOQKn22WlzRC0qaSHpP0bEsq2FydFJtvSXqlIH+AwvOXpEcbXx1kzFNx20ghNpdQndumkZdZPeDuq8Tbygpfng9V+CK9nKRr692j30JzJV0gaW+FN7gl3X0FSYMlfUbSZEnDFZ7zci2rZef6taTDJX1Y0lLuvrykpRQ62W5SiIMfmdnWratiU3VMbJrZ6pKuU+jI+V9JH3H35RSe874KH352kDS+ZZXsbD9T6Mh5S9JBkobF+Bwl6V6FD6Y3dtD/zY6JzYT/VnjOD7W6IljM1xS+IHZq23RUbJrZMpLuVujIeVLSrpKGxM+2QyVtpvBDGT+CtMYvFDpyXpd0lKTlM59tPyRprKSprateU3VMbLr7OZnnuthN0pczm09oUTU7krs/0EPbfETSwrj5hHoeu2lz5rj7bHe/UtKxMWkVSXs26/iN5O4z3f3r7j7R3V9094UxfZ673yLps3HTFSXt1rKKdih3/567/9zd/+Hu82LaQnd/XKEDrvLLxZhW1bGV+nNsSvqGQkfqG5J2d/cnJMnd57v7dep6zl8ys1EtqmNHMrONJe0VH45192vc/V1Jcve/S9pD4YPqSpJObE0tW6ufx+YizGx3hec2UdKtLa4OMjIjpl4QI6ckdURsnqXwA9jTkrZ095sz/5/nuvvD7v5Nd5/W0lp2IDPbV9IBCh1p27v7pe4+S5LcfYG7/9Pdz3P3e1ta0RbpgNgscli8f8zd/9bSmqC7gyQNUhhJ9st67rgVEyD/Rl09U5tWEi0z0ZGZLWdmZ5vZZAtzm8zM7sDCHDVfM7P7zOz1eA3vcxbmrFmv6OBmtku81ndWvK7yQTM7pKhMX7n7vyTNiA9Xa+SxqhGvgR5tZpea2SNm9oqFOX5eNLOJZvapKvezvJmdb2bPxmt1XzCzy6yHeS7MbJiZfdvM/i+2w1wLc2RcGEdSNE3s3PlLfNjytmmx/hibo+P91YkPnVdJek3hf+FBfTxWXZjZNmY23swesq75t141s1vNbJ8q9zHYzL4f22lOLP9LM/twD+Vqbr8a7BLv31QYLr4Id58p6efx4SFmZnU+fpn0x9jM7n+YpIskvS3p+Hrtt546/H3zvyUto9A2bzf4WGXT72LTzEZK+lJ8OLbSUdCuOjA2vx3vL3T3vzZg//1Fv4vNHo43QuFqEKlNRuV0YGwWqXS0/aHuneDuXtebwgvIJU0q2OaVuM1lmbRJMe1ESf+Kf89V+EV9Zma7VSU9HvNdXdfwVh7PkbR34rgnZrZbqNDBsiA+PjdThzE5ZSt5yedV8HxHZY67X7PPd06ZDTP1qVxn+1a3tG8lylbOw1hJ/4x/z+5W/lVJ6yXKrydpSmbbd7uVfV3hV6DU8xyXkzeuUr6G8zdY0r9j+R/XOx7a6daJsRmP6ZKOK3jO98dtHqzz+a68Lqf0osywbnH4RozPbNqlPbTvmQqXlLmkd7qVf1vSNonyNbVf5nlOyMkbkym/Vre8n8T0xwrOx5cy5Ue1OoYadevE2Oy23Xlxu5NrjZ16n++cMh35vilp97jdLfHxdqmY7o+3ToxNSV+JedMkDWi3851TpmNiU9L6mX1v1Or4aOWtE2Ozh/NxbCw3T9KIVpzvnDIdE5s9nIePZo67R73bpukjcyxct1iZnGlmzibfVRiGNFrSUHdfVtLHY9lBkn6nMKnVHxXmWhgct1lNYd6awZKuNLMPdjvuVgrX90rh1/jVPFxfuqKkHypcjrFxnZ6mzGyAma1qZvtL+kNMnqowmVyrzVOYq2JnScPdfbi7D5O0sqRTFf7h/MDMPlGwj1MVfqnbTWGei2EKH/L+rdC+18b2eo+ZDZd0s6Q1JV2r0I6DY9kPSrpGYY6M663Bc2SY2Qpmtp1C26yl8JwvaeQx210/jU2P90WTdlYmgl+/xmPU00KFOX72krSiuy/r7sMV4uJrCm9ER1oYZp1ytMIbx6EKsTlcYX6oRxXmOviNmS2fLdCX9uuD3rSNJG1Qp+OWTj+NzcoxNlH4EPq0QqdOu+q4900zW1phVM5cScfUc9/9RT+NzU/G+79KGmRmp8aRC3PNbJqZ3WJm7TRlQCfFZqVt5kl60swOiiM+3oqjPx4ys2PNbMk6Ha+0+mlsFjks3t/s7XP5YyfFZpFK27wW61Vfze65U/hCUumd+lxOD9w8SRsmylZ+pb1X0qDENpfEbS7qlv7HmH6XJMsp9z+ZetXcU9ptP9nbY5I+1OzzXeM+T437/HnBeVgoaauc/HUVRgO4pIO75Z0e068pOPYtcZsTEs9zXE6Zcaqip1TSwYm2eUXSrvVum3a7dWJsKnxBdEn/k6jTQEnTM/sfVsfzXXldTqnjPg+J+7y7oH1d0kE5+SMUfml1Sd+pY/tVnueEnDJjMnVaq1veyTH9bYWJyfOOeWGm/NcaGR+tvHVibMb8AZL+L26zXc5rqm6x05vzXeM++937prpGTI3LpG2Xiun+eOvE2JT0YMz7febv+Qq/ZC/M7PfCZp/vGvfZb2JTYS4jl/SyFn1/nKEw8qDy+E+Slml2vDTz1omxWXAuPpLZ556tON817rPfxGbBcQbGeHVJFzSibZoyMseCtczsBIVeSUl6TvmjVG7xOElpjsPi/XiPE7HluDrefzpz/BUUllCUpLM9nt1uzkg+AUnuvp27m7tvV7SdwhCyV9Q1R44Uhukd4+7/6KFsu6i0y5YF29znOUv9eZi49Lr4sPv8HpX2O7dgv9fE+08XbNP9mONi2/Q0p8YchbZ5VV3X0U5X6CW/rdrj9ScdEJu3x/sDzex9OflHSFoh83iZomO1gUq7bG5hqeA8z6krjt7j4ZeaS+PDVGz2qv164u4TKrHp7lO6ZVfaZqjCsP5FmNlKWnRS8nZvm7rqgNiUpK8q/BJ6tbtPKtpXCfSr983MiKl/KXyBRNQBsVn5pXpXSf9P0mkKqyWtoDCZ7M9j/jGNnAOkjvpTbFbaZqTCaLnfSlrTw8iPZRXmtZqvMJLkgmqP2V90QGymVOo7TWGV3rLoT7GZsovCSCQprEJXdwN73qRm25pZ3otYkl5S6Dmcl5P3v3kFzGygwpuKJF1qZhcn9l35gpOd2GgTSabwBX6xF4QkufuzZvZ8t3K95u5jFa7vk5ktqzAZ1VmS7jOzc939hL7sv17i8MOjFFaMWV9huFn310PRhMCTCvLukXSgwpKWleOtLun98eHNBa+NytDQuk9M5e7XS7o+1mcphaU1z1QYBvlFM9vL23yivzrppNg8X2FZ+mGSbjWz4yQ9IGlpSftLOkfh16zKEM2FeTtppng+D1NYOn0jhc6m7kOmByvEbN5Q2nsSHyCkEJvflrShmS3p7vP62H41c/fHzOwPCqv9nWlmCxXeXN+Q9AmF5eIHZ4q0vG2aoGNi08xWU/j1bJaktnhf7EmnvG+a2QCFjt8lFH6IYvnpDopNdS2OMkDSVe7+vcx+X5V0uJltqPAZ6luSrqzhGHXVKbGpRdvmWYV5OCurjM2RNN7C6nMnSDrMzE519xfrdOx21UmxuZj4w15lAY9fFnQ+tUQHxWZKpWPpb+7+WCMO0MjOnHcVhmRKXUPpn5V0h8IlDzMS5V5LpGe/0KxYxfGHZP6uXDM5y92LVmL4j+rYqO7+hqRfmdl9kp6SNNbM/uTuE+t1jFpYmP17ksKykxVvK4wmWqjwD2qEwhfelP9UkTcyk5adcXylKqo5tIptaubu70i638y2V/iHu73Cr0/HNfK4baJjYtPdp1iYt+o3ChOx/bHbJlMUlgj8Vnycd11101hY1ec2hV/VKuYonPtKZ0alh39p5XfmVBObSyi8ob6ivrVfX41RGOq6mcKviNlfEhcojAyofJBqads0ScfEpsJcLMtKOtbdX66hfFN12PvmVxVi8gZ3v6VO+yy7TorNtzJ/j09sc75C5/t6Zraqu79Uw3HqosNiM9s2P0l8cT9PoTNnCUnbqs7LILehTorNPDsrjJiT2mQVq4oOi83FWJifsjK/2IRGHaeRnTkP9HKIWMWCRHr2krBN3P3xGvbdEu7+HzObqNA7d7iklnbmKHxh+rDCP7sTFebfeO+fXZzM6591Pma2/Zb3sOxwy7n7fDO7RGEkwOHqjM6cjopNd7/ZzDZQ6BjYTuGf/jSFScjOUddSyM/HTr5WOlWhI2eawgi/W+MvoZLe+wVmfuVhnY7ZsvZz9+lxssAvSPqcpHUUnt8jCl8WXs1sXpbLVPuiI2IzdqLvLelJSVfETsysJbs2fS9vrrvPV+t0xPtmnDjydIVJj7+T0zbZLy5DY/67bfC/s9E6IjajFxVGGEjS3xPbZNNXVxgB0SodEZtRdpRNbtu4+0tm9oZCZ3mzl19uhU6KzTyVkR9PuPujLa3J4jopNvN8XtJSCp9rr+5h25o1sjOn3qYrBN4SktZQmIemWpXe1+FmNtTdZye2Kxrm1VeV3sN6rQRTkzjD/R7x4UHu/mDOZivnpHVXdK4qedle71cyf6+h9vqVvdI2w8xspeyXZ1Sl7WPT3Z9TvPyxOzOrDM/MHXLbZJVVqo5x91/l5NcrNheoa16vvrRfn8Xhz5eqaz6f95jZ3vHP+ZIebma9+ol2jc014/0GKn4vWEPSm/HvL6hFvzp22PtmZe4NKYwoLvJkvP+FFp3fCj1r19iUpCcU5supVuoyhobrsNiUQtv0RsvapsTaOTYXYWElpsrrvyHzsdSqA2Mzz5h4f5u7v1K0YV80fWnyWsWhhJUP86N7WfwxhX9oAyRtlbeBma2t0OiNsna8f6twq8YbodBLKIXzkmfHKvazbRV57/UQu/u/1RVgvW2/Rls783er26d0yhybZraiuiY/W2zS4BaoXOfb6Nh8onINeR/br9EOiPd/iJetohfKHJtthvdN1FWbx+admb/XTWwzKvP3czUepx46LTbvVxg1JyXaJs5HVumQndKEOvUrbR6b3VVGfixQmP+znXRabC7CzEapa+6lCY08Vmk6c6IJ8X6MmW1UtGG8Tk2S5O6vKywRJ0knmVne5QnfrLVScbKsovwPSdozPryv1uPUyZvq6qn/SPfMeH3jMVXsZ1sz26J7YnyulVnFr+2WPSHen2D5KwtV9mGxt7nPqmibIQrLF0rSowW96Cg2Id63VWz2UA9TWNpzsKS/SfpDI47TS5UJuPNic5ikU6rYx1pmdkD3xLjKwpHxYSo2e9V+jWRmOytcerVQ0tnNOGY/NSHet01s+qKrnC12k/T9uOlzmfQJBbtstI5533T3KT20zfaZzdeO6WP6etwONSHet01sRpMkPR//Pj6xTSX94RaPZu6Y2JQkd39LXVM1fMXMBuVs9vV4P1ddrxP0zoR4326x2V3lEqvb2nDuuY6KzRyVtpmh/NXU6qZsnTmXS3pQ4cvXXWZ2hIUVoyRJZraKmR1kZvdo8blPxim8qHaQNMHMVo5lhpvZGQpfcpIrGZnZJDNzM5uUk32hmV1oZluY2eBMmeXMbIzCbNtDFF7Y5+fse1zcd1+GQw4ysxE93Aa5+5sK51CSfmZmG8c6DDCzHWJdq5mL4w1JN5jZZyr/rMxsa4XJTJdSGH79m25lzlK4bnKEpAfMbL/YmaJYfg0zO1Khh3VPVamH83eQmU00s89m/+Ga2VJm9un4fCv/ZE6r9phYTLvGpszsDDPbuVt9NlFY0vNASbMlfcHdF7t+2szGVF5bZrZWTychYUAVsVn59eKOeH+emW2bia3NFCZvrmYyvlmSfhrP98BY/qMKEyuPVJiH5sfdyvSl/ZJ6On9mtq+ZHWVmq2ee60pmdrLCB1aTdE5ieC6q07ax2Re8bwYNet9Ec7RlbMa5qSpfOA+Mr5VlYrmVzOxyhQmyJem7OfsmNtXQ2PyuwsIIa0v6jYXVe2RmQ8zsWHV1tI139+nVHheLaMvY7LbdhyVtHh9WdYkVsRk0+n3TwoqQB8eHv2r4nHLuXtebQm+YS5rUy3KTYrkxPWy3ksIwQ4+3BQrXN76VSXNJ38spe2Imf6HC7Ofz4+Nzi+qQyVvseWWec6U+ryv0xGXr85KkrRLPaVxluz6c72pu28Uyn1D4AltJfyvzeLrCNY659cmch7EKk1Z5LPtmZn+vSlo/Ud91FK7Dr2w7X2Gy12x9XNJhiec5rjfnT+F6xex+34jHm59Jmyvpq/WOhXa7qQNjM+ZPyex7Vmzv7Gt1+4LnlH39rNXL8zZOiz7votuYWOYDCtf+VtLnZM7fbEk7peqTad8zFd5AK6/tWZkyb0vapp7tl3meE3p7/rqdo3cUrm3Ovg7OkWStjh1iszGxWUXsTKkmvvpwvnnf7OX5U5hEvqb/iWW8dXJsSjqj22tuejxO5XhjE+WIzQbHpqTdux3jdUnzMo+vlzSw1fHTyFsnx2bc7geZtl+qyudObDbhfVNh+obK/v9fo2OhbCNz5GE457aSDlJYjeY1ScvE7MmSrpC0n0KvXPeyP1K4fu5uhRfUQIXrIg9199zJUat0lsKvGLcrfHFcUmGps1cUfk3/hqRR7n5/onxlGbWmTPDp7g9J+qTCyIQZkgYpBMSlkjaW9JcqdjNd4VrACxSe55IKs+z/VNLG7p47caK7/1NhlYSvKLTDDEnDFYLsr5IuU5h4r17Xft4k6SiFIXiT43GGK3Tq/Fmh7dZ394uTe0BV2jQ2Jem/JP1OXbE5V6E3fpxCXN5dULYSm/9RE1brcPdnFeLqKoWYXEKhg+NqSZu5++1V7OYdhS9cpynMZbCkQlv8StLH3P3exLFrbr8++L2kSxQuc5ut8CvYFIVfmT7p7id4fGdE7do4NvuC983GvW+iSdo5Nt392wrLHt+o8NpdRtLLCr+Cb+Hu5yaKEpsNjk13/3087s8kTVVY2vkthUt8DpS0j7d29b/Sa+fYjCM/DokPf+3Vj/wgNpvzvnlYvJ/s7n9uwP4XYXxObj0zm6wwkdlu7t4O83YAkGRmtyp8mD3G3S9qdX0ABLxvAu2J2ATaE7HZP9GZ02LxWsqXFSbf3bTV9QEQmFllVMwbkj7Qi18+ADQQ75tAeyI2gfZEbPZfpbvMqh/aJt4z+S7QXj4maZikH9KRA7QV3jeB9kRsAu2J2OynGJkDAAAAAABQIozMAQAAAAAAKBE6cwAAAAAAAEqEzhwAAAAAAIASoTMHAAAAAACgROjMAQAAAAAAKBE6cwAAAAAAAEqEzhwAAAAAAIASoTMHAAAAAACgRAbWWtDMvJ4VAdqZu1ur61AtYhOdhNgE2hOxCbQnYhNoT7XEJiNzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAokYGtrgAAAAAAVJx22mm56TvssEOyzJZbbtmo6gBAW2JkDgAAAAAAQInQmQMAAAAAAFAidOYAAAAAAACUCJ05AAAAAAAAJUJnDgAAAAAAQInQmQMAAAAAAFAiLE0OAAAAoKmOO+64ZN4RRxyRm77ttts2qjoAUDqMzAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKhKXJAQBA2xg5cmQy77XXXmtiTerrlFNOyU0//fTTk2UuueSSZN7RRx/d5zoBjXbFFVck8/bYY49k3oEHHpib/swzz/S5TgDQXzAyBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoETozAEAAAAAACgRliYvmX322SeZt+GGG+am/+1vf6vpWOuss04yb9lll81Nv/7665NlHn300ZrqAQDoX4qWH7/55puTedOmTctNP+SQQ3pdptn23HPP3PSFCxcmy0yePLlR1QF6bcCA9G/Ahx12WG76QQcdlCyTiglJuummm6qvGAB0KEbmAAAAAAAAlAidOQAAAAAAACVCZw4AAAAAAECJ0JkDAAAAAABQInTmAAAAAAAAlAirWTXYBhtskMw75phjctNXW221ZJnPfOYzybyiVQaapWiFkiOPPLKJNQGKLbXUUsm84447Ljf91FNPTZbZb7/9knm33HJL9RUDOsCmm26azPvYxz6WzHv++ecbUZ26WXPNNZN5Q4cOzU2fPXt2ssxtt93W5zoB9TJ8+PBk3uWXX56bfskllyTL8N6I/szMknkrrLBCbvrxxx+fLPP6668n815++eVk3vrrr5+bfsABByTLfPCDH0zm1WKTTTZJ5j3++ON1PVanaf23fwAAAAAAAFSNzhwAAAAAAIASoTMHAAAAAACgROjMAQAAAAAAKBE6cwAAAAAAAEqEzhwAAAAAAIASYWnyKhUtMX744Ycn81LLj0vSwIH1Pf33339/bvqsWbOSZT7wgQ8k89Zbb71k3rvvvpub/sQTTyTLAM1WtPzxmWeemcz71Kc+1etjXXfddcm88847Lze9aKlzoOxOOeWUZN6xxx6bzHP3ZN7Xv/713PRp06ZVX7EGuuKKK5J56667bm76NddckywzefLkPtcJqJcTTjghmbdgwYLc9KJYnz9/fp/rBLSrPfbYI5l3ww03NLEmvVf0Poz2wsgcAAAAAACAEqEzBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoEQ6cmny7bffPpmXWkJ4/fXXT5YZNGhQTfW48cYbc9OvvPLKZJk333wzmXfnnXfmpqeWi5SKl2cuWpo8VY8LL7wwWQZohI033jiZ99vf/jaZt+qqq9a1HoMHD07mff7zn89Nv+CCC5Jlpk+f3uc6Ac2wyy675KafdtppyTJmlswrWhJ14sSJ1VesQbbZZptk3tZbb53MSz2v+++/v891AprhE5/4RDIv9X7G8uPoVLfddlsy7/bbb89N33bbbZNlir5vzpw5s/qKRS+88EIyr+h9eKONNur1sdA4jMwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAokX67mtW4ceOSeSeffHIyb6mllspNL5olvGjFjmuvvTaZ9/LLL+emF60+VYsf/vCHybzjjz8+mVe0ctYWW2zRpzoB9XLggQcm8+q9YtU3v/nNZF7R6jypVeNOOOGEZJmiuJ0xY0YyD2i2PffcMze9aDWMadOmJfPOOOOMPtepkUaNGpXMK3rOqbwbbrihz3UC6qVohciilXa++93vNqI6QGnNmTMnmZdaBbJoxeVVVlklmffLX/6y+opVoSjW77777roeC33DyBwAAAAAAIASoTMHAAAAAACgROjMAQAAAAAAKBE6cwAAAAAAAEqEzhwAAAAAAIASoTMHAAAAAACgRPrt0uTz589P5j388MPJvNQSoddcc02yzCuvvFJ9xRrooIMOyk3ff//9k2WmTJmSzPvqV7+azHvmmWeqrhdQDx//+Mdz048++ui6H2vHHXfMTb/33nuTZZZeeulk3qGHHpqbfuKJJybLfPGLX0zmFf1/W2211ZJ5QCOst956uelmliwzderUZN748eP7XKdG2nrrrZN5Rc/5vvvuy00vWqYdaLaxY8cm80466aRkXmop44ceeihZZsGCBdVXDOgA7bLs9/Dhw3tdpmgp9nfffbcv1UEBRuYAAAAAAACUCJ05AAAAAAAAJUJnDgAAAAAAQInQmQMAAAAAAFAidOYAAAAAAACUCJ05AAAAAAAAJdJvlya/6KKLknlnnHFGMm/hwoWNqE6vDB06NJk3evToZN7Pfvaz3PRBgwYlyxQtiVp0rFTev//972SZq666Kpk3Y8aMZB4gSbvuumtu+rBhw5JliuL5t7/9bTJv0hTpWUcAACAASURBVKRJVder4s0330zmpZZavvTSS5NlVlxxxV7XQZJOOOGE3PRzzjmnpv0BknTKKack80aNGpWb/tRTTyXLFL0Pt4PUc5KkPffcM5nn7sm8dn/O6BxLLbVUMm/jjTdO5hUtPTxkyJDc9Hb4XA2gd4455phel7nrrruSeU8++WRfqoMCjMwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASqTfLk0+c+bMVlehR+9///tz0ydOnJgss+mmm9a1DiNGjEjmHX/88XU91re+9a1k3g9+8IPc9IsvvriudUB5ffnLX85NL1r2dOrUqcm8r33ta32uU7Xuvffe3PSi5cyLllwHGmGXXXZJ5h177LHJvJEjR+amX3bZZckyRe9z7WD06NHJvKFDhybzzCyZlzpPRe/rTz/9dDJv9uzZyTygyOqrr57MW2eddZJ5gwYNSuZ985vfzE139+or1kArr7xyMi9Vx1dffbVR1QGAumBkDgAAAAAAQInQmQMAAAAAAFAidOYAAAAAAACUCJ05AAAAAAAAJUJnDgAAAAAAQIn029WsmmnppZdO5k2YMCGZt8MOO+SmL7fccskyc+fOTeZNnjw5N/2OO+5Ilila9avoWE888URu+uabb54s88UvfjGZN378+Nz0v/zlL8ky999/fzIPkKQZM2Yk81566aWm1eOZZ57JTX/wwQeTZXbccceajjVlypSaygE33XRTMq9oRZobbrghN/3MM8/sc50abdSoUbnpqZV5pNpX5/nFL36Rm160AtZmm22WzHv00Udrqgcwa9asmvLWWmutZN59993XlypVbciQIcm8E088MZn39a9/PZm3YMGC3PTUSquSdP755yfzAKBZGJkDAAAAAABQInTmAAAAAAAAlAidOQAAAAAAACVCZw4AAAAAAECJ0JkDAAAAAABQInTmAAAAAAAAlAhLk9fBgQcemMz73Oc+l8z7+9//nptetNzh73//+2Re0TLezVK0DHqR73//+7npX/3qV5NlWJocPbniiiuadqxtttkmmTd+/Pjc9LXXXrvu9bjuuuvqvk/0L0ceeWRuetES2UVGjhyZm37eeefVtL8iqbo//fTTyTLrrbdeMi+1zHjRuShamryWc3j77bcn86ZOndrr/QE9ee2115J5Ra+5pZdeOpk3ffr0PtWpu1Qs/e53v0uWWX311ZN5qffhIqecckoy7+67707mPf74470+FlB2yy23XDJv8803T+ZtsMEGybzU54utt966+opljB07Njd98uTJNe2vHTAyBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoETozAEAAAAAACgRliavgxtvvDGZ9/zzzyfzHnnkkdz0oiUjy+yvf/1rr8tsueWWDagJOsXRRx+dzHvhhReSeanlV0ePHp0ss+uuuybzhgwZkswD2kXRkttFeVtttVVuetH/71qX/k7lrbvuunXdX5GiMpdddlkyb+LEibnpRUuTA+3k17/+ddOOddRRR+Wmr7jiiskyBxxwQDKvluXCt9hii2Re0bLl++67b6+PBTTbhhtumMxbfvnle72/ovf8Bx54oNf7a4Q77rgjN52lyQEAAAAAANAUdOYAAAAAAACUCJ05AAAAAAAAJUJnDgAAAAAAQInQmQMAAAAAAFAirGZVBy+//HIy79Zbb21iTVovtQqQJP3iF7/o9f5uu+22vlQH/cicOXNy0wcMSPdJr7POOsm8eq/KUVSPhQsX1vVYQF8UrbiUstdeeyXzRowY0Zfq9MpPf/rTXpc5/vjjk3mpVbCKVtu67777knlFK+gBZffQQw/VdX/LLbdcMi/1P2f//fdPlvnnP//Z5zplnXvuucm8Cy64IJk3dOjQ3PTZs2f3uU7oH0aNGpWbvvrqq9e0v7333juZt/766+em13s1q0b405/+lJue+k4gSQ8//HAy7+KLL+5zndoNI3MAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEinF0uSpJf4kaeTIkbnpU6dOTZZx9z7XqdN9+MMfzk0vWqpx2WWXTebdcsstueknnXRS7yqGfmu33XbLTb/rrruSZWpdMnnGjBm56csss0yyTNFSxj/84Q9z03ffffdkmQ022CCZBzRC0ZLltSxn3kypZV4l6bjjjkvmpT4PTJs2LVmmluXRgf7g2Wefrev+vvOd7yTzHnjggdz0ei8/XmTBggXJvKL/OauttlpuejPrjuYYODD9Vbpoafv9998/N32llVbqc51aJbWMuCSdc845ybyXX345mfeXv/wlN33u3LnVV6yfY2QOAAAAAABAidCZAwAAAAAAUCJ05gAAAAAAAJQInTkAAAAAAAAlQmcOAAAAAABAidCZAwAAAAAAUCKlWJr8d7/7XTJv3rx5uel77rlnssy7777b5zp1gmHDhiXztt1229z0XXbZJVnmkUceSeadffbZuempJaLReZ566qnc9G222SZZZtNNN03mLb300sm8Bx98MDd9xRVXTJYpcs899+Smb7bZZskyLE0OVO/KK69M5q233nrJvNdeey03feLEickyW2+9dTJv+vTpybzUUsZXX311skyqfkArfPSjH03m/fGPf8xNL/osucUWWyTz9ttvv+or1gJz5sxJ5qW+m6D/GTJkSDLvmGOO6fX+3n777WTe7bffnswreq9IfY8+6aSTkmVS3/Mk6aGHHspNL/oOWPS80DeMzAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKpBRLk++4447JvLvuuis3/X3ve1+yzJQpU/papX6jaKnln/zkJ8m8ffbZJzf9pZdeSpbZeeedk3mvv/56Mg8o8swzz9SUB6B89tprr9z01LLfkuTuybwbbrghN/2ss85KlkktyypJRxxxRK/rMXv27GSZyy67LJkHNELR6/GUU05J5g0YkP/78CuvvJIss2DBgmRevT8XjhgxIpm3zjrr5KZfdNFFyTKTJk1K5k2dOrXqeqHciuJl7Nixybzp06fnpj/44IPJMvX+TDtjxoxk3gMPPJDMe/PNN3PTWX68NRiZAwAAAAAAUCJ05gAAAAAAAJQInTkAAAAAAAAlQmcOAAAAAABAidCZAwAAAAAAUCKlWM3qtttuS+bttNNOuempFS8k6cc//nEy75133qm+Ym1miSWWyE0/9NBDk2W+9a1vJfNSs/tL0p///Ofc9M985jPJMqxYBQDoi9TqUwsXLkyWueaaa5J5t99+e276LbfckiwzcuTIZN6jjz6azEvVnRWr0E4OPvjgZN6ZZ57Z67yBA9NfNd54441k3vXXX5+bPnTo0GSZIhtuuGEyb/Dgwbnp5557brLM6aefXlM90L8Urch2/vnnN7Em6FSMzAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKpBRLk3/7299O5m255Za56UXLCc6bNy+Z9/Of/zw3ffbs2ckyRVLLhUvp5RWHDx+eLLPvvvsm8/bee+/c9NQ5koqXcz3nnHOSeWeddVZuOsuPA9Uzs5ryav1/BJTBqFGjknmp9yx3r2l/qc8XK664YrLMN77xjWTe+PHjk3lAGbzwwgvJvEMOOSSZl3rtH3300ckyhx9+eDJvlVVWyU1/6qmnkmXeeeedZN6PfvSjZN4jjzySm/7SSy8lywBlt8kmm7S6CqgDRuYAAAAAAACUCJ05AAAAAAAAJUJnDgAAAAAAQInQmQMAAAAAAFAidOYAAAAAAACUCJ05AAAAAAAAJWJFy3kWFjSrrWCdbbzxxrnpDz74YLLMkksumcybPn16bvpNN92ULDNjxoxk3sc//vFkXtGS4bWYP39+bvrf/va3ZJmrrroqmXf++ef3uU79hbun14luM+0Sm+jZrbfemszbcccdk3nnnntuMu/kk0/uU53Khtjsf6688spk3sEHH5ybnlqyXJIGDEj/bnXPPffkphctMT5x4sRkHroQm0B7IjYhFX+3HT16dDLvzjvvzE3faaed+lynTldLbDIyBwAAAAAAoETozAEAAAAAACgROnMAAAAAAABKhM4cAAAAAACAEqEzBwAAAAAAoEQGtroCffX444/npm+++ebJMnvttVcy78QTT8xNP/TQQ3tXsSqkVp+aNWtWssy1116bzLvwwgtz0ydPnty7igFoa2uuuWarqwA0zFNPPZXMS61aVbQy5+mnn57M++lPf5qbPnXq1GQZAACAdsDIHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBErGg5z8KCZrUVbHMbb7xxbvraa69d92Pdfvvtuelvv/123Y+FvnF3a3UdqtVfY7M/uvXWW5N5O+64Y037HDhwYK3VKSViE2hPxCbQnohNSNJNN92UzBs9enQy784778xN32mnnfpcp05XS2wyMgcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAokc5aw7YKjz/+eK/SAaDZHn744VZXAQAAACV1xhlnJPOKlia/7777GlEd1IiROQAAAAAAACVCZw4AAAAAAECJ0JkDAAAAAABQInTmAAAAAAAAlAidOQAAAAAAACVCZw4AAAAAAECJmLvXVtCstoJACbm7tboO1SI20UmITaA9EZtAeyI2gfZUS2wyMgcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBIxd291HQAAAAAAAFAlRuYAAAAAAACUCJ05AAAAAAAAJUJnDgAAAAAAQInQmQMAAAAAAFAidOYAAAAAAACUCJ05GWY2yczczMa0ui7tyswmxHM0rsnHHRePO6GZx0V7IDZ7RmyiFYjNnrXqHLXqfwLaA7HZM2ITrUBs9ozYrF7dO3MyJ6H77Q0ze9zMfmRm76/3cduFme1rZneZ2XQzm21mT5vZ6Wa2TIOOV/kiNaUR++9PzOxDZna5mU01s3fM7EUz+5WZbdrqujVDJ8emma1hZhea2TNmNsfMZpjZn8zsy2bWkE5tYrM6ZjbIzI42s/vNbGZsn3+Y2QVmtmqr69cMnRyb3ZnZ8ZnnP6VBx6ic70mN2H9/ZWabmtn8TPus1eo6NVqnx6aZLWFmXzSzO8zs5fjZ6YX4Ofc7ZjakzscjNnvBzHY0s1/Gz7Vzzew1M3vYzM41sw+0un6N1MmxGb/PXGxmk+N3zXfM7Ln4nWbbBh2T2KyCmQ03s1PN7P/ia/FdM3vVzG43s0Mb8Z1jYL13mPGupNfj3yZppKSN4u1LZrabu9/fwOM3nZldJumI+HC+pLmSRkk6RdIBZra1u7/Yqvp1MjP7tKTfShoak2ZJWkXS/pI+Z2ZfcPerWlW/Juuo2DSzHSVdL2nZmDRL0hBJW8Tbvmb2WXef26IqdiwzW07STQrtIEnzJM2WtI6k4yQdama7uPufW1TFZuuo2OwufvD+r1bXA4szsyUkXSppiVbXpUU6LjbNbDVJN0r6WExaIOkNSatJep+k7SVNkPRCK+rXycxskKTLJR0Sk1zhs83ykkZI2lTSY5KebUkFm6ujYtPM9pT0S0mDY9I8hXOwRrztb2anu/upLapixzKzdSTdJWn1mLRQ0psKr8lPx9vBZrZ7Pb9zNPIyqwfcfZV4W1nSMEmHSpopaTlJ19a7R7+VzOxohY6chZJOlDTM3ZeRtKWk5yR9QNJvWlfDzmVmq0i6TqEj5w5Ja7n7cgqdOVcrdGpebmYbtK6WTdUxsWlmqyu0/bKS/lfSR2LbD5O0r8KHnx0kjW9ZJTvbzxQ6ct6SdJDC/83lFTrB71X4YHpj7PTpBB0Tmwn/rfCcH2p1RbCYryl8QezUtumo2LQwmvxuhY6cJyXtKmmIu6+g8FlqM0lnK/xoieb7hUJHzuuSjpK0fHzvXErShySNlTS1ddVrqo6JTTMbIekKhY6cRyVtrhCXwyR9UOHzriR9x8y2aU0tO9qVCh050xW+YwyJ3zmWl/S9uM2nJZ1Uz4M2bc4cd5/t7ldKOjYmrSJpz2Ydv5HMbClJ4+LD8e5+jru/I0nu/oCkvRR6zbc0s91aU8uO9k2FL/PPS9rb3Z+TJHd/VdJhkh6RtKSk01pWwxbqz7Ep6RuShiv8mri7uz8hSe4+392vU9dz/pKZjWpRHTuSmW2s8L9Rksa6+zXu/q4kufvfJe2h8EF1JYUO8o7Tz2NzEWa2u8Jzmyjp1hZXBxmZEVMviJFTkjoiNs+S9GFJT0va0t1vzvx/nuvuD7v7N919Wktr2YHMbF9JByh0pG3v7pe6+yxJcvcF7v5Pdz/P3e9taUVbpJ/H5m6SKtN27OXuD7n7Qkly92clHSjpnzF/7xbUr2OZ2doKnWuS9HV3v87d50mSu89099MUOmGlOrdNKyZA/o3C6BUp/MojadGJjsxsOTM7O3Mt4MzsDsxsSTP7mpndZ2avZ64V/JmZrVd0cDPbJV7rOytey/agmR1SVKYKOyp84XBJ53bPdPfHJN0ZHx7Ux2PVhZltY2bjzewhC3PHzIvX9N1qZvtUuY/BZvb92E5zYvlfmtmHeyhXc/v1Vrw28fPx4U/c/a1svrsvkHRefPhZM1tWnas/xuboeH914kPnVZJeU/hfSGw2MTYl7RLv31QYLr4Id58p6efx4SFmZnU+fpn0x9jM7n+YpIskvS3p+Hrtt54szB0y2swuNbNHzOyVGJsvmtlEM/tUlftZ3szON7NnLcxx8YKZXWY9zA9lZsPM7NsWrsOfFcv+w8JcYKsXla2D/1b4AnG8QhuhS7+LTTMbKelL8eHYSkdBu+rA2Px2vL/Q3f/agP33F/0uNiWtHO+nu/tiI69ih2vlNbF0H4/VZx0Wmytn/n4ssc0j8b6+bePudb0pXD/rkiYVbPNK3OayTNqkmHaipH/Fv+cq/KI+M7PdqpIej/murmt4K4/nKIy+yDvuiZntFkqaEctXOmEqdRiTU7aSt9jzknROzPtrwXMeG7d5tc7ne1zc75RelBmWOQ8ez9+sbmmX9tC+ZypctuKS3ulW/m1J2yTK19R+mec5ISdvTKb8Wt3yNszkfSxRpxGZbT5T75hol1uHxuacmHdcwXO+P27zYJ3PN7FZHJs/iemPFZyPL2XKj2p1DDXq1omx2W278+J2J9caO/U+3zllsu8lHuPqrW5p30qUrZyHsQq/mrrC3FDZ8q9KWi9Rfj1JUzLbvtut7OsKoydSz3NcTl7lHHsPz3v3uN0t8fF2qZjuj7dOjE1JX4l50yQNaLfznVOmY2JT0vqZfW/U6vho5a1DY/Pzmf2ukZM/UNI/Yv7RzT7fOWU6KTZXyez7kB7O4a/r2TZNH5lj4brFkfHhzJxNvitpkMIv6kPdfVlJH49lB0n6ncKkVn9UmGthcNxmNUkXKFxHeKWZfbDbcbdSuL5XCr/Gr+bh+tIVJf1Q4XKMjWt8WuvH+ycLtnkq3o+0cM1jKy1UuK5yL0kruvuy7j5c4Zq+rym82I+0MJQz5WhJH1W4LnVYLL+JwjWcQyX9xsyWzxboS/v1QaVtXF1tsAgPIzZe7bZ9x+mnsenxvmjSzspE8O3Q9p0Um71pG0nqlDmtFtNPY7NyjE0UhsM/ra5Rku1onsIcTztLGu7uwz3MU7CypFMVPqj/wMw+UbCPUxVGuOymEJvDFDpH/q3QvtfG9nqPmQ2XdLOkNSVdq9COg71rjoRrFP4/XG91nlvKzJZWGJUzV9Ix9dx3f9FPY/OT8f6vkgZZWJllcvxVe5qZ3WLtNWVAJ8VmpW3mSXrSzA6KIz7eiqM/HjKzY81syTodr7T6aWzeKOnl+PdEM/uExdWRLFzmc43CAhJPKMREq3VMbLr7y5L+EB+eb2b7VOIwjgA7VWFqjzfUNTVLfdSzZ6ianjuFLySVnqvP5fTAzZO0YaJs5VfaeyUNSmxzSdzmom7pf4zpd0mynHL/k6lXb3tKH4t55xacl40y+/9IHc/3ONX5F0yFSdVc0t0F7euSDsrJH6Hwa45L+k4d26/yPCfklBmTqdNa3fKOi+nTe3jOPbZh2W8dGptPx7z/SdRpoMJEZZX9D6vj+SY2i2Pz5Jj+tqSlEse8MFP+a42Mj1beOjE2Y/4ASf8Xt9ku5zVVt9jpzfmucZ+nxn3+vOA8LJS0VU7+ugqj6FzSwd3yTo/p1xQc+5a4zQmJ5zkup0zlHHvBfs/rXl6MzOme3+9iU9KDMe/3mb/nK/ySvTCz3wubfb5r3Ge/iU2FuYxc4Qt99v1xhsLIg8rjP0laptnx0sxbJ8ZmzN9UYQ7Qyj7eUdeIk5kKlywPb/b5rnGf/SY2Y/7I+JqptM2C2CYe43OiEiOJ+nJrysgcC9YysxMUeiWlsMLTjTmb3+JxktIch8X78R4nYstxdbz/dOb4KygsoShJZ3s8492ckXwCktx9O3c3d98uJ7ty7ducgl3Mzvw9rOhYbaDSLptbWI40z3MKPZuL8DDK5dL4sPv8HjW1X0/cfUJsG3P3Kd2yq2kbqat92r1t6qoDYvP2eH+gmb0vJ/8ISStkHi+Ts0076U+xWWmboQrD+hdhZispdAZVtHvb1FUHxKYkfVXhl9Cr3X1S0b5KoNIuWxZsc5/nLJHrYcLvyiokqdhcbD6+jEq89yY2x1ViMy8/M2LqXwpfIBF1QGxWfqneVdL/U1gcYnkPK1mtoq65zI6pwxwgzdCfYrPSNiMVRsv9VtKaHkZ+LKswr9V8hZEkF1R7zP6iA2JT7v6IpE+pa/6VJdX1XWdJhddBWeb/7E+xKXd/TdJnFUZkSeEHq+Hx7yUUvmOuWO3xqjWw501qtq2Z5b2IJeklSXt6nOW5m//NK2BmAxXeVCTpUjO7OLHvyhec7MRGm0gyhd69xV4QkuTuz5rZ893K9VvxfB6msHTaRgpfaLsPyxysMAwtb+LYexL/pCTpHoUJ2jY0syXdfV4f2w/11Umxeb6kwxX+gd5qZsdJekDhjW9/hfmu3lUYait1TZbXMp0Sm+7+mJn9QeGN70wzW6jw5vqGpE8oLBc/OFOk5W3TBB0Tm2a2msKvZ7MkndDb8q0Qh+0fpbDS2voKMdj9c9RqBbuYVJB3j8JKJB/LHG91Se+PD28ueG1U/j/UJTbjsP1LFV4Xx7g7y093UGyqa3GUAZKucvfvZfb7qqTDzWxDheXJv6WwHG9LdUpsatG2eVbSft61ytgcSeMtrD53gqTDzOxUd3+xTsduV50UmzKzIyVdrDA66wCFkSCzFT4vnqkwgnsHM9vK3f9dyzHqqYNiU2a2ucLlecso/G+8TuE1+EGFy+sOk7SNme3j7nkdjDVpZGfOuwpDMqWuofTPSrpD4ZKHGYlyryXSs19oqunVGpL5u3LN5Cx3L1qJ4T+qrVEr+xxSsM3QzN9vJbdqAgsrh9ym0HNfMUfh3Fe+MFVm5V5a+V8Y/1NwiEreEgpB+4r61n59UU3bSF3t09K2aZKOiU13n2Jm+yusarChwvDXrCmSfqnwT1fKv666aTosNqUw8uYWhS8FF2jRXxIXKIwMqHyQamnbNEnHxKbCXCzLSjrWw7Xmbc3CqhmTFJZrrnhb4fKGhQoxNULFq1RUE5sjM2nZlTpWqqKaQ3vepCpfVYjJG9z9ljrts+w6KTazn4PGJ7Y5X6HzfT0zW9XdX6rhOHXRYbGZbZufJEaNnKfQmbOEpG0VPuP0Zx0Tm2a2pUJH+xxJn3L3f2Sy7zGz7RWmjVhPYUTl/r09Rj11UmxaWA35RoXnc4i7X5XJ/qukMWa2QOEH5ovM7HZ3f6cex25kZ84DBUOriyxIpGcvCdvE3R+vYd+N8qLCZFZFPYvZvJa96UWnKnxZnKYwS/it8dcWSWEpOYVhmlLoYa6HVrVf5ReJ5c1scMEvjJX2aXXbNEMnxabc/WYz20ChY2A7hX/60xQmRztHXUshP1+vf6x90EmxKXefHicL/IKkzylM3DdfYfjw+eqamFwKKzT0dx0Rm/ED594KiwZcETsxs5bs2vS9vLnuPl+tc4HCB9JnFVYquTv7JSFOgvnPOh8z237Lu3vDOzTjxJGnK0x6/J2ctsl+cRka899tg/+djdYRsRm9qDDCQJL+ntgmm766WvvZqSNiM8qOssltG3d/yczeUOgs74RR7p0Um8fF+5u6deRIktz9HTP7scKPJbuZmRWM1G6GTorNgxXnpuzWkZNVuVpgDYX/sQ/W48CN7Mypt+kKgbeEwknoTXBVel+Hm9lQd5+d2K6oM6bIU5I+o+LVVior5bwW565opcpKOMe4+69y8lfOSeuumo6rBQq9r1Lf2q8vKitYmUIbPNp9Awuri63UbXtUr51jU5Lk7s8pdI4sxswqwzNzh9w2WSfFpiQpDn++VF3z+bzHzPaOf86X9HAz69VPtGtsrhnvN1DxiKs1JL0Z//6CwsSETWdhRYo94sOD3D3vA1i9YjP7a/Ermb/XUHNGp1Xm3pB6fj+srOD5Cy06vxV61q6xKYWVcHbtxfYt+7LYYbEphbbpjVZ+kS+rdo7N9eJ90eVTz8b7IQqv/ZaMfO3A2OxN20jSWqpTZ07TlyavVRxKWPkwP7qXxSsrFQ2QtFXeBhaWdFujxurdHe83iEPK8uwU77tf5tEKlWsJH0vk71jFPratIu+JKOuU8gAAIABJREFUynWqfWy/vnhaXYGdmuSqkj5PiWtckdbmsVnIzFZUV/svNmlwC3RSbFbjgHj/B3d/o6U1KaEyx2abGSFpqfh3o2PzvR8c4nwHlfevdotN9EGbx+admb/XTWwzKvP3czUepx46LTbvVxg1JyXaJs5HVumQndKEOvUrbR6blcvti8qvmfn7zeRWjddpsdmytilNZ040Id6PMbONijY0s+Urf7v76wpLxEnSSWaWd3nCN/tQrz8qXA4wQDm//se6Vl6wV3fPb4FZ8f4j3TPikOlTqtjHWmZ2QPfEOJP7kfHhtd2yJ8T7XrVfX7j7QkmVEQ5fMbNFrsuMEz1+PT68kS+MNZsQ79stNovqYQpLew6W9DdJf2jEcXqpY2KzJ2a2s8KlVwslnd2MY/ZTE+J928SmL7rK2WI3Sd+Pmz6XSZ9QsMtGe1Ndv3DnxeaqCivL9GRbM9uie6KZfUhdq3GkYvMEy1+Rr7IPM7PlUvnVcvcpPbTN9pnN147pY/p63A41Id63TWxGkxSWPpa6LkPurpL+cPZS4BbomNiUJHd/S2F5Yyl8ph2Us1nlM+1cdb1O0DsT4n27xeZf4v3ovNdcvBT/C/Hhkz3M29NoHRWb6mqblc1st8Q2R8R7l/R/dTpu6TpzLlcYkjRY0l1mdkSccEiSZGarmNlBZnaPuq4rrBincPJ2kDTBzFaOZYab2RkKX3JmKcHMJpmZm9mk7nnxWvFx8eHXzWysmS0Vy31S4R/vAEl/cvfFvjCa2Zi4bzeztXo+DbkGmNmIHm6VHtI74v15ZrZt5Z+NmW2m0DFVzYRfsyT9NJ7vgbH8RxUmbx2p0Ln1425l+tJ+SVWcv7MUVshZQ9INZrZGLDdSIeA3UxiV872csqhOW8ZmzD/DzHbuVp9NFJb0PFBhFYAvuPti108Tm0GjYtPM9jWzo8xs9cxzXcnMTlb4v2mSzkkMz0V12jY2+8LMxlVeW33YzaAqYnOQu7+pruHQPzOzjWMdBpjZDgoralQzh9UbCu9Bn8m83rdWmAR8KYXLln7TrcxZCkOzR0h6wMz2s7A6iGL5NSysbvKopD2rfeJ1On/om7aMzTg3VeUL54HxtbJMLLeSmV2u8LlJkr6bs29iUw2Nze8qTIC7tqTfWFi9R2Y2xMyOVVdH23h3n17tcbGItoxNSZfE+2Ul3WZm25nZIAvWlXSDumLzwpx9E5tqWGxep65FSSZY+Pw7LJZbyczOVNdr5Vd17QR397reFL4cu6RJvSw3KZYb08N2KykMM/R4W6BwfeNbmTSX9L2csidm8hcqzH4+Pz4+t6gOmbzk85J0WWb/89TVK+mS/iVptUS5MZnt1urleRvX7XkX3cbEMh9QuL6wkj4nc/5mK1wSllufTPueqRCkrtD7PytT5m1J29Sz/TLPc0It50/hUpq3M9vNjK8BV5gJ/+B6x0K73dShsakwzLiy71nx9Vp5/Kqk7QueU4+vLWKz9tjsdo7eUYjL7OvgHEnW6tghNhv3vtlD7EypJr76cL6ruW0Xy3wixmAl/a3M4+kKcwPk1idzHsYqTPZYiefsZ4RXJa2fqO86CvPXVLadr/ChMVsfl3RY4nmOq9f5U5hEvqb/iWW8dXJsSjqj22tuuro+Ny2UNDZRjthscGxK2r3bMV5X+N5ReXy9pIGtjp9G3jo1NhWWuF6Q2f+7Oa+3SxNlic0GxqbCZV8zu+3/jW6PH5I0vJ6xULaROfLQk7WtpIMUVqN5TWE9d0maLOkKSfsp9Mp1L/sjhevn7lZ4QQ1UuC7yUHfPnRy1l3U7UmEZuOz+J0v6gaSN3f3FRNHKPDv/URNWBHD3ZyX9P0lXKQTCEgovvqslbebut1exm3cUPtSdpnC99JIKbfErSR9z93sTx665/frC3e9QWHHs55JeUJgY7BWF3tzNPT3zOKrUxrH5X5J+p9Cps6RCB8ejCv+UR7n73QVlic2gUbH5e4Vfmv6m8AY7WKGdfiHpk+5+gsd3SNSujWOzLyqx2ZSJsd39IUmfVBjRN0PSIIUYvVThveUv6dLvma4Q3xcovP8sqbA6zU8VPiPkTjjs7v9UWPniKwrtMEPScIUPp39V+CFpV4X/GyiRdo5Nd/+2pJ0VltudHuv1ssLnpi3c/dxEUWKzwbHp7r+Px/2ZpKkKSzu/pXCJz4GS9vHWrv5Xeu0am+5+nkInyc8VOjnmK3xW/I/CyJzR7v7lRHFis4Gx6e73KCzscLbCxNlvKnzfnB7rcJSkrdw9OTKrFsbn5NYzs1sV3jCPcfeLWl0fAAGxCbQnM5usMAHobp5z+TKA1iA2gfZEbPZPdOa0mIXJqmYqDMP6gIf5dwC0GLEJtKc4B8HLkh51901bXR8AAbEJtCdis/8q3WVW/dDHJA2T9EO+LAJthdgE2tM28f60ltYCQHfEJtCeiM1+ipE5AAAAAAAAJcLIHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASoTOHAAAAAAAgBKhMwcAAAAAAKBE6MwBAAAAAAAoETpzAAAAAAAASmRgrQXNzOtZEaCdubu1ug7VIjbRSYhNoD0Rm0B7IjaB9lRLbDIyBwAAAAAAoETozAEAAAAAACgROnMAAAAAAMD/Z+/e4zUb68bxfy7GmZnpYQzCyCFTOesgySGhgzARlW9FJeWpPA6RogiJRE6PBxXJ4ck55dQByU8nITkMSlRSI8w4G8z6/bHu/cxurGvN3ve+99732vf7/Xrt1565Pvta69r3uj97r/uz170+NIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANMi40V4AAADAcNl2222zsf32269yfPPNN8/OmTNnTjb2lre8pXL8V7/6VXYOQDtcmQMAAADQIIo5AAAAAA2imAMAAADQIIo5AAAAAA2imAMAAADQIIo5AAAAAA2iNfkwmzJlSjb21re+tXJ82rRp2Tk77LBDNnbGGWcMfGEt5513XjZ2ww03DHp7ANCr1l133WzssMMOqxy/9dZbBz0HetVKK62UjZ177rnZ2Prrr5+NLbbYYpXjde3Hi6JoKwb8u2222SYb+9SnPlU5vv3222fnXHDBBdnYLrvsMvCFNYQrcwAAAAAaRDEHAAAAoEEUcwAAAAAaRDEHAAAAoEEUcwAAAAAaRDEHAAAAoEG0Ju+AqVOnZmM///nPs7Gll166cjyllJ1T1+5wjz32GPScunZw++67bzZ26aWXZmMAMFa94x3vyMbqfjfOnj27cvykk04a8ppgrPnABz5QOf6lL30pO2eNNdYYruW8zLXXXpuN3XnnnSO2DugWm222WTZ2xBFHZGMbb7xxNpb7vXnfffdl5/ziF7/IxsYiV+YAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANEiq63RUOzGl9iY2VF33iu9+97vZ2KRJk7KxSy65pHJ8p512GvjC+sl1ztpkk02yc+o6Z919993Z2Oabb145/sgjj2TnNFlRFPkHqsv0Wm7S2+Qmw2Hy5MnZ2A033JCN1XXTOeOMMyrH99xzz4EvrEHkJhER48blG+fWPfc//vGPV46vvfbaQ17TvGbOnFk5/vTTT2fnfOITn8jGrrnmmiGvaTjJTeZn4sSJ2diJJ55YOf6+970vO2eRRRbJxo4++uhsLPd78/7778/OabJ2ctOVOQAAAAANopgDAAAA0CCKOQAAAAANopgDAAAA0CCKOQAAAAANopgDAAAA0CD5foE9aurUqZXjV1xxRXZOXXv3I488Mhs76qijBr6wAfjqV79aOV639jprrrlmNnbQQQdVju+7775t7QvatcQSS2Rjde2FP/nJT2Zjq6yySuX4jjvuOOB19bfAAtV18zlz5rS1vbvuuisbu+qqqyrHDzvssOycuvarObfddls2ts0222Rj//znPwe9LwZv/Pjx2dgTTzwxImuoa0n8X//1X9nYN7/5zWzsxRdfHNKaBurggw/OxlZfffW2tvmHP/yh3eVA18v93sydL0ZEfOxjH8vGUqru0Ft3zl3nu9/9bjZ2yimnVI7fcsstbe0LmmDzzTfPxk477bRsbMqUKZXjl112WXbOf/7nf2Zjjz32WDbWbr73ElfmAAAAADSIYg4AAABAgyjmAAAAADSIYg4AAABAgyjmAAAAADSIYg4AAABAg/Rka/K6Vsa5VuK5FokREf/617+ysUMOOWTgCxuiBx98sHL82Wefzc6peyzq5NrKrrzyytk5O+20U1v7oncstdRS2di2225bOb7vvvtm52ywwQbZWDvtDtttkZhrQd7u9l7zmtcMOvamN70pO2eHHXbIxh5//PHK8bq1ayU5+rbeeuts7KKLLhqRNZx88snZ2B577JGN/fWvf83Gvv/97w9pTQP1+te/vuPbvO666zq+TRhJdXmba0Gea2M8PwssUP335rqWyXfddVc2duKJJ7a1DmiCxRdfPBs76qijKsf32muv7JyHHnooG8udj//0pz/NzmH4uDIHAAAAoEEUcwAAAAAaRDEHAAAAoEEUcwAAAAAaRDEHAAAAoEF6spvV1KlTs7Htt9++cryuO8uHPvShIa+pE6ZPnz6o8YiIm2++ORubNm1aNjZp0qTK8bquOHXbu/TSS7Mxxpa6jlVnnnlmNlb33Oq0W265pXL80Ucfzc757W9/m4394he/GPQa6jqAfPrTn87G1lprrcrxTTbZJDvn3e9+dzZ2zjnnZGN0r5HqWBURse6661aOf+QjH8nOqesQ+eY3vzkb63Q3q+WWW65yfLXVVsvOabe75V/+8peBLwyG0XrrrZeNXX755dnY5MmTs7EFF1ywcrzu/Pm2227Lxr797W9Xjp9++unZOS+99FI2Bk230EILZWMXXnhhNvbOd76zcrwul/bff/9s7KmnnsrGGHmuzAEAAABoEMUcAAAAgAZRzAEAAABoEMUcAAAAgAZRzAEAAABoEMUcAAAAgAbpydbke+yxRzaWazn6zDPPZOd0e7vRutbpdW3Lf/zjH2djp512WuX40ksvPeg5ERG/+93vKse7/bFl8L785S9nY+20H7/rrruyseOPPz4bq2sl/uCDD1aOP/nkkwNf2DC6+OKLs7F77rmncrwuN9tx//33Z2PPPfdcR/dFd9tuu+0qxxdeeOHsnLp2xbnfB8Nhiy22qByvy5e6tZ966qnZWLf8/KB3vPGNb6wcr/sdssIKK2Rjdc/93PP7xhtvzM75xCc+kY09/PDD2Rj0or333jsby7Ufj4h4xzveUTn+k5/8JDunLtc7ra7l+rLLLls5/tBDDw3XchrHlTkAAAAADaKYAwAAANAgijkAAAAADaKYAwAAANAgijkAAAAADaKYAwAAANAgY7Y1+dSpU7OxadOmZWO5VmyXXnppdk5de+9u0O766r7nKVOmVI4fe+yx2Tl1rV6PO+64yvGddtopO4futsQSS1SOb7bZZm1tL9eC/O1vf3t2zowZM9raV7d7/PHHs7GTTz65cryuJXw7PvzhD2djTz/9dEf3xehbZ511srH9999/0Nt74IEHsrELL7xw0Ntr14YbbtjR7d17770d3R7MT679eETEBRdcUDm+/PLLt7WvZ555JhvbZ599KsfPOuustvYF/LuPfvSjbc27//77K8dHsv34uHH5kkPuNWBE/nxAa/K5XJkDAAAA0CCKOQAAAAANopgDAAAA0CCKOQAAAAANopgDAAAA0CCKOQAAAAANMmZbk2+66abZ2KRJk7KxdlqT96Jzzz23cnzvvffOzsm1M4/It4uvO4433HBDNsboW2qppSrHN9hgg7a2d+utt1aOj9X244ssskg2dtppp2VjH/rQhyrH61pQPvfccwNfWIv2471lxx13zMZyuV73nLvjjjuysXaej+16z3ve09HtzZ49u6Pbg/k5/vjjs7EVV1xx0Nv7+c9/no19/etfz8auuuqqQe+rHV/60peysfHjx3d0XymlbOwrX/lKNjZr1qyOrgMiIg455JBs7Mwzz8zGfv/731eOn3POOdk5p5xySjZ2++23V44vuOCC2TmXXHJJNpZbX4TXegPhyhwAAACABlHMAQAAAGgQxRwAAACABlHMAQAAAGgQxRwAAACABkl13SZqJ6bU3sQRctxxx2VjdR2Xco/HuHFjtvFXRx100EHZ2BFHHJGN5R73vfbaKzvn9NNPH/jChqgoinxLgy7TLbm59NJLV47fc8892TkTJ07Mxv785z9Xjr/+9a/PzumWjhIrrLBC5fhrX/va7JwDDjggG3vb29426DXceeed2di666476O11C7nZOXUd1H71q19lY7nnT935xRZbbJGNdbp7xTLLLJON5bpyTJ48ua191XXz6DVys3NOOumkbKzuPKkdnX4Ob7vtttnYGmuskY1tttlmleOd7kBXZ4EF8n/znjNnTjZ24oknVo6feuqp2Tn33nvvwBc2RHJz7FlrrbWysX333bdy/L3vfW92zpJLLpmNXXvttZXjdb9rJ0yYkI2ts8462VivdU5tJzddmQMAAADQIIo5AAAAAA2imAMAAADQIIo5AAAAAA2imAMAAADQIIo5AAAAAA3S+NbkU6ZMqRz/zW9+k50zadKkbCz3eGg3OnR1bRxzj/stt9ySnfOGN7xhyGsaKG0cO+ewww7Lxr74xS8Oent1bYy/9rWvDXp77dpwww2zsT322KNyfOWVVx6u5bzM6quvno098MADI7aOTpObnbP00ktnY4888kg2llL1IZg+fXp2zutf//psrNOtSKdNm5aNXXzxxYPe3oUXXpiN7bLLLoPe3lglNwcv1477Bz/4QXZOXQvhnMMPPzwbq/sdvdJKK2Vj5557buX4+uuvn52z2GKLZWO5nyvtvm5pR24N7a7j/vvvz8bqWrh3um253CSi/vXwJz/5yWys7mdEzm677ZaNnX322YPe3lilNTkAAADAGKeYAwAAANAgijkAAAAADaKYAwAAANAgijkAAAAADaKYAwAAANAg40Z7AUO1zDLLVI7XtVitayc4ki0Pe43HnYj6duHLL798NvbRj360cnzTTTfNzsm1eY3ojvamI7mGJrcfZ2TUtQS/8847s7HXve51leOvfvWrs3OuuOKKbOzoo4/Oxh577LFsLOfAAw/MxtrJwTPPPDMbmzhxYja25pprVo5vvPHG2TlTp07Nxvbcc89sjGZ685vfXDm+8MILt7W9559/vnJ8+vTp2TkrrLBCNnbhhRdmY69//esHvrABuOmmmyrH69qqz5o1q619bbfddpXjdecXG264YTaWO16rrrpqds673vWubKzTrckhIuLxxx/PxuqeqzkzZ87Mxk466aRs7OKLL87G6s5LKLkyBwAAAKBBFHMAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBGt/NKifXPWZ+MYZm0qRJ2Vg7j/svfvGLoSyHLvTss89mY5/5zGeysfPPP79y/Ac/+EF2zhJLLDHwhY0Rp5566mgvgYZ67rnnsrG6LnT//d//XTm+5JJLZue89a1vbSuWU/f7pdNd484444y25tV1Ccqp65zF2HPkkUdWjtc9h3MdqyIijjjiiMrxa665Jjvnuuuuy8bWXnvtQa9jxowZ2TmXX355Nrb33ntnY53261//etBz3v3ud2djxx57bOX4GmuskZ1T1wkPhsM+++yTjX3kIx/Jxg466KDK8csuuyw75+67787GFlxwwWyM+XNlDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANMiYbU3ebivSTrcw7TXTpk3Lxuoe21xs+vTpQ14TzVHXYjXXLnX8+PHDtZxBqWuDnlv7hhtu2Na+TjvttGysrr07tOvcc8/Nxh588MHK8T333DM7Z4cddsjGFl988YEvbBTUtRh/7LHHBh37wQ9+kJ2Ta/sOfeqec1deeWXl+MEHH5yds84662Rjdedxhx12WOX4Mccck53TZFdccUU29oUvfGHQ26tr+w5DkWszfsQRR2TnHHfccdnY17/+9crxVVZZZVDrojNcmQMAAADQIIo5AAAAAA2imAMAAADQIIo5AAAAAA2imAMAAADQIIo5AAAAAA2S2m3FnVLq6h7eV111VTa2zTbbZGO5x+O8887LzvnQhz408IWNEVOmTKkc/81vfpOdM2nSpGzskUceqRyfPHny4BY2TIqiSKO9hoHq9twcq772ta9lY5/73Ocqx+t+/ubamUdE7LzzztnY448/no2NRXKzmVZfffW2YjmnnHJKNtZOu9Tbb789GzvooIOysauvvnrQ+xqr5Obg5X4nzJkzZ8TWcOutt2ZjW265ZTY2a9as4VjOqNpwww2zsZ/85CfZ2IQJEyrH//a3v2Xn5M6rh4PcHHumTZuWjZ1++umV4zfddFN2zvbbb5+NLbroopXjV155ZXbOQw89lI3tvvvu2diLL76YjY1F7eSmK3MAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBxo32AobLpZdemo1tvfXW2ViuLeQOO+yQnTN16tRsbPr06dlYk3384x+vHF966aWzc+raMPdie3eaJ/e8j4j47Gc/O+jtPfXUU9nYgQcemI31Wvtxxp4//vGPbcVyVl111Wys7ndPzp577pmN/eY3vxn09mAgDjvssMrxgw8+eMTW8Pe//z0bqzt/zvnlL3+ZjdW16n71q19dOb7uuusOeg0RERMnTszGco/vUkstlZ0zfvz4bCz3M6eu7TvMz5prrpmNnXrqqdnYjBkzKsc/+MEPZucsssgi2dh5551XOT5p0qTsnJ133jkb67X2453myhwAAACABlHMAQAAAGgQxRwAAACABlHMAQAAAGgQxRwAAACABkntdHmIiEgptTexC1x88cXZWK5r1QIL5Otec+bMycZ22mmnbKyu41Y3mDZtWjaWewzrnk9f+tKXsrEjjzxy4AsbBUVRpNFew0A1OTe7XV0nirXXXjsbe/755yvH6zoJ/OAHPxj4wnqY3CSi/vdwO+c5b37zm7Mx3awGRm4O3vLLL185fvrpp2fnbLLJJtlYXTemnJTyh62dXLrvvvuyscceeywbyz0WK6+88qDX0K66x+Lmm2/Oxr7xjW9Ujv/sZz/Lznn00UcHvrAhkpvdq+45d+6552ZjW221VTa20UYbVY7/4x//yM6pOwddYYUVKsff//73Z+fcfvvt2RhztZObrswBAAAAaBDFHAAAAIAGUcwBAAAAaBDFHAAAAIAGUcwBAAAAaBDFHAAAAIAG6cnW5Isvvng2dvbZZ1eO17XprnsMn3322Wws15r8q1/9anbO9OnTs7F2fO9738vGcm3aI/KP4V133ZWdU9e6udtp49hbNt9888rxq6++Ojtn3Lhx2diMGTMqx3PtHRk4uUlE+63JH3jggcrx173uddk5zz333IDX1cvk5sjYYostsrFVV121cvzggw/Ozhk/fnw2NmHChIEvbIhyLZoff/zx7JzZs2dnY88///yg1/C5z30uG7viiiuysbpz/24gN7vXa1/72mzsjjvuyMZOOeWUbOzee++tHD/66KOzc/71r39lYx/84Acrx2+88cbsHAZGa3IAAACAMU4xBwAAAKBBFHMAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBerI1eTu++MUvZmOf//zns7ElllgiG8s99o8++mh2ziWXXJKN5do41rUYnzRpUjb2yCOPZGMnnnhi5fiRRx6ZndNk2jj2lmnTplWOX3TRRW1t78ADD6wcP/bYY9vaHnPJTSLab01+5513Vo6vs846Q15Tr5ObzbTBBhtkY5tuuukIrqTaDTfckI09/PDDbcV6jdzsXu22Jq/z4osvVo6fdNJJ2TmHH354NjZz5sy21sH8aU0OAAAAMMYp5gAAAAA0iGIOAAAAQIMo5gAAAAA0iGIOAAAAQIOMG+0FNEVdl6aLL744G9t7772zsVzHnLoOUx//+MezsVw3q7pOHqeddlo2dsYZZ2Rjt9xySzYGTbfnnntWjrfb/Q8YXn/+85+zsVVWWSUb+/SnPz0Mq4Hmqju/c+4Ho+eFF17Ixq677rps7Igjjqgcv/HGG4e8JkafK3MAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBUrutdlNKevTSM4qiqO773oXk5tBdffXVleNvf/vb29re+9///srxiy66qK3tdYMVVlghG3v00Uezseeff76j65Cb0J3kJnQnuQndqZ3cdGUOAAAAQIMo5gAAAAA0iGIOAAAAQIMo5gAAAAA0iGIOAAAAQIMo5gAAAAA0yLjRXgDAaFh22WWzsSlTpnR0X01uQZ6z5ZZbZmPXXHNNNjZjxozhWA4AAPQUV+YAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDaE0O9KS6FtnHHXdc5fjhhx+enXPggQcOeU0AAAAD4cocAAAAgAZRzAEAAABoEMUcAAAAgAZRzAEAAABoEMUcAAAAgAZJRVG0NzGl9iZCAxVFkUZ7DQMlN+klchO6k9yE7iQ3oTu1k5uuzAEAAABoEMUcAAAAgAZRzAEAAABoEMUcAAAAgAZRzAEAAABoEMUcAAAAgAZpuzU5AAAAACPPlTkAAAAADaKYAwAAANAgijkAAAAADaKYAwAAANAgijkAAAAADaKY009K6fqUUpFS2m2019KtUkpntR6jQ0d4v4e29nvWSO6X7iA352+0HqPR+plAd5Cb8ydQUMQFAAAgAElEQVQ3GQ1yc/6c0zIa5Ob8yc2B63gxp9+DP+/HEyml21JKX08prdjp/Y62lNIDme+7/8f+w7DfvifdA53e9ljR7zGq+7hjtNc53Ho1NyMiUkprpJROSSlNTyk9k1J6PqX0YErpf1NKmw3TPvse7+uHY/tjRUppQkrpkJTSb1vPxRdSSjNSSj9OKX04pTTm/+ggN+VmN5KbPZ+bK6eUTkwp3ZtSejal9HhK6f9LKe05XMfeOe3ApJQWSil9KqV0Y0ppZuv43JdS+mZKafnRXt9I6OXcjIhIKb0vpXRtSunR1u/Ou1NKR6SUlhqm/cnNAWqd13w7pfSX1jnN31vnNBsOx/7GDcdGW16IiMda/04RMSki1m19fDyl9J6iKG4cxv2PlscjYnYm9vRILoSXeS4iZmVi/xrJhYyynsrNlNIOEXF+RCzaGpod5WOwcutjl5TSEUVRHDJKS+xZKaXVI+LaiFipNTQnIp6M8jm5Vevj/6WUtiuK4rnRWeWIkptysyvIzZfptdx8e0RcHBHjW0OzImKxiNi49fG+lNK2PXLsu0pKaWJEXBHlcYgof24+ExGrR8TeEfHhlNI7iqL4zSgtcaT1VG5GRKSUTo+IPVr/fTHK1zdTI+KLEfGBlNJbi6L4+2itr5ellLaKiMsiYvHW0KyIWC4idomIHVNKuxdFcU4n9zmcf1W5qSiK5VofkyNiyYj4cETMjIiJEXFhSmmxYdz/aHlvv+973o9TR3txPe77Ncdm89Fe3AjqmdxMKS0TEWdH+WLxlojYKCIWK4piyYhYLSIuan3pwSmlTUdnlT3te1G+WHw0It4X5bGZGBGviIgvt75mq4g4YHSWN+LkptzsFnLz3/VSbq4UZf6Nj4hfRsTarWO/ZJTPhVkRsWVEnDBqi+xt34mykPNUROwaEUsWRfGKKF/M3xBljv6wVfTpBT2TmxERKaVPRVnImRMRn4vy+C8VEW+JiAcjYtWIuGD0Vti7UkrLRfmzc/GI+ElErNL62blcRJwb5UU0304pva6T+x2xS2SLonimKIrvRcRnW0PLRcQOI7V/oNoYz833RETfJafTiqL4dVEUcyIiiqK4PyI+GBF/bMXfOwrr61kppVdF+QI+ImKfoiguKopidkREURQzi6L4SkR8txXvyWMjN+XmaJCb8zfGc3PfiJgQEU9ExHZFUdwREVEUxYtFUVwUc7/nj6eUpo7SGntSSmm9iJjW+u9+RVGcVxTFCxERRVHcExHbR3mVyrJRvtDvOWM5N1NKi0TEoa3/nlAUxbFFUTwfEVEUxU1RPjeKiHhLSuk9o7PKnvb5KIvgf43y4o4HIyKKopgRER+JiN9FxMIR8ZVO7nQ03u98QZTVxIiI/3vvWOp3M6iU0sSU0tH93kM/s/8GUkoLp5Q+nVL6RUrpsX7vsf9OSuk1dTtPKb2j9R7DWa33Vf4qpfShzn+b3S2ltGlK6YSU0q9b7+Wbncr3wl+dUtppgNtYNKV0WOs4Pduaf35K6dXzmdf28WNYjcXcnNz6/GhRFH+ZN9g6Cbq99d8lhrivIUspLZhSemdK6bSU0u9SSv9s5ebfU0qXppTeNsDtvCKldHxK6f6U0nMppb+llE5P83kvfUppyZTSF1J5j4xZrbn3pfK+CSvVzW3D5H7/vjXzNb9rfR71YzPK5OYok5svIzdLYzE339n6fG5RFFVvQT8nIh6J8jXErkPcV0ek3jmnfUfr85MR8e15g0VRzIyIM1v//VBKKXV4/00yFnPz7VEW6oqI+Ma8waIobo2In7b+KzdHMDdTeR+x97f+e2pRFE/1jxdF8VJEHNf677YppfHRKUVRdPQjIs6K8kl2fc3X/LP1Naf3G7u+Nfa5iPhT69/PRfmXgZn9vm75iLitFS8i4qXW1/T9/9koq2FV+/1cv6+bE+X9bV6KuUnRt4bdKub2xSq/r4h4oBXfvNOP6Xwe70Nb+31gEHOW7Pc4FK3Hb9Y8Y6fN5/geFeXlt0VEPD/P/KcjYtPM/LaOX7/v86yK2G795q8ymLm99NGLuRnlD9a+7a5cER8XEfe14p8a6ce7Ys5a8+ThrCgvpe4/dlBmbt/jsF+UVzQUUb6Pvv/8GRHxmsz818Tcn2NFlO9D7z/3sYh4S833eWhFrC/3iorYcv22/aH5PIbfH+38Gc4PuSk35WZ3fvRobj7biu1d8z3f2PqaX3X48e57Xj4wiDk9c04bEae2xm+teTw+3m/+1NHOoeH66NHcPLYVu73me96v9TUzOvx4y8363Ox/jrBBZk3L9Puad3Xq2Iz4lTmpfN/ipNZ/Z1Z8yZciYqEo/zKweFEU4yPi9a25C0XED6K8qdXPonzP6KKtr1khIr4Z5fvvv5dSWm2e/W4SEUe3/ntORKxQlO8xXToijonystL1OvAtHp9SeqRVefxHSunKlNIHU0oLdmDbnTInyvf0TYuIpYuiGF8UxYQo32f76ShPEj+RUnpfzTY+FRHrRPm+1CVb89eP8t4Hi0fEBSmlV/SfMJTj1yFbtv6S+XyrUv67lNLhKaXJ85869o3R3PxhRPyj9e9LU0pvalXP+95KcF6UNw28I8r3oY+22VGuY5uImFAUxYSivIfI5Ig4JMpfRkemlN5Us41Donz7ynuizM0lI2LziPhzlMf3wtbx+j8ppQkRcWVETImIC6M8josWc+9fcl6UPx8uTh16H35RFP+IiB+1/nt8SmmnlNLCrfVMTCkdEuVlqU/E3MuKe5LclJshN7vSGM3NovW57ry1r4HKa9vcRyf10jntYI5NRERH783RJGM0N/vy7c6ar7mr9XlSKu9NN5p6KTf7jk0Rc4/BvynKKx1nzPP1Q9fJqt08lbTKSmmUB6+vKrVjv/HrW2OzI2KtzNy+avMNEbFQ5mv+p/U1J88z/rPW+LURkSrmfavfugZVKW3FH4h/rxT2r/4VrfkTh+HxPjQGWSkdwDY/1NrmdTXHt4iIXSviy0TZGaqIiIM7ePz6vs+zKubs1m9Nq9Q8RkWUd31/LOZWyIsob/C4ZaePTbd99HBubhjl+1f7tvF8zP2r9syIODnKF2cj+ni3uc1DWts8s+ZxmBMRm1TE12x970VE/L95Yke0xs+r2fdVra/ZP/N9Hlox5/9yL7PNSa3nTP+/nMxs/fuFiLg0MlcrjKUPuSk35WZ3fvRibkbE3a3YtzJrGhfleVPf9pfs4OPd97x8oIPbHDPntBFxYMx9nbFIZp8n9pv/6eHMj9H86NHcvLUV+0bN47Juv+2v3cHHW27W5+berfFH5/M9z/cYDvZjRK7MSaVVUkr7R1mVjCjvuP3Dii+/qmjdbK3CR1qfTyhaN/yqcG7r81b99v8fEbFF679HF61Hcx5fzX4DEVEUxeZFUaQi3/XosojYMSKWKYpiiaKs/k2J8pK4ORGxWTTn7uJ9x2WjmiuKHozyL4L/piirjqe1/jvveyHbOn7zUxTFWa1jk4qieKDiS+6LiP0jYo0oK7P/EeUNqt4fEQ9FxH9ExGXze+/lWNQLuVkUxe8i4m0x9x4PC8fc+zwsHOVzoXPvXR1efcflLTVf84uiog1nUd4csa9DUC43X/Ye7H768n0wuXloX25m4o9ExLZR/vUqorwHw4TWvxeM8hLdpQe6v7FEbsrNFrnZZXogN3/c+vzBlNIrK+J7RHne1Gepiq/pJmPpnLbv2CweEXvNOzeltGyULzj7dPux6ageyM2+34/P1mzimX7/XrJuX11gLOXmQI5NxNzj07FjM27+X9K2zVJKVU/iiIiHI2KHotUdYR6/rJqQUhoXEW9s/fe0lNIpmW33PRn63xBw/YhIURZVXnYiFRFRFMX9KaW/zjNvwIqi+K+Ksb9ExOdSSn+OiFMiYquU0tZFUfz4ZRsYYa3H8yNRtplcN8pfzAvP82WLRnkpXNUN8H6e+SEVEfHziPhCRKyVUlq4KIrZQzx+Q1IUxbkVY09HxPdTSr+M8lK9paOsxn6wU/vtYj2VmymlT0SZf/+IiA9EWal/Jsrn/VFR/mVgy5TSJkVR/LmdfXRS69LgT0bZleK1UebgvD+rV6jZxPU1sZ9H+RzfoN/+VoqIFVv/vbLmudH386FjuZlS2ijKS2GXioiDonxB+3CUbx/ZN8qfUZumlHYqiqLqZGyskZtyU252p17KzeMj4qNRvti4OqW0d0TcFOWLlV2i/CPlC1G+RSVi7k1mR02vnNMWRXFrSulHURZaj0opzYnyhe4TEfGmKNvFL9pvyqgfmxHQS7nZOL2Sm6NpOIs5L0T5dpaIuZcE3h9l3/VvFUXxeGbeI5nx/gd/IH8NWqzfv/veMzmr9SI+56EYnoN6apQ3w1olyvfKj2oxJ6W0ZERcE+V7CPs8G+Vj3/eDv+8+MktEdXI9VLOLvtiCUSbnP2Nox2/YFEXxl1aifyki3p1SWqBotccdw3omN1NKb4mycv9sRLytKIr7+oV/nlLaIspLHl8TEV+L8kR11KSyo831EdH/KrGno7x53pwoc2qZqO8gM5DcnNRvrH8XnWUHsMzFB/A185XKO/n/MMrv50NFUZzTL3x7ROyWUnopyhcVJ6eUfly0WnCOYXKzJDdLcrN79ExuFkXxQEpplyivJl8ryreN9PdARJwfZZEvovp+JCOmB89pd4vyrZVviPLeH9/sF3spypbcfS9gR/XYjJCeyc0ov7d59zmv/r8Hnsp+1QjosdwcyLGJmHt8OnZshrOYc1PNW5LqvJQZ7/+WsPWLoritjW2PiqIoipTSb6Ms5qw6ysuJKN/bv3GUSbNfRFxdFEXfDZmidanbi33/7dA+u/n4/br1eXyUiZ/7AT9W9FJu7t36fMU8LxYjIqIoiudTSv8dESdFxHtSSqnmLwAj4ZtRvli8P8oC8HX9T0RaN2r7Y4f32f/4vaIoW5uOhP8Xrfc8z/Nisb++vxCvHOVfvH41QmsbLXKzRW5GhNzsJr2Um1EUxZUppddFWRjYPMoXqf+K8mbcx0ZE39Xof+2CQl5PndMWRfFoKm+yu3uUt3dYPcrv73dR5uWMfl/+sp+tY1Av5ebfo7x5ct0VoP1jDw/vcuarl3Lz763Pr0gpLVoUxXOZr+s7Ph07NiPezWoIHo25ibfyIOf2vTifkFKq+8tVXXKMJX13Df9MURRn90+sloF0dxrID5KXovyrZcTQjh/drZtz8zWtz3Vv0bi/9XmxGNhzf1ikslvM9q3/7loUxSUVf1HqVG72L1j+s9+/RzI3B3NsIspiOIMjNztAblaSm0PTzbkZERFFUTxYFMV+RVFsWBTFykVRbFAUxcGtomLf2wEr36oywnrunLYoitlFUZxWFMXWRVGsWhTFq4ui+EBRFL+JucfmxYi4eaTXNgZ0c272dUmq61LW1yXpkdZ9ZUZTL+Vm37FJkelUlcruYsvO8/VD1phiTusGRn0/lN45yOl9d45eICI2qfqCVLZCHZaDnlJKUV4OGVF/cjRS+t6Df2sm/vYBbGOzAcTu6Huf6hCP33DrayX7ZJQ/BBiELs/Nvss46+ZP6ffvJ9vcTycsExGLtP493Ll5S99A614kfS8aRzI3m3RsGkludozcfLluOTaN1OW5WSultHTMvWnoy25MOgqc0/67D7Q+/6goiidGdSUN1OW5eV3r8+tab/2tsnXr87xvjxwNvZSbd8fc39e5myr3jc+OzD2V2tGYYk7LWa3Pu6WU1q37wtSv53xRFI9F2SIuIuKAVnFlXp9vd1GZ7fW3Z8z9y9UV7e6ng2a1Pq89b6D1/sYvDmAbq6SUPjDvYOtO7p9o/ffCecJntT4P6vgNxfyOTUppxYj4z9Z/r+qB++UMl7Nan7sqNyPi963P70wVXTlal3ju3vrvnfN5j/NwezLKk4CI6txcPiI+M4DtbJZS2njewZTSGjH3jv+53Ny/6nHqt42UUpo4gDUMRN+xmZxSek/ma/ZofS4i4rcd2m+vOav1WW62T26+nNwcurNan7stN+vWkaJsfb1oRPwhIn40HPsZpJ45p52flNI2Ub71ak5EHD0S+xyjzmp97rbc/FmUb6NbIMq3Lc27lnVjboHkZc1fRkHP5Gbr9eP/tv67V0rp3+6fl1JaICL2af33hx0ttBYd6nHe9xFz+8JfP8h517fm7VbzNQtFeUlnEeUVFHtExPh+8eUiYtco72596DxzN4nyh1sREd+NiMmt8QlRtokrorxRWOUa+q3vZd9XlO/pP6G1j8X6ja8U5Y0bX2zNvTbzfe0Wmb71A3jcDm3N+0uUfz2s+1ikNee8fnM2i4jUGn9DlPeP+VduPf2O78wob960a0SMa8XWifKkroiyOvmKDh6/vu/zrME8fq3v75oo25Av12988YjYOcqWd303THtNp/Ohmz6iN3Nz437PjTuifO//QlFeBrlmlN1a+uKfqMmvYgiP9/8X88/NhVpzbmrNuT0i1muNLRARW0bEvdEvN2seh5lRdgd6V8zN7bdGeT+PvsdhoXnmToyIP7XiD7Zyo//PspWj/KV567zHoN/3eWjFmrKPX5T3qHqk33Nmt4hYshVbNspuRi+14ueNdv4M50fITbkpN7vyI3owN1vxr0bENvOsZ/2Ym5dPR8SGmbm7xdzcXWWQj1vf89I5bebxi/KtK5+M8jVG3/e6bEQcGGU3wCLKttijnj/D+RG9m5ufasVfirKg05cHb47y7a9FRNwoN0clN5eLsoBVRPnac+XW+KSIOLs1/nxEvK6judCk5Gp93bJRXprU92C+1DpQT/UbKyLiyxVzP9cvPifKu5/3FVq+UbeGqD8pPWue9TzW72D2fVwfEf8xjMk1kI/dWnNWjbknakWUdxbve/yeifISvfkl11FR3vCwiIjn5vl+n46ITTt5/KL9Ys7m82z36Sh/eLzYb+xfEbF1p3Oh2z6iB3OzFd835r7wKKLsfPDMPGs6bX75NYTHeyAfm7fmvGmetT3V7/+PRnnfjsr19Hsc9ou5Lw6fiblXFRRR/kXntZn1rh7le3j7vvbFVm7M+1h9JPN9HjrYxy/KX+4z59n+E/P8/9cRMWG082c4P0Juyk252ZUf0bu5+UC/bc+K8jyv/3N1i5rvabd+X7vKIB+3Q+Pfv++6j91ac3rmnLbiMXo+/j1P50R5g+o02rkjN4cnN1tfc3q/7c+Of/9d8qeIWEFujnxutr5mq9a6+r5uZswt7r0QEf+v07nQtLdZRVHePGmzKCtqV0b5JFmqFZ4eZeVr5yiviJl37tejfP/cdVEezHFRvq/uw0VRvOxytUH4nyh/eN4U5d2sF43y/fV/jYhLW+t5W1Feflel732PD8UI3Hm8KIr7I+KNEXFOlL+UF4zyyXZuRLyhKIqBtE5/PspCyVei/GvhwlEei/+NiA2Korghs++2j1+b/hARB0T516Q/RvlDb0KUPwxuivJO668Z4PdMjS7NzSiK4rgoX4idGeVz4MUon/MPRcQlEfHOoij2zEzvy80RuYlgURS/jvKvK5dFeTO3haLM0dOi7GDw+/zs//NolPn9zSj/YrFwlD+XzojyioLKm64VRfHHKP/yuleUx+HxKHPlxSivRjg9It4d5c+NjiiK4udR3sjv6Ii4LcoTksVa38N1Uf71cZOiKGZlN8J8yc2hk5tyczh0a25GxOFRnjc9EOXz9Lko7+d0aERMLYriupq5zmlLw3FOGxFxeZSvO/4Q5QviRaM8Tt+NiDcXRbF/0XpVSfu6ODejKIpPRMQu82x/ekQcGeXvkr9npsrN0nDlZhRF8ZMozwnOjIi/Rfl7858RcUFEbFTkO0S2Lcn30ZdSujrKy1k/UxTFyaO9HqCUUpoe5Vs+3lMURTfcGwAIuQndyjktdCe5OTYp5oyy1k0eZ0Z5+fKqRVE8P8pLAiIipTQ5yvtb3FIUxYajvR6gJDehOzmnhe4kN8euxr3NagzaICKWjIhjJBZ0lU1bn78yqqsA5iU3oTs5p4XuJDfHKFfmAAAAADSIK3MAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBxrU7MaVUdHIh0M2KokijvYaBkpv0ErkJ3UluQneSm9Cd2slNV+YAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDjBvtBfSyTTbZpHJ8mWWWyc55y1veko2tuOKKlePLLbdcds5rX/vabGzLLbfMxu64445sDLrFmmuumY295jWvGfT2Fl544Wzs+9//fjZWFMWg91Xnve99bzZ22WWXdXRf0HS77LJLNrbVVltVjq+zzjrZOa985SuzsUMPPTQbO+OMM7IxAIDBcmUOAAAAQIMo5gAAAAA0iGIOAAAAQIMo5gAAAAA0iGIOAAAAQIMo5gAAAAA0SGq3ZW5KqbO9drvcjjvumI2dfPLJ2diECROysVyb4wUWaK/Gdvvtt1eOP/zww21t78c//nE2dvzxx7e1zaYqiiKN9hoGqtdy8+ijj87Gttxyy2xs/fXXH47ljIjf/va32dhGG200gisZfXJz7Kn7HZhrGX722Wdn56yxxhrZ2KKLLjrwhQ3AE088kY1tuOGGleN//OMfO7qGbiE3oTvJTUbaBhtskI296U1v6ui+7rvvvmzspz/9aUf31Wnt5KYrcwAAAAAaRDEHAAAAoEEUcwAAAAAaRDEHAAAAoEEUcwAAAAAaZNxoL6Ap1ltvvWxs8uTJ2dj555+fjV177bWV45dccsnAF9bP008/XTk+e/bstrYH3eTwww+vHN93332zc9rtDDdSHn/88WzsD3/4Qzb2kY98ZDiWA11hs802y8Zyvzfbdffdd1eOT5kyJTtn8cUXz8bGjx+fjX3961+vHJ82bVp2Doy0ut+bK664Yja21157VY6/8pWvzM657bbbsrHll1++cvyCCy7Izrn11luzsRdeeCEbg160+uqrZ2O77LJLNrbzzjsPel+TJk3KxupeR7dj1qxZ2divf/3ryvGPfvSj2TntdoUeKd39SgcAAACAf6OYAwAAANAgijkAAAAADaKYAwAAANAgijkAAAAADaKYAwAAANAgWpMPs+OPPz4bu/nmm0dwJdD9VltttWzsHe94R+V4t7cfr/Ob3/wmG3vXu96Vja288srZ2LHHHls5/uijj2bnHHXUUdkYDIepU6dmY3Wth9txwAEHZGP/8z//Uzn+rW99KzunnbasERFbb7115fgrXvGK7JzHH3+8rX1Buw488MBsbP3118/G9ttvv8rxv/71r9k5dW3Lt9tuu8rxupbJ559/fjZ22GGHZWNnn312NgZNt+aaa1aOX3755dk5a6yxRjZWFMWQ1zScJkyYkI3lfg9vv/322Tm584Ru0dxXQQAAAAA9SDEHAAAAoEEUcwAAAAAaRDEHAAAAoEEUcwAAAAAaRDEHAAAAoEG0JgdG1NFHH52NffCDH8zGVlhhhY6u409/+lM2dskll1SOn3766R1dw7PPPpuN1X2/P/zhD7OxtdZaq3L8hRdeGPjC+tG2nOGQa2McEbHMMssMenvXX399NnbGGWdkY08++WTl+Be/+MXsnKeffjob23333bOxxRdfvHJ8wQUXzM6B4ZD7PRER8d73vjcb22STTbKx559/ftDreOihh7KxU089tXJ8kUUWyc7JtR2OiNhss82yMa3Jabp99tknG/vMZz5TOT5lypThWs7L3HTTTdnYDTfckI2NHz++cnyvvfYa8pr6q/vZoTU5AAAAAB2jmAMAAADQIIo5AAAAAA2imAMAAADQIIo5AAAAAA2im9UALbXUUqO9BBgTJk2alI11umNVnbrOWTfffPOIrGG11VbLxs4555xsrK4TSc53vvOdbOzWW28d9PZgfqZOnZqNffjDH25rm0899VTl+Pbbb5+d88QTTwx6P/fff382Nnny5EFvLyLi6quvrhx/7LHH2toetCvXKSoi4o477sjG2ulY1Wl1a6jL9SWWWGI4lgMdtdhii2VjJ5xwQjZW10lxgQVG5tqNvffeOxurO6edOXNmNjZuXHWp4t57783OOfbYYwe9vbpziG7nyhwAAACABlHMAQAAAGgQxRwAAACABlHMAQAAAGgQxRwAAACABlHMAQAAAGgQrcnnsdxyy1WO77rrrtk5f/vb37Kxu+++e8hrAjqvrkX6xIkTK8fr2idus8022djhhx9eOT5+/PjsnDXWWCMbu+uuu7KxXHvKO++8Mzvn2WefzcagXXW/NxdeeOG2tnnxxRdXjrfTfrzOgQcemI29613vamubl156aeX4nDlz2toetOs//uM/srG69sfdINdaOKK+rfPaa689HMuBtuSeq1/72teycz72sY8N13Je5qmnnsrG9t1338rxK6+8Mjun7vy5zosvvlg5ftJJJ2XnfOUrX8nG6s67m8qVOQAAAAANopgDAAAA0CCKOQAAAAANopgDAAAA0CCKOQAAAAANopgDAAAA0CBak89jtdVWqxxfeumls3NOP/30bOyll17KxupaQ+bMnj07G0spVY4/+eSTg94PjHU/+tGPsrEbbrihcvyCCy7Iztlwww3birUj1545IuLmm2/u6L6gm0yYMKFyPPf7b37OO++8yvGdd965re3dc8892diZZ57Z1jahXblz2lVXXTU7p90WwiNl0qRJ2dh6662Xjd19993DsRzI2nzzzbOxvfbaq3J8xx13HKbVvNxll12WjdW9tr3mmqedR9cAACAASURBVGuGYzm0yZU5AAAAAA2imAMAAADQIIo5AAAAAA2imAMAAADQIIo5AAAAAA2imAMAAADQIFqTz2PatGmDnvP2t789G7v11luzsTXXXHPQ+/rHP/6RjS244IKV41dddVV2zkUXXZSN/eQnP8nGnn/++WwM6nzjG9/Ixt797ndnYxMnTqwcHzeu8z/GNt1000GNt+ull17Kxk488cRs7KijjuroOmB+tttuu2zs8ssvrxy/9957O76OHXbYoXL8xhtvbGt7G2+88VCW8zJHHHFENvbCCy90dF8wP7nfMXPmzBnhlXTO2muvPdpLgP+z8MILZ2N77rlnNjaSLcjPOuusyvFPfepT2TmzZ88eptXQaa7MAQAAAGgQxRwAAACABlHMAQAAAGgQxRwAAACABlHMAQAAAGgQ3aw6YNVVV21rXu5O4Zdccklb28t15fjwhz+cnVMX+9Of/pSN5TqK3Hnnndk5EFH/HJk8eXI2duyxx1aO77PPPkNe02j56U9/mo3tv//+I7gSqPejH/1o0HMuu+yybGz69OnZ2NSpUwe9r053papT1+nx3HPPHbF1wPw88MADgxqPiFh22WUHvb3hsNBCC1WOH3TQQdk5s2bNysZmzJgx5DXRu5ZYYonK8bruojvvvPNwLedlch2rIvLnyU3oWJXrFnbggQdm5+SOVZ2HHnpo0HO6hStzAAAAABpEMQcAAACgQRRzAAAAABpEMQcAAACgQRRzAAAAABpEMQcAAACgQbQmn0euNVlKKTvnnnvuycaOPvrobOzMM88c+MKGYPnll8/G3ve+92VjdWu/8cYbK8f32GOP7JyLLrooG4P5OeeccyrH//GPf2Tn1D2Hu8G6666bjZ1//vnZ2B//+Mds7JBDDhnSmqDKnDlzBj1ngQU6//eioigqx++8887snDXXXDMbGzeu+jSo7nf+RhttlI1Nnjw5G6v7WQUj6corr8zG6s7Vttpqq2ys7lw4py6XLr/88srxL3/5y9k5b3nLW7KxX/ziFwNfGMxj/fXXrxz/z//8zxFeSbVc+/GIiCeeeGIEV9JZyy23XOX4oYce2tH9fPWrX+3o9kaSK3MAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBFHMAAAAAGkQxBwAAAKBBtCafxwknnFA5/te//jU75/7778/GbrnlliGvaagefvjhbOzEE0/Mxq699tps7LLLLqsc/9a3vpWdc/PNN2djDzzwQDYGERG33XZb5Xhd/q2zzjrZ2K677jrkNQ1VruViRMTOO++cjdW1id5vv/0qxz/2sY9l59S1QYd27b///tnY1KlT29rmMcccUzn++c9/vq3t7b777pXj3/nOd7JzllpqqWxsxRVXzMa0JqdbHHjggdnYyiuvnI3dfvvt2dhDDz1UOf7ggw9m56y00krZ2A477FA5Pnv27OyccePyL2vqzkFhfjbbbLPK8ZRSR/dzxRVXZGMnn3xyNtbk9uOrrLJKNvbDH/6wcrzuca+LPfroo5Xjd911V3ZOt3NlDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDpKIo2puYUnsTGROmTZtWOX7xxRdn53zta1/Lxr7whS8MeU3DqSiKzt6ufhj1Wm6utdZa2djvf//7ju7rpJNOysamT58+6O3luudF1HflaMesWbOysT322CMbq8vpbiA3R99CCy1UOX7HHXdk57z61a/Oxu65555sbIMNNqgcf+aZZ7Jz6iy++OKV43Vrf9WrXpWNHX744dnYl770pYEvbAyQmwyHunPJ973vfdnYaqutNhzLaSS5OXi588m6c9B2bLvtttnYVVdd1dF9dYtf/vKX2dgb3/jGju7r/e9/f+X4hRde2NH9tKud3HRlDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANIhiDgAAAECDKOYAAAAANEhne9/SM+67775Bz8m1r4WBWHnllSvHzz333I7va8aMGZXj559/fnbOr3/960Hv58EHH8zGvvOd72Rjyy677KD3NWHChGzszDPPzMYWXXTRyvHheNxppmOOOaZyvK79eJ0vf/nL2Vi7LcgHu73Zs2d3dD/A4Cy55JKV4x/72Meyc26//fbhWg49YLvttsvGVlpppY7u69Zbb60cz7VAb4Kll146G/vmN7+ZjXW6vfvPfvazbOz666/v6L66gStzAAAAABpEMQcAAACgQRRzAAAAABpEMQcAAACgQRRzAAAAABpEMQcAAACgQbQmH2YTJ07MxmbOnDmCK+msXXfdddBznnjiiWFYCb3is5/9bOV4p1saRkRcdNFFlePttB+vc9VVV2Vjl1xySTb2yU9+sqPrWGKJJbKxXDtJrcnp86pXvaqj27v66qs7ur2RNHXq1NFeAowJCy20UOV4XfvjE044YbiWQw+47LLLsrGiKDq6r7///e+DGh9pG2ywQTa29dZbV47vvPPO2TnrrrvukNfUX12L8boW888991xH19ENXJkDAAAA0CCKOQAAAAANopgDAAAA0CCKOQAAAAANopgDAAAA0CCKOQAAAAANojV5B+Ra90ZEvO1tb8vG3v/+91eO33XXXUNeUydstNFG2dg+++xTOf6Xv/wlO+fYY48d8ppgNC2wQL7+veqqq3Z0X+PHj+/o9qCb/PnPf87GZs+ePYIr6azp06eP9hJgTNh8880rxx999NHsnOuuu26YVgOdteyyy1aOv/vd7+74vl75yldWjn/hC1/Izqk7B50wYcKQ1zRQ11xzTeV47jV0xNhsP17HlTkAAAAADaKYAwAAANAgijkAAAAADaKYAwAAANAgijkAAAAADaKb1QAttNBC2diWW26ZjdV1cOqGrlWrr756NnbeeedlYymlyvHzzz8/O+fZZ58d+MJgFK233nqV4wcccEB2zpFHHjlcy4Ex51WvelU2tvTSS2djf/vb3zq6js9+9rOV42ussUZb23viiSeGshzoKePG5V+GHHjggZXjdd3unnzyySGvid51zDHHZGP/9V//VTle9/qwzhve8IbK8csvv7yt7XVa7nVeRERRFJXjc+bMyc6p+9198MEHZ2M/+clPKsf9rp3LlTkAAAAADaKYAwAAANAgijkAAAAADaKYAwAAANAgijkAAAAADaKYAwAAANAgWpMP0JprrpmNrbrqqtnYU089NRzLGZTdd989GzviiCOysUmTJmVjF154YeX4QQcdNPCFQZfaeOONBzU+lnXDzzC62xVXXFE5vv3227e1veuvvz4bO+WUUyrHZ8yYkZ0zderUbGyPPfaoHF9ggfzfum6++eZs7OSTT87GgH9Xd565wQYbVI7vvffew7UcetznP//5bOyd73xn5fhaa601XMtpnO9973vZ2Ec/+tERXElvcWUOAAAAQIMo5gAAAAA0iGIOAAAAQIMo5gAAAAA0iGIOAAAAQIMo5gAAAAA0iNbkA/Tggw9mY7/61a+ysbo2bZtvvnnl+C233JKdM2XKlGzszW9+c+X42972tuycOsccc0w29sUvfrGtbQLd54477sjGdtxxxxFcCU30ne98p3J8o402ys6pa1O62mqrZWPHHXfcwBc2BP/617+ysfPOOy8be+6554ZjOTAmHXbYYdnYk08+WTl+6qmnDtdyIGu//farHL/mmmtGeCWDN3PmzMrxK6+8MjvnxhtvzMbOP//8yvFnn312cAujI1yZAwAAANAgijkAAAAADaKYAwAAANAgijkAAAAADaKYAwAAANAgqSiK9iam1N7EMejVr351NnbKKadkY1tuuWVH1/HQQw9Vjl9wwQXZOZdeemk2Vncn815TFEUa7TUM1FjNzde97nWV41tssUV2zgknnDBcyxlV9913XzZ2wAEHDHp7f/rTn7KxO++8c9DbG0lys3stssgi2di0adOysS984QvZ2Nprrz3oddSd53z729+uHD/ppJOyc26//fZBr6EXyU3mpy43P/e5z1WOH3vsscO1nJ4hNwdv/PjxleN1v5O23nrrbGz77bevHP/v//7v7Jx2z8eefvrpyvHbbrutre0xfNrJTVfmAAAAADSIYg4AAABAgyjmAAAAADSIYg4AAABAgyjmAAAAADSIYg4AAABAg2hNDgOgjSN0J7kJ3UluEhHxiU98Ihs74YQTsrFcK+gXXnhhyGvqdXITupPW5AAAAABjnGIOAAAAQIMo5gAAAAA0iGIOAAAAQIMo5gAAAAA0iGIOAAAAQINoTQ4DoI0jdCe5Cd1JbhIR8eSTT2Zje+yxRzb2v//7v8OxHEJuQrfSmhz+f/buPF6Oqsz/+PfJRhKykR0SMOIGgkBEFgFJIqDD8GMQBYQENOIA7j8UGHFBEhBRRBAHFRAkbiwCEhnZ9Ae5OIgCCQKDjqJA2LckZCH7cn5/nGpu0alTt7tvL3W6P+/Xq1+drqdP1emqfroqz62qAwAAAABAm6OYAwAAAAAAEBGKOQAAAAAAABGhmAMAAAAAABARijkAAAAAAAARoZgDAAAAAAAQkX6t7gAAAACA9jN06NBWdwEA2hZn5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQEQo5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQETMOdfqPgAAAAAAAKBCnJkDAAAAAAAQEYo5AAAAAAAAEaGYAwAAAAAAEBGKOQAAAAAAABGhmAMAAAAAABARijkpZtZlZs7MZra6L0XVqnVkZnOS5c5q5nJRDORmz8hNtAK52bNW5YiZzUqWO6eZy0UxkJs9Y7+JViA3e8Z+s3J1L+akVn75Y7mZPWhm3zazifVebhGY2VvM7Ptm9jczW2Vma83sSTO7xsymNGiZpfXd1Yj5twszG25mZ5jZ/cl3cb2ZvWRmvzWzj5hZ2xc2OzE3zayfmR1sZv9pZvPNbJmZrTOz583sJjP7QAOXTW72wMx2N7OzkgObl5K8XGJm/21mnzOzga3uYzN0Ym5KkpntYmafMLMrzOxhM9uQfO5rGrzc0sHawkYuJ2ZmNtrMjjSzb5nZnclvZ+l72RF5KXVubkoc0xYZx7Sdm5tmtjDwudOPUxuwXPabPUito7zHI/Vebr96zzBlvaQlyb9N0hhJuyaPfzezQ51zdzdw+U2V/Kfwakmlg5x18utgu+TxYTP7unPujBZ1sWOZ2Zsl3Slp22TSJkkr5L+TByWPY83s35xza1rTy6bqpNz8oaR/T71eL2mNpPGSDpV0qJldL2m6c259C/rXscxshqSfpyZtkrRc0laS9kseJ5nZ+5xzz7agi63QSbkpST+V/2wonmMlXdjqThRIR+Umx7TFxTHtZjoqN1Nekc/LLCub2RFsZo2kZYHYonovrJGV23ucc+OTxzhJQyR9RNJSSSMkXWdmgxq4/KYxs9HyB6UDJT0gaW9Jg5xzQyS9SdL1yVu/amb7t6aXHe1n8ju9xZKOlN82I+T/03hm8p6DJP1Ha7rXdB2Tm5L6S3pO0lmSJkvawjk3TNIESd9P3nOEpHNa072O1l/SKkk/kvReSYOdc1tJGibps/IHI2+XdIOZWct62VydlJuSPwh/UNLlkk6SdHtru4MUJ+kZSTdK+oqkL7e2Oy3XMbnJMW3hcUz7eh2Tm2U+mPrc5Y8ftrpzHe7anG0ztd4La9ppeM65Vc65n0n6XDJpvKSGXeLQZIdKGpr8+3Dn3L3OuU2S5Jx7XNJ0Sf9M4h9sQf86lpm9Uf5ARJI+75y73jm3TpKcc0udc2dJ+kkS78ht0+a5+QNJ2zvnznTOPeicc5LknHvOOfcZSXOS9326TXf2RXaP/LY50Tk3zzm3VpKccyuccxdL+nTyvr0kdeR/GNo8NyVpb+fcZOfcCc65yyS90OoO4TUXO+e2dc590Dn3DUl/bHWHiqTNc5Nj2oLimLZnbZ6bwGZacU3lL+VPCZSk3UsTLXUzKDMbkVynXbpOd2l6BmY2wMw+Y/6+CktS1/H+2Mx2zFu4mf1L6vrv5Wb2JzM7rpefaVzyvNg591R5MLl84+Hk5Za9XFavmVlf8/cRudTMFpjZi+bvI/Kcmd1oZu+tcD5bmdmFZva4ma0xs2fM7DIz27qHdkPM7MvJtb7Lkrb/MLPvmdm2eW1rMC717z8H3rMgeW75tmmxtstN59x9pSJBwJzkebCk3P41QyflpnPuUefcizlvuUrdpxDvnvO+TtB2uSlJzrmNvZ1HM5nZ/mZ2kZndm+TkOvP3qbjNzI6ocB4DzWx2sp1WJ+2vNrO39tCu5u1Xi9i2TQu1Y25yTFvQ/aY4pq1GO+ZmdDppv9kyzrm6PuT/c+QkdeW858XkPZelpnUl006T9Fjy7zXy91BYmnrf1vKnZbvksTF5T+n1avlTz7KWe1rqfZvkrzfcmLz+TqoPMzPalmKbfS5JR6fmu11GvJ+kfyTxTzZ7fWe02TnVXyd/Xd+rZdO+FGhbWg+nyP9lxslfKpFu/5KkHQPtd5S0MPXe9WVtl0jaN+dzzsqIzSq1z4iNT837uB7W4bX1zociPToxN6vMhT2avb576E9b52aF62NR6bvX6vxp5IPc3Gw9XNPg9V36Xi6sos2QsjxcnuRnetqlPXyuc+XPcHGS1pa1Xylp/0D7mrZf6nPOyYjNTLWfVMHnn5p6/8BW50yzHp2Ym+KYtrD7TXFM29G5mcRL37epTV7fpe/lwiradNR+M69tIx9NPzPH/KUMY5KXSzPe8jX5eykcLH8PhWGS3pW07S/p1/I3tbpD0j7yBxXDJG0j6bvy1/j+zMzeVLbc/SR9K3n5c0nbOH9/hlGSzpP0BUm71fix/kvdp4ffaGZ7WXInefOnRF4l6c2SHpH04xqXUU/r5PvxfknDnXPDnb8WepykM+S/8OeY2V458zhD/jTcQyUNSdpPlfSE/Pa9LtlerzGz4ZJukfQGSdfJb8eBrvs67Kvkr/m9wcxG1OODOudekPSb5OWFZnaEmQ1I+jPCzM6Q9FH5BJ9Vj2XGqk1zsydTkuf1kh5t0DKq0TG52RMz20n+OyD5386O1aG5WTSb5O8VcrikUc65Yc654fJ58Rn5/8CdaGZH5szjk5J2kb+fw5Ck/WT5+5IMlvRLM9sq3aA32w+N16a5yTFtQfebHNNWrk1zM+1CM3s5OdPlBTO7xcymm1nfOsy7Xjp1v3lAcnbe2uTMrAVmdraZjeu5aQ0aULmbo/yK4mfUXdX6UGp6VzJtnaSdA23/PXnP7yX1D7znkuQ9F5dNvyOZfqcky2h3eapfMzPipf6FPtfukp5OzWOtuqvzSyVdLL+Taer6rnGeZyTzvDJnPWyStF9G/G3JZ3eSji2LfT2ZflXOsm9N3nNq4HPOymgzq7TeA/Mck3xn0tXZper+K8qNCvzVpZ0enZqbOetjSCpn634mALnZc2728FlvTNo+KWlAvbdPkR7k5mbroXBn5lQwz+OSec7L+VxO0oyM+Gh1n4X21Tpuv9LnnJPRZmaqT5Mq+HxTU+/nzJzueFvmpjimLcULt98Ux7QVfVfaODcXptqv1OvPNnFJ+xENWN+l7+XCOs6zrfab6dyVtEH+zLyNqWmLJR1Q723TlDNzzJtkftz785LJT8pX/8vd6pwL/RX2o8nzRS48jPAvkueDUssfKWla8vJbLlnjZb4R/ACSnHNTnXPmAnehds4tkB+RpXSt6gB1X686QH6ElmF5yyiQ0nbZN+c9/+0yhvpzzv1d3SMdlF8LWdp+38mZ71XJ80E57ylf5qxk22SOeOOce1nS/1H3MMh9JA1P/t1X/j/1ozKatr1OyM0cl0iaKL8jPL3Ktq3SVrkZYmYnqPuGhZ93yQ0eO0mH52aMSttl75y/jD6p7jx6jXNukaRLk5eh3Kxq+/XEOTenlJvOuYWVtkNn5CbHtF4R95sc04Z1Qm5KmivpQ5JGO+e2dP5skzdIOl++KDlF/n5BMWi3/eY/JJ0q6S3yf/QYKf87ebSkZyWNlDS3p3v9VKtfPWdWZoqZZX2JJel5SR8IHKBnjphgZv0k7Zm8vNTMvp/1PvkfMskP21cyWZLJf8k3+7GWJOfc42b2dFm7ipnZifJDHb8g6Rj5auAq+VO8zpWvPh5gZvs5556oZRn1lJx++AlJh8kP/7uVNv8+bJMzi66c2F3yox28M7W8beX/4yxJt+R8NwYkz3W7aZyZ7S1/ut1QSV+S3zE/L38a7Bfkk35/MzvCOZf1g99uOio3s5jZ6ZJmyFfKTyjSf2Y6KTezmNkUSf+ZvPy+c+5XjVxewXR8bhZZsj4/Kj8c8K7yB2YDyt42UD5nF2XM4q7Awb3kc/PLknY2swHOuXW93H6or47KTY5pX6dQ+02OaTfTUbnpnDs5Y9pTkk4zsyfk8/YgM3ufc+63tSyjnjppv+mc+0XGtJWSrjWzP8pfGjZK/gye6fVabiOLOevlTy+Suk8Fe1zS7yRd7px7JdDu5cD09MavpOKcHma4dM3ksmSlhjyrGjaqme0rXx1cLem9zrl/pMJ3mdk0+bvO7yjpm5I+XO0y6sn8nfm7JKUrgyvlb9C1Sf4LPlr5d8J/toLYmNS09GgAYyvo5uAK3tMjMxsmX/kdLX+zuJ+nwg9LmmlmGyUdL+liM/utyx/9qB10TG5mMbOT5A9GJekU51xh/oLRSbmZxczeJekmSVvInyr+fxu1rILq6NwsMjMbIul2+WvvS1bLr/vSiCml6+G3VPZBaSW52Vf+oPZF9W77ob46Jjc5pg3GWr7f5Jg2U8fkZgV+KH/z5Uny939qaTGH/WY359xTSWHpa5IOMbM+zrlNPbWrRCOLOffUeGp1aDjM9CVhk51zD9Yw70Yp/Yfj5rKdniTJObfWzH4g/9fmQ83McqqMzfBd+Z3e4/JJPy/9Y5fcDOqfdV5mevtt5ZzLuhlZIxyr5LrKsp1e2oXyO77t5Kvqf2pS31qlk3LzdcwPC/mD5OUs59yFrexPhk7Kzdcxs13kd/rD5A9AjnadNzxyx+ZmBM6QPyBdJD/yzW3OuZdKweQU8Q2ll3VaJtuvODopNzmm7RnHtMXRSbmZyznnzOx++WLO9i3ujsR+s9y9yfMw+UJTqKBYlaaPZtULi9WdeNtV2ba0soabWV51PO8UzDylserzTjV9PHkepO4qZNOZv+v9YcnLGc65X2VUrSvpX966KsXSX9IXU/+udvv1RjXbRvI/gKhOkXPzNebvln+l/O/ed5xzs3s7z3rqwNx8jZntIP9XtJGS/lvS4YHTolGdKHIzEqXRNj7rnPtp+oA0Ua/c3Ch/RoHUu+2HYitybnJMmx0rwn6TY9rGK3Juxob9ZhNEU8xJbmA0P3l5cJXN/yx/6l0fSftlvcH8cIu1bvTSaVJ57d+Q+veKGpdTD6PlL2GQ/HrJcmAF85lSQeyB0oTkmurSzq/a7dcbMW2bKBU8N0vzOFT+Zmd9JV3inDu1N/NrkE7LTUmv/dX0DvlT1e+XdIhzblWz+9GOYsjNiJTuj9Ho3HykVMjs5fZDgRU8N2M6buq0/WZM2yZKBc/NXGZmkvZIXrb8XlZiv1lur+R5hXzRqS6iKeYk5iTPM81s17w3WmrMeefcEvkh4iTpP5Ive7nejGbzUPJ8sJlNyOhLX0kfS17+pYfrKBtthfwPjSS9ozyYXHv82QrmM8XM9imfaGZvUfddxa8rC89Jnk/NWk+peZiZjaigD5UobZtxyX/os5yQPDv5/0yienOS56LlpszsIPnvYn9JP5H0qd7Mr4E6LTdLN5G8Q/6vKw9Jer9zjoPP+pqTPBcuNyOzLHnOys0hkr5SwTwmmdkxGe1HSjoxeRnKzaq2H6IwJ3kuWm5yTNvdvmj7TY5pm2NO8lyo3AzML+0kdZ+NdXOty6mjjtlv9rRtzGyipE8nL2+t1/1ypPiKOVfIX/s5UNKdZnZCcjMwSZKZjTezGWZ2lza/ceYs+R+2AyTNMbNxSZvhZvYN+S/EMgWYWZeZOTPryghfkjwPk3S7mU01s/7JD/jbJP1K3ZXS72XMe1Yy795cc9zfzEb38Oif/EepdP3sj81st6QPfczsAPk7g1dy3eJySb8ys38tfYHN7D2SbpX/K8lftPnQeN+UP/1ztKR7zOwo8yMQKGm/nfkRFB5Q97DEPeph/V2v7htqzTGzmckPiMxsrJmdq+7vyjUZpwCiMoXMTfM3cpwr/528RtLx1VzbT256jchNMxsr6f/J/xXxr5IOyrlRIWpXyNxM4oPTeaDuv7APKMuPIRltZ5a+W2Y2qZIVkaFPBblZ6tPvkucLzGxKKrf2kC9IVnKjxWWSfpSs735J+9K9osZIeknd9/Qq6c32C+pp/SW/O+ltMzwVHlUWQ22Kmpsc06qY+01xTNssRc3N75nZRWa2X9l3bVsz+6aki5NJ85xzt2bMm/2mGrbf3N/Mbjezo81sfKrNYDM7StIfks+7Sv47Uj/Oubo+5KthTlJXle26knYze3jfWPnh3lzy2Ch/qtKrqWlO0pkZbU9LxTfJ3/18Q/L6O3l9SMUyP5f8cIAbU/Nfn2ywdJ8uDbSdVXpPL9Z3JY+pSZu9yvr2aur1YvnrjzP7k1oPp8jfUM4lbVek5veSpLcH+vtm+f+4ld67QX7HVL6uPhr4nLOqXX/yp+EtLZv/8rLX90oaXu98KNJDHZib8n8hKc33ZflhVkOPD5ObzctN+Tv6l+a5rIdtc1Gr84fcbMh+c1bZ8kOPORltZ6bik6pcb5Uu97XPJX8zyZdT01en1t8qSe8L9Se1fc+VP8B0ktYk3/tSm5WS9q/n9kt9zqrXn/xfdytaR63OH3KzIbnJMW0B95tJfIo4pk2vw47JzbL82JjMd1lZf7okjQx8ppmp902qcr3NKltO3mNmOwrzngAAIABJREFU0qZj9puSppbNd6X8b8GG1LRFkt5X71yI7cwcOV9lniJphqRb5L8kQ5Pw3yT9VNJR8hXz8rbflr9+bp78xuwnf13dR5xzp/SyXxfI71CulN8hbJC/P8ez8n/FONg5d1KgeWmIw/mBeF055+6V9G75MxZekb/85CX5oSh3U/dpnHkWS9pTfhSBF+WHgXtO0o8k7eac+2tg2f+Uv7v+p+S3wyvyf/HbID+s4mWSDpEUukt/1Zxzd0naSdK3JD0ov5MelHyGeZI+IWk/51ywUo6eFTQ3079xo+VvthZ6ZA1PSG42LjfT22aY8rfN8M1ao2IFzc3eKuXms5Keb/TCnHOPy+fVz+Vzsq/8f6h+IWkP51wlQ8CulT/gO0vSk/K5+bL8WYPvdM79PrDsmrcfiq2ouckxbWH3mxzTNklBc/MSSedLukf++zlQ/syxpyXdmPTnvc5f7pWF/abXiP3m/0j6D0m/lv/NXCf/W7BMfnudIWnHCj9zVSypJqGFzOxvkt4m6VDn3G9a3R8AHrkJFJOZ3Sbp/fKjZFzc0/sBNAf7TaCY2G+2J4o5LZZcS/mCpAecc7u3uj8APHITKCbzN2BdKn9pwfbOubUt7hIAsd8Eior9ZvuK7jKrNrR/8nxWS3sBoBy5CRTTOyUNkXQeB6RAobDfBIqJ/Wab4swcAAAAAACAiHBmDgAAAAAAQEQo5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQEQo5gAAAAAAAESkX60NzczVsyNAkTnnrNV9qBS5iU5CbgLFRG4CxURuAsVUS25yZg4AAAAAAEBEKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQEQo5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQEQo5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQEQo5gAAAAAAAESEYg4AAAAAAEBE+rW6AwDQGyeeeGIw9tWvfjUYmzBhQiO6k+m3v/1t5vTf//73wTYLFiyoen4AAAAAOgNn5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQETMOVdbQ7PaGgIRcs5Zq/tQqXbNzR122CFz+h133BFsM27cuEZ1py7Mwl+rdevWBWNz5swJxj75yU/2pkvRITeBYiI3gWIiN4FiqiU3OTMHAAAAAAAgIhRzAAAAAAAAIkIxBwAAAAAAICIUcwAAAAAAACJCMQcAAAAAACAiFHMAAAAAAAAi0q/VHWgHEyZMCMY+/elPB2MTJ07MnD5jxoya+rFixYrM6VdeeWWwzV//+tdg7IorrgjGNm3aVHnHgDrYc889M6cXffjxWvXv3z8Yy/uNeOMb35g5/dhjjw22WbRoUeUdAzrAbrvtFoz94Ac/CMb23nvvYOwd73hH5vS//OUvlXcMaLCpU6cGY2eeeWYwdtddd1XdphbTpk0Lxrq6uuq6LAAoOs7MAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICKNZlRkwYEDm9K9//evBNscff3wwNmLEiKr74Jyruo0kDR06NHP6Zz/72ZrmN3r06GDsvvvuy5w+f/78YJtly5bV1A9Akp566qnM6a+++mqwzZAhQxrVnZYaPHhwMHbggQdmTp80aVKwDaNZoVOFcum6664Lttl+++2Dsbz990EHHZQ5ndGs0Gzz5s0LxvJGs8pTa7tq5fWdka5QJPvss08wNnz48Mzp06dPr2lZecd4zz//fOb0d73rXVW3kcIjGi9cuDDY5tFHHw3GbrrppmAsNLJr3v83Ow1n5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQEQYmrzMvvvumzn9C1/4QpN70npnn3121W1uvfXWYGz27NnBGEPMoSehYUVvuOGGYJuJEyfWtQ9vf/vbg7Gtt966rssC0Hj77bdf5vS84cdr9fe//73u8wTyzJo1K3N6s4YRb7a8z8XQ5OjJiBEjgrEZM2ZkTj/iiCOCbUL/p5Skfv2K/V/wvKHOQ972trcFY6H1J0lf/vKXg7G+fftmTn/iiSeCbX75y18GY3Pnzs2cvmDBgmCbouPMHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiEixx0VrgTPOOKOu87vrrruCsWuvvTZz+s9//vNgm8GDBwdjL774YuUda5CDDz44GNtnn32CsVGjRjWiO+gAH//4x5u2rLyhyceNGxeMHX/88ZnTp0+f3us+lXvllVcyp69ataruywJid8kllzRtWffee2/TlgU02+zZs+s6vzPPPLPqNlOmTAnGQsO058kbzpyhztvP+vXrgzEzy5w+dOjQYJtzzjmn6j4sXLgwGPvBD34QjA0aNCgYC+17vvjFLwbbPPfcc8FYyIoVK4KxI488Mhhbvnx5MLb99ttnTnfOBdsMGTKk6n4wNDkAAAAAAACagmIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQEQo5gAAAAAAAETE8u4GndvQrLaGBRe6o3beKFJ5Tj755GDs4osvrnp+t9xySzD2/ve/P3N63jZ++eWXg7G8Eab69Km+Drhy5cpgbPjw4VXPr5mcc9m3sS+gds3NmIVGNDj99NODbWr9bZ47d27m9COOOKKm+RUduYne2LRpU+b0WvMvb/SS0Gg6oT7EjtxsvdB3rpaRoqT8EZymTZtW0zxDpk6dmjl93rx5dV1OrfI+b9FHuiI347Ro0aJgbOTIkcHY9773vczpef9HRWvUkpucmQMAAAAAABARijkAAAAAAAARoZgDAAAAAAAQEYo5AAAAAAAAEaGYAwAAAAAAEBGKOQAAAAAAABHp1+oOYHMTJkwIxiZPnlz1/ObPnx+MHX300cHYww8/HIzVOlQ70K7GjRsXjB1yyCFN68f111/ftGUBMfjiF79Y1/mtWbMmGLvggguCsXYdghzFFRoiu9ahyUPDhUvhIcNrHbK86MN7562LovcdQPvgzBwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIQ5OX2bBhQ13nN3HixGCsf//+mdMXLVoUbLNkyZJg7M4778yc/rnPfS7YJm94ykGDBgVjQCcaMWJEMHb77bcHYzvvvHNd+3HvvffW1A+gXY0ePToYO+WUU+q6rMsvvzwYW7p0aV2XBfRGaIjsvOHCQ0OM9yQ0VHfe/GbPnh2M1Tp8ej3l9W/WrFnN6wggycxqanfVVVfVuScoEs7MAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiDA0eZmzzz47c/q3v/3tmuZ36qmnBmOhYcbPO++8YJuddtopGNtmm20yp+cNn/jJT34yGKu3a6+9tmnLAhrh8MMPD8bqPfx4nr///e/B2CuvvNK0fgBFMWjQoGBs1KhRVc9v1apVwdiPfvSjqucHFEloyPKeYqHhx/Pktallfo0Q+swMP45mGzFiRDDWt2/fJvYEseDMHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiDCaVZmrrroqc3qto1nl+cpXvpI5/U1velOwzc033xyM/ed//mfm9AkTJlTXsQo88MADmdPvvPPOYJuvfe1rde8H0Ai/+tWvMqcfdthhdV1Onz7hevr9998fjJ188sl17QcQuxkzZtR1fnkjxj3yyCN1XRZQJNOmTQvG5s2bF4wVZWSqkLxRuvI+M9BM7373u4OxYcOGNbEniAVn5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQETMOVdbQ7PaGhbc0KFDM6dfeumlwTZHHXVUo7pTFTPLnF7rNl69enUwNmXKlMzpoSHLY+ecy165BdSuuVmLbbfdNhj71Kc+FYx9/vOfz5zer1+/XvcpLZSzkjR27NhgbPHixXXtR8zIzc4yZsyYzOkPPfRQsM24ceOCsVAO7rnnnsE28+fPD8bQjdzsLKGhyfOGM2+mvOHH84Ytb0fkZnEdfPDBwdjNN99c0zz33nvvzOn33XdfTfND49SSm5yZAwAAAAAAEBGKOQAAAAAAABGhmAMAAAAAABARijkAAAAAAAARoZgDAAAAAAAQEYo5AAAAAAAAEanvOLttYMWKFZnTp0+fHmzz61//Ohg7//zzg7Ftttmm8o5VoE+f7Nrcpk2bgm1++9vfBmP/9m//FoytX7++8o4BDbTddtsFY3Pnzg3Gdtlll0Z0J9MDDzyQOf0DH/hAsM0rr7zSqO4A0XrTm96UOX38+PE1zS80NPkLL7xQ0/wAtM7s2bODsU4bfhyd5dlnnw3GHn300Sb2BM3GmTkAAAAAAAARoZgDAAAAAAAQEYo5AAAAAAAAEaGYAwAAAAAAEBGKOQAAAAAAABFhNKs6uOuuu4Kx2267LRj72Mc+Vtd+hEateuihh4JtjjrqqGCMEasQg+OOOy4Ya+aIVRs2bAjGzj333Mzpzz33XKO6A7SlyZMnZ053ztU0v+effz5z+qpVq2qaH9Cppk6d2uou6MwzzwzGZs2a1byOAE22evXqYGzp0qVN7AmajTNzAAAAAAAAIkIxBwAAAAAAICIUcwAAAAAAACJCMQcAAAAAACAiFHMAAAAAAAAiQjEHAAAAAAAgIgxNXqGxY8cGY7/+9a+Dsd13370R3alK//79g7Fdd901GLv77rsb0R0gaNtttw3Gjj766Mzpp59+eqO6s5m1a9cGY6eeemowduONNzaiO0DHCQ1NXqunnnoqc/qrr75a1+UA7S5vWPAiyBs6vaurq2n9ABrBOdfqLmjgwIHB2E477RSMLViwoBHd6RicmQMAAAAAABARijkAAAAAAAARoZgDAAAAAAAQEYo5AAAAAAAAEaGYAwAAAAAAEBGKOQAAAAAAABFhaPIyQ4YMyZx+2223BdvssssujepOXey4447B2HHHHReMMTQ5mm3u3LnB2K677trEnmRbvnx5MJY3tOKee+5Z13588YtfDMaGDh1a12WF3HDDDcHY/PnzgzGGoERvHHPMMXWd31e/+tXM6evWravrcoB2MGvWrFZ3oWZ5Q6czNDmKotb/U/bv3z8YCw0ZPmrUqGCbvHyZOnVq1X2YOHFiMPbEE08EY2aWOf273/1usM33v//9YKwdcWYOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQEQo5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEhKHJy2y55ZaZ0xsx/Pjq1aszp8+ePTvY5pRTTgnGxo4dW3Ufpk2bFoxtscUWwdjatWurXhYgSR/60IeCsZ122qmJPanemDFjgrF77rmn6vmFhlyUJOdc1fNrpve+973B2MqVK4Ox7bffPhhbvHhxr/qE9nDYYYcFY4MGDarrsu644466zg9AMYWGUwaK5A9/+EMwtmbNmmBs0qRJwdj999+fOX3ChAnBNltttVUwFjo+zTv2y+v7m9/85mAsdJx87rnnBts8/vjjwditt94ajMWKM3MAAAAAAAAiQjEHAAAAAAAgIhRzAAAAAAAAIkIxBwAAAAAAICIUcwAAAAAAACLCaFYt9Nhjj2VOP//884NtPvGJT9S1D3kjy/Tt27euywIk6a1vfWsw1q8fP0ntoE+f8N8Jtt5662CM0aw6R9535Oijjw7G8kaAC7n++uurbgOgvXR1dbW6C0CP7r777mDsJz/5STA2ffr0YKyWkWKfeeaZYCz0/9Sbb7452Gb9+vXB2I477hiM7bHHHpnT80Z+vuKKK4Kx0OjUixYtCrYpOs7MAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiJhzrraGZrU1LLixY8dmTn/qqaeCbWodTnnjxo2Z05csWRJsM2rUqGAsNJR43jbesGFDMDZy5MhgbNWqVcFYO3LOVT8ebosUPTd33333YOx3v/tdMDZs2LBGdKel8oZZrvW3OSQv1+udzzfccEMwdsIJJ9R1WeRmnI466qhg7Oqrr656fnnDiuYNyxrzcKRFR262n6lTpwZj8+bNa15H6ixvX9yOyM32ExpyOy/2xBNPBNs8++yzwdjChQsr7lejXHLJJcHYiSeeGIydcsopmdMvvPDCXvepHmrJTc7MAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiDA0eYWOOeaYYOyyyy4LxgYNGtSI7mQKDa2Yt42vuOKKYOykk07qdZ/aBcM4NkfesOUHHnhg5vSTTz452GbMmDG97lMj5Q2HOnv27GBs7dq1VS/rySefDMauueaaqudXFORmnPK+c0ceeWTV83v66aeDsUmTJlU9P/QeudlZQkOT5w1nXhQMTV5c5Cay7LXXXsHYH//4x6pj++67b6/7VA8MTQ4AAAAAANDmKOYAAAAAAABEhGIOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQET6tboDsbj66quDsX/84x/BWN6IUB/72Md61ad6uPHGG1vdBeA1CxYsqDr2rW99q1HdAdAg//u//9vqLgCoo9AIjM0czaqrqysYyxshEkDvDRgwIBgbPnx4MJY3Quvy5ct71adyb33rW+s6vyLgzBwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIQ5PXwfz584Oxf/7zn8HYpZdemjl9+vTpwTYzZ84Mxh566KHM6TfddFOwzZ133hmMAQDQCH/+859ravfyyy9nTv/2t7/dm+4A6KXQsOB5Q4KfeeaZdVtOT8vKaweg9/r0CZ8j8qUvfSkYO/TQQ4OxJUuWZE4fPHhw5R1LWbduXU3tiowzcwAAAAAAACJCMQcAAAAAACAiFHMAAAAAAAAiQjEHAAAAAAAgIhRzAAAAAAAAIkIxBwAAAAAAICLmnKutoVltDYEIOees1X2oFLmJTkJuAsVEbgLFRG6i2QYMGBCMffzjHw/G9tlnn8zpRxxxRLBN3vDjhxxySOb0u+++O9immWrJTc7MAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiDA0OVABhnEEioncBIqJ3ASKidwEiomhyQEAAAAAANocxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiAjFHAAAAAAAgIiYc67VfQAAAAAAAECFODMHAAAAAAAgIhRzAAAAAAAAIkIxBwAAAAAAICIUcwAAAAAAACJCMQcAAAAAACAiFHNSzKzLzJyZzWx1X4qqVevIzOYky53VzOWiGMjNnrUqR8xsVrLcOc1cLoqB3OwZuYlWIDd7Rm6iFcjNnpGblat7MSe18ssfy83sQTP7tplNrPdyWym14St5zKvzskvru6ue820XZja1im3jWt3fRurE3EwzsyPN7E4zW2xmq8zsf83s62Y2tEHLK/0uLGzE/NuJmb3FzK4ws6fMbK2ZPWdm15jZ7q3uWzN0am6a2cIKfpdPbcByyc0eVHhc80ir+9loHZybu5jZJ5Lf5YfNbEPyua9p8HLJzR6Y2ejkeOZbyTHNstT3cmCr+9csnZqbJRzTFlezj2n7NWKmifWSliT/NkljJO2aPP7dzA51zt3dwOU306uSXsyJ95H//JL0QOO7g5R1yt82kjRaUl91zrbppNyUJJnZZZJOSF5ukLRG0g6SviLpGDN7j3PuuVb1r5OZ2UGS5koanExaJmm8pA9L+pCZfcw59/NW9a/JOi43E6/I/1ZnWdnMjmAza+RzMsuiZnakxTotN38q/9lQPMdKurDVnSiQTstNjmkLrBXHtI28zOoe59z45DFO0hBJH5G0VNIISdeZ2aAGLr9pnHPnpz7rZg9JJ6XePqdF3exIzrl7etg275C0KXn7nNb1tKk6Jjclycw+Kb/T2yTpNElDnHNDJe0r6UlJ20v6Zet62LnMbLyk6+V3er+TNMk5N0J+x/cL+T84XGFmO7Wul03VUbmZ8sGc3+kftrpzHe7anG0ztdWda6JOy831kh6UdLn8Meztre0OUpykZyTdKP+f9y+3tjst11G5yTFtcbXqmLZp98xxzq1yzv1M0ueSSeMlfaBZy2+xjybPf3bO/U9Le4JyMyT1lz9wubrFfWmJds5NM9tC0qzk5UVJ4XWt5At9kg6XPzDa18wObU0vO9rpkoZJelr+P/RPSpJz7iX5380FkgZIOqtlPWyhds5NIGYdkJt7O+cmO+dOcM5dJumFVncIr7nYObetc+6DzrlvSPpjqztUJO2cmxzTFl5LjmlbcQPkX6r7TIjXrh2z1M2gzGxEci3o35LrAJemZ2BmA8zsM2b232a2JLke7Ukz+7GZ7Zi3cDP7l9Q1psvN7E9mdlz9P+Zryxst6V+Tl3MatZxqmFlfMzvYzC41swVm9qKZrUuu6bvRzN5b4Xy2MrMLzexxM1tjZs+Y2WVmtnUP7YaY2ZfN7P5kO6wxs3+Y2ffMbNv6fMqKlQptv3HOddIp41naMTcPlDRWfuf2nfKgc+7Pkv5f8nJGL5dVF2a2v5ldZGb3Jjm5zsxeMrPbzOyICucx0MxmJ9tpddL+ajN7aw/tat5+1TKzPpKOTl7+0Dn3ajrunNso6YLk5f8xs2H1XH5k2jE3o9MpuYmqtGVuJr+/0eik3Ixt27RQO+Ymx7QFzU1r5TGtc66uD/mChZPUlfOeF5P3XJaa1pVMO03SY8m/10haLmlp6n1by5/66ZLHxuQ9pder5athWcs9LfW+TfLX6W9Ud1KU+jAzo20pFvxcgWV+Lmm3TtLoVqzvjDY7p9aDk7+e79WyaV8KtC2th1Mk/TP596qy9i9J2jHQfkdJC1PvXV/WdomkfXM+56yM2KxS+yrX3S6p5R5W721TtEcn5qak85PYwzmf+ZTS97bO67v0vVxYRZshZXm4PMnP9LRLe9i+58r/pc5JWlvWfqWk/QPta9p+qc85JyM2M9V+Ulks/Tv0zkCfRqfe86+tzqFGPToxN5P4wiQ+tcnrm9zMyc2e2nbSo1NzM2c9XNPg9U1u9pCbGe+fmnr/wFbnTLMenZib4pi2sLmpFh7TNv3MHPPXLZZuBrw04y1fk7/s5WBJg51zwyS9K2nbX9Kv5W9qdYekfeR/uIZJ2kbSdyUNlPQzM3tT2XL3k/St5OXPJW3jnNtK0ihJ50n6gqTd6vQx0z6aPN/iinPmxzpJP5b0fknDnXPDnXNDJI2TdIb8F/4cM9srZx5nSBoq6VD56zWHyO9QnpDfvtcl2+s1ZjZc0i2S3iDpOvntODBp+yZJV0naStINZjaiTp81T2nbvJz0q6O1aW6+PXn+S857/po8jzF/Jl0rbZK/3vZwSaOcc8Occ8Pl8+Iz8oXPE83syJx5fFK+UPkR+dwcLmmy/A2+B0v6pZltlW7Qm+3XC6Vt49S9DV4n+c18qez9HadNczPtQjN7OfmL3QtmdouZTTezvnWYd710Um6mHWD+zNm1yV+YF5jZ2WY2rgHLik4H5GYMOjU3kaNNc5Nj2uLmZuuOaetZtSurpGVWSuU3Xqkq9aHU9K5k2jpJOwfa/nvynt9L6h94zyXJey4um35HMv1OSZbR7vJUv6qqlOasi3ek5vmBeq/rStZ3jfM8I5nnlTnrYZOk/TLib5OvnDpJx5bFvp5Mvypn2bcm7zk18DlnZbSZVVrPVXzGfvLXgDtJ323EtinaoxNzU9Kfk9h3ctbLrqn5v6OO67v0vVxYx3kel8xzXs72dZJmZMRHy48+4yR9tY7br/Q552S0mZnq06Sy2P9Npi/u4TP3uA1jf3Ribibxhan2K/X6v5q5pP2IBqxvcrPyM3Oc/GgpS9T9l2UnabGkA1qRL818dGpu5qyHwp2ZU8E82yo3M94/NfV+zszpjrddbopj2nS8ULmpFh7TNuXMHPMmmdmp8lVJyd9x+78y3n6rc+6RwKw+mjxf5JxbH3jPL5Lng1LLHylpWvLyWy5Zm2W+EfwAkpxzU51z5qobvaHU30WSbq6iXauVtsu+Oe/5b5cx1J9z7u/yVVhJKr8WsrQ+NrvOM+Wq5PmgnPeUL3NWsm2s0jaS/kX+TCRJ+kkV7dpKB+Tmlsnz6pxZrEr9e0jesgqgtF32zjlr4Ul159FrnP+LwKXJy1BuVrX9euKcm1PKTefcwrJwJdtG6t4+Rd82ddUBuSn54Ts/JH8J8pbO/9XsDfKnkm+SNEXxjMrRTrkpSf+QdKqkt8j/B3Gk/I0dj5b0rKSRkub2dM+CdtQhudlO2i03EdABuckxbaKAudmyY9p+9ZpRhilmlvUllqTn5c9UWZcRy7wru5n1k7Rn8vJSM/t+YN6lL0P6RrqTJZn8weFmBQhJcs49bmZPl7WrWfKlLN186uqcL1NLJKcffkLSYfKnem2lzb8P2+TMoisndpek6ZLemVretpImJi9vyfluDEieG30j5FIuL0+dAAAdvElEQVSi/4/zNwzrJB2dm0WXrM+PSjpS/i8sI9WdFyUD5XM269LNuwIHEJLPzS9L2tnMBjjn1vVy+6G+Oio3nXMnZ0x7StJpZvaEpO9LOsjM3uec+20ty6inTspN59wvMqatlHStmf1R/hT3UfJ/xZxer+UWWEflZmw6KTexGXKzwMjNxmtkMWe9/Gm5Uvcp1I/Lj7t+uXPulUC7lwPT0xt/VAXLH5T6d+mayWXJwUjIs6rfRn2//HB4UkFGsSoxP9pUl6T0X9RWyt+ga5P8F3y0uquMWZ6tIDYmNS09wtXYCro5uIL31CS5trI0ZN+cRi2nwDopN0vzHJTznvR37dXgu5rAzIZIul3++t6S1fLrvjQqQ+mMsi2VveOrJDf7yu84X1Tvtl9vVLJtpO7t09Jt0ySdlJs9+aH8TSQnyf9et7SY02G5mcs591RygPw1SYeYWR/n3Kae2kWO3CwocrPjdVJuckybHStCbrbsmLaRxZx7ajx9MzTkXvqSsMnOuQdrmHczlc78eMQ590BLe7K578oXch6XP1iel/6xS24G9c86LzO9/bZyzmXdjKxZjpa0hfx9ADb762MH6KTcfE7+RnN5Z5mlY883tjs9OkN+p7dIfkSC25xzpZullc7421B6Wadltmr7PZc8b2VmA51zawLvK22fVm+bZuik3MzlnHNmdr98MWf7FndH6qzcrMS9yfMw+QPm0H+M2gW5WVzkZmfrpNzkmLZnHXdM2/TRrHphsboTb7sq25YOMoabWd4ZH3nJUTHzIzEdlrws1P1YzGyAuvs2wzn3q4yqdSWjVFTyQ5I+uHsx9e9qt1+9zUyeb3fOvZj3RlSkyLlZuqP8TjnvKd1R/mXX+hHnSnf0/6xz7qfpnV6iXrm5Uf5MPKl32683StvGFLirfzISw9iy96NyRc7N2HRSbqLxyM36ITdRT0XOTY5ps2NFyM2WHdNGU8xJ7jkzP3l5cJXNS3eO7iNpv6w3mNkbVb+NXjrzY6P8sHRFMlq+b5JfL1kOrGA+UyqIvXZGknPuCXUXdKrdfnVjZjuo+1rKOa3qRzspeG7OS553Si4vzPK+5PmOGpdRT6X7SjU6Nx8pXUPey+3XG/+r7t+E0A3oStPXKXD9OcIKnpu5zMwk7ZG8fKIRy6hSJ+VmJfZKnlfIHzyjCjHnZgGRm6ibgucmx7TZsSLkZsuOaaMp5iTmJM8zzWzXvDemx5x3zi2RHyJOkv4jOUgsd3pdeuiVLrG63Tn3Qh3nWw8r5H9oJD90+uskPw6frWA+U8xsn/KJZvYWdd9V/Lqy8Jzk+VQzmxCacXI3+hEV9KEWpW3zirLvbo/azEmei5abd0h6Sf637pSMvuyq7p1JES65W5Y8Z+XmEElfqWAek8zsmIz2IyWdmLwM5WZV2683kntsXJO8/JSZve4eXWbWR9Lnk5f/5ZxbXo/ldqA5yXOhcjMwv7ST5C+xkooxGmTH5GZP28bMJkr6dPLy1g64X06jzEmeC5WbEeqY3ETTzEmei5abHNN2ty9UbrbymDa2Ys4Vkv4kf9frO83sBDMbVgqa2Xgzm2Fmd8mP9542S76IcYCkOWY2Lmkz3My+If+FWKYAM+syM2dmXXkdND9M597Jy4ousTKzWcm8Q3frrkR/Mxvdw6O/c26F/DqUpB+b2W5JH/qY2QHydwav5LrF5ZJ+ZWb/WvqxMrP3SLpV/syfv2jzIWW/KX+fntGS7jGzo8yPqqWk/XZmdqL8GT0fqPSDV7r+kkQ6Nnl5jXNubaXLQI8KmZvJNp6VvPy8mZ1iZlsk7d4t6Ub538E/OOd+kzHvmaXvlplN6nk1ZOpTQW6Wzpb7XfJ8gZlNSeXWHvI78Upu5rZM0o+S9d0vab+L/E3oxsgfCPygrE1vtl9QBevvm/K/JdvJ/55sl7QbI78z3kP+LxhnVrpMbKaQuSnpe2Z2kZntV7Yf2NbMvinp4mTSPOfcrRnzJjfVsNzc38xuN7OjzWx8qs1gMztK0h+Sz7tK3b+vqF5Rc7O0rV/LA3Wf0T2gLD82G16X3PQatd80f7ye3jbDU+FRZTHUppC5yTFtsXNTrTqmdc7V9ZF01knqqrJdV9JuZg/vGyt/apJLHhvlT/F9NTXNSTozo+1pqfgm+bufb0hefyevD6lY7ueSdE7yviWStqjws88q9asX67uSx9SkzV7yB2Gl6a+mXi+Wv6dOZn9S6+EU+Zsku6TtitT8XpL09kB/3yx/nWDpvRvkb4yV7o+T9NHA55xV6/qTP72tNP896/3dL/qjk3NT0mWp+a8r+74+JmmbQLuZqfdNqnK9zdLrP3feY2bSZnv5a65L01en1t8q+dNnM/uT2r7nyu/EnKQ18jvCUpuVkvav5/ZLfc45tay/JC9Xpt63NPkOOPlRKo5tde6Qm/XPTb1+37Uxme+ysv50SRpJbjY3NyVNLZvvSvn99IbUtEWS3tfq3CE3G7PfrCJHqvpukZu9y80kPqnSddTq/CE365+byXs4pi1gbibvafoxbWxn5sj5mydNkTRD0i3yX5KhSfhvkn4q6Sj56lh522/LXz83T35j9pO/ru4jzrnNTlerlvkzP45LXl7rKj/zo3Td4/zcd9WJc+5eSe+WNFf+cqP+8gWYS+Xvkv5QBbNZLH/vme/KXyM4QP5O3j+StJtzLvPGTs65f0qaLOlT8tvhFfm/KmyQ9LD8D9Qhasy9hj6aPP/NOXdfA+bf0Yqcm865EyV9uGz+f5Mvvu7mnHsu0LSUm8+qCaMCOOcel8+rn8vnZF/5HcEvJO3hnKtkeOa18v8ZO0vSk/K5+bL86Z/vdM79PrDsmrdfbzjnfif/u3OlpGfkh3V8Uf7Mvr2dc0W771h0Cpqbl0g6X9I98vuOgfJ//X9a/q+LR0l6r/OnrWchN71G5Ob/SPoPSb+W/6PNOvn99DL57XWGpB0r/MzIUdDc7C1y02vYfhONV+Tc5Ji2uLnZimNaS6pIaCEz+5ukt0k61GWcFgegNczsNknvl78T/8U9vR9Ac5CbQDGRm0AxkZvtiWJOiyXXUr4g6QHn3O6t7g8Az8xKf0FYLmn7Ks60A9BA5CZQTOQmUEzkZvuK7jKrNrR/8nxWS3sBoNw7JQ2RdB47PaBQyE2gmMhNoJjIzTbFmTkAAAAAAAAR4cwcAAAAAACAiFDMAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiEi/WhuamatnR4Aic85Zq/tQKXITnYTcBIqJ3ASKidwEiqmW3OTMHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFDMAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIhIv1Z3AAAAAEBn2XLLLYOxrq6uzOmjRo0Kttl3332Dseeff77ifgFALDgzBwAAAAAAICIUcwAAAAAAACJCMQcAAAAAACAiFHMAAAAAAAAiQjEHAAAAAAAgIhRzAAAAAAAAIsLQ5ABQ5rDDDsucPnz48GCbD37wg8HYo48+mjn9kUceCbbZa6+9grHddtstGLv88sszp1955ZXBNgAAxGC77bYLxvKGLWdocgDtiDNzAAAAAAAAIkIxBwAAAAAAICIUcwAAAAAAACJCMQcAAAAAACAiFHMAAAAAAAAiEsVoVjvssEMwdv/992dOX7p0abDNzjvvHIwtW7as8o4BKLTDDz88GDvzzDODsdBvhJkF2yxfvjwY23///TOn542OtXjx4mBs7dq1wVhoNKvJkycH25x66qnB2Lp164IxAABq9Za3vCUYGz9+fBN7AqBSw4YNy5x+/PHHB9tccMEFwdgvfvGLYOySSy7JnP6HP/wh2KbTcGYOAAAAAABARCjmAAAAAAAARIRiDgAAAAAAQEQo5gAAAAAAAESEYg4AAAAAAEBEKOYAAAAAAABEJIqhyZ955plg7NJLL82cfvLJJwfbvPvd7w7Gurq6MqePHDky2GbgwIHB2PPPPx+MrV69OhgD8Hp77rln5vRzzjkn2GbfffcNxrbYYotg7LHHHsucftFFFwXb3HHHHcGYcy5z+sSJE6vugyStXLkyGPvNb36TOf3Tn/50sM0TTzwRjF144YXBGIC47LDDDsFY3rHWq6++2ojuoMNNnz49GNt6662b2BOg82y55ZbB2K677hqMXXXVVZnTt91222Cb0HGwlP87cMghh2ROv+GGG4Jtrr/++mDs9ttvD8ZixZk5AAAAAAAAEaGYAwAAAAAAEBGKOQAAAAAAABGhmAMAAAAAABARijkAAAAAAAARoZgDAAAAAAAQEcsbKiy3oVltDZtk//33D8YuuOCCYGzChAmZ08ePH19TP1555ZVg7KSTTsqcvmjRopqWVYtJkyYFY3nDFYeGcG9XzjlrdR8qVfTczLPTTjsFY3fddVfm9K222irY5umnnw7Gjj766GDskUceyZwew/C8oaEhFy5cGGyzYsWKYGzq1KmZ0x988MFqutUw5CZiMWzYsMzpY8eODbb58Ic/HIwdeeSRmdMHDhwYbJO3z3/ppZeCscceeyxz+rRp04JtyE305LzzzgvGvvCFL1Q9v9122y0YC+3XOxG5CUmaP39+MDZ58uRgzCz761NrTaEWoT5I0tq1a4OxKVOmZE6/7777et2neqglNzkzBwAAAAAAICIUcwAAAAAAACJCMQcAAAAAACAiFHMAAAAAAAAiQjEHAAAAAAAgIv1a3YFG+f3vfx+Mve997wvGpk+fnjn9gAMOCLbZtGlTMJY3qtbVV1+dOb1v377BNs2Ud+f/d7zjHU3sCdrJlltuGYydffbZwVho1Kq5c+cG2xx33HHB2KpVq4KxmIVGpLnnnnuCbfbZZ59gLJTrRRnNCmiEfv3Ch0fvec97grE999wzGAuNYPnGN76x8o5VYPHixcHYn/70p2Ds/vvvD8bOOeecXvUJyJI3Ik0o9uijjwbbvPjii73uE1BUgwcPDsY+//nPB2OHH3545vRdd921131qtCeffDJzet5vx3bbbReMhdbTMcccU13HCoQzcwAAAAAAACJCMQcAAAAAACAiFHMAAAAAAAAiQjEHAAAAAAAgIhRzAAAAAAAAIkIxBwAAAAAAICJtOzR5niVLlgRjF198cVXTe2PatGmZ04cPHx5sc+SRRwZjw4YNy5x+4IEHBtsMHDgwGPvGN74RjAG1Ou2004Kxww47LBi76aabMqd34vDjedauXZs5/YILLgi2yRuafLfddsuc/rOf/ay6jgEtMmDAgGDskEMOyZx++umnB9vkDT+eZ+PGjZnT16xZE2xz7bXXBmPXXXdd5vR58+YF23TibyKKa/To0cGYcy5z+sKFC4NtXn755d52CSisP/7xj8HYzjvv3MSeZHv44YeDsfnz5wdjV111VTC2YMGCqvvx0EMPBWNbbLFF1fMrOs7MAQAAAAAAiAjFHAAAAAAAgIhQzAEAAAAAAIgIxRwAAAAAAICIUMwBAAAAAACICMUcAAAAAACAiFho6L8eG5rV1hANM3Xq1Mzpd9xxR7BNnz7het6YMWOCsUWLFlXcr3bgnLNW96FSRc/NBx98MBhbvXp1MBb6foeG4sbrjR8/Phh74IEHgrF+/fplTt9jjz2CbZ588snKO9ZL5GZnOeusszKn77LLLsE273znO4Ox/9/e3YZWXbdxAP9PM3rSkvWkPSxTSpklgWVYIhS9kLAIZxb2RJaEQkbIMiukpKxJUJZE9iRChZolBMNKJ2SICBuZYmEKDovSymgKqdl2v7jjfnHf/+vcOzs72347n8/L68u1XT3885yrP1yXXHJJ0TMUOn88e/bsMDtw4EBuvSunV1Pg2STLsmz69OlhtmrVqjCLPmc+++yzYc/bb7/d+cEqmGez9w0bNiy3vnr16rBn0qRJYdbV7/ORdevWhVmhZ7ovqK+vD7MXX3wxt17oO+9vv/1W8kyd1ZVn05s5AAAAAAmxzAEAAABIiGUOAAAAQEIscwAAAAASYpkDAAAAkBDLHAAAAICE5N+cJUmjR4/OrRc6P75169Ywa2trK3kmKtfTTz+dW6+trQ17orPDWeYEeal+/vnnMDt69GiYjRw5Mrc+dOjQsKcnT5PT/0SnQ7Msy5544olu/V1ffvllbv3ll18OexobG8Ps5MmTJc8E/UlNTU2YtbS0hNmVV16ZW/fnC/3Bgw8+mFu/4YYbwp5C58cLZbt27cqtF/rMXejPuZRFf59mzpwZ9ixbtqxc43QLb+YAAAAAJMQyBwAAACAhljkAAAAACbHMAQAAAEiIZQ4AAABAQlyz6kduvvnmonsOHToUZidOnChlHCrAGWecEWbTp0/PrW/bti3sWbx4cckz0TMefvjhMJs7d24PTkKKGhoawuzxxx8Ps/b29tz6mjVrwp6lS5eG2e7du3Prx44dC3uAznvkkUfCbMSIEWFW6DoPpGDChAlhNmTIkNz64cOHw57q6uowK3R9Kvq8VujKaaWZMmVKmLlmBQAAAEC3scwBAAAASIhlDgAAAEBCLHMAAAAAEmKZAwAAAJAQyxwAAACAhDhNDnTZggULwmzs2LG59fnz55drHLpo3bp1YVZfX9+Dk1ApbrvttjAbOHBgmL355pu59ULnj4HyO/3003PrgwcPDnsGDIj/n/LXX3+dW//iiy+KGwx6yfDhw8Ns+/btufV58+aFPfv37w+z+++/P8wKnTvn35qamnp7hC7zZg4AAABAQixzAAAAABJimQMAAACQEMscAAAAgIRY5gAAAAAkxDIHAAAAICFOkyfm7LPPDrNp06bl1k+ePBn2NDY2ljwTleupp54quuett94qwySUYv369WEWnSa/8cYbyzUOhKqqqnLr5513Xtjzyy+/lGsc4B91dXW59erq6rCnvb09zA4dOlTyTNCbPvnkkzAbPXp0bn3QoEFhz4oVK8LM+fHOaW1tza3v3LmzhyfpPt7MAQAAAEiIZQ4AAABAQixzAAAAABJimQMAAACQEMscAAAAgIS4ZpWY6JJHoazQJQ+XhSjFgAHxPji6UlHoegW9Y9euXUX3jB07tgyTUClOnDjRpb7Zs2fn1q+99tqwZ/z48WHmv0fQPWpqanLrH330Udhz++23h1lDQ0PJM0FfFV0ELfQ97+OPPy7XOMkZMmRImM2ZMyfMVq9enVvfsGFDyTP1Fm/mAAAAACTEMgcAAAAgIZY5AAAAAAmxzAEAAABIiGUOAAAAQEIscwAAAAAS4jR5Yi644IKiew4ePFiGSSDLjhw5EmZnnnlmbn3ChAlhz+bNm0ueieINHjy4t0egwrzzzjthFp1szbIsq6ury61XV1eHPZdffnmY7d27N8yAzps2bVpufdiwYWHP77//HmabNm0qeSboTaNGjQqzRYsW5da3b98e9uzZs6fkmfqLJUuWhFmhzwOFvrekyps5AAAAAAmxzAEAAABIiGUOAAAAQEIscwAAAAASYpkDAAAAkBDLHAAAAICEOE2emKlTpxbd8+mnn5ZhEsiymTNnhtn69etz68uXLw97XnvttTB79913c+vHjx8Pe+ichQsXFt3T2NhYhknoqwqdF77++utz6xs2bAh7Xn311TDbvHlzmE2ePDm3fumll4Y99fX1YTZ79uwwA0p37rnnhtlzzz3Xg5NAz6qtrQ2z4cOH59ZnzJhRrnGSNGvWrNz6nXfeGfbMnTs3zFatWlXyTH2NN3MAAAAAEmKZAwAAAJAQyxwAAACAhFjmAAAAACTEMgcAAAAgIZY5AAAAAAlxmhzoskKnhx999NHc+vPPPx/2vP7662E2adKk3Pq8efPCnsOHD4fZ33//HWYpGzAgf0d/9dVXhz1333130b9nz549RfeQroaGhjC75557cuvHjx8Pe3bu3Blma9euDbP3338/t/7YY4+FPVOmTAmzkSNH5tb37dsX9kB/Nm7cuDBraWnp1t/lNDmpi06MZ1mW3XvvvWF26NCh3Hpra2vJM6Vm/PjxYfbAAw/k1qPvGFmWZc3NzaWOlBRv5gAAAAAkxDIHAAAAICGWOQAAAAAJscwBAAAASIhlDgAAAEBCXLOqANdcc01vj0A/9ddff4XZ8uXLc+tbtmwJe5YsWRJmM2bMKKqeZVn2wgsvhFk0X5Zl2WWXXZZb37ZtW9jT3UaNGhVm0XxZlmUTJ07MrS9atKhLczQ1NeXWXSGpLFu3bg2z6PrNVVddFfYUul5RKOuKiy++OMzGjBmTW3fNCv5XR0dH0T2NjY1lmAT6hg8++CDMoiusWZZl3333XW79xx9/LHmmvqimpibM6urqwuzWW2/Nrbe1tZU8U3/hzRwAAACAhFjmAAAAACTEMgcAAAAgIZY5AAAAAAmxzAEAAABIiGUOAAAAQEKcJq8Av/76a2+PAP/xzTffhFmhM+NTp07NrS9cuDDsefLJJ8Ns1qxZYTZ06NDc+oEDB8Ke7lZdXR1m55xzTtE/r729PcxWrFgRZgsWLMitHzlypOgZSNcbb7wRZitXrsytX3TRRWFP9O9VlmXZ5MmTw2zUqFFhFjl69GiYDR48uOifB5WqpaUlzK644orc+rfffluucaDXFfoM+tVXX4VZVVVVOcbpVaeddlqY3XTTTWH20ksvlWOciuHNHAAAAICEWOYAAAAAJMQyBwAAACAhljkAAAAACbHMAQAAAEiIZQ4AAABAQpwmrwB//PFHb48AnVLohPCHH35YVD3LsuyOO+4Is7POOivMxo4dm1ufOHFi2NPdDh48GGb79+8Ps88//zy3/tNPP4U9Gzdu7PRc8N/+/PPP3PrevXvDnoceeijMBg0aFGa1tbWdH+wfR44cCbN9+/YV/fOgUm3atCnMzj///Nz6qlWryjUO9LodO3aE2ffffx9mI0aMyK3v3r077Fm5cmWn5+qMzz77LMwK/XVFjh07FmbvvfdemJ166qlhNnDgwKLnqDTezAEAAABIiGUOAAAAQEIscwAAAAASYpkDAAAAkBDLHAAAAICEVHV0dHStsaqqa42UZP78+WG2dOnS3PqJEyfCnoaGhjB75plnOj9YP9fR0VHV2zN0lmeTSuLZhL7Js5mmcePGhVlzc3OYRVd4Cl2BLHTBkvLxbPaM6667Lszuuuuu3Pq8efPCnqqq+B9boe/za9euza3fd999YU+h747draamJsxaW1t7bI6+oCvPpjdzAAAAABJimQMAAACQEMscAAAAgIRY5gAAAAAkxDIHAAAAICGWOQAAAAAJOaW3B6D8BgyId3ZNTU09OAkAAH3Vjh07wuyUU3xtgM7avn17mDU3N+fW16xZE/a0tbWF2bFjx8Lshx9+yK0XOj9e6Fz4LbfcklsfM2ZM2HPhhReG2Zw5c8KM/8+bOQAAAAAJscwBAAAASIhlDgAAAEBCLHMAAAAAEmKZAwAAAJAQyxwAAACAhFR1dHR0rbGqqmuNlKTQqbjNmzfn1pctWxb2vPLKKyXPVAk6OjqqenuGzvJsUkk8m9A3eTahb/Js8v9s2bIlzNrb23PrixcvDns2btxY8kyVoCvPpjdzAAAAABJimQMAAACQEMscAAAAgIRY5gAAAAAkxDIHAAAAICGWOQAAAAAJcZocOsEZR+ibPJvQN3k2oW/ybELf5DQ5AAAAQD9nmQMAAACQEMscAAAAgIRY5gAAAAAkxDIHAAAAICGWOQAAAAAJscwBAAAASIhlDgAAAEBCLHMAAAAAEmKZAwAAAJAQyxwAAACAhFjmAAAAACSkqqOjo7dnAAAAAKCTvJkDAAAAkBDLHAAAAICEWOYAAAAAJMQyBwAAACAhljkAAAAACbHMAQAAAEiIZQ4AAABAQixzAAAAABJimQMAAACQEMscAAAAgIRY5gAAAAAkxDIHAAAAICGWOQAAAAAJ+Rc0q4PsbZql9wAAAABJRU5ErkJggg==\n","text/plain":["
"]},"metadata":{"tags":[],"image/png":{"width":569,"height":573},"needs_background":"light"}},{"output_type":"stream","text":["Done\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"zkVkpsQymNuC","executionInfo":{"status":"ok","timestamp":1609935273694,"user_tz":-540,"elapsed":68894,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}}},"source":[""],"execution_count":10,"outputs":[]}]} -------------------------------------------------------------------------------- /notebook/mdn-cls.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"mdn-cls.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyNn21QiDcBe8s1K9JAV/khs"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"2rZ1UcDezAyc"},"source":["# MDN for Classification"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"28QgYFqVzAP7","executionInfo":{"status":"ok","timestamp":1610108868970,"user_tz":-540,"elapsed":998,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"83c444ad-f74b-4ec0-dd84-fb003061d063"},"source":["import math\n","import matplotlib.pyplot as plt\n","import numpy as np\n","import torch\n","import torch.nn as nn\n","import torch.optim as optim\n","import torch.nn.functional as F\n","import torch.distributions as TD\n","from torch.autograd import Variable\n","from collections import OrderedDict\n","%matplotlib inline\n","%config InlineBackend.figure_format='retina'\n","np.set_printoptions(precision=3)\n","torch.set_printoptions(precision=3)\n","print (\"PyTorch version:[%s].\"%(torch.__version__))\n","device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n","print (\"device:[%s].\"%(device))"],"execution_count":1,"outputs":[{"output_type":"stream","text":["PyTorch version:[1.7.0+cu101].\n","device:[cuda:0].\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"eQ5ZZvu4qoOv"},"source":["### Helper functions"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6o1j8QgKqqIG","executionInfo":{"status":"ok","timestamp":1610108869305,"user_tz":-540,"elapsed":1313,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"7d7589c9-8f22-4b9f-bcfb-a908c0529e94"},"source":["# Codes copied from 'https://github.com/sksq96/pytorch-summary/tree/master/torchsummary' \n","def summary_string(model, input_size, batch_size=-1, device=torch.device('cuda:0'), dtypes=None):\n"," if dtypes == None:\n"," dtypes = [torch.FloatTensor]*len(input_size)\n"," summary_str = ''\n"," def register_hook(module):\n"," def hook(module, input, output):\n"," class_name = str(module.__class__).split(\".\")[-1].split(\"'\")[0]\n"," module_idx = len(summary)\n","\n"," m_key = \"%s-%i\" % (class_name, module_idx + 1)\n"," summary[m_key] = OrderedDict()\n"," summary[m_key][\"input_shape\"] = list(input[0].size())\n"," summary[m_key][\"input_shape\"][0] = batch_size\n"," if isinstance(output, (list, tuple)):\n"," summary[m_key][\"output_shape\"] = [\n"," [-1] + list(o.size())[1:] for o in output\n"," ]\n"," else:\n"," summary[m_key][\"output_shape\"] = list(output.size())\n"," summary[m_key][\"output_shape\"][0] = batch_size\n","\n"," params = 0\n"," if hasattr(module, \"weight\") and hasattr(module.weight, \"size\"):\n"," params += torch.prod(torch.LongTensor(list(module.weight.size())))\n"," summary[m_key][\"trainable\"] = module.weight.requires_grad\n"," if hasattr(module, \"bias\") and hasattr(module.bias, \"size\"):\n"," params += torch.prod(torch.LongTensor(list(module.bias.size())))\n"," summary[m_key][\"nb_params\"] = params\n","\n"," if (\n"," not isinstance(module, nn.Sequential)\n"," and not isinstance(module, nn.ModuleList)\n"," ):\n"," hooks.append(module.register_forward_hook(hook))\n","\n"," # multiple inputs to the network\n"," if isinstance(input_size, tuple):\n"," input_size = [input_size]\n","\n"," # batch_size of 2 for batchnorm\n"," x = [torch.rand(2, *in_size).type(dtype).to(device=device)\n"," for in_size, dtype in zip(input_size, dtypes)]\n","\n"," # create properties\n"," summary = OrderedDict()\n"," hooks = []\n","\n"," # register hook\n"," model.apply(register_hook)\n","\n"," # make a forward pass\n"," # print(x.shape)\n"," model(*x)\n","\n"," # remove these hooks\n"," for h in hooks:\n"," h.remove()\n","\n"," summary_str += \"----------------------------------------------------------------\" + \"\\n\"\n"," line_new = \"{:>20} {:>25} {:>15}\".format(\n"," \"Layer (type)\", \"Output Shape\", \"Param #\")\n"," summary_str += line_new + \"\\n\"\n"," summary_str += \"================================================================\" + \"\\n\"\n"," total_params = 0\n"," total_output = 0\n"," trainable_params = 0\n"," for layer in summary:\n"," # input_shape, output_shape, trainable, nb_params\n"," line_new = \"{:>20} {:>25} {:>15}\".format(\n"," layer,\n"," str(summary[layer][\"output_shape\"]),\n"," \"{0:,}\".format(summary[layer][\"nb_params\"]),\n"," )\n"," total_params += summary[layer][\"nb_params\"]\n"," summary_str += line_new + \"\\n\"\n"," # return summary\n"," return summary_str,summary\n","print (\"Done.\")"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Done.\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"pm-jmaWi9Ovh"},"source":["## $\\color{yellow}{\\text{Mixture Logits Network (MLN) }}$ \n","- \n","`Cross Entropy Loss`\n","$ \\mathcal{L}_{\\text{CE}} = \n"," -\\sum_{d=1}^{D} y_d \\log(\\hat{\\mu}_d)\n","$\n","where $y \\in [0,1]^d$ is the target and $\\hat{\\mu} \\in \\mathbb{S}^d$ is the prediction result.\n","- `Weighted CE Loss`\n","$ \\mathcal{L}_{\\text{WCE}} = \n"," -\n"," \\sum_{k=1}^{K}\n"," \\hat{\\pi}_k\n"," \\sum_{d=1}^{D} y_d \\log(\\hat{\\mu}_d)\n","$\n","where $\\hat{\\pi}$, $\\hat{\\mu}$, and $y$ are mixture weights,\n","output predicitons, and labels, respectively. \n","- \n","`Gal Loss`\n","$\n"," \\mathcal{L}_{\\text{Gal}} \n"," = \\log \\frac{1}{T} \\sum_{t}\n"," \\exp \\left(\n"," \\hat{x}_{t,c} - \\log \\sum_{c'} \\exp \\hat{x}_{t,c'}\n"," \\right)\n","$\n","where $\\hat{x_t} = f^{W} + \\sigma^{W}\\epsilon_t, ~ \\epsilon_t \\sim \\mathcal{N}(0,I)$.\n","- \n","`Mixture of Attenuated CE Loss`\n","$ \\mathcal{L}_{\\text{MACE}} \n"," =\n"," -\n"," \\sum_{k=1}^{K}\n"," \\hat{\\pi}_k\n"," \\sum_{d=1}^{D}\n"," \\frac\n"," {y_d \\log(\\hat{\\mu}_{d,k})}\n"," {\\hat{\\sigma}_{d,k} + \\sigma_{\\text{min}}}\n","$\n","where $\\sigma_{\\text{min}}=1.0$ is the minimum standard deviation.\n"]},{"cell_type":"code","metadata":{"id":"1dhslad3y48u","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1610108873362,"user_tz":-540,"elapsed":5352,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"6423fb0c-7d51-47a0-af99-d1523f3baa3b"},"source":["class MixturesOfLogits(nn.Module):\n"," \"\"\"\n"," Mixture of Logits \n"," \"\"\"\n"," def __init__(self,\n"," in_dim = 64, # input feature dimension\n"," y_dim = 10, # output dimension\n"," k = 5, # number of mixtures\n"," sig_min = 1, # minimum sigma\n"," sig_max = None # maximum signa \n"," ):\n"," super(MixturesOfLogits,self).__init__()\n"," self.in_dim = in_dim\n"," self.y_dim = y_dim\n"," self.k = k\n"," self.sig_min = sig_min\n"," self.sig_max = sig_max\n"," self.fc_pi = nn.Linear(self.in_dim,self.k)\n"," self.fc_mu = nn.Linear(self.in_dim,self.k*self.y_dim)\n"," self.fc_sigma = nn.Linear(self.in_dim,self.k*self.y_dim)\n","\n"," def forward(self,x):\n"," pi_logit = self.fc_pi(x) # [N x K]\n"," pi = torch.softmax(pi_logit,dim=1) # [N x K]\n"," mu = self.fc_mu(x) # [N x KD]\n"," mu = torch.reshape(mu,(-1,self.k,self.y_dim)) # [N x K x D]\n"," sigma = self.fc_sigma(x) # [N x KD]\n"," sigma = torch.reshape(sigma,(-1,self.k,self.y_dim)) # [N x K x D]\n"," if self.sig_max is None:\n"," sigma = self.sig_min + torch.exp(sigma) # [N x K x D]\n"," else:\n"," sigma = self.sig_min + (self.sig_max-self.sig_min)*torch.sigmoid(sigma) # [N x K x D]\n"," return pi,mu,sigma\n","\n","class MixtureLogitNetwork(nn.Module):\n"," def __init__(self,\n"," name='mln',\n"," x_dim = [1,28,28], # iput dimension \n"," k_size = 3, # kernel size\n"," c_dims = [32,64], # channel dimensions for conv layer(s)\n"," p_sizes = [2,2], # pooling sizes\n"," h_dims = [128], # hidden dimensions for dense layer(s)\n"," y_dim = 10, # output dimension\n"," USE_BN = True, # whether to use batch norm \n"," k = 5, # number of mixtures\n"," sig_min = 1, # $\\sigma_{min}$\n"," sig_max = None, # $\\sigma_{max}$\n"," mu_min = -3, # minimum $\\mu$ while initializing bias \n"," mu_max = +3, # maximum $\\mu$ while initializing bias \n"," ):\n"," super(MixtureLogitNetwork,self).__init__()\n"," self.name = name\n"," self.x_dim = x_dim\n"," self.k_size = k_size\n"," self.c_dims = c_dims\n"," self.p_sizes = p_sizes\n"," self.h_dims = h_dims\n"," self.y_dim = y_dim\n"," self.USE_BN = USE_BN\n"," self.k = k\n"," self.sig_min = sig_min\n"," self.sig_max = sig_max\n"," self.mu_min = mu_min\n"," self.mu_max = mu_max\n","\n"," # Build graph\n"," self.build_graph()\n","\n"," # Initialize parameters \n"," self.init_param() \n","\n"," def build_graph(self):\n"," self.layers = []\n"," # Conv layers\n"," prev_c_dim = self.x_dim[0] # input channel \n"," for (c_dim,p_size) in zip(self.c_dims,self.p_sizes):\n"," self.layers.append(\n"," nn.Conv2d(\n"," in_channels = prev_c_dim,\n"," out_channels = c_dim,\n"," kernel_size = self.k_size,\n"," stride = (1,1),\n"," padding = self.k_size//2\n"," ) # conv\n"," )\n"," if self.USE_BN:\n"," self.layers.append(\n"," nn.BatchNorm2d(num_features=c_dim)\n"," )\n"," self.layers.append(nn.ReLU())\n"," self.layers.append(\n"," nn.MaxPool2d(kernel_size=(p_size,p_size),stride=(p_size,p_size))\n"," )\n"," # self.layers.append(nn.Dropout2d(p=0.1)) # p: to be zero-ed\n"," prev_c_dim = c_dim \n"," # Dense layers\n"," self.layers.append(nn.Flatten())\n"," p_prod = np.prod(self.p_sizes)\n"," prev_h_dim = prev_c_dim*(self.x_dim[1]//p_prod)*(self.x_dim[2]//p_prod)\n"," for h_dim in self.h_dims:\n"," self.layers.append(\n"," nn.Linear(\n"," in_features = prev_h_dim,\n"," out_features = h_dim,\n"," bias = True\n"," )\n"," )\n"," self.layers.append(nn.ReLU(True)) # activation\n"," self.layers.append(nn.Dropout2d(p=0.1)) # p: to be zero-ed\n"," prev_h_dim = h_dim\n"," # Final mixture of logits layer\n"," mol = MixturesOfLogits(\n"," in_dim = prev_h_dim, \n"," y_dim = self.y_dim, \n"," k = self.k,\n"," sig_min = self.sig_min,\n"," sig_max = self.sig_max\n"," )\n"," self.layers.append(mol)\n","\n"," # Concatanate all layers\n"," self.net = nn.Sequential()\n"," for l_idx,layer in enumerate(self.layers):\n"," layer_name = \"%s_%02d\"%(type(layer).__name__.lower(),l_idx)\n"," self.net.add_module(layer_name,layer)\n","\n"," def init_param(self): \n"," for m in self.modules():\n"," if isinstance(m,nn.Conv2d): # init conv\n"," nn.init.kaiming_normal_(m.weight)\n"," nn.init.zeros_(m.bias)\n"," if isinstance(m,nn.Linear): # lnit dense\n"," nn.init.kaiming_normal_(m.weight)\n"," nn.init.zeros_(m.bias)\n"," \"\"\"\n"," Heuristic: fc_mu.bias ~ Uniform(mu_min,mu_max)\n"," \"\"\"\n"," self.layers[-1].fc_mu.bias.data.uniform_(self.mu_min,self.mu_max)\n","\n"," def forward(self,x):\n"," return self.net(x)\n","\n","# Instantiate mixture of logits layer \n","M = MixtureLogitNetwork(\n"," name='mln',x_dim=[1,28,28],k_size=3,c_dims=[32,64],p_sizes=[2,2],\n"," h_dims=[128],y_dim=10,USE_BN=True,\n"," k=3,sig_min=1,sig_max=None,\n"," mu_min=-3,mu_max =+3).to(device)\n","print (\"Done.\")"],"execution_count":3,"outputs":[{"output_type":"stream","text":["Done.\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"6GE5bkPJjNG3"},"source":["## $\\color{yellow}{\\text{Loss function}}$ \n","`Mixture of Attenuated CE Loss`\n","$ \\mathcal{L}_{\\text{MACE}} \n"," =\n"," \\sum_{k=1}^{K}\n"," \\hat{\\pi}_k\n"," \\sum_{d=1}^{D}\n"," \\frac\n"," {-y_d \\log(\\hat{\\mu}_{d,k})}\n"," {\\hat{\\sigma}_{d,k} \n"," }\n"," + \n"," \\frac{1}{D}\n"," \\sum_{d=1}^{D}\n"," \\sum_{k=1}^{K}\n"," \\hat{\\pi}_k \\hat{\\sigma}_{d,k}\n","$"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GZwN1fE3jMuk","executionInfo":{"status":"ok","timestamp":1610108873362,"user_tz":-540,"elapsed":5334,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"3c6d8da0-fa72-499c-8876-9ea96fab6820"},"source":["def np2tc(x_np): return torch.from_numpy(x_np).float().to(device)\n","def tc2np(x_tc): return x_tc.detach().cpu().numpy()\n","\n","def mdn_gather(pi,mu,sigma):\n"," \"\"\"\n"," pi: [N x K]\n"," mu: [N x K x D]\n"," sigma: [N x K x D]\n"," \"\"\"\n"," max_idx = torch.argmax(pi,dim=1) # [N]\n"," idx_gather = max_idx.unsqueeze(dim=-1).repeat(1,mu.shape[2]).unsqueeze(1) # [N x 1 x D]\n"," mu_sel = torch.gather(mu,dim=1,index=idx_gather).squeeze(dim=1) # [N x D]\n"," sigma_sel = torch.gather(sigma,dim=1,index=idx_gather).squeeze(dim=1) # [N x D]\n"," out = {'max_idx':max_idx,'idx_gather':idx_gather,\n"," 'mu_sel':mu_sel,'sigma_sel':sigma_sel}\n"," return out\n","\n","def mace_loss(pi,mu,sigma,target,alea_weight=1.0):\n"," \"\"\"\n"," Mixture of attenuated CE loss\n"," pi: [N x K]\n"," mu: [N x K x D]\n"," sigma: [N x K x D]\n"," target: [N x D]\n"," \"\"\"\n"," # softmax \\mu\n"," mu_hat = torch.softmax(mu,dim=2) # logit to prob [N x K x D]\n"," log_mu_hat = torch.log(mu_hat+1e-5) # [N x K x D]\n"," \n"," # Expanded \\pi \n"," pi_usq = torch.unsqueeze(pi,2) # [N x K x 1]\n"," pi_exp = pi_usq.expand_as(sigma) # [N x K x D]\n","\n"," # Expanded target\n"," target_usq = torch.unsqueeze(target,1) # [N x 1 x D]\n"," target_exp = target_usq.expand_as(sigma) # [N x K x D]\n","\n"," # Loss\n"," # ce_loss_exp = -target_exp*log_mu_hat # [N x K x D]\n"," ce_loss_exp = -target_exp*log_mu_hat # [N x K x D]\n"," atte_ce = ce_loss_exp / sigma # attenuated CE loss [N x K x D]\n"," waces = torch.sum(torch.mul(pi_exp,atte_ce),dim=1) # weighted attenuated CE loss [N x D]\n"," wace = torch.mean(waces,dim=1) # N\n"," aleas = alea_weight*torch.sum(pi_exp*sigma,dim=1)# aleatoric uncertainty [N x D]\n"," alea = torch.mean(aleas,dim=1) # [N]\n","\n"," # Accumulate loss \n"," loss = wace + alea # [N]\n","\n"," # Average loss\n"," wace_avg = torch.mean(wace) # [1]\n"," alea_avg = torch.mean(alea) # [1]\n"," loss_avg = torch.mean(loss) # [1]\n","\n","\n"," out = {'mu_hat':mu_hat,'log_mu_hat':log_mu_hat,\n"," 'pi_usq':pi_usq,'pi_exp':pi_exp,\n"," 'sigma':sigma,\n"," 'target_usq':target_usq,'target_exp':target_exp,\n"," 'ce_loss_exp':ce_loss_exp,'atte_ce':atte_ce,\n"," 'waces':waces,'wace':wace,\n"," 'aleas':aleas,'alea':alea,\n"," 'loss':loss,\n"," 'wace_avg':wace_avg,'alea_avg':alea_avg,'loss_avg':loss_avg}\n"," return out\n","\n","def gmm_uncertainties(pi, mu, sigma):\n"," # Compute Epistemic Uncertainty\n"," M = 0.1\n"," # pi = torch.softmax(M*pi,1) # (optional) heuristics \n"," pi_usq = torch.unsqueeze(pi,2) # [N x K x 1]\n"," pi_exp = pi_usq.expand_as(sigma) # [N x K x D]\n","\n"," # For classification problems, we use softmax(mu) instead of me\n"," mu = torch.softmax(mu,dim=2) # logit to prob [N x K x D]\n","\n"," mu_avg = torch.sum(torch.mul(pi_exp,mu),dim=1).unsqueeze(1) # [N x 1 x D]\n"," mu_exp = mu_avg.expand_as(mu) # [N x K x D]\n"," mu_diff_sq = torch.square(mu-mu_exp) # [N x K x D]\n"," epis_unct = torch.sum(torch.mul(pi_exp,mu_diff_sq), dim=1) # [N x D]\n","\n"," # Compute Aleatoric Uncertainty\n"," alea_unct = torch.sum(torch.mul(pi_exp,sigma), dim=1) # [N x D]\n","\n"," # Sqaure root \n"," epis_unct = torch.sqrt(epis_unct) # [N x D]\n"," alea_unct = torch.sqrt(alea_unct) # [N x D]\n"," epis_unct_avg = torch.mean(epis_unct,dim=1) # [N]\n"," alea_unct_avg = torch.mean(alea_unct,dim=1) # [N]\n","\n"," # Out\n"," unct_out = {'epis_unct':epis_unct,'alea_unct':alea_unct,\n"," 'epis_unct_avg':epis_unct_avg,'alea_unct_avg':alea_unct_avg}\n","\n"," return unct_out\n"," \n","# Demo run to check the loss \n","M = MixtureLogitNetwork(\n"," name='mln',x_dim=[1,28,28],k_size=3,c_dims=[32,64],p_sizes=[2,2],\n"," h_dims=[128],y_dim=10,USE_BN=True).to(device)\n","\n","x_np = np.random.rand(2,1,28,28)\n","x_tc = np2tc(x_np)\n","pi_tc,mu_tc,sigma_tc = M.forward(x_tc) # forward path of MLN\n","target_tc = F.one_hot(torch.randint(low=0,high=10,size=(2,)),num_classes=10).to(device) # random one-hot\n","out = mace_loss(pi_tc,mu_tc,sigma_tc,target_tc) # mixture of CE \n","unct_out = gmm_uncertainties(pi_tc,mu_tc,sigma_tc)\n","\n","print ('pi_tc: %s'%(tc2np(target_tc).shape,))\n","print ('mu_tc: %s'%(tc2np(mu_tc).shape,))\n","print ('sigma_tc: %s'%(tc2np(sigma_tc).shape,))\n","print ('target_tc: %s'%(tc2np(target_tc).shape,))\n","print ('=>')\n","print ('mu_hat: %s'%(tc2np(out['mu_hat']).shape,))\n","print ('log_mu_hat: %s'%(tc2np(out['log_mu_hat']).shape,))\n","print ('pi_usq: %s'%(tc2np(out['pi_usq']).shape,))\n","print ('pi_exp: %s'%(tc2np(out['pi_exp']).shape,))\n","print ('target_usq: %s'%(tc2np(out['target_usq']).shape,))\n","print ('target_exp: %s'%(tc2np(out['target_exp']).shape,))\n","print ('ce_loss_exp: %s'%(tc2np(out['ce_loss_exp']).shape,))\n","print ('atte_ce: %s'%(tc2np(out['atte_ce']).shape,))\n","print ('waces: %s'%(tc2np(out['waces']).shape,))\n","print ('wace: %s'%(tc2np(out['wace']).shape,))\n","print ('aleas: %s'%(tc2np(out['aleas']).shape,))\n","print ('alea: %s'%(tc2np(out['alea']).shape,))\n","print ('loss: %s'%(tc2np(out['loss']).shape,))"],"execution_count":4,"outputs":[{"output_type":"stream","text":["pi_tc: (2, 10)\n","mu_tc: (2, 5, 10)\n","sigma_tc: (2, 5, 10)\n","target_tc: (2, 10)\n","=>\n","mu_hat: (2, 5, 10)\n","log_mu_hat: (2, 5, 10)\n","pi_usq: (2, 5, 1)\n","pi_exp: (2, 5, 10)\n","target_usq: (2, 1, 10)\n","target_exp: (2, 5, 10)\n","ce_loss_exp: (2, 5, 10)\n","atte_ce: (2, 5, 10)\n","waces: (2, 10)\n","wace: (2,)\n","aleas: (2, 10)\n","alea: (2,)\n","loss: (2,)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"3xTbbFSlxwI0","executionInfo":{"status":"ok","timestamp":1610108873363,"user_tz":-540,"elapsed":5316,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}}},"source":[""],"execution_count":4,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"JnycQ-fdqyhR"},"source":["### Summarize the model"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_auYKcQ6R-ui","executionInfo":{"status":"ok","timestamp":1610108873364,"user_tz":-540,"elapsed":5302,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"1620a69d-89cd-4e09-a450-9af5e18e0a36"},"source":["summary_str,summary = summary_string(M,input_size=(1,28,28),device=device)\n","print (summary_str)"],"execution_count":5,"outputs":[{"output_type":"stream","text":["----------------------------------------------------------------\n"," Layer (type) Output Shape Param #\n","================================================================\n"," Conv2d-1 [-1, 32, 28, 28] 320\n"," BatchNorm2d-2 [-1, 32, 28, 28] 64\n"," ReLU-3 [-1, 32, 28, 28] 0\n"," MaxPool2d-4 [-1, 32, 14, 14] 0\n"," Conv2d-5 [-1, 64, 14, 14] 18,496\n"," BatchNorm2d-6 [-1, 64, 14, 14] 128\n"," ReLU-7 [-1, 64, 14, 14] 0\n"," MaxPool2d-8 [-1, 64, 7, 7] 0\n"," Flatten-9 [-1, 3136] 0\n"," Linear-10 [-1, 128] 401,536\n"," ReLU-11 [-1, 128] 0\n"," Dropout2d-12 [-1, 128] 0\n"," Linear-13 [-1, 5] 645\n"," Linear-14 [-1, 50] 6,450\n"," Linear-15 [-1, 50] 6,450\n"," MixturesOfLogits-16 [[-1, 5], [-1, 5, 10], [-1, 5, 10]] 0\n","MixtureLogitNetwork-17 [[-1, 5], [-1, 5, 10], [-1, 5, 10]] 0\n","\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"R5XcNuKwO6G7"},"source":["### Check parameters"]},{"cell_type":"code","metadata":{"id":"kyDpsBwk8qr-","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1610108873365,"user_tz":-540,"elapsed":5285,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"2b0a15a3-1829-4367-b8f1-66831a91a9d5"},"source":["n_param = 0\n","for p_idx,(param_name,param) in enumerate(M.named_parameters()):\n"," if param.requires_grad:\n"," param_numpy = param.detach().cpu().numpy() # to numpy array \n"," n_param += len(param_numpy.reshape(-1))\n"," print (\"[%02d] name:[%s] shape:[%s].\"%(p_idx,param_name,param_numpy.shape))\n"," print (\" first 3 values:%s\"%(param_numpy.reshape(-1)[:3]))\n","print (\"Total number of parameters:[%s].\"%(format(n_param,',d')))"],"execution_count":6,"outputs":[{"output_type":"stream","text":["[00] name:[net.conv2d_00.weight] shape:[(32, 1, 3, 3)].\n"," first 3 values:[-0.481 -0.531 -1.441]\n","[01] name:[net.conv2d_00.bias] shape:[(32,)].\n"," first 3 values:[0. 0. 0.]\n","[02] name:[net.batchnorm2d_01.weight] shape:[(32,)].\n"," first 3 values:[1. 1. 1.]\n","[03] name:[net.batchnorm2d_01.bias] shape:[(32,)].\n"," first 3 values:[0. 0. 0.]\n","[04] name:[net.conv2d_04.weight] shape:[(64, 32, 3, 3)].\n"," first 3 values:[-0.028 0.012 -0.068]\n","[05] name:[net.conv2d_04.bias] shape:[(64,)].\n"," first 3 values:[0. 0. 0.]\n","[06] name:[net.batchnorm2d_05.weight] shape:[(64,)].\n"," first 3 values:[1. 1. 1.]\n","[07] name:[net.batchnorm2d_05.bias] shape:[(64,)].\n"," first 3 values:[0. 0. 0.]\n","[08] name:[net.linear_09.weight] shape:[(128, 3136)].\n"," first 3 values:[ 0.023 0.041 -0.054]\n","[09] name:[net.linear_09.bias] shape:[(128,)].\n"," first 3 values:[0. 0. 0.]\n","[10] name:[net.mixturesoflogits_12.fc_pi.weight] shape:[(5, 128)].\n"," first 3 values:[-0.03 0.151 0.103]\n","[11] name:[net.mixturesoflogits_12.fc_pi.bias] shape:[(5,)].\n"," first 3 values:[0. 0. 0.]\n","[12] name:[net.mixturesoflogits_12.fc_mu.weight] shape:[(50, 128)].\n"," first 3 values:[ 0.075 0.026 -0.165]\n","[13] name:[net.mixturesoflogits_12.fc_mu.bias] shape:[(50,)].\n"," first 3 values:[-2.104 1.716 -2.968]\n","[14] name:[net.mixturesoflogits_12.fc_sigma.weight] shape:[(50, 128)].\n"," first 3 values:[-0.089 0.029 -0.171]\n","[15] name:[net.mixturesoflogits_12.fc_sigma.bias] shape:[(50,)].\n"," first 3 values:[0. 0. 0.]\n","Total number of parameters:[434,089].\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"toW9smKjU0tH"},"source":["### Demo forward path"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1fh0_I-fPpid","executionInfo":{"status":"ok","timestamp":1610108873366,"user_tz":-540,"elapsed":5267,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"dd285168-1b4e-43f4-9a16-8101bfccb895"},"source":["# Demo instantiate\n","M = MixtureLogitNetwork(\n"," name='mln',x_dim=[1,28,28],k_size=3,c_dims=[32,64],p_sizes=[2,2],\n"," h_dims=[128],y_dim=10,USE_BN=True,\n"," k=3,sig_min=1,sig_max=None,\n"," mu_min=-3,mu_max =+3).to(device)\n","# Demo forward path \n","x_np = np.random.rand(2,1,28,28)\n","x_tc = np2tc(x_np)\n","pi_tc,mu_tc,sigma_tc = M.forward(x_tc) # forward path of MLN\n","pi_np,mu_np,sigma_np = tc2np(pi_tc),tc2np(mu_tc),tc2np(sigma_tc)\n","out = mdn_gather(pi_tc,mu_tc,sigma_tc)\n","mu_sel_np = tc2np(out['mu_sel'])\n","print ('x_np: %s'%(x_np.shape,))\n","print ('=>')\n","print ('pi_np: %s'%(pi_np.shape,)) # [N x K]\n","print ('mu_np: %s'%(mu_np.shape,)) # [N x K x D]\n","print ('sigma_np: %s'%(sigma_np.shape,)) # [N x K x D]\n","print ('=>')\n","print ('mu_sel_np: %s'%(mu_sel_np.shape,)) # [N x D]"],"execution_count":7,"outputs":[{"output_type":"stream","text":["x_np: (2, 1, 28, 28)\n","=>\n","pi_np: (2, 3)\n","mu_np: (2, 3, 10)\n","sigma_np: (2, 3, 10)\n","=>\n","mu_sel_np: (2, 10)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"89CAV6pAIghW"},"source":["### Dataset"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"j1zE9HLvmi2U","executionInfo":{"status":"ok","timestamp":1610108873368,"user_tz":-540,"elapsed":5251,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"3899b5de-4a7e-438e-cf30-684938ccd489"},"source":["from torchvision import datasets,transforms\n","mnist_train = datasets.MNIST(root='./data/',train=True,transform=transforms.ToTensor(),download=True)\n","mnist_test = datasets.MNIST(root='./data/',train=False,transform=transforms.ToTensor(),download=True)\n","mnist_train.targets = mnist_train.targets # manipulate train labels\n","BATCH_SIZE = 64\n","train_iter = torch.utils.data.DataLoader(mnist_train,batch_size=BATCH_SIZE,shuffle=True,num_workers=1)\n","test_iter = torch.utils.data.DataLoader(mnist_test,batch_size=BATCH_SIZE,shuffle=True,num_workers=1)\n","print (\"Done.\")"],"execution_count":8,"outputs":[{"output_type":"stream","text":["Done.\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"wYyrGiMlKIUp"},"source":["### Evaluation function"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"04YxC4wEKIEy","executionInfo":{"status":"ok","timestamp":1610108873368,"user_tz":-540,"elapsed":5233,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"23bf1468-324f-4fc4-8533-51809fe87b0b"},"source":["def func_eval(model,data_iter,device):\n"," with torch.no_grad():\n"," n_total,n_correct,epis_unct_sum,alea_unct_sum = 0,0,0,0\n"," model.eval() # evaluate (affects DropOut and BN)\n"," for batch_in,batch_out in data_iter:\n"," # Foraward path\n"," y_trgt = batch_out.to(device)\n"," pi,mu,sigma = model.forward(batch_in.view(-1,1,28,28).to(device))\n"," out = mdn_gather(pi,mu,sigma)\n"," model_pred = out['mu_sel']\n","\n"," # Uncertainty \n"," unct_out = gmm_uncertainties(pi,mu,sigma)\n"," epis_unct = unct_out['epis_unct'] # [N]\n"," alea_unct = unct_out['alea_unct'] # [N]\n"," epis_unct_sum += torch.sum(epis_unct)\n"," alea_unct_sum += torch.sum(alea_unct)\n","\n"," # Check\n"," _,y_pred = torch.max(model_pred,1)\n"," n_correct += (y_pred==y_trgt).sum().item()\n"," n_total += batch_in.size(0)\n"," val_accr = (n_correct/n_total)\n"," epis_unct_avg = (epis_unct_sum/n_total).detach().cpu().item()\n"," alea_unct_avg = (alea_unct_sum/n_total).detach().cpu().item()\n"," model.train() # back to train mode \n"," out_eval = {'val_accr':val_accr,\n"," 'epis_unct_avg':epis_unct_avg,'alea_unct_avg':alea_unct_avg}\n"," return out_eval\n","print (\"Done\")"],"execution_count":9,"outputs":[{"output_type":"stream","text":["Done\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"K5luyCj0anq3","executionInfo":{"status":"ok","timestamp":1610108881374,"user_tz":-540,"elapsed":13221,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"369c0f44-ae64-4c32-93d8-cc2d73821f46"},"source":["M.init_param()\n","train_accr = func_eval(M,train_iter,device)['val_accr']\n","test_accr = func_eval(M,test_iter,device)['val_accr']\n","print (\"train_accr:[%.3f] test_accr:[%.3f].\"%(train_accr,test_accr))"],"execution_count":10,"outputs":[{"output_type":"stream","text":["train_accr:[0.169] test_accr:[0.170].\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"vJzFw_SSJWvg"},"source":["### Train with clean data"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZS77J0Hfo8vx","executionInfo":{"status":"ok","timestamp":1610109052179,"user_tz":-540,"elapsed":184010,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"80b6a1d5-c6b6-4042-9021-dc095c861171"},"source":["np.random.seed(seed=0)\n","torch.manual_seed(seed=0)\n","M = MixtureLogitNetwork(\n"," name='mln',x_dim=[1,28,28],k_size=3,c_dims=[32,64],p_sizes=[2,2],\n"," h_dims=[128],y_dim=10,USE_BN=False,k=5,\n"," sig_min=0.01,sig_max=3,\n"," mu_min=-3,mu_max=+3).to(device)\n","M.init_param()\n","optm = optim.Adam(M.parameters(),lr=1e-3,weight_decay=1e-6)\n","M.train() # train mode\n","\n","# Re-define the train iterator\n","mnist_train.targets = mnist_train.targets # manipulate train labels\n","BATCH_SIZE = 64\n","train_iter = torch.utils.data.DataLoader(mnist_train,batch_size=BATCH_SIZE,shuffle=True,num_workers=1)\n","\n","EPOCHS,print_every = 10,1\n","for epoch in range(EPOCHS):\n"," loss_sum,wace_sum,alea_sum = 0,0,0\n"," for batch_in,batch_out in train_iter:\n"," # Forward path\n"," pi,mu,sigma = M.forward(batch_in.view(-1,1,28,28).to(device)) \n"," target = torch.eye(M.y_dim)[batch_out].to(device)\n"," mace_loss_out = mace_loss(pi,mu,sigma,target,alea_weight=0.5) # mixture of CE \n"," loss_out = mace_loss_out['loss_avg']\n"," wace_out = mace_loss_out['wace_avg']\n"," alea_out = mace_loss_out['alea_avg']\n"," # Update \n"," optm.zero_grad() # reset gradient \n"," loss_out.backward() # backpropagate\n"," optm.step() # optimizer update\n"," # Track losses \n"," loss_sum += loss_out\n"," wace_sum += wace_out\n"," alea_sum += alea_out\n"," loss_avg = loss_sum/len(train_iter)\n"," wace_avg = wace_sum/len(train_iter)\n"," alea_avg = alea_sum/len(train_iter)\n"," # Print\n"," if ((epoch%print_every)==0) or (epoch==(EPOCHS-1)):\n"," train_res = func_eval(M,train_iter,device)\n"," test_res = func_eval(M,test_iter,device)\n"," print (\"epoch:[%d] loss:[%.3f]=(wace:%.3f+alea:%.3f) train_accr:[%.3f] test_accr:[%.3f].\"%\n"," (epoch,loss_avg,wace_avg,alea_avg,\n"," train_res['val_accr'],test_res['val_accr'])) \n"," print (\" [Train] alea:[%.3f] epis:[%.3f]\"%\n"," (train_res['alea_unct_avg'],train_res['epis_unct_avg']))\n"," print (\" [Test] alea:[%.3f] epis:[%.3f]\"%\n"," (test_res['alea_unct_avg'],test_res['epis_unct_avg']))\n","\n","print (\"Done\")"],"execution_count":11,"outputs":[{"output_type":"stream","text":["epoch:[0] loss:[0.152]=(wace:0.069+alea:0.084) train_accr:[0.974] test_accr:[0.974].\n"," [Train] alea:[2.257] epis:[0.004]\n"," [Test] alea:[2.244] epis:[0.004]\n","epoch:[1] loss:[0.068]=(wace:0.033+alea:0.035) train_accr:[0.982] test_accr:[0.981].\n"," [Train] alea:[2.061] epis:[0.002]\n"," [Test] alea:[2.014] epis:[0.002]\n","epoch:[2] loss:[0.048]=(wace:0.022+alea:0.026) train_accr:[0.984] test_accr:[0.983].\n"," [Train] alea:[1.724] epis:[0.001]\n"," [Test] alea:[1.700] epis:[0.001]\n","epoch:[3] loss:[0.040]=(wace:0.018+alea:0.021) train_accr:[0.989] test_accr:[0.987].\n"," [Train] alea:[1.572] epis:[0.000]\n"," [Test] alea:[1.555] epis:[0.000]\n","epoch:[4] loss:[0.033]=(wace:0.015+alea:0.019) train_accr:[0.992] test_accr:[0.989].\n"," [Train] alea:[1.429] epis:[0.000]\n"," [Test] alea:[1.404] epis:[0.000]\n","epoch:[5] loss:[0.028]=(wace:0.013+alea:0.016) train_accr:[0.993] test_accr:[0.989].\n"," [Train] alea:[1.341] epis:[0.000]\n"," [Test] alea:[1.319] epis:[0.000]\n","epoch:[6] loss:[0.030]=(wace:0.014+alea:0.017) train_accr:[0.993] test_accr:[0.988].\n"," [Train] alea:[1.296] epis:[0.000]\n"," [Test] alea:[1.278] epis:[0.000]\n","epoch:[7] loss:[0.025]=(wace:0.011+alea:0.014) train_accr:[0.992] test_accr:[0.987].\n"," [Train] alea:[1.354] epis:[0.000]\n"," [Test] alea:[1.333] epis:[0.000]\n","epoch:[8] loss:[0.022]=(wace:0.009+alea:0.013) train_accr:[0.995] test_accr:[0.989].\n"," [Train] alea:[1.200] epis:[0.000]\n"," [Test] alea:[1.185] epis:[0.000]\n","epoch:[9] loss:[0.023]=(wace:0.010+alea:0.013) train_accr:[0.994] test_accr:[0.989].\n"," [Train] alea:[1.280] epis:[0.000]\n"," [Test] alea:[1.269] epis:[0.000]\n","Done\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"HxFB796B3DAn"},"source":["### Train with random shuffle noise"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"-nAEUlM0Toe2","executionInfo":{"status":"ok","timestamp":1610109224031,"user_tz":-540,"elapsed":355846,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"457591d2-a6a4-4b7f-96bd-e1b3a6c81857"},"source":["np.random.seed(seed=0)\n","torch.manual_seed(seed=0)\n","M = MixtureLogitNetwork(\n"," name='mln',x_dim=[1,28,28],k_size=3,c_dims=[32,64],p_sizes=[2,2],\n"," h_dims=[128],y_dim=10,USE_BN=False,k=5,\n"," sig_min=0.01,sig_max=3,\n"," mu_min=-3,mu_max=+3).to(device)\n","M.init_param()\n","optm = optim.Adam(M.parameters(),lr=1e-3,weight_decay=1e-6)\n","M.train() # train mode\n","\n","# Re-define the train iterator\n","mnist_train = datasets.MNIST(root='./data/',train=True,transform=transforms.ToTensor(),download=True)\n","n_train = len(mnist_train)\n","corrupt_rate = 0.5 # random shuffle rate \n","n_corrupt = int(n_train*corrupt_rate)\n","r_idx = np.random.permutation(n_train)[:n_corrupt]\n","mnist_train.targets[r_idx] = torch.randint(low=0,high=10,size=(n_corrupt,)) # random label \n","BATCH_SIZE = 64\n","train_iter = torch.utils.data.DataLoader(mnist_train,batch_size=BATCH_SIZE,shuffle=True,num_workers=1)\n","\n","EPOCHS,print_every = 10,1\n","for epoch in range(EPOCHS):\n"," loss_sum,wace_sum,alea_sum = 0,0,0\n"," for batch_in,batch_out in train_iter:\n"," # Forward path\n"," pi,mu,sigma = M.forward(batch_in.view(-1,1,28,28).to(device)) \n"," target = torch.eye(M.y_dim)[batch_out].to(device)\n"," mace_loss_out = mace_loss(pi,mu,sigma,target,alea_weight=0.5) # mixture of CE \n"," loss_out = mace_loss_out['loss_avg']\n"," wace_out = mace_loss_out['wace_avg']\n"," alea_out = mace_loss_out['alea_avg']\n"," # Update \n"," optm.zero_grad() # reset gradient \n"," loss_out.backward() # backpropagate\n"," optm.step() # optimizer update\n"," # Track losses \n"," loss_sum += loss_out\n"," wace_sum += wace_out\n"," alea_sum += alea_out\n"," loss_avg = loss_sum/len(train_iter)\n"," wace_avg = wace_sum/len(train_iter)\n"," alea_avg = alea_sum/len(train_iter)\n"," # Print\n"," if ((epoch%print_every)==0) or (epoch==(EPOCHS-1)):\n"," train_res = func_eval(M,train_iter,device)\n"," test_res = func_eval(M,test_iter,device)\n"," print (\"epoch:[%d] loss:[%.3f]=(wace:%.3f+alea:%.3f) train_accr:[%.3f] test_accr:[%.3f].\"%\n"," (epoch,loss_avg,wace_avg,alea_avg,\n"," train_res['val_accr'],test_res['val_accr'])) \n"," print (\" [Train] alea:[%.3f] epis:[%.3f]\"%\n"," (train_res['alea_unct_avg'],train_res['epis_unct_avg']))\n"," print (\" [Test] alea:[%.3f] epis:[%.3f]\"%\n"," (test_res['alea_unct_avg'],test_res['epis_unct_avg']))\n","\n","print (\"Done\")"],"execution_count":12,"outputs":[{"output_type":"stream","text":["epoch:[0] loss:[0.619]=(wace:0.297+alea:0.322) train_accr:[0.529] test_accr:[0.957].\n"," [Train] alea:[7.769] epis:[0.027]\n"," [Test] alea:[7.765] epis:[0.027]\n","epoch:[1] loss:[0.589]=(wace:0.289+alea:0.300) train_accr:[0.534] test_accr:[0.964].\n"," [Train] alea:[7.251] epis:[0.009]\n"," [Test] alea:[7.240] epis:[0.009]\n","epoch:[2] loss:[0.583]=(wace:0.287+alea:0.296) train_accr:[0.539] test_accr:[0.972].\n"," [Train] alea:[7.687] epis:[0.011]\n"," [Test] alea:[7.693] epis:[0.011]\n","epoch:[3] loss:[0.578]=(wace:0.285+alea:0.293) train_accr:[0.540] test_accr:[0.972].\n"," [Train] alea:[7.451] epis:[0.007]\n"," [Test] alea:[7.443] epis:[0.007]\n","epoch:[4] loss:[0.573]=(wace:0.283+alea:0.290) train_accr:[0.541] test_accr:[0.969].\n"," [Train] alea:[7.526] epis:[0.007]\n"," [Test] alea:[7.531] epis:[0.007]\n","epoch:[5] loss:[0.566]=(wace:0.279+alea:0.287) train_accr:[0.543] test_accr:[0.967].\n"," [Train] alea:[7.439] epis:[0.004]\n"," [Test] alea:[7.440] epis:[0.004]\n","epoch:[6] loss:[0.558]=(wace:0.276+alea:0.282) train_accr:[0.545] test_accr:[0.960].\n"," [Train] alea:[7.397] epis:[0.003]\n"," [Test] alea:[7.399] epis:[0.003]\n","epoch:[7] loss:[0.549]=(wace:0.271+alea:0.278) train_accr:[0.547] test_accr:[0.953].\n"," [Train] alea:[7.361] epis:[0.003]\n"," [Test] alea:[7.376] epis:[0.003]\n","epoch:[8] loss:[0.536]=(wace:0.265+alea:0.272) train_accr:[0.551] test_accr:[0.941].\n"," [Train] alea:[7.262] epis:[0.003]\n"," [Test] alea:[7.290] epis:[0.003]\n","epoch:[9] loss:[0.524]=(wace:0.258+alea:0.265) train_accr:[0.558] test_accr:[0.934].\n"," [Train] alea:[7.079] epis:[0.002]\n"," [Test] alea:[7.110] epis:[0.002]\n","Done\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"rbfrSSuN9msB"},"source":["### Train with random permutation noise"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NqA6YD4W1Q-f","executionInfo":{"status":"ok","timestamp":1610109396420,"user_tz":-540,"elapsed":528217,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"c2564487-88fd-4b00-a16f-f45f35f95dd6"},"source":["np.random.seed(seed=0)\n","torch.manual_seed(seed=0)\n","M = MixtureLogitNetwork(\n"," name='mln',x_dim=[1,28,28],k_size=3,c_dims=[32,64],p_sizes=[2,2],\n"," h_dims=[128],y_dim=10,USE_BN=False,k=5,\n"," sig_min=0.01,sig_max=3,\n"," mu_min=-3,mu_max=+3).to(device)\n","M.init_param()\n","optm = optim.Adam(M.parameters(),lr=1e-3,weight_decay=1e-6)\n","M.train() # train mode\n","\n","# Re-define the train iterator\n","mnist_train = datasets.MNIST(root='./data/',train=True,transform=transforms.ToTensor(),download=True)\n","n_train = len(mnist_train)\n","corrupt_rate = 0.3 # random permutation rate \n","targets_bu = mnist_train.targets\n","permute_targets = [1,2,3,4,5,6,7,8,9,0] # shift label \n","for idx in range(10):\n"," sel_idx = torch.where(targets_bu==idx)[0]\n"," n_sel = sel_idx.shape[0]\n"," corrupt_idx = np.random.permutation(n_sel)[:int(n_sel*corrupt_rate)]\n"," mnist_train.targets[sel_idx[corrupt_idx]] = permute_targets[idx]\n","BATCH_SIZE = 64\n","train_iter = torch.utils.data.DataLoader(mnist_train,batch_size=BATCH_SIZE,shuffle=True,num_workers=1)\n","\n","EPOCHS,print_every = 10,1\n","for epoch in range(EPOCHS):\n"," loss_sum,wace_sum,alea_sum = 0,0,0\n"," for batch_in,batch_out in train_iter:\n"," # Forward path\n"," pi,mu,sigma = M.forward(batch_in.view(-1,1,28,28).to(device)) \n"," target = torch.eye(M.y_dim)[batch_out].to(device)\n"," mace_loss_out = mace_loss(pi,mu,sigma,target,alea_weight=0.5) # mixture of CE \n"," loss_out = mace_loss_out['loss_avg']\n"," wace_out = mace_loss_out['wace_avg']\n"," alea_out = mace_loss_out['alea_avg']\n"," # Update \n"," optm.zero_grad() # reset gradient \n"," loss_out.backward() # backpropagate\n"," optm.step() # optimizer update\n"," # Track losses \n"," loss_sum += loss_out\n"," wace_sum += wace_out\n"," alea_sum += alea_out\n"," loss_avg = loss_sum/len(train_iter)\n"," wace_avg = wace_sum/len(train_iter)\n"," alea_avg = alea_sum/len(train_iter)\n"," # Print\n"," if ((epoch%print_every)==0) or (epoch==(EPOCHS-1)):\n"," train_res = func_eval(M,train_iter,device)\n"," test_res = func_eval(M,test_iter,device)\n"," print (\"epoch:[%d] loss:[%.3f]=(wace:%.3f+alea:%.3f) train_accr:[%.3f] test_accr:[%.3f].\"%\n"," (epoch,loss_avg,wace_avg,alea_avg,\n"," train_res['val_accr'],test_res['val_accr'])) \n"," print (\" [Train] alea:[%.3f] epis:[%.3f]\"%\n"," (train_res['alea_unct_avg'],train_res['epis_unct_avg']))\n"," print (\" [Test] alea:[%.3f] epis:[%.3f]\"%\n"," (test_res['alea_unct_avg'],test_res['epis_unct_avg']))\n","\n","print (\"Done\")"],"execution_count":13,"outputs":[{"output_type":"stream","text":["epoch:[0] loss:[0.411]=(wace:0.192+alea:0.219) train_accr:[0.659] test_accr:[0.930].\n"," [Train] alea:[5.737] epis:[0.022]\n"," [Test] alea:[5.717] epis:[0.022]\n","epoch:[1] loss:[0.335]=(wace:0.162+alea:0.174) train_accr:[0.657] test_accr:[0.918].\n"," [Train] alea:[5.203] epis:[0.012]\n"," [Test] alea:[5.164] epis:[0.012]\n","epoch:[2] loss:[0.317]=(wace:0.154+alea:0.163) train_accr:[0.681] test_accr:[0.962].\n"," [Train] alea:[4.874] epis:[0.005]\n"," [Test] alea:[4.881] epis:[0.005]\n","epoch:[3] loss:[0.303]=(wace:0.147+alea:0.156) train_accr:[0.690] test_accr:[0.976].\n"," [Train] alea:[4.483] epis:[0.002]\n"," [Test] alea:[4.490] epis:[0.002]\n","epoch:[4] loss:[0.293]=(wace:0.143+alea:0.150) train_accr:[0.691] test_accr:[0.973].\n"," [Train] alea:[4.628] epis:[0.003]\n"," [Test] alea:[4.638] epis:[0.003]\n","epoch:[5] loss:[0.283]=(wace:0.138+alea:0.145) train_accr:[0.695] test_accr:[0.982].\n"," [Train] alea:[4.059] epis:[0.001]\n"," [Test] alea:[4.067] epis:[0.001]\n","epoch:[6] loss:[0.275]=(wace:0.134+alea:0.141) train_accr:[0.696] test_accr:[0.976].\n"," [Train] alea:[4.136] epis:[0.001]\n"," [Test] alea:[4.145] epis:[0.001]\n","epoch:[7] loss:[0.271]=(wace:0.132+alea:0.139) train_accr:[0.697] test_accr:[0.976].\n"," [Train] alea:[4.090] epis:[0.001]\n"," [Test] alea:[4.103] epis:[0.001]\n","epoch:[8] loss:[0.261]=(wace:0.127+alea:0.134) train_accr:[0.697] test_accr:[0.970].\n"," [Train] alea:[4.101] epis:[0.001]\n"," [Test] alea:[4.129] epis:[0.001]\n","epoch:[9] loss:[0.257]=(wace:0.125+alea:0.132) train_accr:[0.700] test_accr:[0.974].\n"," [Train] alea:[3.959] epis:[0.001]\n"," [Test] alea:[3.981] epis:[0.001]\n","Done\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"NnGQhAdr2vKV","executionInfo":{"status":"ok","timestamp":1610109396425,"user_tz":-540,"elapsed":528163,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}}},"source":[""],"execution_count":13,"outputs":[]}]} -------------------------------------------------------------------------------- /notebook/mha.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"mha.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyNtKYtrAk1tHkKhTt/4wK5a"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"rsnoyeYbS7i6"},"source":["\n"," \n"," \n","
\n"," Colab\n"," \n"," View Source\n","
"]},{"cell_type":"markdown","metadata":{"id":"xRwj6fIzVc-s"},"source":["# Multi-Headed Attention"]},{"cell_type":"code","metadata":{"id":"G9N9n94EVWy9","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1609913294780,"user_tz":-540,"elapsed":5521,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"24a09030-6f5a-4e62-cd26-a642cc2bac57"},"source":["import numpy as np\n","import matplotlib.pyplot as plt\n","import torch\n","import torch.nn as nn\n","import torch.optim as optim\n","import torch.nn.functional as F\n","%matplotlib inline\n","%config InlineBackend.figure_format='retina'\n","print (\"PyTorch version:[%s].\"%(torch.__version__))\n","device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n","print (\"device:[%s].\"%(device))"],"execution_count":1,"outputs":[{"output_type":"stream","text":["PyTorch version:[1.7.0+cu101].\n","device:[cuda:0].\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"TRRvnKZ0XlJT"},"source":["### Scaled Dot-Product Attention (SDPA)\n","- Data $X \\in \\mathbb{R}^{n \\times d}$ where $n$ is the number data and $d$ is the data dimension\n","- Query and Key $Q, K \\in \\mathbb{R}^{n \\times d_K}$ \n","- Value $V \\in \\mathbb{R}^{n \\times d_V} $\n","\n","$\\text{Attention}(Q,K,V) = \\text{softmax} \\left( \\frac{QK^T}{\\sqrt{d_K}} \\right)V \\in \\mathbb{R}^{n \\times d_V} $"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"K-Z3Vd_VV5Pm","executionInfo":{"status":"ok","timestamp":1609913693623,"user_tz":-540,"elapsed":1321,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"15debb01-d0cd-4df9-b636-e19de5c46d40"},"source":["class ScaledDotProductAttention(nn.Module):\n"," def forward(self,Q,K,V,mask=None):\n"," d_K = K.size()[-1] # key dimension\n"," scores = Q.matmul(K.transpose(-2,-1)) / np.sqrt(d_K)\n"," if mask is not None:\n"," scores = scores.masked_fill(mask==0, -1e9)\n"," attention = F.softmax(scores,dim=-1)\n"," out = attention.matmul(V)\n"," return out,attention\n","\n","# Demo run of scaled dot product attention \n","SPDA = ScaledDotProductAttention()\n","n_batch,d_K,d_V = 3,128,256 # d_K(=d_Q) does not necessarily be equal to d_V\n","n_Q,n_K,n_V = 30,50,50\n","Q = torch.rand(n_batch,n_Q,d_K)\n","K = torch.rand(n_batch,n_K,d_K)\n","V = torch.rand(n_batch,n_V,d_V)\n","out,attention = SPDA.forward(Q,K,V,mask=None)\n","def sh(x): return str(x.shape)[11:-1] \n","print (\"SDPA: Q%s K%s V%s => out%s attention%s\"%\n"," (sh(Q),sh(K),sh(V),sh(out),sh(attention)))\n","\n","# It supports 'multi-headed' attention\n","n_batch,n_head,d_K,d_V = 3,5,128,256\n","n_Q,n_K,n_V = 30,50,50 # n_K and n_V should be the same\n","Q = torch.rand(n_batch,n_head,n_Q,d_K)\n","K = torch.rand(n_batch,n_head,n_K,d_K)\n","V = torch.rand(n_batch,n_head,n_V,d_V)\n","out,attention = SPDA.forward(Q,K,V,mask=None)\n","# out: [n_batch x n_head x n_Q x d_V]\n","# attention: [n_batch x n_head x n_Q x n_K] \n","def sh(x): return str(x.shape)[11:-1] \n","print (\"(Multi-Headed) SDPA: Q%s K%s V%s => out%s attention%s\"%\n"," (sh(Q),sh(K),sh(V),sh(out),sh(attention)))"],"execution_count":4,"outputs":[{"output_type":"stream","text":["SDPA: Q[3, 30, 128] K[3, 50, 128] V[3, 50, 256] => out[3, 30, 256] attention[3, 30, 50]\n","(Multi-Headed) SDPA: Q[3, 5, 30, 128] K[3, 5, 50, 128] V[3, 5, 50, 256] => out[3, 5, 30, 256] attention[3, 5, 30, 50]\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"uLbi13pDi3No"},"source":["### Multi-Headed Attention (MHA)\n","\n","$\\text{head}_{\\color{red}i} = \\text{Attention}(Q {\\color{green}W}^Q_{\\color{red}i},K {\\color{green}W}^K_{\\color{red}i}, V {\\color{green}W}^V_{\\color{red}i}) $"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Hf7j24l1dnSF","executionInfo":{"status":"ok","timestamp":1609913914502,"user_tz":-540,"elapsed":1159,"user":{"displayName":"Sungjoon Choi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiFkMNCaA4zshD2C87LC6X0Y7ohjLlu0sIiLepLnQ=s64","userId":"10728677910935649939"}},"outputId":"3866ebf2-b462-46f3-dffe-09414727afca"},"source":["class MultiHeadedAttention(nn.Module):\n"," def __init__(self,d_feat=128,n_head=5,actv=F.relu,USE_BIAS=True,dropout_p=0.1,device=None):\n"," \"\"\"\n"," :param d_feat: feature dimension\n"," :param n_head: number of heads\n"," :param actv: activation after each linear layer\n"," :param USE_BIAS: whether to use bias\n"," :param dropout_p: dropout rate\n"," :device: which device to use (e.g., cuda:0)\n"," \"\"\"\n"," super(MultiHeadedAttention,self).__init__()\n"," if (d_feat%n_head) != 0:\n"," raise ValueError(\"d_feat(%d) should be divisible by b_head(%d)\"%(d_feat,n_head)) \n"," self.d_feat = d_feat\n"," self.n_head = n_head\n"," self.d_head = self.d_feat // self.n_head\n"," self.actv = actv\n"," self.USE_BIAS = USE_BIAS\n"," self.dropout_p = dropout_p # prob. of zeroed\n","\n"," self.lin_Q = nn.Linear(self.d_feat,self.d_feat,self.USE_BIAS)\n"," self.lin_K = nn.Linear(self.d_feat,self.d_feat,self.USE_BIAS)\n"," self.lin_V = nn.Linear(self.d_feat,self.d_feat,self.USE_BIAS)\n"," self.lin_O = nn.Linear(self.d_feat,self.d_feat,self.USE_BIAS)\n","\n"," self.dropout = nn.Dropout(p=self.dropout_p)\n"," \n"," def forward(self,Q,K,V,mask=None):\n"," \"\"\"\n"," :param Q: [n_batch, n_Q, d_feat]\n"," :param K: [n_batch, n_K, d_feat]\n"," :param V: [n_batch, n_V, d_feat] <= n_K and n_V must be the same \n"," :param mask: \n"," \"\"\"\n"," n_batch = Q.shape[0]\n"," Q_feat = self.lin_Q(Q) \n"," K_feat = self.lin_K(K) \n"," V_feat = self.lin_V(V)\n"," # Q_feat: [n_batch, n_Q, d_feat]\n"," # K_feat: [n_batch, n_K, d_feat]\n"," # V_feat: [n_batch, n_V, d_feat]\n","\n"," # Multi-head split of Q, K, and V (d_feat = n_head*d_head)\n"," Q_split = Q_feat.view(n_batch, -1, self.n_head, self.d_head).permute(0, 2, 1, 3)\n"," K_split = K_feat.view(n_batch, -1, self.n_head, self.d_head).permute(0, 2, 1, 3)\n"," V_split = V_feat.view(n_batch, -1, self.n_head, self.d_head).permute(0, 2, 1, 3)\n"," # Q_split: [n_batch, n_head, n_Q, d_head]\n"," # K_split: [n_batch, n_head, n_K, d_head]\n"," # V_split: [n_batch, n_head, n_V, d_head]\n","\n"," # Multi-Headed Attention\n"," d_K = K.size()[-1] # key dimension\n"," scores = torch.matmul(Q_split,K_split.permute(0,1,3,2)) / np.sqrt(d_K)\n"," if mask is not None:\n"," scores = scores.masked_fill(mask==0,-1e9)\n"," attention = torch.softmax(scores,dim=-1)\n"," x_raw = torch.matmul(self.dropout(attention),V_split) # dropout is NOT mentioned in the paper\n"," # attention: [n_batch, n_head, n_Q, n_K]\n"," # x_raw: [n_batch, n_head, n_Q, d_head]\n","\n"," # Reshape x\n"," x_rsh1 = x_raw.permute(0,2,1,3).contiguous()\n"," # x_rsh1: [n_batch, n_Q, n_head, d_head]\n"," x_rsh2 = x_rsh1.view(n_batch,-1,self.d_feat)\n"," # x_rsh2: [n_batch, n_Q, d_feat]\n","\n"," # Linear\n"," x = self.lin_O(x_rsh2)\n"," # x: [n_batch, n_Q, d_feat]\n"," out = {'Q_feat':Q_feat,'K_feat':K_feat,'V_feat':V_feat,\n"," 'Q_split':Q_split,'K_split':K_split,'V_split':V_split,\n"," 'scores':scores,'attention':attention,\n"," 'x_raw':x_raw,'x_rsh1':x_rsh1,'x_rsh2':x_rsh2,'x':x}\n"," return out\n","\n","# Self-Attention Layer\n","n_batch = 128\n","n_src = 32\n","d_feat = 200\n","n_head = 5\n","src = torch.rand(n_batch,n_src,d_feat)\n","self_attention = MultiHeadedAttention(\n"," d_feat=d_feat,n_head=n_head,actv=F.relu,USE_BIAS=True,dropout_p=0.1,device=device)\n","out = self_attention.forward(src,src,src,mask=None)\n","\n","Q_feat,K_feat,V_feat = out['Q_feat'],out['K_feat'],out['V_feat']\n","Q_split,K_split,V_split = out['Q_split'],out['K_split'],out['V_split']\n","scores,attention = out['scores'],out['attention']\n","x_raw,x_rsh1,x_rsh2,x = out['x_raw'],out['x_rsh1'],out['x_rsh2'],out['x']\n","\n","# Print out shapes\n","def sh(_x): return str(_x.shape)[11:-1] \n","print (\"Input src:\\t%s \\t= [n_batch, n_src, d_feat]\"%(sh(src)))\n","print ()\n","print (\"Q_feat: \\t%s \\t= [n_batch, n_src, d_feat]\"%(sh(Q_feat)))\n","print (\"K_feat: \\t%s \\t= [n_batch, n_src, d_feat]\"%(sh(K_feat)))\n","print (\"V_feat: \\t%s \\t= [n_batch, n_src, d_feat]\"%(sh(V_feat)))\n","print ()\n","print (\"Q_split: \\t%s \\t= [n_batch, n_head, n_src, d_head]\"%(sh(Q_split)))\n","print (\"K_split: \\t%s \\t= [n_batch, n_head, n_src, d_head]\"%(sh(K_split)))\n","print (\"V_split: \\t%s \\t= [n_batch, n_head, n_src, d_head]\"%(sh(V_split)))\n","print ()\n","print (\"scores: \\t%s \\t= [n_batch, n_head, n_src, n_src]\"%(sh(scores)))\n","print (\"attention:\\t%s \\t= [n_batch, n_head, n_src, n_src]\"%(sh(attention)))\n","print ()\n","print (\"x_raw: \\t%s \\t= [n_batch, n_head, n_src, d_head]\"%(sh(x_raw)))\n","print (\"x_rsh1: \\t%s \\t= [n_batch, n_src, n_head, d_head]\"%(sh(x_rsh1)))\n","print (\"x_rsh2: \\t%s \\t= [n_batch, n_src, d_feat]\"%(sh(x_rsh2)))\n","print ()\n","print (\"Output x: \\t%s \\t= [n_batch, n_src, d_feat]\"%(sh(x)))\n"],"execution_count":5,"outputs":[{"output_type":"stream","text":["Input src:\t[128, 32, 200] \t= [n_batch, n_src, d_feat]\n","\n","Q_feat: \t[128, 32, 200] \t= [n_batch, n_src, d_feat]\n","K_feat: \t[128, 32, 200] \t= [n_batch, n_src, d_feat]\n","V_feat: \t[128, 32, 200] \t= [n_batch, n_src, d_feat]\n","\n","Q_split: \t[128, 5, 32, 40] \t= [n_batch, n_head, n_src, d_head]\n","K_split: \t[128, 5, 32, 40] \t= [n_batch, n_head, n_src, d_head]\n","V_split: \t[128, 5, 32, 40] \t= [n_batch, n_head, n_src, d_head]\n","\n","scores: \t[128, 5, 32, 32] \t= [n_batch, n_head, n_src, n_src]\n","attention:\t[128, 5, 32, 32] \t= [n_batch, n_head, n_src, n_src]\n","\n","x_raw: \t[128, 5, 32, 40] \t= [n_batch, n_head, n_src, d_head]\n","x_rsh1: \t[128, 32, 5, 40] \t= [n_batch, n_src, n_head, d_head]\n","x_rsh2: \t[128, 32, 200] \t= [n_batch, n_src, d_feat]\n","\n","Output x: \t[128, 32, 200] \t= [n_batch, n_src, d_feat]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"QgqsHUT-OuJA"},"source":[""],"execution_count":null,"outputs":[]}]} --------------------------------------------------------------------------------