├── CNN Multichannel Text Classification.ipynb ├── Convolutional Attention Text Classification.ipynb ├── GRU Feedforward Attention Text Classification.ipynb ├── Hierarchical Attention Network Text Classification.ipynb ├── Hierarchical Convolution Attention Network.ipynb ├── Multichannel Convolutional Text Classification.ipynb ├── Packed Padded Sequences, Masking and Inference.ipynb ├── README.md ├── Reading Material ├── Convolutional Neural Networks for Sentence Classification.pdf ├── Feed-Forward Networks With Attention Can Solve Some Long-Term Memory Problems.pdf ├── Hierarchical Attention Networks for Document Classification.pdf └── Hierarchical Convolutional Attention Networks for Text Classification.pdf └── text-classification-with-rnn.ipynb /CNN Multichannel Text Classification.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"CNN Multichannel Text Classification.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"metadata":{"id":"1Xc8EKCGWo_V","colab_type":"code","outputId":"d17c4acb-0f9a-4f9d-fc81-b875aee410bd","executionInfo":{"status":"ok","timestamp":1546011501065,"user_tz":-330,"elapsed":85946,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":1079}},"cell_type":"code","source":["!pip install torch\n","!pip install torchtext\n","!pip install -U spacy\n","!python -m spacy download en\n","!pip install nltk"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Collecting torch\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/7e/60/66415660aa46b23b5e1b72bc762e816736ce8d7260213e22365af51e8f9c/torch-1.0.0-cp36-cp36m-manylinux1_x86_64.whl (591.8MB)\n","\u001b[K 100% |████████████████████████████████| 591.8MB 26kB/s \n","tcmalloc: large alloc 1073750016 bytes == 0x620be000 @ 0x7ff203a3e2a4 0x591a07 0x5b5d56 0x502e9a 0x506859 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x504c28 0x502540 0x502f3d 0x507641\n","\u001b[?25hInstalling collected packages: torch\n","Successfully installed torch-1.0.0\n","Collecting torchtext\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/c6/bc/b28b9efb4653c03e597ed207264eea45862b5260f48e9f010b5068d64db1/torchtext-0.3.1-py3-none-any.whl (62kB)\n","\u001b[K 100% |████████████████████████████████| 71kB 3.3MB/s \n","\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from torchtext) (4.28.1)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchtext) (1.14.6)\n","Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from torchtext) (1.0.0)\n","Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from torchtext) (2.18.4)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2018.11.29)\n","Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (3.0.4)\n","Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (1.22)\n","Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2.6)\n","Installing collected packages: torchtext\n","Successfully installed torchtext-0.3.1\n","Requirement already up-to-date: spacy in /usr/local/lib/python3.6/dist-packages (2.0.18)\n","Requirement already satisfied, skipping upgrade: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.1)\n","Requirement already satisfied, skipping upgrade: thinc<6.13.0,>=6.12.1 in /usr/local/lib/python3.6/dist-packages (from spacy) (6.12.1)\n","Collecting numpy>=1.15.0 (from spacy)\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/ff/7f/9d804d2348471c67a7d8b5f84f9bc59fd1cefa148986f2b74552f8573555/numpy-1.15.4-cp36-cp36m-manylinux1_x86_64.whl (13.9MB)\n","\u001b[K 100% |████████████████████████████████| 13.9MB 3.0MB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.18.4)\n","Requirement already satisfied, skipping upgrade: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.1)\n","Requirement already satisfied, skipping upgrade: dill<0.3,>=0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.2.8.2)\n","Requirement already satisfied, skipping upgrade: ujson>=1.35 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.35)\n","Requirement already satisfied, skipping upgrade: regex==2018.01.10 in /usr/local/lib/python3.6/dist-packages (from spacy) (2018.1.10)\n","Requirement already satisfied, skipping upgrade: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.9.6)\n","Requirement already satisfied, skipping upgrade: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.2)\n","Requirement already satisfied, skipping upgrade: msgpack<0.6.0,>=0.5.6 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.5.6)\n","Requirement already satisfied, skipping upgrade: cytoolz<0.10,>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.9.0.1)\n","Requirement already satisfied, skipping upgrade: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (4.28.1)\n","Requirement already satisfied, skipping upgrade: wrapt<1.11.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (1.10.11)\n","Requirement already satisfied, skipping upgrade: six<2.0.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (1.11.0)\n","Requirement already satisfied, skipping upgrade: msgpack-numpy<0.4.4 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.4.3.2)\n","Requirement already satisfied, skipping upgrade: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.22)\n","Requirement already satisfied, skipping upgrade: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.6)\n","Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n","Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2018.11.29)\n","Requirement already satisfied, skipping upgrade: toolz>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from cytoolz<0.10,>=0.9.0->thinc<6.13.0,>=6.12.1->spacy) (0.9.0)\n","\u001b[31mfeaturetools 0.4.1 has requirement pandas>=0.23.0, but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n","\u001b[31mcufflinks 0.14.6 has requirement plotly>=3.0.0, but you'll have plotly 1.12.12 which is incompatible.\u001b[0m\n","Installing collected packages: numpy\n"," Found existing installation: numpy 1.14.6\n"," Uninstalling numpy-1.14.6:\n"," Successfully uninstalled numpy-1.14.6\n","Successfully installed numpy-1.15.4\n","Requirement already satisfied: en_core_web_sm==2.0.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz#egg=en_core_web_sm==2.0.0 in /usr/local/lib/python3.6/dist-packages (2.0.0)\n","\n","\u001b[93m Linking successful\u001b[0m\n"," /usr/local/lib/python3.6/dist-packages/en_core_web_sm -->\n"," /usr/local/lib/python3.6/dist-packages/spacy/data/en\n","\n"," You can now load the model via spacy.load('en')\n","\n","Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (3.2.5)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.11.0)\n"],"name":"stdout"}]},{"metadata":{"id":"EopWaTjVW2lo","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch\n","from torchtext import data\n","from torchtext import datasets\n","import random\n","import nltk\n","import re\n","import numpy as np\n","from tqdm import tqdm\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","%matplotlib inline"],"execution_count":0,"outputs":[]},{"metadata":{"id":"LHeE87yJXbq_","colab_type":"code","colab":{}},"cell_type":"code","source":["SEED = 10\n","torch.manual_seed(SEED)\n","torch.cuda.manual_seed(SEED)\n","torch.backends.cudnn.deterministic = True #for reproducible results"],"execution_count":0,"outputs":[]},{"metadata":{"id":"7It70WIDXf3X","colab_type":"code","colab":{}},"cell_type":"code","source":["def clean_text(text):\n"," text = text.lower()\n"," text = re.compile('<.*?>').sub(\" \", text)\n"," text = re.sub(r\"i'm\", \"i am\", text)\n"," text = re.sub(r\"he's\", \"he is\", text)\n"," text = re.sub(r\"she's\", \"she is\", text)\n"," text = re.sub(r\"that's\", \"that is\", text)\n"," text = re.sub(r\"what's\", \"what is\", text)\n"," text = re.sub(r\"where's\", \"where is\", text)\n"," text = re.sub(r\"\\'ll\", \" will\", text)\n"," text = re.sub(r\"\\'ve\", \" have\", text)\n"," text = re.sub(r\"\\'re\", \" are\", text)\n"," text = re.sub(r\"\\'d\", \" would\", text)\n"," text = re.sub(r\"won't\", \"will not\", text)\n"," text = re.sub(r\"can't\", \"cannot\", text)\n"," text = re.sub(r\"[-()\\\"#/@;:<>{}+=~|.?,]\", \"\", text)\n"," text = text.split()\n"," return text\n","\n","TEXT = data.Field(tokenize=clean_text,lower=True, fix_length=800)\n","LABEL = data.LabelField(dtype = torch.float)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"hetNKN7xXicX","colab_type":"code","colab":{}},"cell_type":"code","source":["from torchtext import datasets\n","train_data, test_data = datasets.IMDB.splits(text_field=TEXT,label_field=LABEL,)\n","train_data, valid_data = train_data.split(split_ratio= 0.8, random_state=random.seed(SEED))"],"execution_count":0,"outputs":[]},{"metadata":{"id":"36xWKUorYYQw","colab_type":"code","outputId":"e0536fe9-6749-47dd-edef-2f0424f5a461","executionInfo":{"status":"ok","timestamp":1546011931951,"user_tz":-330,"elapsed":19074,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":89}},"cell_type":"code","source":["print('Number of training examples: {}'.format(len(train_data)))\n","print('Number of validation examples: {}'.format(len(valid_data)))\n","print(vars(train_data[0]))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Number of training examples: 20000\n","Number of validation examples: 5000\n","{'text': ['wow', 'this', 'movie', 'bored', 'the', 'pants', 'off', 'me', 'when', 'i', 'saw', 'it', 'bland', 'pointless', 'and', 'unmoving', 'apparently', 'ash', 'and', 'co', 'can', 'travel', 'through', 'time', 'with', 'the', 'help', 'of', 'the', 'spirit', 'of', 'the', 'forest', \"'princess\", \"mononoke'\", 'much', 'there', 'they', 'meet', 'a', 'dorky', 'kid', 'named', 'sam', 'and', 'the', 'plot', 'begins', 'so', 'tom', 'ash', 'and', 'huck', 'sam', 'get', 'high', 'with', 'nature', 'become', 'hippies', 'and', 'try', 'to', 'free', 'celebi', 'the', 'spirit', 'from', 'some', 'weirdo', 'hunter', 'guy', 'i', \"don't\", 'even', 'know', 'what', 'else', 'went', 'on', 'it', 'all', 'went', 'by', 'in', 'a', 'blur', \"ash's\", 'friends', 'were', 'hardly', 'in', 'it', 'and', 'all', 'the', 'fight', 'scenes', 'were', 'boring', 'after', 'saving', 'the', 'day', 'ash', 'and', 'his', 'infamous', 'friends', 'must', 'return', 'to', 'their', 'time', 'while', 'watching', 'sam', 'float', 'away', 'with', 'celebi', 'that', 'scene', 'was', 'just', 'creepy', 'oo', 'then', 'after', 'returning', 'to', 'their', 'time', 'ash', 'learns', 'that', 'his', 'new', 'friend', 'is', 'actually', 'his', \"rival's\", 'grandpa', 'and', 'i', 'think', 'that', 'is', 'it', 'pretty', 'retarded', \"isn't\", 'it', 'if', 'you', 'love', 'your', 'children', 'you', 'will', 'not', 'expose', 'them', 'to', 'this', '1', 'out', 'of', '10'], 'label': 'neg'}\n"],"name":"stdout"}]},{"metadata":{"id":"1GMrHuUxXlg-","colab_type":"code","outputId":"1b6e4687-abd9-4000-e73b-d3f4138a0fed","executionInfo":{"status":"ok","timestamp":1546011932567,"user_tz":-330,"elapsed":19531,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":379}},"cell_type":"code","source":["train_data_dist = [len(train_data[i].text) for i in range(len(train_data))]\n","valid_data_dist = [len(valid_data[i].text) for i in range(len(valid_data))]\n","test_data_dist = [len(test_data[i].text) for i in range(len(test_data))]\n","\n","sns.set(style='whitegrid')\n","f, axes = plt.subplots(1, 3)\n","\n","sns.distplot(a = train_data_dist,ax=axes[0], axlabel='Train')\n","sns.distplot(a = valid_data_dist,ax=axes[1], axlabel='Validation')\n","sns.distplot(a = test_data_dist,ax=axes[2], axlabel='Test')\n"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":19},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAe0AAAFZCAYAAAC173eYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xt8lOWd+P3PPaecJuQAmRDl4IgH\naJS2RFptLKiF0rJd+9vdVtgW+vT52T10sUd4lGb9CX0W8ER5WtG26rK+trXSuJbt9mBFu00rSyJo\npKhBUBBiwiGZYUJgcpzD/fwxuSeTZCYzSeZ03/N9v159lcnkTq7Bi/t7X9f1vb6XoqqqihBCCCGy\nninTDRBCCCFEYiRoCyGEEDohQVsIIYTQCQnaQgghhE5I0BZCCCF0QoK2EEIIoROWRL5p27ZtHD58\nGEVRqKurY+HCheH3Ghsb2bFjB2azmSVLlrBu3bqY1/h8PjZu3EhraytFRUU88sgjlJSUpOaTCSGE\nEAYTN2gfPHiQ1tZW6uvrOXHiBHV1ddTX14ff37JlC7t27aKyspI1a9awYsUKPB5P1GueffZZysrK\n+N73vkd9fT2vvfYan/jEJ2L+7ubm5uR8SmEYNTU1Kf8d0u9EpHT0OZB+J0aK1e/iBu2mpiaWLVsG\nwLx58+ju7sbr9WK322lra6OkpISqqioAli5dSlNTEx6PJ+o1DQ0NfP3rXwdg1apVU2o4hDp5uv5B\nJZu0fXK/N13ifT49//ebrFz9zOlk1PvdaPJZ4v/MWOKuabvdbsrKysKvy8vLcblcALhcLsrLy8e8\nF+ua06dP8/LLL7N27Vq+9a1vceHChUl9ICGEECIXJbSmHWkyVU+1a1RVxel0ctddd/HDH/6Qxx9/\nnHvuuWfca+M96ep5SknaLoQQYiLiBm2Hw4Hb7Q6/7uzspKKiIup7HR0dOBwOrFZr1GtmzJjB4sWL\nAbj55pvZuXNn3AYadbpI2j653yuEELks7vR4bW0te/fuBaClpQWHw4Hdbgdg1qxZeL1e2tvb8fv9\nNDQ0UFtbG/OaJUuWsG/fvvDXnU5nqj6XEEIIYThxR9qLFi2iurqa1atXoygKmzZtYs+ePRQXF7N8\n+XI2b97M+vXrAVi5ciVOpxOn0znmGoC1a9dyzz338Nxzz1FYWMiDDz6Y2k8nhBBCGEhCa9obNmwY\n8Xr+/PnhPy9evHjEFrBY1wAUFBTwyCOPTLSNQgghhEAqogkhhBC6IUFbCCGE0AkJ2kIIIYROSNAW\nQgghdEKCthBCCKETErSFEELHfvL8EX7XdCrTzRBpMuEyptnuhaZT4T9/6qYrMtUMkeO0fih9UKSS\nzx/kP/77XQA+LX0tJ8hIWwghdOp8d1+mmyDSTIK2EELolPuCBO1cI0FbCCF0yiVBO+dI0BZCCJ1y\ndUnQzjUStIUQQqdkpJ17JGgLIYROubp6ATCblAy3RKSLBG0hhNApbaRts8qtPFfIf2khhNCpSz2D\nAPgDaoZbItJFgrYQQuhUUA0F60AgmOGWiHQxXEU0gIs9g+TZzJluhpiCbdu2cfjwYRRFoa6ujoUL\nF4bfa2xsZMeOHZjNZpYsWcK6detiXuPz+di4cSOtra0UFRXxyCOPUFJSkqmPJURSBYOhoB1UQ382\nydq24RlupK2qKj/93dv85LdHMt0UMUkHDx6ktbWV+vp6tm7dytatW0e8v2XLFnbu3Mnu3bvZv38/\nx48fj3nNs88+S1lZGc899xwrV67ktddey8RHEiIlghGz4oGgjLZzgeFG2r6haaJBv3RgvWpqamLZ\nsmUAzJs3j+7ubrxeLwBtbW2UlJRQVVUFwNKlS2lqasLj8US9pqGhga9//esArFq1KgOfRojUCUZE\nbZ8/iNUiM4xGZ7yg7ZNgrXdut5vq6urw6/LyclwuFwAul4vy8vIR77W1tdHV1RX1mtOnT/Pyyy/z\n8MMPM2PGDDZt2kRpaWncNjQ3N0/pe1rfDz1kNNvOx/05epLI34tIH21NGyAQlGS0XGC8oC0jbMNR\n1YnfjLRrVFXF6XRy11138cMf/pDHH3+ce+65J+71NTU1477f3Nw87ve4Bk8N/ZwrEm5ztov3mY0o\n2x9SIkfafklGywmGW9P2+QOZboKYIofDgdvtDr/u7OykoqIi6nsdHR04HI6Y18yYMYPFixcDcPPN\nN3P8+PE0fQohUi8yaAdk21dOMFzQlrVs/autrWXv3r0AtLS04HA4sNvtAMyaNQuv10t7ezt+v5+G\nhgZqa2tjXrNkyRL27dsX/rrT6czMhxIiBSJnoWSknRtkelxknUWLFlFdXc3q1atRFIVNmzaxZ88e\nOjs7qampYfPmzaxfvx6AlStX4nQ6cTqdY64BWLt2Lffccw/PPfcchYWFPPjgg5n8aEIkjaqqI7LH\nJWjnBgMGbZkeN4INGzaMeD1//vzw+uLixYupr6+Pew1AQUEBjzzySGoaKUQGjc47k+nx3GC86XHJ\nHhdC5IDgqKgtI+3cYLigLdPjQohcEBy1q0K2fOUGAwZtmR4XQhifKiPtnGS4oC3Z40KIXDB6pC1B\nOzcYLmjL9LgQIheMXdOW6fFcYLigPegbnh4f3amFEMIoRq9hy/GcucFwQTtypC2n3gghjGrs9LgM\nUnKBoYO2dGIhhFGNLskvg5TcYMCgPTw9LtNFQgijkjXt3GS4imiDMtIWQkzStm3bOHz4MIqiUFdX\nN+K9xsZGduzYgdlsZsmSJaxbty7qNQsXLmTjxo20tLSEj4G98847ueWWW5La1jFBW5Jwc4Lhgras\naQshJuPgwYO0trZSX1/PiRMnqKur4+677w6/v2XLFnbt2kVlZSVr1qxhxYoVeDyeMddoJXa//e1v\nc+utt6asvdqatsWs4A+ocr/LEQYM2oGIP0snFkIkpqmpiWXLlgEwb948uru76e3tBaCtrY2SkhKq\nqqoAWLp0KU1NTXg8njHXeL3etLRXC9pWiwl/ICAziznCcGvakckZUtZPCJEot9tNWVlZ+HV5eTnd\n3d0AuFwuysvLR7zncrmiXuNyuQB4+umn+dKXvsS3vvUtPB5P0turTY9bLWZAcnhyheFG2nK+rBAi\nGdTR6dkTuOazn/0spaWlLFiwgCeeeIJHH32U++67L+712kl2ibzf2e0b+qWh2cVT77fR3Hxhwm3O\nlHifVU/S+VkMF7QjB9dyVJ0QIlEOhwO32x1+3dnZGU4kG/1eR0cHDocDq9U65pqKigqcTmf4a7fd\ndhubN29OqA01NTUx32tubh7x/qmzF+G3HRQV5nOxt4eZMy+jpuaahH5Ppo3+LHqWis8y3kOA4abH\nkZG2EGISamtr2bt3LwAtLS04HA4KCgoAmDVrFl6vl/b2dvx+Pw0NDdTW1ka9xm6387WvfY22tjYA\nDhw4wNVXX5309u7782lguAqkTI/nhoRG2tG2NGiybhuEjLSFEJOwaNEiqqurWb16NYqisGnTJp5/\n/nk8Hg/Lly9n8+bNrF+/HoCVK1fidDpxOp1jrgH44he/yDe/+U0KCgooLCzk/vvvT36DhwYoZpMC\ngF9yeHJC3KAdbRuEtqUBsmsbxOg1KL9sgRBCTMCGDRtGvO7p6QlPfS5evHjEvS/WNQA33ngjv/jF\nL1LTyCFajDabQhOmMtLODXGDdrRtEF6vF7vdnnXbIEY/aEonFkIYlTZIMZuHRtoys5gT4gZtt9tN\ndXV1+LW2pcFut0fdBtHW1kZXV1fUayC0DeKpp55i+vTp/J//839GXB/NRLIpR3fao8feIXCpLd5H\nzBg9Z0/que1CGIE2sWgxh0baksOTGyacPZ7ubRATyaYc9AWg/nT4tdM5j5rrqibc3nTQc/Zkptou\nDwpCDBs70pagnQviZo9H2wZRUVER9T1tG0Ssa2666SYWLFgAhLZBvPPOO0n7ICAF9IUQuUMds6Yt\n97tcEDdox9rSANm3DWLs+bLy5CnS7/Vjnfz3q+9L/xMpFVl7HCTxNlfEnR6Ptg1iz549FBcXZ902\niDGJaNKJRQY89txhOj295OdZ+MzNV2a6OcKghkfaoaAtI+3ckNCa9ugtDfPnzw//OZu2Qcj0uMgG\nM8sL6fT08ud3XPgDwXCikBDJNLymLYloucRQd5PRQVu2fIlMyLOZw3/u6fNlsCXCyCR7PDcZKmhr\nT55KaLZIRtoiIyKPhB30yY1UpIbKyIpoMj2eGwwVtIPhsn5D2ZSypi0yYETQjjjfXYhkCq9py5av\nnGKsoD3UZy1SIUhkkH/ESFuCtkiN4Kja4wGpPZ4TjBW0RyVmyJq2yITI0bUEbZEqw8uBCiaTgvtC\nX4ZbJNLBWEE7OPLJ0ydBW2TAyOlx6YMiNbTpcUVRMCnKmERcYUyGCtqjy/pJYobIBJ9Mj4s00IK2\nSQkNVEYXlxLGZKigra3pWEyyBUJkjm/E9Lj0QZEakedAmEwy0s4VhgrawdEjbenEIgNkpC3SQbvf\nmRQFkyL3u1xhqKA9uoC+jLRFJkQGbZ9s+RIpMmJNW6bHc4ahgnY4EU3WtEUGSSKaSIfIYlIyPZ47\nJnyedjYLypq2YWzbto3Dhw+jKAp1dXUsXLgw/F5jYyM7duzAbDazZMkS1q1bF/OajRs30tLSQmlp\nKQB33nknt9xyS8raHQiqI6YpZXpcpIrWy8IjbQnaOcFYQVuyxw3h4MGDtLa2Ul9fz4kTJ6irqxtx\nKM2WLVvYtWsXlZWVrFmzhhUrVuDxeGJe8+1vf5tbb701LW3XpsMVQjdVSUQTqTJipC1bvnKGoYO2\nnC+rT01NTSxbtgyAefPm0d3djdfrBaCtrY2SkhKqqqoAWLp0KU1NTXg8npjXpJNWDc1mNTPgC8hI\nW6TM8JYvRbZ85RBDrWmrQzE6nIgm64m65Ha7KSsrC78uLy/H5XIB4HK5KC8vH/PeeNc8/fTTfOlL\nX+Jb3/oWHo8npW3X1rC1k75kTVukSlBG2jnJkCNti9TiNRR1EiMI7ZrPfvazlJaWsmDBAp544gke\nffRR7rvvvrjXNzc3T+p7urz+oQaERtjtp8/S3Nw/gZZnt0T+XkR6jM0eD/V7RTvmUBiSsYJ2OHtc\nEtH0zOFw4Ha7w687OzupqKjA4/GMea+jowOHw4HVao16jdPpDH/ttttuY/PmzQm1oaamZtz3m5ub\no35Pe+cl+NU5iosKuNjrpbRsOjU1H07od2a7WJ/ZyLL5IWV09jiEDkmyWiRoG5mhpsfHnHojiWi6\nVFtby969ewFoaWnB4XBgt9sBmDVrFl6vl/b2dvx+Pw0NDdTW1sa85mtf+xptbW0AHDhwgKuvvjql\nbfeF17RD/7QkEU2kSnikjRIO2nJIkvEZcqRtMikoyEhbrxYtWkR1dTWrV69GURQ2bdrEnj176Ozs\npKamhs2bN7N+/XoAVq5cidPpxOl0jrkG4Itf/CLf/OY3KSgooLCwkPvvvz+lbdeCdp5VW9OWRDSR\nGqOzxwH8siRoeMYK2qOmiwKSPa5bGzZsGPF6/vz54anKxYsXj9gCFusagBtvvJFf/OIXqWlkFKOD\ntk8S0USKBEdlj4OMtHOBoabHR08X+WV6XKSZtk/bpo20ZcuXSJHoa9oStI3OUEFbmx4Pj7SlA4s0\n07Z4WSwmTIoEbZE6o7PHARmo5ABjBe3wk2foUHjpwCLdtOlws0nBbDZJIppImWhr2jJQMT5jBe2I\nkbZZ1rRFBoSDttmExWySRDSRMtFH2nLPMzpjBe3IkbZJkYpoIu18Q9PhFpOC2azI9LhIGTV8nvbw\nmrYUlDI+QwVtrYypaWi6SLY/iHTzBbSRtoLFZJIypiJltJ6lKArmoelx2a1gfIYK2oHh+SJJRBMZ\nMbymbcJsVsIjbyGSLkr2uBSUMj5DBe1wcRUFTCbJpBTppwVtk0nBYjYxIIloIkWC0da0JY/H8AwV\ntCMPljApJhlpi7QLr2mbQwUv/IGgnL4kUkKyx3OToYL28EhbwWySkn4i/cJr2kPT4yAZvSI1RmaP\nh/4ss4vGZ6ygPfqouqA6qWMdhZisEfu0h+6kkhwkUiEye1zra/KAaHwGC9pRCujLk6dIo8g1bbPs\nnRUpFG2kLYloxmesoB2Mlk0pN0yRPoGIk+a0PigjbZEK4UEKkYMU6WtGZ6igrY4qrgLSiUV6aQ+J\npqHiKiBBW6RGtIpoUgXS+AwVtEeMtGV6XGSA9pAYSoaUdUaROtFP+ZL7ndEZK2hHGWnLk6dIJ21N\n0Ryxpi0jbf3Ytm0bq1atYvXq1bzxxhsj3mtsbORzn/scq1at4rHHHkvomn379nHttdempK3aSNs0\ndEASyHJgLrBkugHJpMVnBSKSgOTJU6SPVtzCNCJoS1U0PTh48CCtra3U19dz4sQJ6urquPvuu8Pv\nb9myhV27dlFZWcmaNWtYsWIFHo9nzDX19fUADAwM8MQTT1BRUZGS9qoMD1LCfU2CtuEZf6QtnVik\nkTbSNiky0tabpqYmli1bBsC8efPo7u6mt7cXgLa2NkpKSqiqqsJkMrF06VKampqiXuP1egH48Y9/\nzBe+8AVsNltK2htRtVnKmOYQQwXtEafeSDalyAAte1wxgUnWtHXF7XZTVlYWfl1eXk53dzcALpeL\n8vLyEe+5XK6o17hcLk6ePMnRo0f59Kc/nbL2Rt/iKn3N6Aw2PR5tTVuePEX6aDdNsyLZ43o3mcJM\n2jX3338/995774Svb25uTvj9vt4+ANra2/Bc8of+3Haa5uZLE/69mRDvs+pJOj9LQkF727ZtHD58\nGEVRqKurY+HCheH3Ghsb2bFjB2azmSVLlrBu3bq41+zbt4+vfOUrHDt2LKkfJnr2uNwwRfqEp8cl\nEU13HA4Hbrc7/Lqzs5PS0tKo73V0dOBwOLBarWOusdlsvPfee2zYsCH8tTVr1vD000/HbUNNTU3M\n95qbm0e8n/f7/4aLXubOmUve+R44dgLHzJnU1CxI/ENnyOjPomep+CzjPQTEnR6PTM7YunUrW7du\nHfH+li1b2LlzJ7t372b//v0cP3583GtSmZwR7dQbWeMR6eQPBDEpI5OD5MFRH2pra9m7dy8ALS0t\nOBwOCgoKAJg1axZer5f29nb8fj8NDQ3U1tZGvebyyy/n97//Pc8++yzPPvssDocjoYA9UcPZ4xGJ\nt/KAaHhxR9qxEi3sdvuI5AwgnJzh8XhiXqMlZzz88MNJ/zAjKgRJNqXIgGBQxWwOPQtLRTR9WbRo\nEdXV1axevRpFUdi0aRPPP/88Ho+H5cuXs3nzZtavXw/AypUrcTqdOJ3OMdeky8hTDWU5MFfEDdpu\nt5vq6urway3Rwm63R03OaGtro6urK+o1LpeLo0eP8o1vfCM1QTtyTVv2LYoM8AeD4VGPFrwlaOuH\nNqWt6enpCU99Ll68OLyda7xrRvvDH/6QvAZGUFUVBakAmWsmnIiW7uSMiSRmnD17AYCOznNcvBhK\nzDh67B0Cl9om9DvTRc+JGHpueyoFAsMjbZkeF6kUVEMBG2TLVy6JG7SjJWdo69HpSM6YSGLGn0+/\nBW97qZpZBZYeaD+D0zmPmuuq4n3MtNNzIkam2q6HBwV/IIhlKGtcEtFEKqmqylDMlsTbHBI3ES1a\nooXdbgeyLzkjMjEjfCi8rPGINAoE1HDNcQnaIpVUxo60JWgbX9yRdrTkjD179lBcXJx1yRnBiBJB\nsqYtMiEQjBxpS3EVkTqRI22zTI/njITWtEcnWsyfPz/852xKztAS0Uxy6o3IEH9AJc9mBpDiKiKl\nVHV4Wjx8v5MDkgzPUGVMpfa4yLRARPa4bPkSqRSMMtKWvmZ8xgrawYh92lpihqxpizTyB1Qskj0u\n0kCNyB6XoJ07jBm0ZaQtMiQQCIanxbU1bbmRilRQVRWTlj0uQTtnGCpoRzuqTkY5Ip0CQRWLaXRF\nNDlPWyRfMDg80tYGKlLG1PgMFbQj17TNiiSiifQbMdI2Sx8UqaOqavjBEEJT5DLSNj6DBm1kelyk\nXTCoElSJsk9bRtoi+SIT0WAoaAekrxmdsYJ2lDVtGeWIdAkMbbeRNW2RDpGJaBCqdS99zfgMGrQj\nT72RTqxH27ZtY9WqVaxevZo33nhjxHuNjY187nOfY9WqVTz22GMJXbNv3z6uvfbalLZZe0AcnT0u\nN1KRCqFENJkezzUTPjAkmw2XMZWRtp5Fnsd+4sQJ6urqRhTw2bJlC7t27aKyspI1a9awYsUKPB5P\nzGtSeYZ7JO1YxMh92iZFkiFFakSdHpegbXjGGmlHOU9b1rT1J9YZ7sCIM9xNJlP4DPfxrtHOcLfZ\nbCltt9bXtJE2gMVilhupSInIimgQWpaRvmZ8xgraUc7TllGO/rjdbsrKysKvtfPYgahnuLtcrpjX\nnDx5kqNHj/LpT3865e3W+po5IqPXKjdSkSLB4OiRtqxp5wJDTY9HZo+HC+hLRTTdS/cZ7pDYMaCj\nv+dCT+gM9wsXumh9v2foq0G8Pb26OFY0EUb5HEYwZqRtUvAHgkMHiSjjXCn0zFhBOzh2y5c8eepP\nrDPcPR5PWs5wh/HPcYfoZ4qfcXvhv85R6ZjB3DmhUX/BO+9gNim6PTs9kp7PgJ+sbH5ICY4KzpFl\nc60Wc6aaJVLMUNPjwxXRFKn7rGN6OsM9knYsojliTdtqkSlLkXyRAxSN1u+kvxmb4UfaErT1J9YZ\n7p2dndTU1GTVGe6RtKUYS8SatsVsoifgS3tbhLFpS4GjK6KBBG2jM1bQjixjKh1Y16Kd4a5NVWbT\nGe6RwoloY0baUqVKJFe0kbYsCeYGQ02PR275Ck+PSwcWaRKIkj3e2+djYFD6oEguLWiP3vIFErSN\nzlhBO0oZU59Mj4s0CRdXMY+8kQZVNdw3hUiGQMS9TjNcNldmdozMcEFbG+QoioLFLEfVifQJjCpj\nChE3Unl4FEkUXtOOkj0uI21jM1TQHl1AX1EUznf3Z7BFIpcMr2lHmbL0yehHJI/2gDi6jCnIA6LR\nGSpoB0bV4jWZFCmuItJmOHs8ykhbRj8iiaJmj8uWr5xgqKA9uhKQ2aTIKV8ibaKNtC1Dfx6UG6lI\nonFH2j7pa0ZmqKAdDKpEFu8zmxRJABJpM3zK1/A/q+FtODI9LpIncnurxix9LScYL2hHdGKZHhfp\nNHzK18jiKiBTliK5xt3yJWvahmasoK2Oni4ySdAWaeOPUsZUG/0MSiKaSCJt2W/0/Q7kAdHojBW0\ng1HWtAMStEV6RBtpS3KQSIWoI23Z8pUTjBW01dHny8qatkgff5Q17fBIW26kIokCUQ8MkaCdCwwV\ntFVVHfHkaTJJNSqRPkHZpy3SJPpIW2Z1coGhgnYwqEKUJ0/Z9iXSIdpI2yIV0UQKRCtjKjsVcoPh\ngrY8eYpMib6mrSWiSR8UyTO85Wv4a3JIUm4wVNAOjAnassYj0id69rg8OIrk0xJsR1ZEky1fucBw\nQVuJ+ERah/ZLJxZpMN5IW6YsRTJFK65ikQfEnGC4oC0jbZEp42aPy/S4SKLhRLThr0n2eG4wVtAO\nBMdkj4N0YpEe2kjbaolIRNP2aQdkpC2SZ/zztOV+Z2TGCtqji6sM3TBlelykg7aWaDZFme2RkbZI\novHKmA7KUoyhGS5oR+QAYVZkpC3SR8vatUSMtLUHRymuIpIpGKW4ijarIyVzjc0wQVsdKqKiRHny\nlJG2SAdtytIapfa4JKKJZNL6WmT2uJYAOTAofc3ILJluQLIEo3RiWdMW6aT1sxFbviQ5SFe2bdvG\n4cOHURSFurq6Ee81NjayY8cOzGYzS5YsYd26dVGvWbhwIYcOHeKhhx7CYrFgs9l4+OGHKS8vT1o7\nxx9pS18zMsOMtP3jFNCXkbZIB3+0LV8mmbLUi4MHD9La2kp9fT1bt25l69atI97fsmULO3fuZPfu\n3ezfv5/jx4/HvOapp57ioYce4qc//Skf/vCHefbZZ5Pa1mCURDRFUbCYTQz4/En9XSK7GGakrWXu\njig2INmUIo1Od3oBePnQaQryQv+0LDLS1o2mpiaWLVsGwLx58+ju7qa3txeAtrY2SkpKqKqqAmDp\n0qU0NTXh8XjGXOP1ennkkUeA0LJdR0cHNTU1SW2rVpo5cpACkGczy0jb4BIK2tGmfzTZMmUUbbpI\nRtoinaKtM8qDo3643W6qq6vDr8vLy+nu7gbA5XKNuFeVl5fT1tZGV1fXmGtcLhd2u52XX36ZrVu3\ncuWVV3L77bcn1Ibm5uaE3j/xXg8Anq7ztL7vDb+vEODipd64Pycb6KGNiUrnZ4kbtCOnf06cOEFd\nXR319fXh97ds2cKuXbuorKxkzZo1rFixAo/HE/Uabcpo9uzZPProozz77LP84z/+Y1I+SCDK9Lis\naYt0CoaLq0SriCZ9UG9UdeKnA0Zes2TJEj7+8Y+zfft2nnjiiYTudeONyJubm8PvdwVa4ZUuKqbP\nYO6c4YeJ4hPHGfAFkj6yT7bIz6J3qfgs4z0ExF3TjjZl5PWGnuwip4xMJlN4yijWNY888gizZ88O\nTxnNnDkzGZ8PiDHK0bLH5YYp0iD6SFv2zuqFw+HA7XaHX3d2dlJaWhr1vY6ODhwOR9RrKioqeOml\nl4DQOvOKFSuSPhKLVlwFwGY1Sf6EwcUN2m63m7KysvBrbfoHok8ZuVyuca95+eWX+dSnPoXb7U54\nyigR2hR41DKmMj0u0kCrB20alRxkMilSXEUHamtr2bt3LwAtLS04HA4KCgoAmDVrFl6vl/b2dvx+\nPw0NDdTW1ka9xm63s3PnTt5++20ADh8+jNPpTGpbh3fLjPy6zWqWoG1wE05ES/eUUaJrPB5vKGOy\nt7eH1vdbATjvGQTg5MlWmvM8E253qul5TUfPbU+VYFAdMTWusZgUmR7XgUWLFlFdXc3q1atRFIVN\nmzbx/PPP4/F4WL58OZs3b2b9+vUArFy5EqfTidPpHHMNwNatW/nud7+L2WwmPz+fhx56KKltjTXS\nzrOZGfQHQ8cUR+mLQv/iBu1Y0z/R3tOmjKxWa8wpo+XLl4enjHbu3Bm3gYmu8Zx2eeFX5ygutjN3\nzmwAVOtFOH6SmVWXU1NzddzflU56XtPJVNuz/UEhEONGaTabZHpcJzZs2DDidU9PT7ivL168eEQ+\nT6xrAK6//np+/vOfp6aRRE9klaBXAAAgAElEQVS8hdBIG0LLMfk2w2wOEhHiTo/Hmv6B7JoyCowz\nPS7Z4yIdYo1uzCZFypiKpIqWeAuQNxS0pSqaccV9FIs2ZbRnzx6Ki4uzaspovCQgmZoU6RBzetxs\nwifrjCKJohVXgeGgLXu1jSuh+ZPR0z/z588P/zlbpowCgbGd2CQjbZFGsafHFfoGpEqVSJ7hpMeR\nX+/sChWDkeUY4zJMGdPhCkHDX5ORtkinoBp9pG01m2S6UiRVrEQ0re699DfjMlDQjrZPW45FFOkT\nDKpj1hghdFRnIKjKjI9ImmjnaQNYtTO1ZTnGsAwXtCOfPC3hp06ZmhSpFwgGo06PW2T0I5IsECN7\nPDzSlqBtWIbZE6Blj5ujnC8rSRn6o4d696ONl4gGodFPUYE1Jb9b5JZoRxFDxAOiBG3DMuBIe/hr\n0oH1SU9HJEaKteXLYgl9TfqhSJZYa9qRD4jCmIwz0o6yxiMdWJ8SrXcPmT8iURMMqgRVxh1py/S4\nSJZYxVUssqZteMYZaQfGTheZTAomRZERjs7opd59pPDuhfGCtvRDkSSxztOWB0TjM8xIO1Y2pcWs\nSAfWuUwckZhIydTI7xkYypsY6O8P177X9Hj7AHjjrSNccuUn1P5sle2lZHNF7JG29oAoeTxGZZig\nrW2nUUYnZljkqDq9iVXv3uPxpKXePYxf8x7G1l+/1DsI/3EGe1ERc+fMHfG953s7effMWa5wXkXN\ngsqEfn820nO9/MnK1oeUoZgddZACMOCTHTNGZZzp8WD0CkEWswRtvdFLvftI2pntkogm0kHbLRM7\nEU1G2kZlmJF2cJw1HrlZ6kusevednZ3U1NRkTb37SNqZ7bJPW6RDuIzpqGGXJN8an2GCdrSKaBCa\nLvL2yVOn3kSrd69NVWZLvftIWiLkuNnjciMVSSJlTHOXYabH/VEODIGhs4x9gUklMwmRKL+MtEUa\nxTrlyypLMYZnmKA9vOVm5NetUn9cpEEiQVumLEWyxMrhMUtfMzzDBO1YW77MUmxApIEWtM1RDwyR\n0Y9IrpgjbVmKMTzDBO1o52mDTE2K9PD7o+dUQMSNVPqgSJLYgxQZaRudcYJ2MHoikExNinRIaE1b\n+qBIkpinfJkUFEUeEI3MOEE75r5FmZoUqeeLcsqcxmKRI2JFcg1v+Yr+kCj7tI3LOEF7nOIqIEFb\npFYgoUQ0uZGK5AgGoo+0QWpTGJ3xgrZMj4sM8I830pbZHpFk4ZF2tMRHsxySZGTGC9qSiCYyYLxE\nNOmDItliFVcBKd1sdAYK2rEODNG2fMnUpEgdfzD2SFtRFGwWkxziIJImGFSjTo2DBG2jM07QDsia\ntsic8Q4MAcizmemXkbZIkkF/APPoSlJDtOlxqQJpTMYJ2nH2LUrQFqkU3vIVY/hTkGehf0BG2iI5\n+gcCWC2xgrYJVR3uk8JYjBO0Y2TvWiURTaTBeFu+IBS0e/slaIvkGPAFwrOIow0PVCRoG5FxgnbM\nU2+GMndlalKk0HiJaBAK2n0DfpmyFEkxMOiPOdK2Sl0AQzNO0A5Ev2lKLV6RDoP+UP+KNfopyLMQ\nCKr45OAakQT9g+ONtCX51siME7S1U75GDXSsVjMAvf2+dDdJ5BBtJidm0M4PHV3fJ+vaYoq0h7/x\n1rRBBipGZaCgHT0RLc8a+oiynihSScuZ0AqpjFaQJ0FbJIc27R0vaEsejzEZJmgHY1REs1lkpC1S\nTxvVWGLcSCVoi2TpjzOrIyNtYzNM0I6ViDY8PS43S5E64aA9zpo2SD8UU9cfd6QtybdGZpigPXw0\n4sivm00KeTazjLRFSg3GCdqF+VZARtpi6sL5EzI9npMME7RjHQoPUJhnoUdGOCKFhhPR4qxpSz8U\nU9Q/EOpr1hh9TYK2sRkmaAeCKiYlegH9wnyrjLRFSmnba+JOj8tIW0yRNj1uMZujvi+nyhmbcYJ2\nIIgpRi3eogILPX1ysxSpM+DzYzYpUR8aQRLRRPJoiWiy5Ss3GSZo+4NquKjAaIV5VvyBID6/dGKR\nGoO+YMxRNoSWaECCtpg6bctXrDVtsxwFa2iGCdrBgBqz7vOl3kEAGW2LlBkYDMRczwYpriKSJzzS\njvGQKFUgjc0wQdsXiD3SsUlVNJFiA75AeIQTzfCWL+mDYmriTY9bhwpKaQlrwlgsmW5Asly41B/z\nPZtURRMpNuALkGeNnhgEcKDlHCAj7Wy3bds2Dh8+jKIo1NXVjXivsbGRHTt2YDabWbJkCevWrYt6\nzcKFCzl79izf+c538Pv9WCwWHn74YSoqKpLSxvD0eKyR9lAw75cDQwzJOCNtfzDmGo9WFa1HRjki\nRQbHOSoRwDbUNyVoZ6+DBw/S2tpKfX09W7duZevWrSPe37JlCzt37mT37t3s37+f48ePx7zm+9//\nPnfccQdPP/00y5cv56mnnkpaO+OOtIfud9LXjCmhkXa0J0lNtjx9+gPBmGs8Mj0uUik4dIDDeGva\nVgnaWa+pqYlly5YBMG/ePLq7u+nt7QWgra2NkpISqqqqAFi6dClNTU14PJ4x13i9XjZt2kReXh4A\nZWVltLS0JK2d8UrmygOiscUdaevh6TMYVPEH1HHWtGV6XKTOYJybKITqB1gtJnr65MExW7ndbsrK\nysKvy8vL6e7uBsDlclFeXj7iPZfLFfUal8tFYWEhZrOZQCDAM888w1/+5V8mrZ3H2y4AsRPRtH4o\na9rGFHekHe3p0+v1Yrfbs+bpU3vyjDVdpI20ZXpcpEK8uuOaPJsZrwRt3VBVdUrXBAIB7r77bm68\n8UZuuummhK5vbm6O+373xUsAdHScobsrdv3x813dcX9eJmVz2yYqnZ8lbtB2u91UV1eHX2tPkna7\nPerTZ1tbG11dXVGvcTqdAOGnT20qfar64+xbzJNDQ0QKJRy0rWYZaWcxh8OB2+0Ov+7s7KS0tDTq\nex0dHTgcDqxW65hrtCW/73znO8ydO5e77ror4TbU1NTEfK+5uZmamhpsDQ2AjyvmzomZ/Gg/chTF\nbB3352WS9lmMIBWfZbyHgAlnj6f76TORJ0+PNxSMB/r7aH2/dcz3dA29f7K1neZm70SanlJ6ftLU\nc9uTLd5Z2po8q5nz3f0EgrFrCojMqa2tZefOnaxevZqWlhYcDgcFBQUAzJo1C6/XS3t7OzNnzqSh\noYHt27fT1dU15hq73c6vfvUrrFYrX//615PeTp8/VDI31vQ4QH6eWbLHDSpu0I729Kk9Sabj6TOR\nJ89TZy8C5ygrKWbunFljvm/axX72HzlGccl0amo+lNDvTTU9P2lmqu3Z+qAwEOd8Y02ebWiZps/H\ntCJbytslJmbRokVUV1ezevVqFEVh06ZNPP/883g8HpYvX87mzZtZv349ACtXrsTpdOJ0OsdcA/DM\nM88wMDDA2rVrgdAy4ebNm5PSTn8giMmkYBrnwS/fZuFST29Sfp/ILnGDdrSnT7vdDmTP02e8fYvh\nNW2ZmhQpEO+wEI0E7ey3YcOGEa97enrCD6iLFy+mvr4+7jUAP//5z1PTQEIj7fFG2RAq5tM34EdV\n1Zj18IU+xQ3a0Z4+9+zZQ3FxcdY8ffbHOV82vOVLtkCIFBjwaQ+N8abHQ//cvH2DQFGqmyUMyh8I\nxky61RTkWQiqMOgPjlv0R+hPQmvao58k58+fH/5zNjx9DsSpxWsxK5gU6JWRtkiBhEfaQzdPb6/0\nQzF5vgQCcX5e6P3+Ab8EbYMxRBlTLeEi1tOnoihYrWZ6JHtcN/RQ0EczkS1fgGz7ElPi8wcpKrCO\n+z2RR8GW2PPS0SyRJoYoYxpvehxCo5w+2aetC3oo6BNpIIH+ByPXtIWYDFVVx63+qCmwyalyRmWs\nkfZ4tZ+tJjmaUydiFfSB7ConqRn0J77lC2SkLSYvXiEpTf7QSFuqohmPIYJ2IiMdm8WMe0D2yOpB\nrII+EL2cZCoK+iSyvUz7nhPvhSpUeTznaVUvxvz+rkuhh8bjJ9tobo79fdksW7fd5YpEZ3Uip8eF\nsRgiaMc7FB6GM8j7BvzY46wHieySiXKS8fahR+5V/93hA0A3l82sZHZlccxrii/20/i2Vi/ggwm1\nI5voubbAZGXbQ0oi9zoYTkTrkwIrhmOQNe3xy5iCnPSlJ5Mp6DPeNZMpJzkRWoWqxLPHB1PSDmF8\nidzrYHhNu19G2oZjiKAdb8sXyElfelJbW8vevXsBxi3o4/f7aWhooLa2NuY1qSwnqQkEtKA9/rJL\n/lAi2iUJ2mKSErnXARTky/S4URliejzRNW2QzF09iFXQp7Ozk5qamqwo6BPJNxS0zXFupGazicJ8\nC91eCdpichIdaRcXhCruXeqRvmY0hgja8fZpw/DUpDx56kO0gj7a+mI2FPSJFAiE1s/jTY8DlBTl\ncbFnINVNEgYVHmnHCdpvvRdaKnrjhJu/TXmrRDoZYno8sUS00Hsy0hbJ5k9wehxgmt1Gt3dwUsl1\nQiSeiCbT40ZliKA9MBhAgXFPvQkfGiKJaCLJwkE7zugHQiPtQFCVh0cxKQMTTETrkxwewzFE0PYF\ngpjNyrin2WhJQFL3WSSbfyLT4/bQWmO3rDWKSehPcHrcZFLIt5llpG1AxgjavgBmU7wSkqEnT8nc\nFckWPt84gSMQtTrQ3V5Z1xYT15/g2e0wfDynMBZjBG1/cNypcZCRtkgdfyCY0Ho2RIy0JYNcTEIi\nSbeagjwL/YOB8JZEYQzGCNoJ3DTzZI+sSJFQ0E7sn9K0otBIWzLIxWQkuk8bhkuZXpR7nqEYI2j7\ng3Gnx21WM4oihzWI5PMH1ISDdml4elxupGLiEjnRUBMO2tLXDMU4QTvOSNukKBTlW2WkLZLO7098\npN2i7Z9915XKJgmDChdXmcBI+4LkTxiKcYJ2Aid3FRfaZE1bJN1E1rS1G2mvJAiJSfD5Eq8JoJUy\nvXBJgraRGCJo+/2BhIK2vdAqhzWIpAoGVQLBxKfHC/OtKIrkVojJ0WoCxFsOhOFSpuc8PSltk0gv\n3QftQCBIUAVTAp3YXmBl0B8MHyQvxFQN+hPfggOh/bP2AiuXZMZHTIIWtOPtlgEomxbKn2jv8Ka0\nTSK9dB+0B/3aYQ2JTY+DHI0okmfQl9ixnJGKC2309PnCR3oKkajhkXYC97siG2aTQlvnpVQ3S6SR\n7oO2duNLdHockFGOSJrhE+YSW9OG4YfH8919KWmTMC6t+l4CdXwwKQqlxXm0dXgJBqXWvVEYIGgn\nPj2p3SxlPVEky0SnxwGKhx4eO7t6U9ImYVxa9b3xSjZHKp+Wz6AvgOuCPCAahQGCduIj7fbO0NrO\ny4faU9omkTu0kXa8s7QjFReFHh47PXIjFRPjDyS2U0ZTVpwPQFuHTJEbhWGCdiKJGYVDWyB6+mS7\njUiOQZ9WoWri0+MuGWmLCfInULI5klY2t8Mjfc0odB+0w4kZCYx0tDXtnj6ZHhfJMZmR9rShkfZp\nl2zFERPjDwQTOphGoz0gdkrQNgzdB+0JJaIVaGvakogmkmNgEmva04psWC0mTp3tTlWzhEH5A+qE\npse1pZg3jksFPqPQfdDWpicT6chFBaGRttQfF8mi9b+JBG1FUZg+LZ+2Tm/4eiESoSWiJaoo34JJ\nUbjYI/c8o9B90B4eacf/KGaTQlG+hR4J2iJJwlu+JrCmDTC9tIBgUKX+pWOpaJYwqIkmoimKgr1Q\nzlwwEv0H7UDixVUAigpC9cdl36JIhsmMtAFmlISyet3d/UlvkzCuiSaiQWg5pm/AL5UgDUL/QXsC\na9oQ2iMbVFW65eQbkQTajTCRoxIjzSgtAMAt+2fFBPiDE1vTBtmtYDTGCdoJjnTsQ+vabqlGJZIg\nHLQTWJ6JNH1opC1V0cRETGakPVzMR/qaEeg+aPv9iSeiwXA25bnz8tQppm6ya9pWi5kSuw33hX5U\nVZZqRHxBNXSi3ERH2loCbtdFWYoxAt0H7YlOj5fYQyffnHHLyTdi6sIHhkxwehxgRkkBA74A7gty\nMxXxBYfOl5noSLswPxS0PRK0DcE4QTvBkU6pFrSlsIVIgoFJJqLB8Lr2SdmvLRIQGEqenUhxFQht\n+wLouiR5PEag+6A9OIEtXxCaHlcUOOuWoC2mbrLZ4zC8rn3yjARtEd/QRpmE73WawqHpcY/sVDAE\n3QftiU6Pm00K04psnHbJ9LiYuuGR9sRGPxAx0j5zMaltEsYUHmlPcHq8IM+CgkyPG4UBgvbEEtEg\nNEV+sWcQrxQcEFM0nIg28X9K9gIreVYzp2SkLRKgBe2JJqKZFIWCfAtdlyRoG4EBgvbEtnzB8Lp2\nu4y2xRRNZXpcURSml+Rzxt1D/4CcPCfGN9lENAhlkHu6ZaeCEeg+aPsnOD0OUD60lth6VqYlxdQM\n+AKYFGVSN1IITZGrKrSek74oxjfZ6XGAonwrg/4gPf3ycKh3ug/aEy1jCqGtNgCnZC1RTNGgLzCp\n9WzNcDKa9MVssG3bNlatWsXq1at54403RrzX2NjI5z73OVatWsVjjz0W95qf/OQnVFdX09OTnKTX\nyU6PAxQOZZB7pJiP7iUUtLO5I0/kwBBN2bR8FKD5WGdS2iBy18BgYEJLM6MNJ6PJunamHTx4kNbW\nVurr69m6dStbt24d8f6WLVvYuXMnu3fvZv/+/Rw/fjzmNb/85S85f/48Docjae0LTGF6PHyutlRF\n0z1LvG+I7JQnTpygrq6O+vr68Ptbtmxh165dVFZWsmbNGlasWIHH44l6TSo68kSzxwGsFhMlxXmc\n7+5DVVWUCe57FEIz6AtgnURhFU35tHxMJkVG2lmgqamJZcuWATBv3jy6u7vp7Q1VTmxra6OkpISq\nqioAli5dSlNTEx6PZ8w1Xq+XZcuWYbfb+fWvf5209k12nzZAiT0UtGWrq/7FDdrROrLX68Vut2dF\nRw6fpz3BKcoZJfkcvzSAq6sPR3lh0tojcsuALzCpamgai9nE5RV2Tp29SDCoTnptXEyd2+2muro6\n/Lq8vJzu7tAMiMvlory8fMR7bW1tdHV1jbnG5XLhdDon1Ybm5uaY72lB+9Klblrfn9jOl15vaC37\n8Nsnuaywa1JtS7bxPqvepPOzxA3a0Tqyy+XCbrenpSPH+8s47wl1wNOn2ye0tqgEQ9sfGvYf4qrL\n8ifcrmTQc6fVc9uTacAXpCAv7j+jcTkvm0ZbxyU6u3qZOb0oSS0TUzWZTOupZmfX1NTEfO/4b/YD\nUF5Wxtw5lRP6uZWDfv7nSAtBc9G4vyNdmpubs6IdyZCKzzLe/XXCd5t0d+Tx/jKam5spLCoGBnDO\nnTuhUcoAXbx75n0KSmdSUzNv0u2bLD132nS0fdu2bRw+fBhFUairq2PhwoXhjtzY2MiOHTswm80s\nWbKEdevWxbwGQnkUDz74IAcPHqSoKHlBUVVVBn1TW9OG4SWek2e6JWhnkMPhwO12h193dnZSWloa\n9b2Ojg4cDgdWq3XMNRUVFSlp31QS0fJtFuwFVs6dl+lxvYt7t4nWkbVOGasjj3dNsvmGjqqb6LRi\nWXFor/bpTtmrnW2yPSFIo5XQtU4xaM+QDPKsUFtby969ewFoaWnB4XBQUBBKFJw1axZer5f29nb8\nfj8NDQ3U1tZGvcZut6ekfVPZpw2hDPIzrh6CQdmrrWdxR9q1tbXs3LmT1atXj+mUkR155syZNDQ0\nsH37drq6umJek2w+/+QSgUq0oC0FVrJOrDwKyI6EIM1k8ylGmy4Z5Flh0aJFVFdXs3r1ahRFYdOm\nTTz//PN4PB6WL1/O5s2bWb9+PQArV67E6XTidDrHXAPwox/9iMbGRlwuF3/3d3/Hhz70Ie6+++4p\ntW8qI20InXDY2dUnyzA6FzdoR+vIe/bsobi4OCs6ss8fnNRIx2YxU1RglZF2FoqVRwHpSwhKxFRK\nmEYqyrdSkGeRkXYW2LBhw4jXPT094aWgxYsXj9g5E+sagK9+9at89atfTWrbplJcBaCyvJB32y7w\nxnG3BG0dS2hNe3SnnD9/fvjPme7IPn9w0ltuSu15nHZ56R/wkz/FZCKROplICEok0e71Q4cB6O/v\npfX91in9vqI86PD0sr/pVfJt2VvzSBIQM2f4lK/JBe05M4vhMLx+tJNPfnRuElsm0kn3kcoXmELQ\nLg4F7XOeXq6ompbklonJipUT4fF40pYQFC/Rrrm5mauvXQC/6aB0WjFz58ya9O8CaL9wBvdFFyUO\nJ9fNmzGln5Uqek6enKxsekiZ6ki71J5HcaGNP7/rIhAITjmBUmSG7v+reXt9DPqCk7p22lCVoA7J\nqMwq4yX3ZENCkGZgCoeFjFY5VCvgWGt27KEV2WeqQVtRFGZX2unp83FKzl3QLd2PtAPBIGbz5D5G\ncdFQ0Pb0JrNJYopi5VF0dnZSU1OT8TwKzVRO+BqtamiN8chJD38z5Z8mjCg8Pa5Mvr/NnF7EkZMe\n3j7lYd6s0iS1TKST/oN2QJ30Gs80CdpZK1oehTZVmek8Ck2/log2hYpomqICK9OKbLxx3CWV0URU\nUx1pw/DD4dunPHzm5iuT0i6RXrqeHldVlUBQndBhIZEkaIup0M7Ankrt8UgzpxfRPxiQbYgiquAU\nE9EgVIN8WpGNt095ktQqkW66DtpajYDJ7pPNt5kpyDNL0BaToo20kxW0q2ZoU+Tnk/LzhLEkY6St\nKAoLrijH1dWH+4Kc+KVHug7a/sDUig0oikJhvpXTLu+UtwiJ3NM/ODTSTlIWbuS6thCjJSNoAyy4\nIlTnQEbb+qTroD1cIWjyH6O0OA+fP8i58zLaFhPTP5DckXb5tDzyrGbelqAtogiEutuUpscBunsG\nAHih6dTUGiQyQtdB26914imUkbxsRmhb0Jsn3HG+U4iRtJF2MhLRIDTzM3N6IWfP99B1sT8pP1MY\nR0BNzkjbUVaISVHk8BCd0nfQnmItXoDLK0JTkhK0xUQNr2mbk/Yzw+vaMnUpRtGWAydyBHE0FrOJ\nirICXBf6wsmUQj90HbQDgak/eZZPyyffZubN425Z1xYTEs4eT2JlqeF1bUlGEyP5/EODlCTVBVBV\neLftwpR/lkgvXQft8Eh7Cp1YURTmzJzG+e5+3nlfqlGJxCU7exzAUV6IxazIurYYY3ikPfX+NnNG\nqALfkVPycKg3ug7a2kjbMsU1nquHKgPt+/OZKbdJ5I5w9ngSg7bFbOKqWaWcON1Nb78vaT9X6J8v\nSdPjQPiUr6OnZKCiN7oO2v4kFBsAmD3TTlGBlf2HT8sUuUjYQBIrokW6YUElwaDKnw6dTurPFfrm\nD6iYFDApUw/aRfmhCnxHTp7H55/c2Q0iM3QdtLWR9lTXeMwmE4sXVOLu7pdC+iJhfQN+bBZTUm6i\nkZZ9ZA4mk8LvGk/KQ6QI8wVUbFYzSpL629yZxfT2+znynkyR64mug7Y/ScUGAGzW0F/FwSPnpvyz\nRG7oHwyQZ0t++f7pJQV87PoqTp65yJ9eb0/6zxf65B8K2snivKwEkHue3ug6aAemWBEt0uzKYhQF\nXnnzrIxuREL6B/0U5CXvJhrpy5+pxmY1s+vXLeG1c5HbfP7kBu3LKoqwWkz88fV2uefpiK6D9nBx\nlal/jHybhbkzp3G8vVvKSIqE9A+kZqT9QtMpDh3r5Pp507lwaYBH6g8l/XcI/fEHVPKsybtlm00m\n5sws5mLPIO93XErazxWppe+gnYTiKpEWXesA4BcN7ybl5wljG0jhSBvgg1dXYDErHDrmkmQhQb8v\nGN5mmCzOqmkAHGyRKXK90HXQDiQ5aFfNKOLq2aU0H+2k65KUkRSxBYMqg/4g+SkYaWsK8ix8wDkd\nb59P1rYFgWBy9mhHmjtzGgrwyltnk/pzReroOmgns9iAZub0QoJBlSf+882k/UxhPINDfS/PlrqR\nNsCHrqnApCjs+eO7su6YwwJBFVVNftDOz7Mwy2HnnfcvcMYt57jrga6DdrKOqot09ewyTAoca5Wi\nAyK2QV+o7xWkcKQNUFxo46rZpbR1ePnX/3orpb9LZK9B31BNgCQUVhntmrllAPyxWWZz9EDXQduf\npKPqIhXkWZgzcxquC328f072bIvotDXmVI+0Aa6fNx2QQ21y2XDQTv4t+8rLSrCYTfx633sEApI7\nke10HbTDa9pJfvq8Zk7oyfMPr7Ul9ecK4xgcOrwhPy+1I22AyvJCZpTm03r2It3egZT/PpF9BoaC\ndjJ2yoxms5pZcEUZ3j4fTbK2nfV0HbT94X3ayf0YzsumkW8z80LTKbx9Uv9ZjNXvC41IzrpTfyax\noijMn1tOUIU/HZIpzFyUyulxgIVXVQDwzN6jslMhyxkkaCe3I1vMJj58jYOefj+/3f9eUn+2MIbe\ngdCNLT8N0+MAV88uRVFk9idXDQ49JKZiehygtDiP666cTluHl6d+00IwKEmP2UrXQVtbfkn29DjA\ndfOmYzGb+K8/nZAOLMbo04J2GqbHAQrzrcydOY0T7d20Sn38nJPqkTbAjddVcXmFnV/ve4+nX3g7\nZb9HTI2ug3aqpschtM5z9exSLvX6ePO4JACJkXoH0zvSBrh2KMv396++n7bfKbJDKte0NXk2Mytu\nnMu0Ihu/+MO7HGuVypDZSNdBe2Bo7cWWxNJ+kRZcUQ7Az/YeDSe9CQERI+0Ub/mKdEXVNMqK8/hd\n0yk8F6X4Ty5JZfZ4pII8C7fVzEYFHvj3VyXxMQvpOmhrN85kFtGPVDWjiHmzSnj7lIdf/EFKm4ph\n6V7ThtAN+29XzGdgMMAPfn6I3n5JkswVw2vaqZse11zusPORD8zE3d3P/7vrQMp/n5gYfQftwSB5\nVnPSzzOOtORDlzOjJJ+f/u5tXnu7I2W/R+hLute0NWpQZU5lMa8f6+RL393L937WzCtvnZX9tQY3\nkKaRtmbRfAcVpQW8834Xb8sBSllF10G7fzCY8uIWhflWbls8B5Oi8Nhzh2V0I4DQmrZJAZslvf+E\nTCaFlbVX8NHqmeTbzNLa+1oAABQ8SURBVPzx9Xa2PnWQ//tfXuR424W0tkWkz2Aa1rQjmRSFmz90\nGQDPvHg0Lb9TJEbXQbtvMJiW6cmK0gIWXVuB+0If/9/u1yWbXNA3ECTPZkFJ4SxPLGaTiRsWVPLF\nFfP53G1X8wFnOV2XBrj70X3sfeWUPFgaUDqyx0e7bIadWQ47f37HJbOMWUS3QXvAF8AfSE8ZSYAb\nPlDJwqtm8Mpb53jhlVNp+Z0ie/Wm6YFxPIqiUFleyK01s/mLWic2i4lH/+Mwq+99ng0/eJmW985n\ntH0iecLT4ynYKTOej11fhcVs4ns/a6ZVyjpnBd0GbW/vIAB51vSsKZpNJhbNd2C1mHjq1y14LvbL\nOmKOCgRV+gdTeyznRF1RNY2/uuUqPnxNBVXTizj2fhd1P9ov5yQbRKqLq8RSUVbI0g9fjrfPxz2P\n/g+PPXeY8919aW2DGEnHQTs0BZjO0U5RvpUbFlTSPxjg//ruXj5f91t2v3hMpstzTE+fD1WF/LzM\njrRHK7Hn8bGFl/FXt1zFZ5fMw2ox8cBPXuWN465MN01MUe9A6H5nTXMOBcD8K8q5tWYWgUCQF5pO\n8bXtDXKYUgbpNmhf0kbaaZ6i/PA1Fdww30FRgRWrxcQze4/ywE9epcPTm9Z2iMzR9q5m00h7tFkO\nO5/8yFwCQZUt/3ZACmXoXOfQ/aW4yJaR3/8B53T+919Wc/MHL+NSr4/1P3iZh59+jfbOSxlpTy7L\n3rtOHNpBHukO2oqi8NHrqvjodVX0D/p5fv8pmt48y2tvd/DXt17FB6+qoKKsgJnTi6Je33r2IheH\nHjiEPr1/LnSjKivOy3BLxjdnZjGf/OgcXnyllc1PvsK2f6rFeVlJppslJuHc+V7MpvTOLI6mKAof\nvLoCi9nEoXc6efnQaV556xyrl19DWXE+H3CWc1mFfdyfcbTVw1l3D9PS1GYj0m/QHgp8mRzt5Nss\n/K9b5vHu+xc49E4n9S+9Q/1L72AyKaxefi2rl18zIrv4mb1H2f3iMQDW3DqDmkw1XEzJqaHa39NL\nCjLckvjmXV7KrTVB/vu1Nr65449MLy3gUs8gc6um8befvJaa+ZXh733vdDcvHWilqMDK/1o6D3uh\njUAgyKP/8WeCKnzUKTkcmaCqKh2eXgrzTBnZrTBa9ZXT+YCznOPtF2hobucnz4fqlFstJu78y2pW\n1jqjtvOd97v4fx7ZB8Da2+T+N1m6DdqXejMz0h7NpChcO7cM5+XTeONdN/0Dft47080ze4/i6upl\nzacXUD4tn5cPtYcDNsDBY15WfSaDDReTdupsNwDTS/Mz3JLEzL+iHJvVzMEj5+gf8FNUYOWd1i42\nP/kKf1Hr5Muf+QDvvN/Fln87QN9AKEv51//zHquWXUNDc3v4IeXYSSsfXRxM215hEXKp10ffgJ/K\n0uy5XSuKwtWzy7i8wk5bp5e+fj+vH+vkx//5JvvfOMsNCyr58LUVDAwGeO9MN/0Dfn62d3i/90uH\nuvn8SjUrHkL0JqFesG3bNg4fPoyiKNTV1bFw4cLwe42NjezYsQOz2cySJUtYt25dzGvOnj3L3Xff\nTSAQoKKigocffhibbXJrNNr0eH6KSphOlM1i5oYFoVHLB6+p4Nf/8x4vHXyfPx06zZxKOydOd2O1\nmPirpfP406HTvHOml3Pne2JOo+e6bOxzmlNnL1KYZ6IwzdXQpuLKy0u48vLhqXHXhT5eOtjKb/ef\n5NUj5+i6NICqwrLFc7jUO8irb3fw1G+OADD/ijIGBgOcPHORPX88zuc/cU2mPkbKje5DkTLV7zo8\noTPbC/Ky72GpMN/KtXNCB9nMm1XCH15r480Tbt484eap34z8XkWB5R+ZQ0+/j8Y3znLidDdXzSrN\nQKv1LW4vOHjwIK2trdTX17N161a2bt064v0tW7awc+dOdu/ezf79+zl+/HjMax555BG+8IUv8Mwz\nzzB37lyee+65STdcO56wID/7bpzFhTZWLbuWWxbNojDPwvH2bhxlhfzV0nlUlBVy/bzpALzQdCqj\n7cxW2drnADwX+zl3vhdHqVXXo4SK0gLu+MQ1LLxqBq4LfdisZlZ+7AqunVvGDQsq+cInr2X9F2t4\nZP0tfOKGOdx2w2zs+Sae2XvMsJnD2drvtCTXwiwM2pGKC218dsk8vrRyAcsWz2bBFeVcf9UMPrF4\nNp+6aS5rP72Aa+aUUVIUygX5998eyXCL9SluxGtqamLZsmUAzJs3j+7ubrxeL3a7nba2NkpKSqiq\nqgJg6dKlNDU14fF4ol5z4MABvvvd7wJw66238m//9m984QtfmHCjj5w8z4GWc1SVWSm1Z2cykNmk\nUH3ldKqvnI6qjpwGmjerlJcPtfOb/zlJ1Qw704psVJYXkp9nxucLMugPMOgLUpBnocRuQ1EUgkGV\nQFAd+v8gwaCKqu00U4b/L/L3aH9UFIXwVxXQXkXGnPG+Z/T3XeoL0HWxP+bPmgqzSYnZ54CM9blA\nIIjrQh9P/aYFgOvmZP96djwWs4mPf+hyahdehsk08j9giT2P/gE/x1q7gFD+xmc+UsbPXz7Ppidf\n4Su3X0dpcR6KEloiCvWT4f/Xvhb5nskU+bWhr6NgMg1fa1IUVBV8/iC+QICBwdD/UMBqNmG1mPD5\ng/QPBjCbFWwWM1ZL6Os2qxmzSSGohv5tBIMqQVUN/1vR/jzgC+DzBbFaTOTZzOTZzEwrskXtd729\noYCZqX7n7QuNSgGKsmyLYSzFhTaunVvOtXPLo74/d2YxFjMcPeXhtbc7mF6Sjz8QxGIO/ffQzpIw\nmRQsZhNmU3Y/HPf0ByZ9Gtq0ItuEH/7jBm232011dXX4dXl5OS6XC7vdjsvlory8fMR7bW1tdHV1\nRb2mr68vPEU0ffp0XK6J7x89eOQc/zJ08sxHr7XrYrQzuo0Ws4mrL8+npbWPR//jzxlq1RT959mU\n/ejiruP87yj9B8hInwO474km3hg6V/2q2aUsmlfEef+kflTWGR2wY/H2B7jxupm88tY5HvjJqylu\nVXoV5Jm5Itgxpg91d4fyFzLR795t6+LunfvwB1Sunl3KjBJj1IMwm00smF3Am6f6+O6/vpLp5iTH\nnsndDz990xX80+c+OKFrJjy3rKoT7zjRrkn05zQ3N494bQY2f2FWxFf0Warx87XT+XxtpluRnZ58\n0seJEycoKwutlV26dImWlpbwKCcRU+lzMLbf/fVH8vnrj8wa8bUKmz773mRVXBXazvOphbPifKc+\nPfmkZ0y/m6hk97t7V10+6juM0ef+5mPT+ZuPZboV2cA/5r95PHGDtsPhwO12h193dnZSUVER9b2O\njg4cDgdWqzXqNYWFhfT395Ofnx/+3vHU1MimgFzU2NhISUlJ+L9/b28vt9xyC3a7nfb29pT2OZB+\nl6ti9TtI/b0OpN+JxMTNbKitrWXv3r0AtLS04HA4sNtDT9yzZs3C6/XS3t6O3++noaGB2tramNd8\n7GMfC3/9xRdf5OMf/3iqPpfQMelzIhOk3wk9UNQE5m62b9/Oa6+9hqIobNq0iSNHjlBcXMzy5ct5\n9dVX2b59OwCf/OQnufPOO6NeM3/+fDo7O7nnnnsYGBjgsssu4/7778dqtab2Ewpdkj4nMkH6nch2\nCQVtIYQQQmRedm/8E0IIIUSYBG0hhBBCJ7KvnFiCxitzmWnvvPMO//RP/8SXv/xl1qxZE7Ok4a9+\n9Sv+/d//HZPJxB133MHnP/95fD4fGzdu5MyZM5jNZu6//35mz56dlnY/9NBDNDc34/f7+Yd/+Aeu\nv/56XbQ73bK5702WXvvsVOilv+utvxmpL2VlH1F16MCBA+rf//3fq6qqqsePH1fvuOOODLdoWE9P\nj7pmzRr13nvvVX/605+qqqqqGzduVJ9//nlVVVX1e9/7nvqzn/1M7enpUT/5yU+qFy9eVPv6+tS/\n+Iu/ULu6utQ9e/aomzdvVlVVVfft26d+4xvfSEu7m5qa1K985Suqqqqqx+NRly5dqot2p1s2973J\n0mufnQq99He99Tcj9aVs7SO6nB4fr8xlptlsNp588skR+zIPHDjAJz7xCSBU0rCpqYnDhw9z/fXX\nU1xcTH5+PosWLeL111+nqamJ5cuXA/Cxj32M119/PS3tXrx4MT/4wQ8AmDZtGn19fbpod7plc9+b\nLL322anQS3/XW38zUl/K1j6iy6DtdrvDVYtgZJnLTLNYLOTnjzyyMVpJQ7fbPaYs4uivm0yh83MH\nBwdT3m6z2UxhYSEAzz33HEuWLNFFu9Mtm/veZOm1z06FXvq73vqbkfpStvYRXQbt0VQd7VqL1daJ\nfj1Vfv/73/Pcc89x3333JdSObGl3puTC5zTyf3u99Xc9/d1Gk61/r+PJtj6iy6A9XmnVbKSVNITh\n8ofRPoP2de1J2ufzoarqlM9/TtS+ffv48Y9/zJNPPklxcbFu2p1Oeut7k5UL/+310N+N0N+y8e81\nUdnYR3QZtMcrN5iNopU0/OAHP8ibb77JxYsX6enp4fXXX+eGG26gtraWF154AYCGhgY++tGPpqWN\nly5d4qGHHuLxxx+ntLRUN+1ON731vcky+n97vfR3I/S3bPx7TUS29hHdVkSLVjowG7z11ls8+OCD\nnD59GovFQmVlJdu3b2fjxo1jShq+8MIL7Nq1C0VRWLNmDbfffjuBQIB7772XU6dOYbPZeOCBByZ0\nutVk1dfXs3PnTpxOZ/hrDzzwAPfee29WtzsTsrXvTZZe++xU6Km/66m/GakvZWsf0W3QFkIIIXKN\nLqfHhRBCiFwkQVsIIYTQCQnaQgghhE5I0BZCCCF0QoK2EEIIoRMStDPgoYceYu3atdxxxx1cd911\nrF27lrVr1/LLX/4yoeufeOIJ/vjHP6a2kSKrffGLX+T3v//9iK/19/ezePFizp49G/WatWvX0tjY\nyNtvv82//Mu/jHm/tbWV2267bdzf29fXx4svvgjAyy+/zI9+9KNJfgKRS6Z6z9O89tprtLe3p6iV\nOpGUY0fEpLS1takf//jHM90MoUN79uxR161bN+Jrv/rVr9Q777wz5jVr1qxR9+/fH/P9U6dOqbfe\neuu4v/e1115T169fP7HGCjFkqve8jRs3qq+88koSW6Q/uj1P24h27txJe3s7Z86c4Z577qG/v5/t\n27djs9no7+9n06ZNVFdXs3HjRmpqarjpppv46le/ys0338wbb7xBT08Pjz/+OJWVlZn+KCLFPvWp\nT/Hggw/S1dUVPlDil7/8JZ///Od56aWX+Nd//VdsNhuBQICHHnqIWbNmha89cOAA3//+99m9ezev\nv/46mzZtory8nOrq6vD3nDhxgk2bNmE2m/F6vXzzm99k8eLF/PM//zMXL17koYce4qqrrqKxsZHt\n27dz+PBhHnjgASwWC4qicN9993HVVVexdu1abrrpJg4dOsSpU6f42te+xu233572vy+RnQYHB9m8\neTNtbW309PRw++238+Uvf5mjR4+yefNmrFYrAwMD3HXXXfT29vLiiy9y5MgR/vmf/5mPfOQjmW5+\nRsj0eJZpb2///9u7g5Cm2ziA499mkzyUFttlu4iyIFBTB6FdChUNFMH6Y5empBjiIUfaX9YkHDHR\nHdrJQIzU/fVQBDJjaYfw5hZ4KPCgoQfx4KGFazLB3PA9hOP19bXL25tb/j6n7Xme/Xie8bDf/s9/\nex58Ph8FBQVEIhH6+vrw+Xw0NTUxPDx8qP3q6io3b95kcnKSS5cuMTMzcwy9Fr9bVlYW1dXVBAIB\n4Md+x0tLS1RUVBCNRvF6vWiaxrVr15icnDwyjsfjobu7m/Hx8QN7WofDYTo7OxkfH6e3txev18uZ\nM2e4d+8eV69eRVXVA3FUVcXhcKBpGnfv3sXlciXrtre3GRkZwe128/z581/8Toh0Njo6itlsRtM0\nXr16hd/vZ2VlhZcvX1JdXY2maQwNDbG5ucmNGze4ePEijx49OrEJG0CutFPM5cuXOXXqFAAGgwGP\nx8POzg5bW1tkZ2cfan/+/HksFgsAJpOJSCTyW/srjo+iKLhcLu7cucP09DR1dXVkZmZiMBjo6elh\nb2+PL1++UFJScmSM5eVlrFYrAGVlZWiaBoDRaMTj8eD1etnd3f3pvIpGo3z9+pWioiIArly5woMH\nD5L1+x+wJpOJb9++/edxiz/Hhw8fCIfDhEIhAOLxOGtra9TU1OB0OllfX+f69euyOvM3krRTjF6v\nTz5WVRWXy0V5eTlzc3O8ePHiUPuMjIwDz/dkV9oTo6ioiO/fv7O6uorf7+fp06fs7u5it9uZmpoi\nNzeXiYkJFhcXfxpHp/ux4JZIJJJlT548oba2FkVR+Pz5M+3t7Ue+fv9L5r5/zsHTp08fWSdOtszM\nTO7fv09VVdWhujdv3hAMBnn9+jWBQACPx3MMPUw9sjyewsLhMBaLhUQiwezs7LEdBi9S161bt3j2\n7BlZWVlYLBZisRg6nQ6z2czOzg7v37//6bzJz8/n48ePAMzPzyfL9+cewNu3b5MxdDod8Xj8QIyz\nZ89iNBr59OkTAMFgkOLi4l86TvFnKi0tTd7SSyQSuN1uotEoY2NjhMNhKisrcbvdybn1b/PvpJGk\nncLa2tpobm6mvb2dhoYGNjY2GBsbO+5uiRRSX1/Pu3fvUBQFgJycHOrq6lAUBbvdTmtrK6FQ6Mjf\nOjx8+JD+/n7a2tqIxWLJ8paWFlRVpbW1FavVSnZ2NgMDAxQWFrKwsIDD4TgQZ3BwkMHBQWw2GxMT\nEzx+/Pj/G7T4Y9hsNvR6Pbdv36axsZELFy5w7tw58vLysNvt2Gw2Ojo66OrqAn4cjel0Og/93fEk\nkVO+hBBCiDQhV9pCCCFEmpCkLYQQQqQJSdpCCCFEmpCkLYQQQqQJSdpCCCFEmpCkLYQQQqQJSdpC\nCCFEmpCkLYQQQqSJvwDE03d2iIU0YgAAAABJRU5ErkJggg==\n","text/plain":[""]},"metadata":{"tags":[]}}]},{"metadata":{"id":"Zy_iTThEXzjR","colab_type":"code","colab":{}},"cell_type":"code","source":["TEXT.build_vocab(train_data, max_size=25000, vectors=\"glove.6B.100d\")\n","LABEL.build_vocab(train_data)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"mqVeGRU0YSpI","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch.nn as nn\n","import torch.nn.functional as F"],"execution_count":0,"outputs":[]},{"metadata":{"id":"1T8S07AVY-I0","colab_type":"code","colab":{}},"cell_type":"code","source":["\n","class CNN(nn.Module):\n"," def __init__(self,vocab_size, embedding_dim, n_filters, filter_sizes, output_dim, dropout):\n"," super().__init__()\n"," \n"," self.embedding = nn.Embedding(vocab_size, embedding_dim)\n"," self.embedding.weight.requires_grad = False\n"," self.conv_0 = nn.Conv2d(in_channels=1, out_channels=n_filters, kernel_size=(filter_sizes[0],embedding_dim))\n"," self.conv_1 = nn.Conv2d(in_channels=1, out_channels=n_filters, kernel_size=(filter_sizes[1],embedding_dim))\n"," self.conv_2 = nn.Conv2d(in_channels=1, out_channels=n_filters, kernel_size=(filter_sizes[2],embedding_dim))\n"," self.fc = nn.Linear(len(filter_sizes)*n_filters, output_dim)\n"," self.dropout = nn.Dropout(dropout)\n"," \n"," def forward(self, x):\n"," #x = [sen_len, batch_size]\n"," x = x.permute(1,0)\n"," #x = [batch_size, sen_length]\n"," embedded = self.embedding(x)\n"," \n"," # embedded - [batch_size, sent_len, emb_dim]\n"," embedded = embedded.unsqueeze(1)\n"," #embedded = [batch size, 1, sent len, emb dim]\n"," \n"," conved_0 = F.relu(self.conv_0(embedded).squeeze(3))\n"," conved_1 = F.relu(self.conv_1(embedded).squeeze(3))\n"," conved_2 = F.relu(self.conv_2(embedded).squeeze(3))\n"," \n"," #conv_n = [batch size, n_filters, sent len - filter_sizes[n]]\n"," \n"," pooled_0 = F.max_pool1d(conved_0, conved_0.shape[2]).squeeze(2)\n"," pooled_1 = F.max_pool1d(conved_1, conved_1.shape[2]).squeeze(2)\n"," pooled_2 = F.max_pool1d(conved_2, conved_2.shape[2]).squeeze(2)\n"," \n"," #pooled_n = [batch size, n_filters]\n"," \n"," cat = self.dropout(torch.cat((pooled_0, pooled_1, pooled_2), dim=1))\n","\n"," #cat = [batch size, n_filters * len(filter_sizes)]\n"," \n"," return self.fc(cat)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"LqJezieMZSIc","colab_type":"code","colab":{}},"cell_type":"code","source":["INPUT_DIM = len(TEXT.vocab)\n","EMBEDDING_DIM = 100\n","N_FILTERS = 100\n","FILTER_SIZES = [3,4,5]\n","OUTPUT_DIM = 1\n","DROPOUT = 0.5\n","BATCH_SIZE = 128\n","\n","model = CNN(INPUT_DIM, EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"6DkugEsaZru9","colab_type":"code","outputId":"f6a6e01e-d6f8-4ee6-ffec-54f6c1d99843","executionInfo":{"status":"ok","timestamp":1546011934704,"user_tz":-330,"elapsed":20648,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":139}},"cell_type":"code","source":["pretrained_embeddings = TEXT.vocab.vectors\n","model.embedding.weight.data.copy_(pretrained_embeddings)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([[ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n"," [ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n"," [-0.0382, -0.2449, 0.7281, ..., -0.1459, 0.8278, 0.2706],\n"," ...,\n"," [ 0.3581, -0.3348, -0.4465, ..., -0.7128, -0.1006, -0.2799],\n"," [ 0.3896, 0.0446, -0.3506, ..., 0.1727, 0.0145, -0.9884],\n"," [ 0.0994, -0.7076, -0.0949, ..., -0.0898, -1.1580, -0.1628]])"]},"metadata":{"tags":[]},"execution_count":24}]},{"metadata":{"id":"GCoIXAZfZsj6","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch.optim as optim\n","device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","\n","optimizer = optim.Adam(model.parameters(),lr=0.01)\n","criterion = nn.BCEWithLogitsLoss()\n","model = model.to(device)\n","criterion = criterion.to(device)\n","\n","\n","train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits((train_data,valid_data,test_data), batch_size=BATCH_SIZE,device=device)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"QSzKWu6HZ4iP","colab_type":"code","colab":{}},"cell_type":"code","source":["def binary_accuracy(preds,y):\n"," \n"," rounded_preds = torch.round(torch.sigmoid(preds))\n"," correct = (rounded_preds==y).float()\n"," acc = correct.sum()/len(correct)\n"," return acc\n","\n","\n","def precision_recall_f1(preds, y):\n"," rounded_preds = torch.round(torch.sigmoid(preds))\n"," tp = y*rounded_preds\n"," fp = (y==0).float()*rounded_preds\n"," tn = (y==0).float()*(rounded_preds==0).float()\n"," fn = y*(rounded_preds==0).float()\n"," \n"," return sum(tp), sum(fp), sum(tn), sum(fn)\n"," \n"," \n"," \n"," \n"," \n","def train(model, iterator, optimizer, criterion):\n"," \n"," epoch_loss= 0\n"," epoch_acc = 0\n"," tp = 0\n"," fp = 0\n"," tn = 0\n"," fn = 0\n"," \n"," model.train()\n"," \n"," for batch in iterator: \n"," optimizer.zero_grad()\n"," \n"," predictions = model(batch.text).squeeze(1)\n"," loss = criterion(predictions, batch.label)\n"," acc = binary_accuracy(predictions, batch.label)\n"," a, b, c, d = precision_recall_f1(predictions, batch.label)\n"," \n"," loss.backward() \n"," optimizer.step()\n"," \n"," epoch_loss += loss.item()\n"," epoch_acc +=acc.item()\n"," tp += a.item()\n"," fp += b.item()\n"," tn += c.item()\n"," fn += d.item()\n"," \n"," return epoch_loss/len(iterator), epoch_acc/len(iterator), tp, fp, tn, fn\n"," \n"," \n","\n","def evaluate(model, iterator, criterion):\n"," \n"," epoch_loss = 0\n"," epoch_acc = 0\n"," tp = 0\n"," fp = 0\n"," tn = 0\n"," fn = 0\n"," \n"," model.eval()\n"," \n"," with torch.no_grad():\n"," \n"," for batch in iterator:\n"," predictions = model(batch.text).squeeze(1)\n"," \n"," loss = criterion(predictions, batch.label)\n"," \n"," acc = binary_accuracy(predictions, batch.label)\n"," a, b, c, d = precision_recall_f1(predictions, batch.label)\n"," \n"," tp += a.item()\n"," fp += b.item()\n"," tn += c.item()\n"," fn += d.item()\n"," epoch_loss += loss.item()\n"," epoch_acc += acc.item()\n"," \n"," return epoch_loss / len(iterator), epoch_acc / len(iterator), tp, fp, tn, fn"],"execution_count":0,"outputs":[]},{"metadata":{"id":"sPxLvKHKabWc","colab_type":"code","outputId":"2a31e49f-3e4c-41e0-817a-9dfef619726c","executionInfo":{"status":"ok","timestamp":1546012023566,"user_tz":-330,"elapsed":85046,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":176}},"cell_type":"code","source":["N_EPOCHS = 3\n","\n","for epoch in tqdm(range(N_EPOCHS)):\n"," model.embedding.weight.requires_grad=True\n"," if epoch>0:\n"," lr = 0.001\n"," for param_group in optimizer.param_groups:\n"," param_group['lr'] = lr\n"," model.embedding.weight.requires_grad=False\n"," train_loss, train_acc, train_tp, train_fp, train_tn, train_fn = train(model, train_iterator, optimizer, criterion)\n"," valid_loss, valid_acc, valid_tp, valid_fp, valid_tn, valid_fn = evaluate(model, valid_iterator, criterion)\n"," valid_precision = valid_tp/(valid_tp+valid_fp)\n"," valid_recall = valid_tp/(valid_tp+valid_fn)\n"," valid_f1= (2*valid_precision*valid_recall)/(valid_precision+valid_recall)\n"," print('| Epoch: {} | Train Loss: {} | Train Acc: {} | Val. Loss: {} | Val. Acc: {} | Val. F1 {} |'.format(epoch+1,train_loss, train_acc*100,valid_loss, valid_acc*100, valid_f1))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["\n"," 0%| | 0/3 [00:00=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (1.22)\n","Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2.6)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2018.11.29)\n","Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (3.0.4)\n","Installing collected packages: torchtext\n","Successfully installed torchtext-0.3.1\n","Requirement already up-to-date: spacy in /usr/local/lib/python3.6/dist-packages (2.0.18)\n","Requirement already satisfied, skipping upgrade: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.2)\n","Requirement already satisfied, skipping upgrade: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.1)\n","Requirement already satisfied, skipping upgrade: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.1)\n","Requirement already satisfied, skipping upgrade: regex==2018.01.10 in /usr/local/lib/python3.6/dist-packages (from spacy) (2018.1.10)\n","Requirement already satisfied, skipping upgrade: ujson>=1.35 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.35)\n","Collecting numpy>=1.15.0 (from spacy)\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/ff/7f/9d804d2348471c67a7d8b5f84f9bc59fd1cefa148986f2b74552f8573555/numpy-1.15.4-cp36-cp36m-manylinux1_x86_64.whl (13.9MB)\n","\u001b[K 100% |████████████████████████████████| 13.9MB 3.2MB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: dill<0.3,>=0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.2.8.2)\n","Requirement already satisfied, skipping upgrade: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.18.4)\n","Requirement already satisfied, skipping upgrade: thinc<6.13.0,>=6.12.1 in /usr/local/lib/python3.6/dist-packages (from spacy) (6.12.1)\n","Requirement already satisfied, skipping upgrade: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.9.6)\n","Requirement already satisfied, skipping upgrade: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.6)\n","Requirement already satisfied, skipping upgrade: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.22)\n","Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2018.11.29)\n","Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n","Requirement already satisfied, skipping upgrade: wrapt<1.11.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (1.10.11)\n","Requirement already satisfied, skipping upgrade: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (4.28.1)\n","Requirement already satisfied, skipping upgrade: msgpack<0.6.0,>=0.5.6 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.5.6)\n","Requirement already satisfied, skipping upgrade: six<2.0.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (1.11.0)\n","Requirement already satisfied, skipping upgrade: msgpack-numpy<0.4.4 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.4.3.2)\n","Requirement already satisfied, skipping upgrade: cytoolz<0.10,>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.9.0.1)\n","Requirement already satisfied, skipping upgrade: toolz>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from cytoolz<0.10,>=0.9.0->thinc<6.13.0,>=6.12.1->spacy) (0.9.0)\n","\u001b[31mfeaturetools 0.4.1 has requirement pandas>=0.23.0, but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n","\u001b[31mcufflinks 0.14.6 has requirement plotly>=3.0.0, but you'll have plotly 1.12.12 which is incompatible.\u001b[0m\n","Installing collected packages: numpy\n"," Found existing installation: numpy 1.14.6\n"," Uninstalling numpy-1.14.6:\n"," Successfully uninstalled numpy-1.14.6\n","Successfully installed numpy-1.15.4\n","Requirement already satisfied: en_core_web_sm==2.0.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz#egg=en_core_web_sm==2.0.0 in /usr/local/lib/python3.6/dist-packages (2.0.0)\n","\n","\u001b[93m Linking successful\u001b[0m\n"," /usr/local/lib/python3.6/dist-packages/en_core_web_sm -->\n"," /usr/local/lib/python3.6/dist-packages/spacy/data/en\n","\n"," You can now load the model via spacy.load('en')\n","\n"],"name":"stdout"}]},{"metadata":{"id":"eszw4RVDGrMM","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch\n","from torchtext import data\n","from torchtext import datasets\n","import random\n","import re\n","import numpy as np\n","from tqdm import tqdm\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","%matplotlib inline"],"execution_count":0,"outputs":[]},{"metadata":{"id":"slI8poiBHG9O","colab_type":"code","colab":{}},"cell_type":"code","source":["SEED = 10\n","torch.manual_seed(SEED)\n","torch.cuda.manual_seed(SEED)\n","torch.backends.cudnn.deterministic = True #for reproducible results"],"execution_count":0,"outputs":[]},{"metadata":{"id":"Cn6DVNLzHJas","colab_type":"code","colab":{}},"cell_type":"code","source":["# Initializing Hyperparameters\n","#MAX_DOCUMENT_LEN = 900"],"execution_count":0,"outputs":[]},{"metadata":{"id":"JnN7xdUQHSOk","colab_type":"code","colab":{}},"cell_type":"code","source":["def clean_text(text):\n"," text = text.lower()\n"," text = re.compile('<.*?>').sub(\" \", text)\n"," text = re.sub(r\"[-()\\\"#/@;:<>{}+=~|.?,]\", \"\", text)\n"," text = re.sub(r\"i'm\", \"i am\", text)\n"," text = re.sub(r\"he's\", \"he is\", text)\n"," text = re.sub(r\"she's\", \"she is\", text)\n"," text = re.sub(r\"that's\", \"that is\", text)\n"," text = re.sub(r\"what's\", \"what is\", text)\n"," text = re.sub(r\"where's\", \"where is\", text)\n"," text = re.sub(r\"\\'ll\", \" will\", text)\n"," text = re.sub(r\"\\'ve\", \" have\", text)\n"," text = re.sub(r\"\\'re\", \" are\", text)\n"," text = re.sub(r\"\\'d\", \" would\", text)\n"," text = re.sub(r\"won't\", \"will not\", text)\n"," text = re.sub(r\"can't\", \"cannot\", text)\n"," #text = re.sub(r\"[-()\\\"#/@;:<>{}+=~|.?,]\", \"\", text)\n"," text = text.split()\n"," return text"],"execution_count":0,"outputs":[]},{"metadata":{"id":"SS2TDc8qHTv_","colab_type":"code","colab":{}},"cell_type":"code","source":["TEXT = data.Field(tokenize=clean_text,lower=True) \n","LABEL = data.LabelField(dtype = torch.float)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"XWgxURnvHWY0","colab_type":"code","outputId":"833cb4d9-9ba9-441e-ce68-d846a42d9821","executionInfo":{"status":"ok","timestamp":1546106336342,"user_tz":-330,"elapsed":55476,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":52}},"cell_type":"code","source":["from torchtext import datasets\n","train_data, test_data = datasets.IMDB.splits(text_field=TEXT,label_field=LABEL)\n","train_data, valid_data = train_data.split(split_ratio= 0.8, random_state=random.seed(SEED))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["downloading aclImdb_v1.tar.gz\n"],"name":"stdout"},{"output_type":"stream","text":["aclImdb_v1.tar.gz: 100%|██████████| 84.1M/84.1M [00:07<00:00, 10.9MB/s]\n"],"name":"stderr"}]},{"metadata":{"id":"9XM9SBVBHZsO","colab_type":"code","outputId":"273fc257-6173-4542-cb7d-baca12c75568","executionInfo":{"status":"ok","timestamp":1546106336347,"user_tz":-330,"elapsed":20966,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":89}},"cell_type":"code","source":["print('Number of training examples: {}'.format(len(train_data)))\n","print('Number of validation examples: {}'.format(len(valid_data)))\n","print(vars(train_data[0]))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Number of training examples: 20000\n","Number of validation examples: 5000\n","{'text': ['wow', 'this', 'movie', 'bored', 'the', 'pants', 'off', 'me', 'when', 'i', 'saw', 'it', 'bland', 'pointless', 'and', 'unmoving', 'apparently', 'ash', 'and', 'co', 'can', 'travel', 'through', 'time', 'with', 'the', 'help', 'of', 'the', 'spirit', 'of', 'the', 'forest', \"'princess\", \"mononoke'\", 'much', 'there', 'they', 'meet', 'a', 'dorky', 'kid', 'named', 'sam', 'and', 'the', 'plot', 'begins', 'so', 'tom', 'ash', 'and', 'huck', 'sam', 'get', 'high', 'with', 'nature', 'become', 'hippies', 'and', 'try', 'to', 'free', 'celebi', 'the', 'spirit', 'from', 'some', 'weirdo', 'hunter', 'guy', 'i', \"don't\", 'even', 'know', 'what', 'else', 'went', 'on', 'it', 'all', 'went', 'by', 'in', 'a', 'blur', \"ash's\", 'friends', 'were', 'hardly', 'in', 'it', 'and', 'all', 'the', 'fight', 'scenes', 'were', 'boring', 'after', 'saving', 'the', 'day', 'ash', 'and', 'his', 'infamous', 'friends', 'must', 'return', 'to', 'their', 'time', 'while', 'watching', 'sam', 'float', 'away', 'with', 'celebi', 'that', 'scene', 'was', 'just', 'creepy', 'oo', 'then', 'after', 'returning', 'to', 'their', 'time', 'ash', 'learns', 'that', 'his', 'new', 'friend', 'is', 'actually', 'his', \"rival's\", 'grandpa', 'and', 'i', 'think', 'that', 'is', 'it', 'pretty', 'retarded', \"isn't\", 'it', 'if', 'you', 'love', 'your', 'children', 'you', 'will', 'not', 'expose', 'them', 'to', 'this', '1', 'out', 'of', '10'], 'label': 'neg'}\n"],"name":"stdout"}]},{"metadata":{"id":"uteu4JwKHiHh","colab_type":"code","outputId":"b61523c8-1ab4-47ca-9a15-17c1c46bd709","executionInfo":{"status":"ok","timestamp":1546106337874,"user_tz":-330,"elapsed":20711,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":379}},"cell_type":"code","source":["train_data_dist = [len(train_data[i].text) for i in range(len(train_data))]\n","valid_data_dist = [len(valid_data[i].text) for i in range(len(valid_data))]\n","test_data_dist = [len(test_data[i].text) for i in range(len(test_data))]\n","\n","sns.set(style='whitegrid')\n","f, axes = plt.subplots(1, 3)\n","\n","sns.distplot(a = train_data_dist,ax=axes[0], axlabel='Train')\n","sns.distplot(a = valid_data_dist,ax=axes[1], axlabel='Validation')\n","sns.distplot(a = test_data_dist,ax=axes[2], axlabel='Test')"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":9},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAe0AAAFZCAYAAAC173eYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xt8lOWd+P3PPaecJuQAmRDl4IgH\naJS2RFptLKiF0rJd+9vdVtgW+vT52T106RkepVl/Qp8FVKQ8LWhbdVlf21ppXMt2e7Ci3aaVJRFq\npKhRUBBiwiGZYUJgcpzD/fwxuSeTZCYzSeZ03/N9v159lcnkTq7Bi/t7f6/re12XoqqqihBCCCGy\nninTDRBCCCFEYiRoCyGEEDohQVsIIYTQCQnaQgghhE5I0BZCCCF0QoK2EEIIoROWRL5p27ZtHD16\nFEVRqKurY+HCheH3Ghsb2blzJ2azmSVLlrBu3bqY1/h8PjZu3EhraytFRUXs2rWLkpKS1HwyIYQQ\nwmDiBu3Dhw/T2tpKfX09J0+epK6ujvr6+vD7W7ZsYc+ePVRWVrJmzRpWrFiBx+OJes0zzzxDWVkZ\n3/3ud6mvr+eVV17hYx/7WMzf3dzcnJxPKQyjpqYm5b9D+p2IlI4+B9LvxEix+l3coN3U1MSyZcsA\nmDdvHt3d3Xi9Xux2O21tbZSUlFBVVQXA0qVLaWpqwuPxRL2moaGBr33tawCsWrVqSg2HUCdP1z+o\nZJO2T+73pku8z6fn/36TlaufOZ2Mer8bTT5L/J8ZS9w5bbfbTVlZWfh1eXk5LpcLAJfLRXl5+Zj3\nYl1z5swZXnrpJdauXcs3v/lNLl68OKkPJIQQQuSihOa0I01m11PtGlVVcTqdfOUrX+EHP/gBjz32\nGPfee++418Z70tXzkJK0XQghxETEDdoOhwO32x1+3dnZSUVFRdT3Ojo6cDgcWK3WqNfMmDGDxYsX\nA3Drrbeye/fuuA006nCRtH1yv1cIIXJZ3OHx2tpa9u/fD0BLSwsOhwO73Q7ArFmz8Hq9tLe34/f7\naWhooLa2NuY1S5Ys4cCBA+GvO53OVH0uIYQQwnDiZtqLFi2iurqa1atXoygKmzZtYt++fRQXF7N8\n+XI2b97M+vXrAVi5ciVOpxOn0znmGoC1a9dy77338uyzz1JYWMhDDz2U2k8nhBBCGEhCc9obNmwY\n8Xr+/PnhPy9evHjEErBY1wAUFBSwa9euibZRCCGEEMiOaEIIIYRuSNAWQgghdEKCthBCCKETErSF\nEEIInZCgLYQQQuiEBG0hhNCxHz/3Jr9tOp3pZog0mfA2ptnu+abT4T9/4parMtUMkeO0fih9UKSS\nzx/kP/77HQA+KX0tJ0imLYQQOnWhuy/TTRBpJkFbCCF0yn1RgnaukaAthBA65ZKgnXMkaAshhE65\nuiRo5xoJ2kIIoVOSaeceCdpCCKFTrq5eAMwmJcMtEekiQVsIIXRKy7RtVrmV5wr5Ly2EEDp1uWcQ\nAH9AzXBLRLpI0BZCCJ0KqqFgHQgEM9wSkS6G2xEN4FLPIHk2c6abIaZg27ZtHD16FEVRqKurY+HC\nheH3Ghsb2blzJ2azmSVLlrBu3bqY1/h8PjZu3EhraytFRUXs2rWLkpKSTH0sIZIqGAwF7aAa+rNJ\n5rYNz3CZtqqq/OS3b/Hj37yZ6aaISTp8+DCtra3U19ezdetWtm7dOuL9LVu2sHv3bvbu3cvBgwc5\nceJEzGueeeYZysrKePbZZ1m5ciWvvPJKJj6SECkRjBgVDwQl284Fhsu0fUPDRIN+6cB61dTUxLJl\nywCYN28e3d3deL1eANra2igpKaGqqgqApUuX0tTUhMfjiXpNQ0MDX/va1wBYtWpVBj6NEKkTjIja\nPn8Qq0VGGI3OeEFbgrXuud1uqqurw6/Ly8txuVwAuFwuysvLR7zX1tZGV1dX1GvOnDnDSy+9xMMP\nP8yMGTPYtGkTpaWlcdvQ3Nw8pe9pfS/0kNFsuxD35+hJIn8vIn20OW2AQFCK0XKB8YK2T4K20ajq\nxG9G2jWqquJ0OvnKV77CD37wAx577DHuvffeuNfX1NSM+35zc/O43+MaPD30c65KuM3ZLt5nNqJs\nf0iJzLT9UoyWEww3p+3zBzLdBDFFDocDt9sdft3Z2UlFRUXU9zo6OnA4HDGvmTFjBosXLwbg1ltv\n5cSJE2n6FEKkXmTQDsiyr5xguKAtc9n6V1tby/79+wFoaWnB4XBgt9sBmDVrFl6vl/b2dvx+Pw0N\nDdTW1sa8ZsmSJRw4cCD8dafTmZkPJUQKRI5CSaadG4w3PC5BW/cWLVpEdXU1q1evRlEUNm3axL59\n++js7KSmpobNmzezfv16AFauXInT6cTpdI65BmDt2rXce++9PPvssxQWFvLQQw9l8qMJkTSqqo6o\nHpegnRsMGLRleNwINmzYMOL1/Pnzw/OLixcvpr6+Pu41AAUFBezatSs1jRQig0bXncnweG4w3vC4\nFKIJIXJAcFTUlkw7NxguaMvwuBAiFwRHraqQJV+5wYBBW4bHhRDGp0qmnZMMF7SlelwIkQtGZ9oS\ntHOD4YK2DI8LIXLB2DltGR7PBYYL2oO+4eHx0Z1aCCGMYvQcthzPmRsMF7QjM2059UYIYVSjd/eV\nTDs3GDpoSycWQhjV2OpxSVJygQGD9vDwuAwXCSGMSua0c5PhdkQblExbCDFJ27Zt4+jRoyiKQl1d\n3Yj3Ghsb2blzJ2azmSVLlrBu3bqo1yxcuJCNGzfS0tISPgb27rvv5rbbbktqW8cEbSnCzQmGC9oy\npy2EmIzDhw/T2tpKfX09J0+epK6ujnvuuSf8/pYtW9izZw+VlZWsWbOGFStW4PF4xlyjbbH7rW99\ni9tvvz1l7dWGxy1mBX9AlftdjjBg0A5E/Fk6sRAiMU1NTSxbtgyAefPm0d3dTW9vLwBtbW2UlJRQ\nVVUFwNKlS2lqasLj8Yy5xuv1pqW9WtC2Wkz4AwEZWcwRhpvTjqzNkG39hBCJcrvdlJWVhV+Xl5fT\n3d0NgMvlory8fMR7Lpcr6jUulwuAp556ii984Qt885vfxOPxJL292vC41WIGpIYnVxgu05bzZYUQ\nyaCOXlM1gWs+/elPU1payoIFC3j88cd55JFHuP/+++Ner51kl8j7nd2+oV8aGl08/V4bzc0XJ9zm\nTIn3WfUknZ/FcEE7MrmWo+qEEIlyOBy43e7w687OznAh2ej3Ojo6cDgcWK3WMddUVFTgdDrDX7vj\njjvYvHlzQm2oqamJ+V5zc/OI90+fuwS/6aCoMJ9LvT3MnHkFNTXXJfR7Mm30Z9GzVHyW8R4CDDc8\njmTaQohJqK2tZf/+/QC0tLTgcDgoKCgAYNasWXi9Xtrb2/H7/TQ0NFBbWxv1Grvdzle/+lXa2toA\nOHToENdee23S23vgz2eA4V0gZXg8NySUaUdb0qDJumUQkmkLISZh0aJFVFdXs3r1ahRFYdOmTTz3\n3HN4PB6WL1/O5s2bWb9+PQArV67E6XTidDrHXAPw+c9/nm984xsUFBRQWFjIAw88kPwGDyUoZpMC\ngF9qeHJC3KAdbRmEtqQBsmsZxOg5KL8sgRBCTMCGDRtGvO7p6QkPfS5evHjEvS/WNQA333wzP//5\nz1PTyCFajDabQgOmkmnnhrhBO9oyCK/Xi91uz7plEKMfNKUTCyGMSktSzOahTFtGFnNC3KDtdrup\nrq4Ov9aWNNjt9qjLINra2ujq6op6DYSWQTz55JNMnz6d//N//s+I66OZSDXl6E577PjbBC63xfuI\nGaPn6kk9t10II9AGFi3mUKYtNTy5YcLV4+leBjGRaspBXwDqz4RfO53zqLmhasLtTQc9V09mqu3y\noCDEsLGZtgTtXBC3ejzaMoiKioqo72nLIGJdc8stt7BgwQIgtAzi7bffTtoHAdlAXwiRO9Qxc9py\nv8sFcYN2rCUNkH3LIEYfVSdPniITXj3eyX+/8p70P5FSkXuPgxTe5oq4w+PRlkHs27eP4uLirFsG\nMaYQTTqxyIBHnz1Kp6eXfJuFT916daabIwxqONMOBW3JtHNDQnPao5c0zJ8/P/znbFoGIcPjIhvM\nLC+k09PLn992EQgEMZuNt4eRyLzhOW0pRMslhrqbjA7asuRLZEKezRz+s7fPl8GWCCOT6vHcZKig\nrT15KqHRIsm0RUZEHgk76JMbqUgNlZE7osnweG4wVNAOhrf1G6qmlDltkQEjgnbE+e5CJFN4TluW\nfOUUYwXtoT5rkR2CRAb5R2TaErRFagRH7T0ekL3Hc4KxgvaowgyZ0xaZEJldS9AWqTI8HahgMim4\nL/ZluEUiHYwVtIMjnzx9ErRFBowcHpc+KFJDGx5XFAWToowpxBXGZKigPXpbPynMEJngk+FxkQZa\n0DYpoURl9OZSwpgMFbS1OR2LSZZAiMzxjRgelz4oUiPyHAiTSTLtXGGooB0cnWlLJxYZIJm2SAft\nfmdSFEyK3O9yhaGC9ugN9CXTFpkQGbR9suRLpMiIOW0ZHs8Zhgra4UI0mdMWGSSFaCIdIjeTkuHx\n3DHh87SzWVDmtA1j27ZtHD16FEVRqKurY+HCheH3Ghsb2blzJ2azmSVLlrBu3bqY12zcuJGWlhZK\nS0sBuPvuu7nttttS1u5AUB0xTCnD4yJVtF4WzrQlaOcEYwVtqR43hMOHD9Pa2kp9fT0nT56krq5u\nxKE0W7ZsYc+ePVRWVrJmzRpWrFiBx+OJec23vvUtbr/99rS0XRsOVwjdVKUQTaTKiExblnzlDEMH\nbTlfVp+amppYtmwZAPPmzaO7uxuv1wtAW1sbJSUlVFVVAbB06VKamprweDwxr0knbTc0m9XMgC8g\nmbZImeElX4os+cohhprTVodidLgQTeYTdcntdlNWVhZ+XV5ejsvlAsDlclFeXj7mvfGueeqpp/jC\nF77AN7/5TTweT0rbrs1hayd9yZy2SJWgZNo5yZCZtkX24jUUdRIZhHbNpz/9aUpLS1mwYAGPP/44\njzzyCPfff3/c65ubmyf1PV1e/1ADQhl2+5lzNDf3T6Dl2S2RvxeRHmOrx0P9XtGOORSGZKygHa4e\nl0I0PXM4HLjd7vDrzs5OKioq8Hg8Y97r6OjA4XBgtVqjXuN0OsNfu+OOO9i8eXNCbaipqRn3/ebm\n5qjf0955GX55nuKiAi71eiktm05NzQcT+p3ZLtZnNrJsfkgZXT0OoUOSrBYJ2kZmqOHxMafeSCGa\nLtXW1rJ//34AWlpacDgc2O12AGbNmoXX66W9vR2/309DQwO1tbUxr/nqV79KW1sbAIcOHeLaa69N\nadt94Tlt04jXQiRbONNGCQdtOSTJ+AyZaZtMCgqSaevVokWLqK6uZvXq1SiKwqZNm9i3bx+dnZ3U\n1NSwefNm1q9fD8DKlStxOp04nc4x1wB8/vOf5xvf+AYFBQUUFhbywAMPpLTtWpDOs4bmtAekEE2k\nyOjqcQC/TAkanrGC9qjhooBUj+vWhg0bRryeP39+eKhy8eLFI5aAxboG4Oabb+bnP/95ahoZxeig\nLZm2SJXgqOpxkEw7FxhqeHz0cJFfhsdFmmnrtG1DQVuWfIlUiT6nLUHb6AwVtLXh8XCmLR1YpJm2\nxMtiMWFSJGiL1BldPQ5IopIDjBW0w0+eoUPhpQOLdNOGw80mBbPZJDuiiZSJNqctiYrxGStoR2Ta\nZpMiQ0Ui7cJB22zCYjYxKKd8iRSJnmnLPc/ojBW0IzNtGR4XGeAbGg4PZdqKDI+LlFHD52kPz2nL\nhlLGZ6igrW1jahoaLpLlDyLdfEMPihazgsVkkm1MRcpoPUtRFMxDw+OyWsH4DBW0A8PjRZJpi4wY\nntM2YTYr4cxbiKSLUj0uG0oZn6GCdnhzFQVMJqmkFOmnBW2TScFiNjEghWgiRYLR5rRlbwrDM1TQ\njjxYwqSYJNMWaadl1hazEi6GlNOXRCpI9XhuMlTQHs60Fcwm2dJPpJ82p60Nj4NU9IrUGFk9Hvqz\njC4an7GC9uij6oLqpI51FGKyRqzTNsmhISJ1IqvHtb4mD4jGZ7CgHWUDfXnyFGkUOadtlrWzIoWi\nZdpSiGZ8xgrawWjVlHLDFOkTiDhpTuuDkmmLVAgnKUQmKdLXjM5QQVsdtbkKSCcW6aU9JJqGNlcB\nCdoiNaLtiCYnGxqfoYL2iExbhsdFBmgPiaFiSJlnFKkT/ZQvud8ZnbGCdpRMW548RTppc4rmiDlt\nybT1Y9u2baxatYrVq1fz2muvjXivsbGRz3zmM6xatYpHH300oWsOHDjA9ddfn5K2apm2aeiAJJDp\nwFxgyXQDkkmLzwpEFAHJk6dIH21zC9OIoC27ounB4cOHaW1tpb6+npMnT1JXV8c999wTfn/Lli3s\n2bOHyspK1qxZw4oVK/B4PGOuqa+vB2BgYIDHH3+cioqKlLRXZThJCfc1CdqGZ/xMWzqxSCMt0zYp\nijw46kxTUxPLli0DYN68eXR3d9Pb2wtAW1sbJSUlVFVVYTKZWLp0KU1NTVGv8Xq9APzoRz/ic5/7\nHDabLSXtjdi1WbYxzSGGCtojTr2RakqRAVr1uGICU3idtmTaeuB2uykrKwu/Li8vp7u7GwCXy0V5\nefmI91wuV9RrXC4Xp06d4tixY3zyk59MWXujL3GV+53RGWx4PNqctjx5ivTRbppmRarH9W4yGzNp\n1zzwwAPcd999E76+ubk54ff7evsAaGtvw3PZH/pz2xmamy9P+PdmQrzPqifp/CwJBe1t27Zx9OhR\nFEWhrq6OhQsXht9rbGxk586dmM1mlixZwrp16+Jec+DAAb70pS9x/PjxpH6Y6NXjcsMU6RMeHpdC\nNN1xOBy43e7w687OTkpLS6O+19HRgcPhwGq1jrnGZrPx7rvvsmHDhvDX1qxZw1NPPRW3DTU1NTHf\na25uHvF+3u/+Gy55mTtnLnkXeuD4SRwzZ1JTsyDxD50hoz+LnqXis4z3EBB3eDyyOGPr1q1s3bp1\nxPtbtmxh9+7d7N27l4MHD3LixIlxr0llcUa0U29kjkekkz8QxKSMLA6SB0d9qK2tZf/+/QC0tLTg\ncDgoKCgAYNasWXi9Xtrb2/H7/TQ0NFBbWxv1miuvvJLf/e53PPPMMzzzzDM4HI6EAvZEDVePRxTe\nygOi4cXNtGMVWtjt9hHFGUC4OMPj8cS8RivOePjhh5P+YUbsECTVlCIDgkEVszn0LCw7ounLokWL\nqK6uZvXq1SiKwqZNm3juuefweDwsX76czZs3s379egBWrlyJ0+nE6XSOuSZdRp5qKNOBuSJu0Ha7\n3VRXV4dfa4UWdrs9anFGW1sbXV1dUa9xuVwcO3aMr3/966kJ2pFz2rJuUWSAPxgMZz1a8JagrR/a\nkLamp6cnPPS5ePHi8HKu8a4Z7fe//33yGhhBVVUUZAfIXDPhQrR0F2dMpDDj3LmLAHR0nufSpVBh\nxrHjbxO43Dah35kuei7E0HPbUykQGM60ZXhcpFJQDQVskCVfuSRu0I5WnKHNR6ejOGMihRl/PvMG\nvOWlamYVWHqg/SxO5zxqbqiK9zHTTs+FGJlqux4eFPyBIJahqnEpRBOppKoqQzFbCm9zSNxCtGiF\nFna7Hci+4ozIwozwofAyxyPSKBBQw3uOS9AWqaQyNtOWoG18cTPtaMUZ+/bto7i4OOuKM4IRWwTJ\nnLbIhEAwMtOWA0NE6kRm2mYZHs8ZCc1pjy60mD9/fvjP2VScoRWimeTUG5Eh/oBKns0MIJuriJRS\n1eFh8fD9Tg5IMjxDbWMqe4+LTAtEVI/Lki+RSsEombb0NeMzVtAORqzT1gozZE5bpJE/oGKR6nGR\nBmpE9bgE7dxhzKAduW5ROrFIo0AgGB4WN5tknbZIHVVVMWnV4xK0c4ahgnbUo+pkjkekUSCoYjGN\n3hFNTvkSyRcMDmfaWqIiSYrxGSpoR85pmxUpRBPpNyLTNksfFKkTqh5Xwq/NJkUy7Rxg0KCNFKKJ\ntAsGVYIqUdZpS6Ytki+oquH9KGAoaAekrxmdsYJ2tDltyXJEmmhTMTKnLdIhshANQnvdS18zPoMG\n7chTb6QT69G2bdtYtWoVq1ev5rXXXhvxXmNjI5/5zGdYtWoVjz76aELXHDhwgOuvvz6lbdYeEEdX\nj8uNVKRCqBBNhsdzzYQPDMlmw9uYSqatZ5HnsZ88eZK6uroRG/hs2bKFPXv2UFlZyZo1a1ixYgUe\njyfmNak8wz2Sdixi5DptkyJLvkRqRK7TBgnaucJYmXaU87RlTlt/Yp3hDow4w91kMoXPcB/vGu0M\nd5vNltJ2a31Ny7QBLBaz3EhFSkTuiAahaRnpa8ZnrKAd5TxtyXL0x+12U1ZWFn6tnccORD3D3eVy\nxbzm1KlTHDt2jE9+8pMpb7fW17RMG8AqN1KRIsHg6Exb5rRzgaGGxyOrx8Mb6MuOaLqX7jPcIbFj\nQEd/z8We0BnuFy920fpez9BXg3h7enVxrGgijPI5jGBMpm1S8AeCY5aCCWMxVtAOjl3yJU+e+hPr\nDHePx5OWM9xh/HPcIfqZ4ufcPfBf56l0zGDunFDWX/D225hNim7PTo+k5zPgJyubH1KCUdZpQ2jE\nx2oxZ6pZIsUMNTw+vCOaIvs+65ieznCPFB4ej5jTtlpkyFIkX2SCotH6nfQ3YzN8pi1BW39ineHe\n2dlJTU1NVp3hHkmbirFEzGlbzCZ6Ar60t0UYmzYVaDKNzbQlaBubsYJ25Dam0oF1LdoZ7tpQZTad\n4R4pdqYtu1SJ5IqWacuUYG4w1PB45JIvs5zyJdIsEKV6vLfPx8Cg9EGRXFrQHr3kCyRoG52xgnaU\nbUx9Mjwu0iS8uYp55I00qKrhvilEMgQi7nWa4W1zZWTHyAwXtLUkR1EULGY5qk6kT2DUNqYQcSOV\nh0eRROE57SjV45JpG5uhgvboDfQVReFCd38GWyRyyfCcdpQhS59kPyJ5tAfE0duYgjwgGp2hgnZg\n1F68JpMim6uItBmuHo+SaUv2I5IosuhWI0u+coOhgna0Q+HllC+RLtEybcvQnwflRiqSSMu0R5+n\nDeDzSV8zMkMF7WBQJXLzPrNJkQIgkTbDp3wN/7MaXoYjw+MieaJm2tLXcoLxgvaoakoZHhfpMnzK\n18jNVUCGLEVyjbvkS+a0Dc1YQVsdu9mABG2RLtrZ7eYR1eNDw+NSiCaSSJv2G33KF8gDotEZK2iP\nybSV8NyPEKkWLdOW4iCRClEzbVnylROMFbTV0efLypy2SB9/lDntcKYtN1KRRIGoB4ZI0M4Fhgra\nqqqOePI0mWQ3KpE+QVmnLdIkeqYtozq5wFBBOxhUIcqTpyz7EukQLdO2yI5oIgWibWMqKxVyg+GC\ntjx5ikyJPqetFaJJHxTJM7zka/hrckhSbjBU0A6MCdoyxyPSJ3r1uBziIJJveHMVWfKVawwXtJWI\nT6R1aL90YpEG42Xa8uAokina5ioWGVnMCYYL2pJpi0wZt3pchsdFEg0Xog1/TR4Qc4OxgnYgOKZ6\nHKQTi/TQMm2rJaIQTVunHZDhcZE845+nLfc7IzNW0B69ucrQDVOGx0U6aHOJZlOU0R7JtEUSjbeN\n6aDUTxia4YJ2RA0QZkUybZE+WtWuJSLT1h4cZXMVkUzBKJuraKM6smWusRkmaKtDm6goUZ48JdMW\n6aANWVqj7D0u1eMimaINj2sFkAOD0teMzJLpBiRLeLjIJHPaIjO0fjZiyZcUB+nKtm3bOHr0KIqi\nUFdXN+K9xsZGdu7cidlsZsmSJaxbty7qNQsXLuTIkSNs374di8WCzWbj4Ycfpry8PGntHL7fDX9t\nONOWvmZkhsm0/eNsoC+ZtkgHf7QlXyYZstSLw4cP09raSn19PVu3bmXr1q0j3t+yZQu7d+9m7969\nHDx4kBMnTsS85sknn2T79u385Cc/4YMf/CDPPPNMUtsajJJpK4qCxWxiwOdP6u8S2cUwmbZWuTti\nswGpphRpdKbTC8BLR85QkBf6p2WRTFs3mpqaWLZsGQDz5s2ju7ub3t5eANra2igpKaGqqgqApUuX\n0tTUhMfjGXON1+tl165dQGjarqOjg5qamqS2NaCOTVIA8mxmybQNLqGgHW34R5MtQ0bRCjMk0xbp\nFIgyRSMPjvrhdruprq4Ovy4vL6e7uxsAl8s14l5VXl5OW1sbXV1dY65xuVzY7XZeeukltm7dytVX\nX82dd96ZUBuam5sTev/kuz0AeLou0PqeN/y+QoBLl3vj/pxsoIc2JiqdnyVu0I4c/jl58iR1dXXU\n19eH39+yZQt79uyhsrKSNWvWsGLFCjweT9RrtCGj2bNn88gjj/DMM8/wj//4j0n5IIEow+Mypy3S\nKRjeXEV2RDMCVZ346YCR1yxZsoSPfvSj7Nixg8cffzyhe914GXlzc3P4/a5AK7zcRcX0GcydM/ww\nUXzyBAO+QNIz+2SL/Cx6l4rPMt5DQNw57WhDRl5v6MkucsjIZDKFh4xiXbNr1y5mz54dHjKaOXNm\nMj4fECPL0arH5YYp0iB6pi1rZ/XC4XDgdrvDrzs7OyktLY36XkdHBw6HI+o1FRUVvPjii0BonnnF\nihVJz8SiVY8D2KwmqZ8wuLhB2+12U1ZWFn6tDf9A9CEjl8s17jUvvfQSn/jEJ3C73QkPGSVCGwKP\nuo2pDI+LNAhGmWdUFAWTSZHNVXSgtraW/fv3A9DS0oLD4aCgoACAWbNm4fV6aW9vx+/309DQQG1t\nbdRr7HY7u3fv5q233gLg6NGjOJ3OpLY1WvU4gM1qlqBtcBMuREv3kFGiczweb6hisre3h9b3WgG4\n4BkE4NSpVprzPBNud6rpeU5Hz21PlWBQHTE0rrGYFBke14FFixZRXV3N6tWrURSFTZs28dxzz+Hx\neFi+fDmbN29m/fr1AKxcuRKn04nT6RxzDcDWrVv5zne+g9lsJj8/n+3btye1rbEy7TybmUF/MHRM\ncZS+KPQvbtCONfwT7T1tyMhqtcYcMlq+fHl4yGj37t1xG5joHM8Zlxd+eZ7iYjtz58wGQLVeghOn\nmFl1JTU118b9Xemk5zmdTLU92x8UAjFulGazSYbHdWLDhg0jXvf09IT7+uLFi0fU88S6BuDGG2/k\nZz/7WWoaSfTCWwhl2hCajskxTg9xAAAgAElEQVS3GWZxkIgQd3g81vAPZNeQUWCc4XGpHhfpECu7\nMZsU2cZUJFW0wluAvKGgLbuiGVfcR7FoQ0b79u2juLg4q4aMxisCkqFJkQ4xh8fNJnwyzyiSKNrm\nKjActGWttnElNH4yevhn/vz54T9ny5BRIDC2E5sk0xZpFGt43GJR6O2XXapE8gwXPY78emdXaDMY\nmY4xLsNsYxoIasPjw1+TTFukU1CNVYhmkuFKkVSxCtG0fe+lvxmXgYJ2tHXaciyiSJ9gUB0zxwih\nozoDQVVGfETSRDtPG8Cqnakt0zGGZbigPfKoOu2pU4YmReoFgsHow+OS/YgkC8SoHg9n2hK0Dcsw\nawK06vHI4UlL+KlTMhy90cN+96ONV4gGoeynqMCakt8tcku0o4gh4gFRgrZhGTDTHv6adGB90tMR\niZFiLfmyWEJfk34okiXWnHbkA6IwJuNk2lHmeKQD61Oi+91D5o9I1ASDKkGVqJm2VYbHRZLF2lzF\nInPahmecTDswdrjIZFIwKYpkODqjl/3uI4VXL8TYEQ0k0xbJM7xaJsbwuDwgGpZhMu1Y1ZQWsyId\nWOcycURiIlumRn7PwFDdxEB/f3jve02Ptw+A1954k8uu/ITan62yfSvZXBE709YeEKWOx6gME7S1\n5TTK6MIMixxVpzex9rv3eDxp2e8ext/zHsbuv365dxD+4yz2oiLmzpk74nsv9HbyztlzXOW8hpoF\nlQn9/myk5/3yJytbH1KGYnbUJAVgwCcrZozKOMPjweg7BFnMErT1Ri/73UfSzmyXQjSRDtpqmdiF\naJJpG5VhMu3gOHM8crPUl1j73Xd2dlJTU5M1+91H0s5sl3XaIh3C25iOSruk+Nb4DBO0o+2IBqHh\nIm+fPHXqTbT97rWhymzZ7z6SVgg53jpteXgUySLbmOYuwwyP+6McGAJDZxn7ApMqZhIiUX7JtEUa\nxTrlyypTMYZnmKA9vORm5Netsv+4SINEgrYMWYpkiVXDY5a+ZniGCdqxlnyZZbMBkQZa0DZHPTBE\nsh+RXDEzbZmKMTzDBO1o52mDDE2K9PD7o9dUgOyIJpIvdpIimbbRGSdoB6MXAsnQpEiHhOa0pQ+K\nJIl5ypdJQVHkAdHIjBO0Y65blKFJkXq+KKfMaSwWOSJWJNfwkq/oD4myTtu4jBO0x9lcBSRoi9QK\nJFSIJjdSkRzBQPRMG2RvCqMzXtCW4XGRAf7xMm0Z7RFJFs60oxU+muWQJCMzXtCWQjSRAeMVokkf\nFMkWa3MVkK2bjc5AQTvWgSHaki8ZmhSp4w/GzrQVRcFmMckhDiJpgkE16tA4SNA2OuME7YDMaYvM\nGe/AEIA8m5l+ybRFkgz6A5hH7yQ1RBsel10gjck4QTvOukUJ2iKVwku+YqQ/BXkW+gYk0xbJ0T8Q\nwGqJFbRNqOpwnxTGYpygHaN61yqFaCINxlvyBUNBu1+CtkiOAV8gPIo42nCiIkHbiIwTtGOeejNU\nuStDkyKFxitEg+FMW4YsRTIMDPpjZtpW2RfA0IwTtAPRb5qyF69Ih0F/qH/Fyn4K8iwEgio+ObhG\nJEH/4HiZthTfGplxgrZ2yteoRMdqNQPQ2+9Ld5NEDtFGcmIG7fzQ0fUyry2mSnv4G29OGyRRMSoD\nBe3ohWh51tBH7JX5RJFCWs2EtpHKaAV5ErRFcmjD3vGCttTxGJNhgnYwxo5oNotk2iL1tKzGEuNG\nKkFbJEt/nFEdybSNzTBBO1Yh2vDwuNwsReqEg3aMG2lhvhWQfiimrj9upi3Ft0ZmmKA9fDTiyK+b\nTQp5NjM9kmmLFBqME7Ql0xbJEq6fkOHxnGSYoB3rUHiAwjyLZDgipYYL0eLMaUs/FFPUPxDqa9YY\nfU2CtrEZJmgHgiomJfoG+oX5VpnTFimlLa+Jl2n3SqYtpkgbHreYzVHfl1PljM04QTsQxBRjL96i\nAgs9fXKzFKkz4PNjNilRHxpBhsdF8miFaLLkKzcZJmj7g2p4U4HRCvOt+ANBfH7pxCI1Bn3BmFk2\nhKZoQIK2mDptyVesOW2zHAVraIYJ2sGAGnPf58s9gwCSbYuUGRgMxJzPBtlcRSRPONOO8ZAou0Aa\nm2GCti8QO9Oxya5oIsUGfIFwhhNNeE5b+qCYonjD49ahDaW0gjVhLJZMNyBZLl7uj/meTXZFEyk2\n4AuQZ41eGARwqOU8IJl2ttu2bRtHjx5FURTq6upGvNfY2MjOnTsxm80sWbKEdevWRb1m4cKFnDt3\njm9/+9v4/X4sFgsPP/wwFRUVSWljeHg8VqY9FMz75cAQQzJOpu0Pxpzj0XZFk7XaIlUGxzkqEcA2\n1DclaGevw4cP09raSn19PVu3bmXr1q0j3t+yZQu7d+9m7969HDx4kBMnTsS85nvf+x533XUXTz31\nFMuXL+fJJ59MWjvjZtpD9zvpa8aUUKYd7UlSky1Pn/5AMOYcjwyPi1QKDh3gMN6ctlWCdtZrampi\n2bJlAMybN4/u7m56e3sBaGtro6SkhKqqKgCWLl1KU1MTHo9nzDVer5dNmzaRl5cHQFlZGS0tLUlr\nZ7wtc+UB0djiZtp6ePoMBlX8AXWcOW0ZHhepMxjnJgqh/QOsFhM9ffLgmK3cbjdlZWXh1+Xl5XR3\ndwPgcrkoLy8f8Z7L5Yp6jcvlorCwELPZTCAQ4Omnn+Yv//Ivk9bOE20XgdiFaFo/lDltY4qbaUd7\n+vR6vdjt9qx5+tSePGMNF2mZtgyPi1SIt++4Js9mxitBWzdUVZ3SNYFAgHvuuYebb76ZW265JaHr\nm5ub477ffekyAB0dZ+nuir3/+IWu7rg/L5OyuW0Tlc7PEjdou91uqqurw6+1J0m73R716bOtrY2u\nrq6o1zidToDw06c2lD5V/XHWLebJoSEihRIO2lazZNpZzOFw4Ha7w687OzspLS2N+l5HRwcOhwOr\n1TrmGm3K79vf/jZz587lK1/5SsJtqKmpiflec3MzNTU12BoaAB9XzZ0Ts/jR/uYxFLN13J+XSdpn\nMYJUfJbxHgImXD2e7qfPRJ48Pd5QMB7o76P1vdYx39M19P6p1naam70TaXpK6flJU89tT7Z4Z2lr\n8qxmLnT3EwjG3lNAZE5tbS27d+9m9erVtLS04HA4KCgoAGDWrFl4vV7a29uZOXMmDQ0N7Nixg66u\nrjHX2O12fvnLX2K1Wvna176W9Hb6/KEtc2MNjwPk55mletyg4gbtaE+f2pNkOp4+E3nyPH3uEnCe\nspJi5s6ZNeb7pl3q5+CbxykumU5NzQcS+r2ppucnzUy1PVsfFAbinG+sybMNTdP0+ZhWZEt5u8TE\nLFq0iOrqalavXo2iKGzatInnnnsOj8fD8uXL2bx5M+vXrwdg5cqVOJ1OnE7nmGsAnn76aQYGBli7\ndi0QmibcvHlzUtrpDwQxmRRM4zz45dssXO7pTcrvE9klbtCO9vRpt9uB7Hn6jLduMTynLUOTIgXi\nHRaikaCd/TZs2DDidU9PT/gBdfHixdTX18e9BuBnP/tZahpIKNMeL8uG0GY+fQN+VFWNuR++0Ke4\nQTva0+e+ffsoLi7OmqfP/jjny4aXfMkSCJECAz7toTHe8Hjon5u3bxAoSnWzhEH5A8GYRbeagjwL\nQRUG/cFxN/0R+pPQnPboJ8n58+eH/5wNT58DcfbitZgVTAr0SqYtUiDhTHvo5untlX4oJs+XQCDO\nzwu93z/gl6BtMIbYxlQruIj19KkoClarmR6pHtcNPWzoo5nIki9Aln2JKfH5gxQVWMf9nsijYEvs\neelolkgTQ2xjGm94HEJZTp+s09YFPWzoE2kggf4HI+e0hZgMVVXH3f1RU2CTU+WMyliZ9nh7P1tN\ncjSnTsTa0AeyaztJzaA/8SVfIJm2mLx4G0lp8ocybdkVzXgMEbQTyXRsFjPuAVkjqwexNvSB6NtJ\npmJDn0SWl2nfc/Ld0A5VHs8FWtVLMb+/63LoofHEqTaam2N/XzbL1mV3uSLRUZ3I4XFhLIYI2vEO\nhYfhCvK+AT/2OPNBIrtkYjvJeOvQI9eq//boIaCbK2ZWMruyOOY1xZf6aXxL2y/g/Qm1I5voeW+B\nycq2h5RE7nUwXIjWJxusGI5B5rTH38YU5KQvPZnMhj7jXTOZ7SQnQtuhKvHq8cGUtEMYXyL3Ohie\n0+6XTNtwDBG04y35AjnpS09qa2vZv38/wLgb+vj9fhoaGqitrY15TSq3k9QEAlrQHn/aJX+oEO2y\nBG0xSYnc6wAK8mV43KgMMTye6Jw2SOWuHsTa0Kezs5Oampqs2NAnkm8oaJvj3EjNZhOF+Ra6vRK0\nxeQkmmkXF4R23LvcI33NaAwRtOOt04bhoUl58tSHaBv6aPOL2bChT6RAIDR/Hm94HKCkKI9LPQOp\nbpIwqHCmHSdov/FuaKrotZNu/jblrRLpZIjh8cQK0ULvSaYtks2f4PA4wDS7jW7v4KSK64RIvBBN\nhseNyhBBe2AwgALjnnoTPjRECtFEkoWDdpzsB0KZdiCoysOjmJSBCRai9UkNj+EYImj7AkHMZmXc\n02y0IiDZ91kkm38iw+P20Fxjt8w1iknoT3B43GRSyLeZJdM2IGMEbV8AsyneFpKhJ0+p3BXJFj7f\nOIEjELV9oLu9Mq8tJq4/wbPbYfh4TmEsxgja/uC4Q+MgmbZIHX8gmNB8NkRk2lJBLiYhkaJbTUGe\nhf7BQHhJojAGYwTtBG6aebJGVqRIKGgn9k9pWlEo05YKcjEZia7ThuGtTC/JPc9QjBG0/cG4w+M2\nqxlFkcMaRPL5A2rCQbs0PDwuN1IxcYmcaKgJB23pa4ZinKAdJ9M2KQpF+VbJtEXS+f2JZ9ot2vrZ\nd1ypbJIwqPDmKhPItC9K/YShGCdoJ3ByV3GhTea0RdJNZE5bu5H2SoGQmASfL/E9AbStTC9elqBt\nJIYI2n5/IKGgbS+0ymENIqmCQZVAMPHh8cJ8K4oitRVicrQ9AeJNB8LwVqbnPT0pbZNIL90H7UAg\nSFAFUwKd2F5gZdAfDB8kL8RUDfoTX4IDofWz9gIrl2XER0yCFrTjrZYBKJsWqp9o7/CmtE0ivXQf\ntAf92mENiQ2PgxyNKJJn0JfYsZyRigtt9PT5wkd6CpGo4Uw7gftdkQ2zSaGt83KqmyXSSPdBW7vx\nJTo8DkiWI5Jm+IS5xOa0Yfjh8UJ3X0raJIxL230vgX18MCkKpcV5tHV4CQZlr3ujMEDQTnx4UrtZ\nynyiSJaJDo8DFA89PHZ29aakTcK4tN33xtuyOVL5tHwGfQFcF+UB0SgMELQTz7TbO0NzOy8daU9p\nm0Tu0DLteGdpRyouCj08dnrkRiomxh9IbKWMpqw4H4C2DhkiNwrDBO1ECjMKh5ZA9PTJchuRHIM+\nbYeqiQ+PuyTTFhPkT2DL5kjatrkdHulrRqH7oB0uzEgg09HmtHv6ZHhcJMdkMu1pQ5n2GZcsxRET\n4w8EEzqYRqM9IHZK0DYM3QftCRWiFWhz2lKIJpJjYBJz2tOKbFgtJk6f605Vs4RB+QPqhIbHtamY\n107IDnxGofugrQ1PJtKRiwpCmbbsPy6SRet/EwnaiqIwfVo+7Z3ecCGlEInQCtESVZRvwaQoXOqR\ne55R6D5oD2fa8T+K2aRQlG+hR4K2SJLwkq8JzGkDTC8tIBBU+dkLx1PRLGFQEy1EUxQFe6GcuWAk\n+g/agcQ3VwEoKgjtPy7rFkUyTCbTBphREqrqdXf3J71NwrgmWogGoemYvgG/7ARpEPoP2hOY04bQ\nGtmgqtItJ9+IJNBuhIkclRhpRmkBAG5ZPysmwB+c2Jw2yGoFozFO0E4w07EPzWu7ZTcqkQThoJ3A\n9Eyk8qFMW3ZFExMxmUx7eDMf6WtGoPug7fcnXogGw9WU5y/IU6eYusnOadssZkrsNtwX+1FVmaoR\n8QXV0IlyE820tQLcrksyFWMEug/aEx0eL7GHTr4565aTb8TUhQ8MmeDwOMCMkgIGfAHcF+VmKuIL\nDp0vM9FMuzA/FLQ9ErQNwThBO8FMp1QL2rKxhUiCgUkWosHwvPYpWa8tEhAYKp6dyOYqEFr2BdB1\nWep4jED3QXtwAku+IDQ8rihwzi1BW0zdZKvHAaYPzWufOitBW8Q3tFAm4XudpnBoeNwjKxUMQfdB\ne6LD42aTwrQiG2dcMjwupm44055Y9gMRmfbZS0ltkzCmcKY9weHxgjwLCjI8bhQGCNoTK0SD0BD5\npZ5BvLLhgJii4UK0if9TshdYybOaOS2ZtkiAFrQnWohmUhQK8i10XZagbQQGCNoTW/IFw/Pa7ZJt\niymayvC4oihML8nnrLuH/gE5eU6Mb7KFaBCqIPd0y0oFI9B90PZPcHgchtfItp6TYUkxNQO+ACZF\nmdSNFEJD5KoKreelL4rxTXZ4HKAo38qgP0hPvzwc6p3ug/ZEtzGF0FIbgNMylyimaNAXmNR8tma4\nGE36YjbYtm0bq1atYvXq1bz22msj3mtsbOQzn/kMq1at4tFHH417zY9//GOqq6vp6UlO0etkh8cB\nCocqyD2ymY/uJRS0s7kjT+TAEE3ZtHwUoPl4Z1LaIHLXwGBgQlMzow0Xo8m8dqYdPnyY1tZW6uvr\n2bp1K1u3bh3x/pYtW9i9ezd79+7l4MGDnDhxIuY1v/jFL7hw4QIOhyNp7QtMYXg8fK627Iqme5Z4\n3xDZKU+ePEldXR319fXh97ds2cKePXuorKxkzZo1rFixAo/HE/WaVHTkiVaPA1gtJkqK87jQ3Yeq\nqigTXPcohGbQF8A6iY1VNOXT8jGZFMm0s0BTUxPLli0DYN68eXR3d9PbG9o5sa2tjZKSEqqqqgBY\nunQpTU1NeDyeMdd4vV6WLVuG3W7nV7/6VdLaN9l12gAl9lDQlqWu+hc3aEfryF6vF7vdnhUdOXye\n9gSHKGeU5HPi8gCurj4c5YVJa4/ILQO+wKR2Q9NYzCaurLBz+twlgkF10nPjYurcbjfV1dXh1+Xl\n5XR3h0ZAXC4X5eXlI95ra2ujq6trzDUulwun0zmpNjQ3N8d8Twvaly930/rexFa+9HpDc9lH3zrF\nFYVdk2pbso33WfUmnZ8lbtCO1pFdLhd2uz0tHTneX8YFT6gDnjnTPqG5RSUYWv7QcPAI11yRP+F2\nJYOeO62e255MA74gBXlx/xmN6+orSmjruExnVy8zpxclqWViqiZTaT3V6uyampqY75349UEAysvK\nmDunckI/t3LQz/+82ULQXDTu70iX5ubmrGhHMqTis4x3f53w3SbdHXm8v4zm5mYKi4qBAZxz504o\nSxmgi3fOvkdB6UxqauZNun2TpedOm462b9u2jaNHj6IoCnV1dSxcuDDckRsbG9m5cydms5klS5aw\nbt26mNdAqI7ioYce4vDhwxQVJS8oqqrKoG9qc9oAg0N7DZw62y1BO4McDgdutzv8urOzk9LS0qjv\ndXR04HA4sFqtY66pqKhISfumUoiWb7NgL7By/oIMj+td3LtNtI6sdcpYHXm8a5LNN3RU3USHFcuK\nQ2u1z3TKWu1sk+0FQRptC13rFIP2DKkgzwq1tbXs378fgJaWFhwOBwUFoULBWbNm4fV6aW9vx+/3\n09DQQG1tbdRr7HZ7Sto3lXXaEKogP+vqIRiUtdp6FjfTrq2tZffu3axevXpMp4zsyDNnzqShoYEd\nO3bQ1dUV85pk8/knVwhUogVt2WAl68Sqo4DsKAjSTLaeYrTpQxXk756RCvJMWrRoEdXV1axevRpF\nUdi0aRPPPfccHo+H5cuXs3nzZtavXw/AypUrcTqdOJ3OMdcA/PCHP6SxsRGXy8Xf/d3f8YEPfIB7\n7rlnSu2bSqYNoRMOO7v6ZBpG5+IG7Wgded++fRQXF2dFR/b5g5PKdGwWM0UFVsm0s1CsOgpIX0FQ\nIqayhWmkonwrBXkWTslmPxm3YcOGEa97enrCU0GLFy8esXIm1jUAX/7yl/nyl7+c1LZNZXMVgMry\nQt5pu8hrJ9wStHUsoTnt0Z1y/vz54T9nuiP7/MFJL7kptedxxuWlf8BP/hSLiUTqZKIgKJFCu1eP\nHAWgv7+X1vdap/T7ivKg09PLwaY/kW/L3j2PpAAxc4ZP+Zpc0J4zsxiOwqvHOvn4h+cmsWUinXQf\nqXyBKQTt4lDQPu/p5aqqaUlumZisWDURHo8nbQVB8Qrtmpubufb6BfDrDkqnFTN3zqxJ/y6A9otn\ncV9yUeJwcsO8GVP6Wami5+LJycqmh5SpZtql9jyKC238+R0XgUBwygWUIjN0/1/N2+tj0Bec1LXT\nhnYJ6pCKyqwyXnFPNhQEaQamcFjIaJVDewUcb82ONbQi+0w1aCuKwuxKOz19Pk7LVIxu6T7TDgSD\nmM2T+xjFRUNB29ObzCaJKYpVR9HZ2UlNTU3G6yg0Uznha7SqoTnGN095+Jsp/zRhROHhcWXy/W3m\n9CLePOXhrdMe5s0qTVLLRDrpP2gH1EnP8UyToJ21otVRaEOVma6j0PRrhWhT2BFNU1RgZVqRjaMn\nXLIzmohqqpk2DD8cvnXaw6duvTop7RLppevhcVVVCQTVCR0WEkmCtpgK7Qzsqew9Hmnm9CIGBgOy\nDFFEFZxiIRqE9iCfVmTjrdOeJLVKpJuug7a2R8Bk18nm28wU5JklaItJ0TLtZAXtqhnaEPmFpPw8\nYSzJyLQVRWHBVeW4uvpwX5QTv/RI10HbH5jaZgOKolCYb+WMyzvlJUIi9/QPDmXaSarCjZzXFmK0\nZARtgAVXhfY5kGxbn3QdtId3CJr8xygtzsPnD3L+gmTbYmL6B5KbaZdPyyPPauYtCdoiikCou01p\neBygu2cAgOebTk+tQSIjdB20/VonnsI2klfMCC0Lev2kO853CjGSlmknoxANQiM/M6cXcu5CD12X\n+pPyM4VxBNTkZNqOskJMiiKHh+iUvoP2FPfiBbiyIjQkKUFbTNTwnLY5aT8zPK8tQ5dilKlOB2os\nZhMVZQW4LvaFiymFfug6aAcCU3/yLJ+WT77NzOsn3DKvLSYkXD2exJ2lhue1pRhNjOTzh+5PydgX\nYOb0IlQV3mm7OOWfJdJL10E7nGlPoRMrisKcmdO40N3P2+/JblQiccmuHgdwlBdiMSsyry3G0DLt\npGzmMyO0A9+bp+XhUG90HbS1TNsyxeGia4d2Bjrw57NTbpPIHeHq8SQGbYvZxDWzSjl5ppvefl/S\nfq7QP184aE994x3tlK9jpyVR0RtdB21/EjYbAJg9005RgZWDR8/IELlI2EASd0SLdNOCSoJBlT8e\nOZPUnyv0zR9QUZSpF6JB6DjYaUU23jx1AZ9/cmc3iMzQddDWMu2pnlZjNplYvKASd3e/bKQvEtY3\n4MdmMWFSkrvl6LIPzcFkUvht4yl5iBRhvoCKzWpGSVJ/mzuzmN5+P2++K0PkeqLroO1P0mYDADZr\n6K/i8Jvnp/yzRG7oHwyQZ0v+9v3TSwr4yI1VnDp7iT++2p70ny/0yR9QsSVxpYLzihJA7nl6o+ug\nHUjSEgiA2ZXFKAq8/Po5yW5EQvoH/RTkJe8mGumLn6rGZjWz51ct4blzkdt8fpU8a/Ju2VdUFGG1\nmPjDq+1yz9MRXQft4c1Vpv4x8m0W5s6cxon2btlGUiSkfyA1mfbzTac5cryTG+dN5+LlAXbVH0n6\n7xD64w+o5NmS95BoNpmYM7OYSz2DvNdxOWk/V6SWvoN2EjZXibToegcAP294Jyk/TxjbQAozbYD3\nX1uBxaxw5LhLioUE/b5geJlhsjirpgFwuEWGyPVC10E7kOSgXTWjiGtnl9J8rJOuy7KNpIgtGFQZ\n9AfJT0GmrSnIs/A+53S8fT6Z2xYEgslZox1p7sxpKMDLb5xL6s8VqaProJ3MzQY0M6cXEgyqPP6f\nryftZwrjGRzqe8kcrozmA9dVYFIU9v3hHZl3zGGBoIqqJj9o5+dZmOWw8/Z7FznrlnPc9UDXQTtZ\nR9VFunZ2GSYFjrfKpgMitkFfqO8VpDDTBigutHHN7FLaOrz863+9kdLfJbLXoG9oT4AkbKwy2nVz\nywD4Q7OM5uiBroO2P0lH1UUqyLMwZ+Y0XBf7eO+8rNkW0WlzzKnOtAFunDcdkENtctlw0E7+Lfvq\nK0qwmE386sC7BAJSO5HtdB20w3PaSX76vG5O6Mnz96+0JfXnCuMYHDq8IT8vtZk2QGV5ITNK82k9\nd4lu70DKf5/IPgNDQTsZK2VGs1nNLLiqDG+fjyaZ2856ug7aw0fVJfdjOK+YRr7NzPNNp/H2yf7P\nYqx+XygjOedO/ZnEiqIwf245QRX+eESGMHNRKofHARZeUwHA0/uPyUqFLGeQoJ3cjmwxm/jgdQ56\n+v385uC7Sf3Zwhh6B0I3tvw0DI8DXDu7FEWR0Z9cNTj0kJiK4XGA0uI8brh6Om0dXp78dQvBoBQ9\nZitdB21t+iXZw+MAN8ybjsVs4r/+eFI6sBijTwvaaRgeByjMtzJ35jROtnfTKvvj55xUZ9oAN99Q\nxZUVdn514F2eev6tlP0eMTW6DtqpGh6H0DzPtbNLudzr4/UTUgAkRuodTG+mDXD9UJXv7/70Xtp+\np8gOqZzT1uTZzKy4eS7Timz8/PfvcLxVdobMRroO2gNDcy+2JO7HG2nBVeUA/HT/sXDRmxAQkWmn\neMlXpKuqplFWnMdvm07juSSb/+SSVFaPRyrIs3BHzWxU4MF//5MUPmYhXQdt7cZps6Ym26maUcS8\nWSW8ddrDz38vW5uKYeme04bQDftvV8xnYDDA9392hN5+KZLMFcNz2qkbHtdc6bDzoffNxN3dz/+7\n51DKf5+YGH0H7cEgeVZz0s8zjrTkA1cyoySfn/z2LV55qyNlv0foS7rntDVqUGVOZTGvHu/kC9/Z\nz3d/2szLb5yT9bUGN5CmTFuzaL6DitIC3n6vi7fkAKWsouug3T8YTPnmFoX5Vu5YPAeTovDos0cl\nuxFAaE7bpIDNkt5/QhFJ340AABRJSURBVCaTwsraq/hw9UzybWb+8Go7W588zP/9Ly9wou1iWtsi\n0mcwDXPakUyKwq0fuAKAp184lpbfKRKj66DdNxhMy/BkRWkBi66vwH2xj/9v76tSTS7oGwiSZ7Og\npHCUJxazycRNCyr5/Ir5fOaOa3mfs5yuywPc88gB9r98Wh4sDSgd1eOjXTHDziyHnT+/7ZJRxiyi\n26A94AvgD6RnG0mAm95XycJrZvDyG+d5/uXTafmdInv1pumBcTyKolBZXsjtNbP5i1onNouJR/7j\nKKvve44N33+JlncvZLR9InnCw+MpWCkzno/cWIXFbOK7P22mVbZ1zgq6Ddre3kEA8qzpmVM0m0ws\nmu/AajHx5K9a8Fzql3nEHBUIqvQPpvZYzom6qmoaf3XbNXzwugqqphdx/L0u6n54UM5JNohUb64S\nS0VZIUs/eCXePh/3PvI/PPrsUS5096W1DWIkHQft0BBgOrOdonwrNy2opH8wwP/1nf18tu437H3h\nuAyX55iePh+qCvl5mc20Ryux5/GRhVfwV7ddw6eXzMNqMfHgj//EaydcmW6amKLegdD9zprmGgqA\n+VeVc3vNLAKBIM83nearOxrkMKUM0m3Qvqxl2mkeovzgdRXcNN9BUYEVq8XE0/uP8eCP/0SHpzet\n7RCZo61dzaZMe7RZDjsf/9BcAkGVLf92SDbK0LnOoftLcZEtI7//fc7p/O+/rObW91/B5V4f67//\nEg8/9QrtnZcz0p5clr13nTi0gzzSHbQVReHDN1Tx4Ruq6B/089zB0zS9fo5X3urgr2+/hvdfU0FF\nWQEzpxdFvb713CUuDT1wCH1673zoRlVWnJfhloxvzsxiPv7hObzwciubn3iZbf9Ui/OKkkw3S0zC\n+Qu9mE3pHVkcTVEU3n9tBRaziSNvd/LSkTO8/MZ5Vi+/jrLifN7nLOeKCvu4P+NYq4dz7h6mpanN\nRqTfoD0U+DKZ7eTbLPyv2+bxznsXOfJ2J/Uvvk39i29jMimsXn49q5dfN6K6+On9x9j7wnEA1tw+\ng5pMNVxMyemhvb+nlxRkuCXxzbuylNtrgvz3K218Y+cfmF5awOWeQeZWTeNvP349NfMrw9/77plu\nXjzUSlGBlf+1dB72QhuBQJBH/uPPBFX4sFNqODJBVVU6PL0U5pkyslphtOqrp/M+Zzkn2i/S0NzO\nj58L7VNutZi4+y+rWVnrjNrOt9/r4v/ZdQCAtXfI/W+ydBu0L/dmJtMezaQoXD+3DOeV03jtHTf9\nA37ePdvN0/uP4erqZc0nF1A+LZ+XjrSHAzbA4eNeVn0qgw0Xk3b6XDcA00vzM9ySxMy/qhyb1czh\nN8/TP+CnqMDK261dbH7iZf6i1skXP/U+3n6viy3/doi+gVCV8q/+511WLbuOhub28EPK8VNWPrw4\nmLa1wiLkcq+PvgE/laXZc7tWFIVrZ5dxZYWdtk4vff1+Xj3eyY/+83UOvnaOmxZU8sHrKxgYDPDu\n2W76B/z8dP/weu8Xj3Tz2ZVqVjyE6E1CvWDbtm0cPXoURVGoq6tj4cKF4fcaGxvZuXMnZrOZJUuW\nsG7dupjXnDt3jnvuuYdAIEBFRQUPP/wwNtvk5mi04fH8FG1hOlE2i5mbFoSylvdfV8Gv/uddXjz8\nHn88coY5lXZOnunGajHxV0vn8ccjZ3j7bC/nL/TEHEbPddnY5zSnz12iMM9EYZp3Q5uKq68s4eor\nh4fGXRf7ePFwK785eIo/vXmerssDqCosWzyHy72D/OmtDp789ZsAzL+qjIHBAKfOXmLfH07w2Y9d\nl6mPkXKj+1CkTPW7Dk/ozPaCvOx7WCrMt3L9nNBBNvNmlfD7V9p4/aSb10+6efLXI79XUWD5h+bQ\n0++j8bVznDzTzTWzSjPQan2L2wsOHz5Ma2sr9fX1bN26la1bt454f8uWLezevZu9e/dy8OBBTpw4\nEfOaXbt28bnPfY6nn36auXPn8uyzz0664drxhAX52XfjLC60sWrZ9dy2aBaFeRZOtHfjKCvkr5bO\no6KskBvnTQfg+abTGW1ntsrWPgfgudTP+Qu9OEqtus4SKkoLuOtj17Hwmhm4LvZhs5pZ+ZGruH5u\nGTctqORzH7+e9Z+vYdf62/jYTXO446bZ2PNNPL3/uGErh7O132lFroVZGLQjFRfa+PSSeXxh5QKW\nLZ7NgqvKufGaGXxs8Ww+cctc1n5yAdfNKaOkKFQL8u+/eTPDLdanuBGvqamJZcuWATBv3jy6u7vx\ner3Y7Xba2tooKSmhqqoKgKVLl9LU1ITH44l6zaFDh/jOd74DwO23386//du/8bnPfW7CjX7z1AUO\ntZynqsxKqT07i4HMJoXqq6dTffV0VHXkMNC8WaW8dKSdX//PKapm2JlWZKOyvJD8PDM+X5BBf4BB\nX5CCPAsldhuKohAMqgSC6tD/BwkGVVRtpZky/H+Rv0f7o6IohL+qgPYqMuaM9z2jv+9yX4CuS/0x\nf9ZUmE1KzD4HZKzPBQJBXBf7ePLXLQDcMCf757PjsZhNfPQDV1K78ApMppH/AUvsefQP+Dne2gWE\n6jc+9aEyfvbSBTY98TJfuvMGSovzUJTQFFGonwz/v/a1yPdMpsivDX0dBZNp+FqToqCq4PMH8QUC\nDAyG/ocCVrMJq8WEzx+kfzCA2axgs5ixWkJft1nNmE0KQTX0byMYVAmqavjfivbnAV8Any+I1WIi\nz2Ymz2ZmWpEtar/r7Q0FzEz1O29fKCsFKMqyJYaxFBfauH5uOdfPLY/6/tyZxVjMcOy0h1fe6mB6\nST7+QBCLOfTfQztLwmRSsJhNmE3Z/XDc0x+Y9Glo04psE374jxu03W431dXV4dfl5eW4XC7sdjsu\nl4vy8vIR77W1tdHV1RX1mr6+vvAQ0fTp03G5Jr5+9PCb5/mXoZNnPny9XRfZzug2Wswmrr0yn5bW\nPh75jz9nqFVT9J/nUvaji7tO8L+j9B8gI30O4P7Hm3ht6Fz1a2aXsmheERf8k/pRWWd0wI7F2x/g\n5htm8vIb53nwx39KcavSqyDPzFXBjjF9qLs7VL+QiX73TlsX9+w+gD+gcu3sUmaUGGM/CLPZxILZ\nBbx+uo/v/OvLmW5Ocuyb3P3wk7dcxT995v0TumbCY8uqOvGOE+2aRH9Oc3PziNdmYPPnZkV8RZ9b\nNX62djqfrc10K7LTE0/4OHnyJGVlobmyy5cv09LSEs5yEjGVPgdj+91ffyifv/7QrBFfq7Dps+9N\nVsU1oeU8n1g4K8536tMTT3jG9LuJSna/u2/VlaO+wxh97m8+Mp2/+UimW5EN/GP+m8cTN2g7HA7c\nbnf4dWdnJxUVFVHf6+jowOFwYLVao15TWFhIf38/+fn54e8dT02NLArIRY2NjZSUlIT/+/f29nLb\nbbdht9tpb29PaZ8D6Xe5Kla/g9Tf60D6nUhM3MqG2tpa9u/fD0BLSwsOhwO7PfTEPWvWLLxeL+3t\n7fj9fhoaGqitrY15zUc+8pHw11944QU++tGPpupzCR2TPicyQfqd0ANFTWDsZseOHbzyyisoisKm\nTZt48803KS4uZvny5fzpT39ix44dAHz84x/n7rvvjnrN/Pnz6ezs5N5772VgYIArrriCBx54AKvV\nmtpPKHRJ+pzIBOl3ItslFLSFEEIIkXnZvfBPCCGEEGEStIUQQgidyL7txBI03jaXmfb222/zT//0\nT3zxi19kzZo1Mbc0/OUvf8m///u/YzKZuOuuu/jsZz+Lz+dj48aNnD17FrPZzAMPPMDs2bPT0u7t\n27fT3NyM3+/nH/7hH7jxxht10e50y+a+N1l67bNToZf+rrf+ZqS+lJV9RNWhQ4cOqX//93+vqqqq\nnjhxQr3rrrsy3KJhPT096po1a9T77rtP/clPfqKqqqpu3LhRfe6551RVVdXvfve76k9/+lO1p6dH\n/fjHP65eunRJ7evrU//iL/5C7erqUvft26du3rxZVVVVPXDggPr1r389Le1uampSv/SlL6mqqqoe\nj0ddunSpLtqdbtnc9yZLr312KvTS3/XW34zUl7K1j+hyeHy8bS4zzWaz8cQTT4xYl3no0CE+9rGP\nAaEtDZuamjh69Cg33ngjxcXF5Ofns2jRIl599VWamppYvnw5AB/5yEd49dVX09LuxYsX8/3vfx+A\nadOm0dfXp4t2p1s2973J0mufnQq99He99Tcj9aVs7SO6DNputzu8axGM3OYy0ywWC/n5I49sjLal\nodvtHrMt4uivm0yh83MHBwdT3m6z2UxhYSEAzz77LEuWLNFFu9Mtm/veZOm1z06FXvq73vqbkfpS\ntvYRXQbt0VQdrVqL1daJfj1Vfve73/Hss89y//33J9SObGl3puTC5zTyf3u99Xc9/d1Gk61/r+PJ\ntj6iy6A93taq2Ujb0hCGtz+M9hm0r2tP0j6fD1VVp3z+c6IOHDjAj370I5544gmKi4t10+500lvf\nm6xc+G+vh/5uhP6WjX+vicrGPqLLoD3edoPZKNqWhu9///t5/fXXuXTpEj09Pbz66qvcdNNN1NbW\n8vzzzwPQ0NDAhz/84bS08fLly2zfvp3HHnuM0tJS3bQ73fTW9ybL6P/t9dLfjdDfsvHvNRHZ2kd0\nuyNatK0Ds8Ebb7zBQw89xJkzZ7BYLFRWVrJjxw42btw4ZkvD559/nj179qAoCmvWrOHOO+8kEAhw\n3333cfr0aWw2Gw8++OCETrearPr6enbv3o3T6Qx/7cEHH+S+++7L6nZnQrb2vcnSa5+dCj31dz31\nNyP1pWztI7oN2kIIIUSu0eXwuBBCCJGLJGgLIYQQOiFBWwghhNAJCdpCCCGETkjQFkIIIXRCgnYG\nbN++nbVr13LXXXdxww03sHbtWtauXcsvfvGLhK5//PHH+cMf/pDaRoqs9vnPf57f/e53I77W39/P\n4sWLOXfuXNRr1q5dS2NjI2+99Rb/8i//Mub91tZW7rjjjnF/b19fHy+88AIAL730Ej/84Q8n+QlE\nLpnqPU/zyiuv0N7enqJW6kRSjh0Rk9LW1qZ+9KMfzXQzhA7t27dPXbdu3Yiv/fKXv1TvvvvumNes\nWbNGPXjwYMz3T58+rd5+++3j/t5XXnlFXb9+/cQaK8SQqd7zNm7cqL788stJbJH+6PY8bSPavXs3\n7e3tnD17lnvvvZf+/n527NiBzWajv7+fTZs2UV1dzcaNG6mpqeGWW27hy1/+MrfeeiuvvfYaPT09\nPPbYY1RWVmb6o4gU+8QnPsFDDz1EV1dX+ECJX/ziF3z2s5/lxRdf5F//9V+x2WwEAgG2b9/OrFmz\nwtceOnSI733ve+zdu5dXX32VTZs2UV5eTnV1dfh7Tp48yaZNmzCbzXi9Xr7xjW+wePFi/vmf/5lL\nly6xfft2rrnmGhobG9mxYwdHjx7lwQcfxGKxoCgK999/P9dccw1r167llltu4ciRI5w+fZqvfvWr\n3HnnnWn/+xLZaXBwkM2bN9PW1kZPTw933nknX/ziFzl27BibN2/GarUyMDDAV/7/9u4vpOkuDOD4\n15WjXeQy3M12E8kEodQcjOqmqDDBIVgjb5qiYzEkavRnYQtpyKLtol0ZhKKbP3dhBGGyyiC8cwu6\nUIjIyIso6MKFazJh6fC9CMfr66s3b2/b9PlcbeecPZwzDnv2O7/tnMuXWVpa4tWrV7x//x6Px4PZ\nbM539/NClscLzNevXxkeHubQoUMkk0nu3r3L8PAwbW1tPHr0aEP7ubk5zp07RyQSobq6mhcvXuSh\n1+JP02g0NDQ0EI1GgV/7HX/48IFTp06RSqUIBoMoisKJEyeIRCKbxgkEAty4cYNwOLxuT+tEIsHV\nq1cJh8PcuXOHYDDInj17uHTpEsePH8ftdq+L43a76e7uRlEUOjo68Hq9ubqlpSX6+/vx+XwMDAz8\n5ndCFLOhoSEMBgOKovD48WPGxsb49OkTo6OjNDQ0oCgKfX19LCws0NjYSFVVFbdv396xCRtArrQL\nTG1tLSUlJQBUVFQQCATIZDIsLi6i1Wo3tC8vL8doNAKg1+tJJpN/tL8if6xWK16vl4sXL/Ls2TMs\nFgtqtZqKigpu3brF6uoq8/PzHDlyZNMYs7OzmEwmAI4ePYqiKADodDoCgQDBYJDl5eUt51UqleL7\n9+/U1NQAYDabuXbtWq5+7QNWr9fz48eP/zxusX28efOGRCJBPB4HYGVlhc+fP3P27Fk8Hg9fvnzh\n5MmTsjrzN5K0C0xpaWnusdvtxuv1cuzYMSYnJxkcHNzQfteuXeuer8qutDtGTU0NP3/+ZG5ujrGx\nMR48eMDy8jIul4unT59y4MABRkZGePfu3ZZxVKpfC27ZbDZX1tvbS1NTE1arlY8fP+J0Ojd9/dqX\nzDX/nIO7d+/etE7sbGq1mitXrnDmzJkNdePj48RiMZ48eUI0GiUQCOShh4VHlscLWCKRwGg0ks1m\nefnyZd4OgxeF6/z58zx8+BCNRoPRaCSdTqNSqTAYDGQyGV6/fr3lvKmsrGR6ehqAqampXPna3AN4\n/vx5LoZKpWJlZWVdjL1796LT6ZiZmQEgFotRV1f3W8cptqf6+vrcLb1sNovP5yOVShEKhUgkEpw+\nfRqfz5ebW/82/3YaSdoFzOFw0N7ejtPppKWlhW/fvhEKhfLdLVFAmpubmZiYwGq1ArBv3z4sFgtW\nqxWXy4Xdbicej2/6W4ebN29y7949HA4H6XQ6V97Z2Ynb7cZut2MymdBqtdy/f5/Dhw/z9u1buru7\n18Xx+/34/X5sNhsjIyP09PT8f4MW24bNZqO0tJTW1lYuXLjA/v37KSsr4+DBg7hcLmw2G11dXVy/\nfh34dTSmx+PZ8HfHnURO+RJCCCGKhFxpCyGEEEVCkrYQQghRJCRpCyGEEEVCkrYQQghRJCRpCyGE\nEEVCkrYQQghRJCRpCyGEEEVCkrYQQghRJP4CA/J3dSw7oWgAAAAASUVORK5CYII=\n","text/plain":[""]},"metadata":{"tags":[]}}]},{"metadata":{"id":"Ks1lkojJHikC","colab_type":"code","outputId":"9becc23f-78d5-4a49-8bc7-a6158362e51f","executionInfo":{"status":"ok","timestamp":1546106622260,"user_tz":-330,"elapsed":292076,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":52}},"cell_type":"code","source":["TEXT.build_vocab(train_data, max_size=25000, vectors=\"glove.6B.100d\")\n","LABEL.build_vocab(train_data)"],"execution_count":0,"outputs":[{"output_type":"stream","text":[".vector_cache/glove.6B.zip: 862MB [03:59, 3.61MB/s] \n","100%|█████████▉| 399379/400000 [00:20<00:00, 19526.79it/s]"],"name":"stderr"}]},{"metadata":{"id":"bkLqRI0GHlux","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch.nn as nn\n","import torch.nn.functional as F"],"execution_count":0,"outputs":[]},{"metadata":{"id":"7G5_15PzI5AY","colab_type":"code","colab":{}},"cell_type":"code","source":["class MLP(nn.Module):\n"," def __init__(self,hidden_size, bidirectional=False):\n"," super().__init__()\n"," if bidirectional:\n"," num_directions = 2\n"," else:\n"," num_directions = 1\n"," \n"," self.MLPHidden = nn.Linear(num_directions*hidden_size,num_directions*hidden_size)\n"," self.tanh = nn.Tanh()\n"," def forward(self, x):\n"," #x = [N, ..., emd_dim]\n"," e = self.MLPHidden(x)# a = [N, ..., num_directions*hidden_size]\n"," u = self.tanh(e) # a =[N, ..., num_directions*hidden_size]\n"," return u\n"," \n","class FAtten(nn.Module):\n"," def __init__(self, vocab_size, embedding_dim, hidden_size, num_layers, bidirectional=False):\n"," super().__init__()\n"," if bidirectional:\n"," bi_flag = 2\n"," else:\n"," bi_flag = 1\n"," \n"," self.embedding = nn.Embedding(vocab_size, embedding_dim,)\n"," self.rnn = nn.GRU(input_size=embedding_dim, hidden_size=hidden_size,\n"," num_layers=num_layers, bidirectional=bidirectional, batch_first=True,dropout=0.5)\n"," self.MLP = MLP(hidden_size=hidden_size, bidirectional=bidirectional)\n"," self.fc1 = nn.Linear(bi_flag*hidden_size,1)\n"," self.context = nn.Linear(bi_flag*hidden_size,1,bias = False)\n"," def forward(self,x):\n"," #x = [sen_len, batch_size]\n"," x = x.permute(1,0)\n"," #x = [batch_size, sen_len]\n"," embedded = self.embedding(x)\n"," # embedding = [batch_size, sen_len,emd_dim]\n"," h = self.rnn(embedded)[0]\n"," #h = F.leaky_relu(h)\n"," # h = [batch_size, sen_len, num_directions*hidden_size]\n"," u = self.MLP(h)\n"," # u = [batch_size, sen_len, num_directions*hidden_size]\n"," \n"," c = self.context(u) # c [batch_size, sen_len, 1]\n"," a = F.softmax(c, dim=1) # a [batch_size, sen_len, 1]\n"," s = h*a #s = [batch_size, sen_len, num_direction*hidden_size]\n"," s = s.sum(dim=1)\n"," s = self.fc1(s)\n"," return s \n"," "],"execution_count":0,"outputs":[]},{"metadata":{"id":"wcodYs-OJCzR","colab_type":"code","outputId":"c68a979e-74a6-4cd4-9faa-768e23383039","executionInfo":{"status":"ok","timestamp":1546107480437,"user_tz":-330,"elapsed":1911,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":72}},"cell_type":"code","source":["INPUT_DIM = len(TEXT.vocab)\n","EMBEDDING_DIM = 100\n","HIDDEN_SIZE = 50\n","NUM_LAYERS = 1\n","BIDIRECTIONAL = True\n","BATCH_SIZE = 128\n","model = FAtten(INPUT_DIM, EMBEDDING_DIM, hidden_size=HIDDEN_SIZE, num_layers=NUM_LAYERS, bidirectional=BIDIRECTIONAL)\n"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:46: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n"," \"num_layers={}\".format(dropout, num_layers))\n"],"name":"stderr"}]},{"metadata":{"id":"s4nN7k63Jko3","colab_type":"code","outputId":"c8d39e60-5d09-473c-fb60-3654e2dece66","executionInfo":{"status":"ok","timestamp":1546107480440,"user_tz":-330,"elapsed":1661,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":139}},"cell_type":"code","source":["pretrained_embeddings = TEXT.vocab.vectors\n","model.embedding.weight.data.copy_(pretrained_embeddings)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([[ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n"," [ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n"," [-0.0382, -0.2449, 0.7281, ..., -0.1459, 0.8278, 0.2706],\n"," ...,\n"," [ 0.3581, -0.3348, -0.4465, ..., -0.7128, -0.1006, -0.2799],\n"," [ 0.3896, 0.0446, -0.3506, ..., 0.1727, 0.0145, -0.9884],\n"," [ 0.0994, -0.7076, -0.0949, ..., -0.0898, -1.1580, -0.1628]])"]},"metadata":{"tags":[]},"execution_count":34}]},{"metadata":{"id":"f4fuFyPaJrI3","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch.optim as optim\n","device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","\n","optimizer = optim.Adam(model.parameters(),lr=0.01)\n","criterion = nn.BCEWithLogitsLoss()\n","model = model.to(device)\n","criterion = criterion.to(device)\n","\n","\n","train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits((train_data,valid_data,test_data), batch_size=BATCH_SIZE,device=device)\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"1RymBkeJJzn6","colab_type":"code","colab":{}},"cell_type":"code","source":["def binary_accuracy(preds,y):\n"," \n"," rounded_preds = torch.round(torch.sigmoid(preds))\n"," correct = (rounded_preds==y).float()\n"," acc = correct.sum()/len(correct)\n"," return acc\n","\n","\n","def precision_recall_f1(preds, y):\n"," rounded_preds = torch.round(torch.sigmoid(preds))\n"," tp = y*rounded_preds\n"," fp = (y==0).float()*rounded_preds\n"," tn = (y==0).float()*(rounded_preds==0).float()\n"," fn = y*(rounded_preds==0).float()\n"," \n"," return sum(tp), sum(fp), sum(tn), sum(fn)\n"," \n"," \n"," \n"," \n"," \n","def train(model, iterator, optimizer, criterion):\n"," \n"," epoch_loss= 0\n"," epoch_acc = 0\n"," tp = 0\n"," fp = 0\n"," tn = 0\n"," fn = 0\n"," \n"," model.train()\n"," \n"," for batch in iterator:\n"," \n"," optimizer.zero_grad()\n"," \n"," \n"," predictions = model(batch.text).squeeze(1)\n"," \n"," loss = criterion(predictions, batch.label)\n"," acc = binary_accuracy(predictions, batch.label)\n"," a, b, c, d = precision_recall_f1(predictions, batch.label)\n"," \n"," loss.backward()\n"," \n"," optimizer.step()\n"," \n"," epoch_loss += loss.item()\n"," epoch_acc +=acc.item()\n"," tp += a.item()\n"," fp += b.item()\n"," tn += c.item()\n"," fn += d.item()\n"," \n"," return epoch_loss/len(iterator), epoch_acc/len(iterator), tp, fp, tn, fn\n"," \n"," \n","\n","def evaluate(model, iterator, criterion):\n"," \n"," epoch_loss = 0\n"," epoch_acc = 0\n"," tp = 0\n"," fp = 0\n"," tn = 0\n"," fn = 0\n"," \n"," model.eval()\n"," \n"," with torch.no_grad():\n"," \n"," for batch in iterator:\n"," \n"," predictions = model(batch.text).squeeze(1)\n"," \n"," loss = criterion(predictions, batch.label)\n"," \n"," acc = binary_accuracy(predictions, batch.label)\n"," a, b, c, d = precision_recall_f1(predictions, batch.label)\n"," \n"," tp += a.item()\n"," fp += b.item()\n"," tn += c.item()\n"," fn += d.item()\n"," epoch_loss += loss.item()\n"," epoch_acc += acc.item()\n"," \n"," return epoch_loss / len(iterator), epoch_acc / len(iterator), tp, fp, tn, fn"],"execution_count":0,"outputs":[]},{"metadata":{"id":"lUFbK7jJKCxy","colab_type":"code","outputId":"a83ec99a-0fcb-41c6-ab47-441eb49ade6b","executionInfo":{"status":"ok","timestamp":1546107549268,"user_tz":-330,"elapsed":65585,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":139}},"cell_type":"code","source":["N_EPOCHS = 1\n","\n","for epoch in tqdm(range(N_EPOCHS)):\n"," model.word.embedding.weight.requires_grad=True\n"," if epoch>0:\n"," model.word.embedding.weight.requires_grad=False\n"," lr = 0.0002\n"," for param_group in optimizer.param_groups:\n"," param_group['lr'] = lr\n"," train_loss, train_tp, train_fp, train_tn, train_fn = train(model, train_iterator, optimizer, criterion)\n"," valid_loss, valid_tp, valid_fp, valid_tn, valid_fn = evaluate(model, valid_iterator, criterion)\n"," train_acc = (train_tp+train_tn)/(train_tp+train_fp+train_fn+train_tn)\n"," valid_acc = (valid_tp+valid_tn)/(valid_tp+valid_fp+valid_fn+valid_tn)\n"," valid_precision = valid_tp/(valid_tp+valid_fp)\n"," valid_recall = valid_tp/(valid_tp+valid_fn)\n"," valid_f1= (2*valid_precision*valid_recall)/(valid_precision+valid_recall)\n"," print('| Epoch: {} | Train Loss: {:0.2f} | Train Acc: {:0.2f} | Val. Loss: {:0.2f} | Val. Acc: {:0.2f} | Val. F1 {:0.2f} |'.format(epoch+1,train_loss, train_acc*100,valid_loss, valid_acc*100, valid_f1))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["\n","\n"," 0%| | 0/1 [00:00=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2.6)\n","Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (1.22)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2018.11.29)\n","Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (3.0.4)\n"],"name":"stdout"}]},{"metadata":{"id":"Vt3ao3xbwwTm","colab_type":"code","colab":{}},"cell_type":"code","source":["import math\n","import torch\n","from torchtext import data\n","from torchtext import datasets\n","import random\n","import re\n","import numpy as np\n","from tqdm import tqdm\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","\n","%matplotlib inline"],"execution_count":0,"outputs":[]},{"metadata":{"id":"PVRet5S0xmdp","colab_type":"code","colab":{}},"cell_type":"code","source":["SEED = 10\n","torch.manual_seed(SEED)\n","torch.cuda.manual_seed(SEED)\n","torch.backends.cudnn.deterministic = True #for reproducible results"],"execution_count":0,"outputs":[]},{"metadata":{"id":"YiOZBqsqxp7l","colab_type":"code","colab":{}},"cell_type":"code","source":["MAX_DOCUMENT_LEN = 900"],"execution_count":0,"outputs":[]},{"metadata":{"id":"yS73kUvRxtCT","colab_type":"code","colab":{}},"cell_type":"code","source":["def clean_text(text):\n"," text = text.lower()\n"," text = re.compile('<.*?>').sub(\" \", text)\n"," text = re.sub(r\"[-()\\\"#/@;:<>{}+=~|.?,]\", \"\", text)\n"," text = re.sub(r\"i'm\", \"i am\", text)\n"," text = re.sub(r\"he's\", \"he is\", text)\n"," text = re.sub(r\"she's\", \"she is\", text)\n"," text = re.sub(r\"that's\", \"that is\", text)\n"," text = re.sub(r\"what's\", \"what is\", text)\n"," text = re.sub(r\"where's\", \"where is\", text)\n"," text = re.sub(r\"\\'ll\", \" will\", text)\n"," text = re.sub(r\"\\'ve\", \" have\", text)\n"," text = re.sub(r\"\\'re\", \" are\", text)\n"," text = re.sub(r\"\\'d\", \" would\", text)\n"," text = re.sub(r\"won't\", \"will not\", text)\n"," text = re.sub(r\"can't\", \"cannot\", text)\n"," #text = re.sub(r\"[-()\\\"#/@;:<>{}+=~|.?,]\", \"\", text)\n"," text = text.split()\n"," return text"],"execution_count":0,"outputs":[]},{"metadata":{"id":"tlH0gYy_xtmc","colab_type":"code","colab":{}},"cell_type":"code","source":["TEXT = data.Field(tokenize=clean_text,lower=True,fix_length=MAX_DOCUMENT_LEN) \n","LABEL = data.LabelField(dtype = torch.float)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"f7r83HH-xwYD","colab_type":"code","colab":{}},"cell_type":"code","source":["from torchtext import datasets\n","train_data, test_data = datasets.IMDB.splits(text_field=TEXT,\n"," label_field=LABEL)\n","train_data, valid_data = train_data.split(split_ratio= 0.8,\n"," random_state=random.seed(SEED))"],"execution_count":0,"outputs":[]},{"metadata":{"id":"Kgdhoklwx--P","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":89},"outputId":"211b3bfc-861a-481d-beba-6fe422743c2a","executionInfo":{"status":"ok","timestamp":1546969042292,"user_tz":-330,"elapsed":18889,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}}},"cell_type":"code","source":["print('Number of training examples: {}'.format(len(train_data)))\n","print('Number of validation examples: {}'.format(len(valid_data)))\n","print(vars(train_data[0]))"],"execution_count":31,"outputs":[{"output_type":"stream","text":["Number of training examples: 20000\n","Number of validation examples: 5000\n","{'text': ['this', 'is', 'not', 'an', 'entirely', 'bad', 'movie', 'the', 'plot', 'new', 'house', 'built', 'next', 'door', 'seems', 'to', 'be', 'haunted', 'is', 'not', 'bad', 'the', 'mood', 'is', 'creepy', 'enough', 'and', 'the', 'acting', 'is', 'okay', 'the', 'big', 'problem', 'i', 'had', 'is', 'that', 'being', 'familiar', 'with', 'lara', 'flynn', 'boyle', 'from', 'twin', 'peaks', 'and', 'other', 'shows', 'i', \"couldn't\", 'get', 'over', 'how', 'different', 'she', 'looks', 'with', 'her', 'apparently', 'new', 'big', 'lips', 'i', 'kept', 'staring', 'at', 'them', 'they', 'look', 'so', 'out', 'of', 'place', 'on', 'her', 'face!', 'they', 'make', 'her', 'look', 'completely', 'different', 'and', 'not', 'better', 'markpaul', 'gosselaar', 'the', 'actor', 'who', 'plays', 'kim', 'the', 'architect', 'who', 'designs', 'and', 'pours', 'his', 'heart', 'and', 'soul', 'into', 'the', 'house', 'does', 'a', 'fine', 'job', 'and', 'lara', 'as', 'col', 'is', 'also', 'quite', 'good', 'but', 'those', 'lips!', 'as', 'the', 'owner', 'of', 'the', 'house', 'next', 'door', 'her', 'husband', 'walker', 'colin', 'ferguson', 'is', 'appropriately', 'wooden', 'the', 'various', 'characters', 'who', 'live', 'in', 'the', 'house', 'were', 'also', 'fine', 'i', 'particularly', 'liked', 'pie', 'charlotte', 'sullivan', 'and', 'her', 'husband', 'buddy', 'stephen', 'amell', 'the', 'first', 'people', 'to', 'move', 'into', 'the', 'house', 'the', 'attempt', 'to', 'involve', 'us', 'in', 'the', 'overall', 'neighborhood', 'vibe', 'fails', 'unfortunately', 'as', 'the', 'other', 'neighbors', 'are', 'not', 'particularly', 'likable', 'for', 'some', 'reason', 'the', 'director', 'was', 'unable', 'to', 'make', 'the', 'haunted', 'house', 'particularly', 'ominous', 'other', 'movies', 'such', 'as', 'amityville', 'horror', 'the', 'legend', 'of', 'hell', 'house', 'manage', 'to', 'achieve', 'that', 'spooky', 'feel', 'but', 'it', 'just', \"doesn't\", 'happen', 'here', 'the', 'closest', 'is', 'when', 'col', 'paints', 'a', 'depiction', 'of', 'the', 'house', 'another', 'thing', 'that', \"didn't\", 'work', 'for', 'me', 'is', 'the', 'plot', 'twist', 'that', 'occurs', 'with', 'kim', 'the', 'architect', 'initially', 'he', 'appears', 'to', 'be', 'a', 'victim', 'of', 'the', 'house', 'like', 'the', 'others', 'it', 'has', 'sucked', 'him', 'dry', 'of', 'inspiration', 'but', 'later', 'he', 'seems', 'to', 'have', 'joined', 'forces', 'with', 'it', 'in', 'evil', 'overall', 'not', 'a', 'bad', 'movie', 'for', 'horror', 'fans', 'if', 'you', 'can', 'take', 'your', 'eyes', 'off', 'those', 'big', 'lips!'], 'label': 'neg'}\n"],"name":"stdout"}]},{"metadata":{"id":"iKo3Y3dxyCrM","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":433},"outputId":"dac8ae6f-4004-4241-a500-1dae2565f4fc","executionInfo":{"status":"ok","timestamp":1546969043360,"user_tz":-330,"elapsed":19708,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}}},"cell_type":"code","source":["train_data_dist = [len(train_data[i].text) for i in range(len(train_data))]\n","valid_data_dist = [len(valid_data[i].text) for i in range(len(valid_data))]\n","test_data_dist = [len(test_data[i].text) for i in range(len(test_data))]\n","\n","sns.set(style='whitegrid')\n","f, axes = plt.subplots(1, 3)\n","\n","sns.distplot(a = train_data_dist,ax=axes[0], axlabel='Train')\n","sns.distplot(a = valid_data_dist,ax=axes[1], axlabel='Validation')\n","sns.distplot(a = test_data_dist,ax=axes[2], axlabel='Test')"],"execution_count":32,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/scipy/stats/stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.\n"," return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":32},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAe0AAAFYCAYAAAB+s6Q9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xt8XFW5+P/Pnltuk+bWpFdaQrm0\nRkCJRUugRU4L0qP4+noQerTlh188Hj2AeoRfgcrXlmNbpCAqFRUQ+XETww97lHNEChyqYBtaCdBC\nL0BLmyalJJNOmmRyncv+/jHZk0kyk5lJ9szsved5/2OSyc6ssYv97PWsZ62lqKqqIoQQQgjDs2W7\nAUIIIYRIjgRtIYQQwiQkaAshhBAmIUFbCCGEMAkJ2kIIIYRJSNAWQgghTMKR7QaMp7GxMdtNEAZT\nW1ub9veQfieiZaLPgfQ7MVK8fpdU0N64cSO7d+9GURTWrFnDOeecE3ltx44d3HvvvdjtdhYvXsz1\n118f9xq/38+tt95KU1MTRUVF3HfffZSUlEyo4RDu5Jn6D0pv0vaJvW+mJPP5jPhvKG1KTrJtynQg\nter9bjT5LIn/ZjwJ0+O7du2iqamJ+vp6NmzYwIYNG0a8vn79ejZv3sxTTz3F9u3bOXjwYNxrnn76\nacrKynjmmWdYvnw5r7/++iQ/mhBCCJE7Eo60GxoaWLp0KQDz5s2js7MTn8+H2+2mubmZkpISZsyY\nAcCSJUtoaGjA6/XGvGbbtm18+9vfBuDqq69O12cSQgghLClh0G5vb6empibyfXl5OR6PB7fbjcfj\noby8fMRrzc3NdHR0xLzm2LFjvPLKK9x9991MnTqVtWvXUlpaOu77J0pPmXkeSNouhBAiFSkXok1k\nq3LtGlVVqa6u5oYbbuAXv/gFDzzwALfccsu411p1jkfaPrH3FUKIXJZwTruqqor29vbI921tbVRW\nVsZ8rbW1laqqqrjXTJ06lYULFwJw4YUXcvDgQd0+iBBCCGF1CYN2XV0dW7duBWDv3r1UVVXhdrsB\nmD17Nj6fj5aWFgKBANu2baOuri7uNYsXL+bVV1+N/Ly6ujpdn0sIIYSwnITp8fPOO4+amhpWrFiB\noiisXbuWLVu2UFxczLJly1i3bh033XQTAMuXL6e6uprq6uox1wCsWrWKW265hWeeeYbCwkLuuuuu\n9H46IYQQwkKSmtO++eabR3w/f/78yNcLFy6kvr4+4TUABQUF3Hfffam2UQghhBDINqZCCCGEaUjQ\nFkIIIUxCgrYQQghhEhK0hRDCxB57bh9/bjiS7WaIDDH0KV8T8XzDkcjXn1t0araaIUSkL0o/FOni\nD4T4///nfQAul36WE2SkLYQQJnWisy/bTRAZJkFbCCFMqv2kBO1cY7n0uLCGbJ7hLoRZeCRo5xwZ\naQvDkTPchUiOp0OCdq6RkbYwnHhnuANyhrsQUWSknXskaAvDiXeGO5CRM9wh+WNAx/u9pqPhB41G\n14mk/pZejHiEqbQpPTwdvQDYbUqWWyIyRYK2MLxMn+EO45/jrkl0rrhn8MjQ3zo1qTbrwYjntJu5\nTUYP7NpI2+WUmc5cIf/SwnDkDHchktPdMwhAIJj6g60wJwnawnDkDHchkhMayigFg6Est0RkiiXT\n4y/sbKKqrFB2ojKpeGe4t7W1UVtbK2e4CzEkFAoH7ZAa/tomc9uWZ7mgHQyGeL/5JO83n8x2U8Qk\nxDrDXZtflDPchQgLRWXFg6EQNps9e40RGWG59PiAP5jtJgghREaEoqK2PyAp8lxguaA9KB1XCJEj\nQlErK4IhKUbLBZYL2n4ZaQshckT0SDsgxWg5wXJBe9AvHVcIkRuig3ZQln3lBMsF7YGAjLSFELkh\neuMhGWnnBssFbUmPCyFygaqqI6rHJWjnBssFbUmPCyFywei6M0mP5wbrBW1JjwshckBoVNSWkXZu\nsF7QlvS4ECIHhEYdpCNLvnKDBYO2PG0KIaxPlZF2TrJe0Jb0uBAiB4weaUvQzg3WC9oy0hZC5ICx\nc9qSHs8FFgzawyNtmeMRQljV6PubHM+ZG6wXtKPS46GQdGIhhDWNyo7LSDtHWC9oR6XHpRMLIaxq\nbPW4DFJygQWDdlR6XNJFQgiLkjnt3GS5oB0cceqNdGIhhDWNCdpyLHFOcGS7AXqLzhhJukgIkYqN\nGzeye/duFEVhzZo1I17bsWMH9957L3a7ncWLF3P99dfHvOacc87h1ltvZe/evZSWlgJw3XXXcfHF\nF+vaVi097rArBIKq3O9yhAWDtoy0hRCp27VrF01NTdTX13Po0CHWrFnD6tWrI6+vX7+ehx9+mGnT\nprFy5Uouu+wyvF7vmGvq6+sB+N73vsdnP/vZtLVXC9pOh41AMCj3uxxhwaA9/LXMaQshktXQ0MDS\npUsBmDdvHp2dnfT29gLQ3NxMSUkJM2bMAGDJkiU0NDTg9XrHXOPz+TLSXi097nTY6RsIyv0uR1gw\naMv5skKI1LW3t1NTUxP5vry8nM7OTgA8Hg/l5eUjXmtubqajo2PMNR6PB4AnnniCRx55hIqKCv7P\n//k/I66Pp7GxMenX2zr94S/UcPHtkaPNNDaeTPgeRpHos5pJJj+LpYK2qqpEJ4hkcxUhxESpoxdC\np3DNF7/4RUpLS1mwYAEPPvggP//5z/nBD36Q8Pra2tq4rzU2No54/cjxLvhTK0WF+XT19jB9+kxq\na89Muc3ZMPqzmFk6Pst4DwGWqh4fu9mAjLSFEMmpqqqivb098n1bW1ukkGz0a62trVRVVcW8prKy\nkkWLFrFgwQIALrnkEt577z3d2/vqW8eA4WWukh7PDZYK2mM2G5DCDCFEkurq6ti6dSsAe/fupaqq\nioKCAgBmz56Nz+ejpaWFQCDAtm3bqKuri3mN2+3mxhtvpLm5GYCdO3dyxhln6N/gofud3aYAEJDM\nYk5IKj0ea0mDxkjLIEans2SkLYRI1nnnnUdNTQ0rVqxAURTWrl3Lc889h9frZdmyZaxbt46bbroJ\ngOXLl1NdXU11dfWYawC++tWv8t3vfpeCggIKCwu58847dW+vFqPttvDYS0bauSFh0I61DEJb0gDG\nWgYxdgN9efIUQiTv5ptvHvF9T09PZL5y4cKFI+598a4B+MxnPsPvf//79DRyiDZIsduHRtpyv8sJ\nCYN2rGUQPp8Pt9ttuGUQY3YIks0GhBAWpSUWHfbwSFsyi7kh4Zx2e3s7ZWVlke+jlzTEWgbh8XjG\nveaJJ57gmmuu4d///d/xer26fRAYW4gm1eNCCKsaO9KWoJ0LUl7ylellEKmsW+wdGNlp33vvIPa+\nYym3N1PMvE7RzG0XwgrUMXPaMkjJBQmDdrwlDbFe05ZBOJ3OmNdUV1dHfnbJJZewbt26hA1MZd1i\np28Afv9h5Pu5p1ZT+4lZCd8jG8y8TjFbbTfjg8KJzj72H/ayoDrxxhpCpCJ673GQ6cBckTA9Hm9J\nAxhvGcTYJV/SiUV2/e7F91j981ez3QxhQcMj7XDQlpF2bkg40o61DGLLli0UFxcbbhmEnC8rjKSn\n3x/5OhgMYbdbalsEkWXDc9pSiJZLkprTHr2kYf78+ZGvjbQMYmwhmnRikT3H23siX/cOBCgudGWx\nNcJqpHo8N1nq0V8baUd2CJKRtsiiDz3Dyxx7+vzj/KYQqdNOWpD0eG6xVtAetQRC5rRFNp3o7I98\nLUFb6C0ypy1LvnKKNYO2TUsXyZOnyJ7om2j0/LYQegiN2ntc9qXIDdYK2qFRI22Z0xZZFP3Q2NMX\nyGJLhBVphWiKomCzKbSf7Mtyi0QmWOo8bZnTtg6zHFIznhEjbUmPC51p6XFFUbApypjVM8KaLBW0\nR1dTypy2OZnpkJrxRAftXkmPC51p9zubEh6ojN6nQliTpYL26DkeKcwwp3iH1ACGO6RmPMER6XEJ\n2kJf0VtK22wy0s4V1pzT1vbilU5sSmY6pGY8gWAIJfz8SE+/zGkLfWmDFJuiYFPkfpcrrDnStks1\npZVk+pAaSH6f83i/FwqpBEMq+S6F/kGVppbjNDYOJP8BJsGIe7RLm/Q3Yk5b0uM5w1pBO6RtoC87\nBJlZvENqvF5vRg6pgfEPqtGMd3BK/0AAfneMKUX59A/2UVBUkpFDVox4EI2Z22TkwD5cPR5Oj/sD\ncr/LBZZKj8sG+tZgpkNq4hnwBwEozHcCMqct9Kfd3SIjbcks5gRLjbS1dLhNCtFMLd4hNW1tbdTW\n1hrqkJp4Bv3hvudy2nDYbRw/0ZPgCiFSM2KkLUu+coalgvbYbUylE5tVrENqtFSlkQ6piWcwEB5p\nO+w2XE4bA4PBjLdBWNvwki9FlnzlEIulx4fmtLVtTGVHNJElg0Ppcbvdhsthl/lGobuQjLRzkqWC\n9phtTCU9LrJEm9N22BTsdkX6otDd2Orxia20EOZiraAdKUSTA0NEdmnpcIfdhtNuk/oKobvR1eMg\n97xcYK2gLSNtYRBaetxht+Fw2AipUhgp9BUZaaNEgrbc86zPWkF71Jy2bK4iskWrHnc4lMi+AVog\nF0IPo6vHAQJyz7M8SwVtVZZ8CYMYiB5pD2V+pIJc6Ck0qnocZKSdCywVtEOj5nhkyZfIlujqcW2k\nPSAjbaGj2HPaErStzmJBO/y/WrpIlnyJbBme01YkaIu0GF09DlKIlgusFbSHorZCOF0kI22RLcNL\nvqJG2pIeFzqKNact6XHrs2bQ1tLjMtIWWRIpRBuqHgcZaQt9xR5pyz3P6iwVtIefPMPny0qqSGTL\nQGROW4kUokn1uNCTGjlPe3hOW1bMWJ+lgvbYQjR56hTZMWKdtqTHRRpodzdFUbAPpcdlu1zrs1bQ\nDkWNtG2KjLRF1kSCtkOqx0WaxKgelzoe67NW0B5VPS5z2iJbBmKMtCU9LvQUijWnLfc8y7NW0JaR\ntjCIEUu+HLK5itCfVI/nJksF7RGFGYrMaYvs0R4Y7TZZpy3SY2T1ePhrGahYn6WC9vB5sjLSFtnl\nD4QDtM0mc9oiPaIHKcMnG8pAxeosFbSDo0faMr8jskR7YLQpSPW4SItYI20pRLM+SwXt0ZsNqKqs\nWxTZEQiEsNkUFEXS4yI9IktciTrlS0balmepoD1yR7Twz2ReW2SDPxiKnLzkdMjmKkJ/sXZEk+yi\n9Vk0aCvy5CmyKhAVtO2SHhdpEPuUL8ksWp21gnZ0uki29RNZpKXHAZySHjeNjRs3cvXVV7NixQr2\n7Nkz4rUdO3Zw5ZVXcvXVV3P//fcndc2rr77KWWedlZa2aiNtW9QgRTKL1ufIdgP0FFJHrtMGKcwQ\n2RFrpC3pcWPbtWsXTU1N1NfXc+jQIdasWcPq1asjr69fv56HH36YadOmsXLlSi677DK8Xu+Ya+rr\n6wEYGBjgwQcfpLKyMi3tVRm+32l9zS9B2/IsNdJWR+2IBjLHI7IjEBweadtt4QNsJD1ubA0NDSxd\nuhSAefPm0dnZSW9vLwDNzc2UlJQwY8YMbDYbS5YsoaGhIeY1Pp8PgF/96ld85StfweVypaW9I+53\nMkjJGZYK2tFz2naZ4xFZ5A+okbWzEF72JelxY2tvb6esrCzyfXl5OZ2dnQB4PB7Ky8tHvObxeGJe\n4/F4OHz4MAcOHODyyy9PW3tDMXZEkxoe67NWejwU46g66cQiCwLBIHkue+R7h8MmI22T0Qq9JnLN\nnXfeye23357y9Y2NjUm/3tfbB0BzSzPe7kD46+ZjNDZ2p/y+2ZDos5pJJj+LtYK2KtXjwhjCI20l\n8r3DbpM5bYOrqqqivb098n1bWxulpaUxX2ttbaWqqgqn0znmGpfLxQcffMDNN98c+dnKlSt54okn\nErahtrY27muNjY0jXs976X+gy8fcOXPJO9ED7x6iavp0amsXJP+hs2T0ZzGzdHyW8R4CkkqPm6Wi\ncuQ6bakeF9mhquqIOW2Q9LgZ1NXVsXXrVgD27t1LVVUVBQUFAMyePRufz0dLSwuBQIBt27ZRV1cX\n85pZs2bx0ksv8fTTT/P0009TVVWVVMBO1XD1OMPTgXKetuUlHGnHqqjUqiPBWBWVI46qk5G2yBLt\nQXHkSFvB1ytB28jOO+88ampqWLFiBYqisHbtWp577jm8Xi/Lli1j3bp13HTTTQAsX76c6upqqqur\nx1yTKdHp++HCWxmkWF3CoB2vOtLtdo+oqAQiFZVerzfuNVpF5d133637h1FjrdOWQjSRYdpoZ0x6\nPBAiFFJHjMCFsWgpbU1PT08k9blw4cIRA5Z414z28ssv69fAKKqqojDqPG0ZpFhewvR4vOpIMF5F\nZaz0uHRikWlanxudHgcYDMhoW+gjpIYDNsggJZekXIiW6YrKVKopP2o9OfS/H9HV5Qdg/4F3GTjZ\nlNJ7ZoqZqyfN3PZ00za4GDHSdgxvZZrvslT9p8gSVVUZitkyHZhDEt49YlVUavPRmaioTKWacteR\n3fCej5kzZjBIFxz7iNPmnU7t/GmJPmbGmbl6MlttN8uDQiAwtPRwxDpt7dAQuakKfaiMHWlL0La+\nhOnxWNWRbrcbMF5FZaxCNEkXiUwLxBppR/YfD2SlTcJ6okfadkmP54yEI+1YFZVbtmyhuLjYcBWV\nWhreJoUZIov8Q/PWsea0ZYMVoRdVHU6LR+53sm2z5SU1uTa6OnL+/PmRr41UURmKDLWlMENkj7Z1\nbqyRtqTHhV5CMUbaflmnbXmW2ns8GL2NqRwYIrJkvOpxSY8LvahR1eMStHOHpYL2cGW7IofCm5xZ\nduGLxR9rnbYj/LWkx4VeVFVF62I2Cdo5w1JrT7RB9YgDQ2SkbTpm2oUvlvEK0SQ9LvQSCg2PtLUN\nVmQbU+uz5Eh75DamMtI2m/HOKDbiucajSXpcZEK4eny4j9ltioy0c4ClgnYw6nzZ4SUQ0onNxky7\n8MUyvI3pyPO0QdLjQj8hVSWqi4WDdlD6l9VZLD0uI20rMuK5xuP93rtHewE42dlB09DXHR3hHfoO\nHTlKY0FHyu1JhRE3oZE26S+6EA3AbrfJSDsHWCpoq2r03uPhn8mctvnE24XP6/Ua4lxjTbyd4bqV\nFvibl8qKCubOqQDAltcN739AZdUMamvTVxBnxJ32zNwmIwf2cCGapMdzjaXS41p8HnnqjYy0zcZM\nu/DFEoixuYozsve4zGkLfUSv0wYJ2rnCUiPtUPRIW5E5bbOKtwtfW1sbtbW1htqFLxb/eJuryE1V\n6CR6RzQAu12hX2omLM+aQRvZxtTsYu3Cp6UqjbQLXyxaIZpsYyrSKRQaPdKWOe1cYLH0eNSOaJF1\n2pIeF5kVe522bK4i9DVmpG1TCARDEyrcFOZhqaAde522PHmKzIoZtB1aelyCttBHKMY6bZB7ntVZ\nKmgPF6LJgSEie4bT47JOW6TH8PLW4Z/Zh/qYpMitzVpBO8ZIW9LjItP8MUba2tcDfgnaYvK0e50t\nRh+ToG1t1graUYUZUogmskVbZhh9Q1UUhTyXXYK20EWskbYcGpIbrBW0VRWFkUfVyQb6ItNizWkD\nuBx2SY8LXQwX3Y5c8gUStK3OUkFbjXUovIy0RYb5Yyz5AmSkLXQTjNqyWaPtde+XYkdLs1TQDqfH\nw53YJk+dIksCMc7TBshz2hmUoC10EJnTjlE9Lvc8a7NY0CZqpB3+aJIeF5kW62hOCAdtSY8LPWir\nYkZvYwqSXbQ6awXtqA30tfuldGCRabGqxwF6B/z0y97jQgfRK2U0suQrN1guaA/VoaEoCnabQvvJ\nvuw2SuScWOdpQ3ittqrKigYxecHICoXhn0VG2n7pX1ZmraAdGntUnRzNKTItXnpcNlgReok50o7M\naUv/sjJLBW111FF1NpsiO6KJjIu35CsStKUYTUzSuEu+JJNjaZYK2qFQJDsOhOd4ZEc0kWmBYPjh\nURkZs3E4wj+QCnIxWVoGcfQpXyBz2lZnraAdYwP9kARtkWGBQAiH3TaiL4Kkx4V+Yo60ZclXTrBg\n0B7+PjynLUFbZJZ/KGiPJulxoZdgzANDJGjnAksF7WBw5EjbJoVoIgv8QQnaIr1ij7QlPZ4LLBW0\nQyF11Kk3NilEExkXCIZwOpQxP3cMjYQkPS4mK9Y2pjapHs8JlgrawVAoxpIvFVWVwC0yJxAM4XDY\nx/xcRtpCL8NLvoZ/Jock5QaLBW115GYDdjlTW2ReIBDCaR870nY6wp1TqsfFZAVjHP8qS75yg/WC\ndoxqSnnyFJkUiDOnbZfqcaGTWJurOGROOydYK2jHKEQDefIUmRVOj4/9T8sp6XGhk+FCtOGfSfV4\nbrBW0B4zpy1PniLz/AE1TvW4bK4i9DH+edpyv7MyywTtUEhFVWPP8Uh6XGSKqqpx0+OyuYrQy3jb\nmA5K9bilWSZoB2OliyQ9LjIsMFQg5JR12iKNQjE2V9H6l2RyrM1CQXtoL17b2EI0SReJTNEOC4k1\np639TIK2mKxY6XHZByA3OLLdAL3EShfZhuZ4JD0uMiUStGMs+ZL0uPFt3LiR3bt3oygKa9asGfHa\njh07uPfee7Hb7SxevJjrr78+5jXnnHMOb775Jps2bcLhcOByubj77rspLy/XrZ2R+13Us+HwSFvu\nd1ZmoZG2bKAvsk97QHTG3FxlaCQkI21D2rVrF01NTdTX17NhwwY2bNgw4vX169ezefNmnnrqKbZv\n387BgwfjXvPII4+wadMmHn/8cT75yU/y9NNP69rWUIyRtqIoOOw2BvwBXd9LGItlRtraCEeJ3lwl\nMqctN0mRGf4kRtoy52hMDQ0NLF26FIB58+bR2dlJb28vAM3NzZSUlDBjxgwAlixZQkNDA16vd8w1\nPp+P++67DwgXJra2tlJbW6trW4Pq2EEKQJ7LLiNti7NM0B73UHgZaYsMGU6PS/W42bS3t1NTUxP5\nvry8nM7OTgA8Hs+I9HZ5eTnNzc10dHSMucbj8eB2u3nllVfYsGEDp512GldccUVSbWhsbEzq9UMf\n9ADg7ThB01Ff5HWFIF3dvQn/jhGYoY3JyuRnSSpox5qz0Rhlnifmtn4ypy0yTOtrsQrRbDYFm6JI\netwkJnJmQfQ1ixcv5qKLLuKee+7hwQcf5Jvf/GbC68cbkTc2NkZe7wg2wWsdVFZMZe6c4Xto8aGD\nDPiDuo/s9Rb9WcwuHZ9lvIeAhHPaZpnniTWnLTuiiUwbb8kXgMOhSHrcoKqqqmhvb49839bWRmlp\naczXWltbqaqqinlNZWUlL774IhCeZ77ssst0H4nFqh4HcDlt0r8sLmHQjjXP4/OF0zHR8zw2my0y\nzxPvmvvuu49TTjklMs8zffp03T5IZMlXrHXaMtIWGTJeelz7uaTHjamuro6tW7cCsHfvXqqqqigo\nKABg9uzZ+Hw+WlpaCAQCbNu2jbq6upjXuN1uNm/ezP79+wHYvXs31dXVurY1VvU4gMtpl6BtcQnT\n47HmebQ5m0zN8yQjMtKWHdFEFvnHSY/DUNCWm6ohnXfeedTU1LBixQoURWHt2rU899xzeL1eli1b\nxrp167jpppsAWL58OdXV1VRXV4+5BmDDhg3ccccd2O128vPz2bRpk65tjTfSznPZGQyECIXUEfdC\nYR0pF6Jlep4n2cKM4x2DAPT4umk62gSA1xv+2aHDRyh3tMf+A1lk5kIMM7c9nSJz2uOMtGUkZFw3\n33zziO97enoi85ULFy6kvr4+4TUAZ599Nr/73e/S00hi74gG4ZE2hLcyzXdZps5YREn4rxpvzibW\na9o8j9PpjDvPs2zZssg8z+bNmxM2MNnCjPebO+DPbZRMKWHunJkABO2dcOgIM2bOprZ2XsL3yiQz\nF2Jkq+1meFAYb8lX+Oc2fL2DmWySsKBYNTwAeUNBe2BQgrZVJZzTjjdnA8aa5wnGmOOxyXnaIsO0\nOe1Ym6tAOJhr6UshJirW5iowHLRlrbZ1JXwUizXPs2XLFoqLiw01z6Mt+VJi7Ygm1eMiQyJBO95I\ne2iuW9KXYjJC6tgDkgDaOsKbwchJX9aV1F1j9JzN/PnzI18bZZ4n5uYqUj1uWmbYGyCW6HXasco/\nnFEbrEjQFhMVrxDNLhv4WJ6F9h4P3yxHVo/LofBmZJa9AWJJtOTLLsdzCh3EGqTAcIZHih2tyzKP\n+tqmFjE3V5FUkakkuzcAZH8P6NH8Q/3QYbfFfFh0yE1V6CAYp3pcHgqtzzJBO1KYETXAccpI25Ti\n7Q0AxtkDOt7vHT4cfrg42nSY3sGx/a6vtw+At3a/Q2u5K6n3SJURq+ylTfoKxdiXAqL2t5egbVmW\nCdqR9HiMA0NkfsfcjLYHtCbW0remroPASc468wy8Xf1jrvmw8ziHW9s47fQz+Vh1ReIPkiIjLiU0\nc5uMGtjjzWnLSXLWZ6E57bHpcac8dZrSRPYGyNYe0KMlmtN2OuSmKiYv3uYqMv1ifdYJ2tqSrxiF\naBK0zcUsewPEEgnajvibq4Bkf8TkxMosgvSvXGCh9PjYdYsOSY+bUry9Adra2qitrTXM3gCxDK/T\njr+5CsiDpJic+CNtbaAidTxWZZ2gHRy75EtRFBx2Ob/YjGLtDaClto2yN0AswweGjD/SlvSlmAxt\nQ72xI23toTCQ6SaJDLFOejzOukW7HNAgMijhgSEOSV+KydMGKfEL0WSkbVWWC9qjO7FTzi8WGTQ4\nFLS105ZGc9ikzkJMXmQb01F3cMnkWJ+FgraWHh/5czm/WGSStuezK96BIQ4tfSkjITFxso1p7rJM\n0I63rZ/drshTp8gYra+5nPHP0wYYGJQ5RzFx8U75cjqk0NHqLBO0gzG2MQVJj4vM0uYS46bHZRmi\n0EGs1TIwPNKWgYp1WSZoB2IcGALhwp9gSI0sxREinSIjbcf4I20pFBKTEXekLQ+FlmeZoB2KcZ42\nSGGGyCx/IITdpkRGPKNpO6L1D0h6XExc/OlAud9ZnWWCdrx0kWywIjJpwB+MO58N4Bx6rVeCtpiE\nuKd82RQURe53Vma9oC2n3ogs8geCOONUjkM4fWmzKfT0+TPYKmE1w0u+xm7i47DbZPrFwqwXtOPt\nxStBW2TAgD8UtwgNwtM3hXkOevslaIuJG54OHPuaLHO1NgsF7aEdguKNtCVdJDLA7w/GLULTFBY4\n6emX9LiYuMhIO0bUlq2brc3fOwiFAAAgAElEQVQ6QTsYZ07bISNtkTmDgfFH2gBF+TLSFpMTb3MV\n0NLjcr+zKusE7bjpcTlfVmTOYIJCNIDCfCd9A4FIBbAQqQqF1JipcZCgbXXWCdpBSY+L7FJVFX9S\nI20nqgp9UkEuJmgwEMQ+es/mIVp6XFXlodCKrBO0pRBNZFnksJBxqscBCgvCJ+L2SIpcTFD/QDCy\n5n80h92GqiIbSlmUZYJ2KMGSL0kXiXTzJ9h3XFOU7wSgV4rRxAQN+INxj3+NHBoiy74syTJBO+7m\nKg7ZXEVkxoB//BO+NIX5QyNtWastJmhgMBB3pO2MnNkuD4VWZJmgraWCxqTH5fxikSH+BGdpa7SR\ntqTHxUT1D4430taKb2WkbUWWCdqRDfRjHBgCMtIW6ac9GDoTVY8XDKXHZaQtJiAYChc8jjenDTJQ\nsSrLBO146XFto4s+SRWJNPMPjWzyklinDcgGK2JCtLR3oqAtdTzWZJ2gHYx9NKe2D3Sf3CBFmg0G\nhkbaiXZEixSiyUhbpK5/KGsYLz0uI21rs07QjrPkS05VEpmijWwSj7SH5rQlPS4moD/hSFuKb63M\nUkFbUcZu6xdJj0vQFmmmrdMe75QvgKICSY+LidOCsUPS4znJOkE7GMIe55g6myLpcZF+wyPt8f+z\nmlKUB0BXz0Da2ySsp39gaBrGHnsfUwna1madoB1SscXY1k9RFPLzHDLSFmmnLbFxJkiPFxe5UBTo\n9A1molnCYrT0uMMeu59F0uMStC3JUkE71kgboECCtsiAwciOaOMHbbtNwV3gkpG2mBCtEE2WfOUm\n6wTtYCjyhDlaYb4EbZF+WvV4ovO0AaYUuWSkLSZEW/IVb07bLockWZplgnZH9wDxDrWRkbbIBC09\nnmik/XzDEUIhFV/vYGTVgxDJioy04yz5cspI29IsE7QDwVDcdYu9/QH8gZCceiPSSjswJNE6bQg/\nSIZU8PXKaFukJlF6XFvmqhWsCWtxZLsBegkEQpGDGEZzRi37Ki50ZbJZIof0Dd1MC/IS/2eVnxce\njXf1DFLizktru0TyNm7cyO7du1EUhTVr1ox4bceOHdx7773Y7XYWL17M9ddfH/Oac845h+PHj3Pb\nbbcRCARwOBzcfffdVFZW6tLGSHo83kh76H7XL7tAWlJOjLS1U5fkKESRTv1DUzDJBG3tdzp9Uoxm\nFLt27aKpqYn6+no2bNjAhg0bRry+fv16Nm/ezFNPPcX27ds5ePBg3Gt++tOfctVVV/HEE0+wbNky\nHnnkEd3amXCkre0CKVOClmSJkXYgGCKkjvPk6ZQNVkT6af0r35V80O7qkfS4UTQ0NLB06VIA5s2b\nR2dnJ729vQA0NzdTUlLCjBkzAFiyZAkNDQ14vd4x1/h8PtauXUteXjiDUlZWxt69e3VrpzZXHa8Q\nTTaUsrakRtobN27k6quvZsWKFezZs2fEazt27ODKK6/k6quv5v777x/3muPHj3PttdeycuVKrr32\nWjwejy4fYiDBXryRTiwjbZFG2k2yIM40TbR8baQtQdsw2tvbKSsri3xfXl5OZ2cnAB6Ph/Ly8hGv\neTyemNd4PB4KCwux2+0Eg0F++9vf8oUvfEG3dh5sPgnEL0TTgrnMaVtTwrtLdPrn0KFDrFmzhvr6\n+sjr69ev5+GHH2batGmsXLmSyy67DK/XG/MaLWW0fPlynnzySR555BFWr1496Q8xkKAASNJFIhMi\nQds1fvV4+HeGRtqSHjcsNd5ylCSvCQaDrF69ms985jMsWrQoqesbGxsTvt7Z1Q1Aa+uHdHbE33/8\nREdnwr+XTUZuW6oy+VkSBu1YKSOfz4fb7TZMyijhSFvS4yIDegcCOOwKL+46mvB3taLJjm4J2kZR\nVVVFe3t75Pu2tjZKS0tjvtba2kpVVRVOp3PMNVrB2W233cbcuXO54YYbkm5DbW1t3NcaGxupra3F\ntW0b4OfUuXPiHk7j3ncAxe4c9+9lk/ZZrCAdn2W8h4CE6fF46R8wTsoo0RzP8EhbTlUS6dM/EEh4\nWIimqCB80tf+w950NkmkoK6ujq1btwKwd+9eqqqqKCgoAGD27Nn4fD5aWloIBAJs27aNurq6mNe4\n3W6effZZnE4n3/72t3Vvp187mCbOIAXCqxOketyaUi5Ey3TKKJl0UXN7eLTS29NN09GmMb/TeTI8\nb/juwcOU2dvHvJ4tZk4Pmbnt6dI3EEhqjTZAvsuOTVHokTO1DeO8886jpqaGFStWoCgKa9eu5bnn\nnsPr9bJs2TLWrVvHTTfdBMDy5cuprq6murp6zDUAv/3tbxkYGGDVqlVAOOO4bt06XdoZCIaw2RRs\ncbZthnAxZHdPry7vJ4wlYdCOlTLS0j+ZSBklky5yvO8BPFSUlTJ3zvQxv2fL66bx4AdUVM6gtvas\npN433cycHspE22OtfdUeFIyyXna0voFAUsu9IHyQTVGBQ87UNpibb755xPc9PT2Rvr5w4cIR9Tzx\nrgH43e9+l54GEh5pjzfKhuFdIFVVHXNcsTC3hMOCeOkfME7KKJIejzunHU5Zyg3SHMyyXjaaqqr0\nDwQiewIkoyjfSU+/n5BsZSpSEAiGEmZ0tB33tDPehXUkHBbEShlt2bKF4uJiw6SMEh0KnydB21Ti\nFT+CsdbLRhvwBwmpw3sCJKOowInqhc6eAcqK89PSLmE9/kAobgGaRttxr38gkPB3hbkklcsbnf6Z\nP39+5GsjpIySrR6XHdHMob29nZqamsj3iYofm5ub6ejoiHlNdXU1QKT4UUul6+257YeB5PYd12jF\naCc6+yVoi6T5A6FI34lHm6bpGwjINrkWY4kd0RKlx2WkbW5GXC87+veOHA1vwjHY3xezGDKWwf5+\nAHa98Q6drQVJXZNKm4xE2qQPVVXD6fFEc9qu4aAtrMUaQTvBUXV2uw2HXSp1zSJe8aPX6zXEellN\ndEHevtZ9QDdlpVOYO2dWUu/Rr3ZwoOUo5ZWzqa09Nem2JdsmozBzm4wW2BNtJKXRdtyTXdGsxxIH\nhgyv045fJely2mWkbRJmKH4cTVs760opPR6+sZ7o7E9Lm4T1JKrf0USnx4W1WGSkPf5RdRBOkctI\n2xziFT+2tbVRW1triOLH0bQq3WQ3V4HhOW1vlwRtkZz+BFlFjVaI1icbrFiORYL2+HPaEB5pn+js\nl3WLJhGr+FFLVRqh+HE0f2DoZppK9Xi+VojWl5Y2CevRdjlLONJ2aelxCdpWY7H0+Pgj7UAwJOsW\nRVpMJD3uctpxOmwy0hZJS1S/o9FOmpP0uPVYI2gn0ZG1DVZ6ZV5bpMGgP/X0OIRT5BK0RbKSHWkX\nF7gA6JajXy3HGkE7wZIvGF6r7ZOgLdIgkh5PYaQN4RR5p28wcr0Q44kMUBL0s3c+CK+k2HPIOGct\nCH1YI2gPJq4e19Zq90oxmkiDyMlLqQbtoQryji45olMklnwhmqTHrcoaQdsfRAFs4xSYDe8/Lp1Y\n6E+rlXCluGWkVowmKXKRjIEUC9H6ZBdIy7FE0PYHgtjttnGrwvNcQ0FbRtoiDSacHo/aylSIRPqT\nTI/bbAr5LruMtC3IIkE7hH2cs2VhOD3u65XCDKE/rRAtlVO+ICpod8myL5FYfxLLWzXa8ZzCWqwT\ntO3jB21th6BOqaYUaTDhOW0tPS4jbZEErXo8mX5WkOegfzBIMCjLXK3EGkE7mHikHQnaPin4Efrz\nB4LYbQq2BP1wtOGRtgRtkViy67Rh+J7XJdlFS7FG0A6EsNuS24u30ycdWOjPHwilXIQGUDS0CYaM\ntEUy+pPcexyigrbc8yzFOkE7QXo8X0baIo0GA6GUU+MQPoFuSpFLqsdFUvqTOGdBowXtk3LPsxTr\nBO0EaUm7TcFd4JSgLdLC7w9OKGgDlE/Jl+pxkRT/UMGjI8EgBYa3Mj3ZLfc8KzF90FZVNTKfmEiJ\n2yWFaEJ34T4YSmnf8WjlJfn0DQRk4x+RUGCoqCzRdCAMb2X6kbcnrW0SmWX6oB0Mqagq2JLoxCXu\nPLp6BgmF1Ay0TOSKAX8QldT3HddUTMkHZIMVkZgWtJMpeCybkgdAS6svrW0SmWX6oK0ttUkmXVTi\nziMUUmX/caErbS1sKsdyRisvkaAtkjM80k58vysucmG3KTS3dae7WSKDLBO0k0kXlbjDT54yry30\npAXtVDdW0VSUFADQflI2WBHjCwTDWcJxNn+MsCkKpcV5NLf6JLtoIRYI2uElEImqxwFKisJzPBK0\nhZ60/Z0nWog2rbwQgFavBG0xvkAwhM2mjLtlc7TyKfkM+oN45IHQMiwQtJNPF33YHp7b2dbYktY2\nidyS7H7Q8UyPBG0pGBLjCySxkVS0suLw1Etzq6TIrcIyQTuZwgz3UDVlj8xpCx1NNj1eWVaAokCr\nt1fPZgkLCgRCKe26V+IO3/Okb1mH6YN2pDAjic0G3ENbRsqhIUJPkfT4BAvRnA47FVPy+eiE3FjF\n+ALB0LhHEI9WXBgO2m0StC3D9EF70D80p53MSLswHLS7ZaQtdNQ7MLk57ecbjuBy2mk/2RfJHAkR\nSyCoppQeLx6q49lz0JOuJokMM33QTmVOuyDPgd2myEhb6Gqy6XGAKUM3V89JGRGJ+LRCtGQV5Tuw\nKQpdPTJQsQoLBe3EH0VRFIoKnHT3SgcW+knluMR4tKD9UbsEbRFfqoVoiqLgLnTSLQMVyzB/0I7M\naSfXkYsLXfQNBCJLxYSYrMku+YLhDVYOf9ipS5uENaVaiAbhB8K+gQADfrnnWYH5g3YK6XEYnteW\nAxqEXiLp8QkczamZOrTBygfHJGiL+AKh1Oa0YbgYzdMhWRwrsE7QTqJ6HKB4qILc0yGbDQh99E2y\nEA3CoyGX08YHMtIW45jISLt4aKDSJvc8SzB90A4Ekq8eB3BrT52yQ5DQiR5BW1EUppYUcMzjo3/o\n7wkRLaSqBCcw0i4aGqh0yN72lmD6oJ1yelwbaUuVrtCJHtXjAJWlBagqvN9yUo9mCYsJDa0GTHWk\nXZgfvufJgTTWYJ2gnWQhmjbSbj8pHVjoo28ggN2mpHwzHW12VTEAjftb9WiWsJjg0KEfqWyuAuFl\nXwAd3XLmghWYPmgPprDkC4bnd6QoQ+ilbyAwqdS4ZvY0Ny6nnb9L0BYxDC2USfpepykcyi56pfjW\nEkwftFNNj7ucdlxOmxyDKHTTNxCYVOW4xmG3ce4ZUzn6UTfH2+XwEDFSZKSdYkanIM+BgqTHrcIC\nQTu1QjQIHxwihWhCL3qNtAEuOHsmAK+8JSfRiZG0oJ1qIZpNUSjId9DRLUHbCiwQtFNb8gXhtdq9\n/QE57UtMmqqq9A8EcOkUtHv6/dhtCn/622H+vOOwLn9TWMNEC9EgXEHu7exHVVWdWyUyzfRBO5Bi\nehyiNhuQ0baYpAF/kJAaPqlLD3lOO3NnTKGje0AKh8QIE02PAxTlOxkMhOjpl+WEZmf6oJ3qNqYA\nJUP7PB9r86WlTSJ36LFGe7TqGVMAaDrepdvfFMnZuHEjV199NStWrGDPnj0jXtuxYwdXXnklV199\nNffff3/Cax577DFqamro6dGnPmGi6XGAwqEKcm+nDFTMLqk7jZE7cioHhmgqhvZ5PiI3RTFJkaA9\nwbO0Y5kzPbz0q+kj6Z+ZtGvXLpqamqivr2fDhg1s2LBhxOvr169n8+bNPPXUU2zfvp2DBw/GveYP\nf/gDJ06coKqqSrf2BSeRHo+cqy27opmeI9EvRHfKQ4cOsWbNGurr6yOvr1+/nocffphp06axcuVK\nLrvsMrxeb8xr0tGRU60eB6gY2udZbopisvoHwoWQk91YJVphvpOqsgKOt/fQNxCgIC/hf6ZCBw0N\nDSxduhSAefPm0dnZSW9veGloc3MzJSUlzJgxA4AlS5bQ0NCA1+sdc43P52Pp0qW43W7+67/+S7f2\nTXSdNkCJOxy0ZVWC+SW8G8TqyD6fD7fbbYiOHKkeTyE9XpjvIM9lZ9/hE7q1Q+SmdKTHAWZXuWnr\n6OO9pg7OPbNS178tYmtvb6empibyfXl5OZ2d4b3gPR4P5eXlI15rbm6mo6NjzDUej4fq6uoJtaGx\nsTHua1rQ7u7upOloakdt9vrC/XT3/sPMLOyYUNv0Nt5nNZtMfpaEQTtWR/Z4PLjd7ox15PEMTmCk\nrSgKFVPy+bC9h/7BAPkuGckYzcaNG9m9ezeKorBmzRrOOeecyGs7duzg3nvvxW63s3jxYq6//vpx\nr3nssce466672LVrF0VFRbq2M11Be3pFEeBh3+ETErSzZCKV1pOtzq6trY372sH/3g5AeVkZc+dM\nS+nvThsM8Ld9ewnZi8Z9j0xpbGw0RDv0kI7PMt5DQMrRKtMdOdETjLcj/CTccqw5pbSR0xZe7vXC\nX/7OrArXhNs3GWZ+0kxn2/fv38+ePXtYvXo1x44d47bbbuM//uM/Iq9ne0ommnaWtp7pcYAZFeGH\ni32Hvbr+XRFfVVUV7e3tke/b2tooLS2N+VpraytVVVU4nc4x11RWpuchazKFaPkuB+4CJx+dkPS4\n2SUM2rE6stYpM9GRx3uCaWxspKCgCJsyQPXcU1P6u92BEzS1tVBYOova2jkTattkmPlJM91t/9vf\n/sY//dM/UVtbS21tLb/4xS8466yzePfddw0xJROtNw2FaAD5eQ7KivN496iXYDCU0j4EYmLq6urY\nvHkzK1asYO/evVRVVVFQEK5/mT17Nj6fj5aWFqZPn862bdu455576OjoGHON2+1OS/sms04bwtOC\nH3p6CIXUSe+TL7InYdCO1ZG1TmmEjuwPBHFOYAvJiinhCnIpRjOeeFMyYIy5xWjvHwpvgHKy4wRN\nSveE3ised75KR3eQ517eyczy5LNBRszgmKVNFRUVfP7zn0dRFL72ta/x17/+Fa/Xy7Jly1i3bh03\n3XQTAMuXL6e6uprq6mpqampYsWIFiqKwdu1aAH75y1+yY8cOPB4P//Iv/8InPvEJVq9ePan2Tmak\nDVDizqOto4+2jt6h6RdhRgmD9nnnnTemU27ZsoXi4mJDdGR/IIRzAqOQcln2ZRpGm1vUNDY2MrVq\nBtDJrBnTOWVa8aTec7S+kJdmTzP7P3LwhWXJZTaMmMExU5tG/yz69xYuXDhi5Yzm5ptvHvOzb33r\nW3zrW9/SqbVhk9lcBWBaeSHvN59kz8F2CdomltSc9uhOOX/+/MjX2e7I/kBoQkVAeU477kInR2Wk\nbTjxpmS8Xq8h5hajaXPaeheiAUyfGr6xyjIdAdGnfE0saM+ZXgy74Y0DbVz66bk6tkxkkuknyvzB\niQVtCKfIvV0DdPemtnxCpFddXR1bt24FGHdKJhAIsG3bNurq6sa9Jp2Gq8f1LUSD8M59BXkOjrf3\nyJ7RYtIj7VJ3HsWFLt5630NQewIQpmP6tU7+QChyyHuqSt15NBE+BrF4TnYqyMVY8aZk2traqK2t\nzfqUTDQtaLt0LkSD8NLEGRVFfPBhJ56TfVSVFer+HsI8Jhu0FUXhlGlu9h32cuR4F/Nml+rZPJEh\npg/avUOnIk3ElKFdgj460cOZc8r0bJaYpFhTMlrhULanZKKla522ZnpFIR982MmBI14J2jkukh5X\nJt7XplcUse+wl/1HvBK0Tcr06fFgUJ140C7KA+CjE716NknkkHSmx4FIwdD+I7JeO9dNdqQNw+v/\npT+Zl6mDtqqqBEMTD9raaV+y4YCYqL6BAE6HbcJ9MJHKsgJsNoUDcpPNeaFJFqJBeA/yKUUuCdom\nZuqgPfTgiS2FE76iFUeCtoy0xcSk+0APh91GVWkBH3zYRf+AnIWcy/QYaSuKwoJTy/F09NF+Uk78\nMiNTB+1AMNyJHSkcFhLNYbdRVODk8PFOPZslckgmTuGaXlFEKKTyfvPJtL6PMDY9gjbAglPDmxPJ\naNucTB20h3cImvjHKCly4ev1yyhGTEh/JoL21HABmtxkc1swfKDhpKdiOnsGAHi+4cjkGiSywtRB\nO6B14gmOtCG8SxDAgSa5IYrUqKqamZF2uRQPCQiq+oy0q8oKsSmK1PKYlKmD9mT34gWYWRnegOPt\nQ3K2tkiNP6gSUkl70C4qcDKrsoh3DrUz6A+m9b2EcWnTgZMdaTvsNirLCvCc7JMMowmZOmhrnXgy\nT54zpxahAG8fbE/4u0JEG/SH+1+6gzbAwo9Np38wyNuHpJ/mKn9Aq+GZ/G17ekURqorUSZiQuYO2\nNtKeRCd2Oe1UlRfy7tEO2jqkilwkb3DoJpqfl5412tHO/9h0AHbt/Sjt7yWMabjwdvK37RlDdRL7\njkiG0WxMHbSDOqWLPn5aBaGQyh//ekiPZokcMRAIL5zNxEh7QXU5U4pcvPrWMQYkRZ6T/JNcLRNN\n27TnwJGOSf8tkVmmDtoBHTYbADhjTikVJfm8uOso/oBspC+Sk8n0uMNu49JPz6W718+rbx5L+/sJ\n4wkEVRRl8oVoAEX5TqYUudh3+ITc80zG1EF7eKQ9uY9ht9lYdPYM+gYC7Jd0kUjSgD9zI22Ayxed\nigL89oUD/HnH4Yy8pzAOf1DF5bSjKPrsvjd3ejG9/QH2fSD3PDMxddAentPWbwvJ1/e36fa3hLX1\nD4b7X9NH3Rl5v6ryQk6dOQVPRx+tXqm/yDWBoIpLxz3uq2eWALBrn9RJmImpg7Zec9oAsyrdOOw2\ndu09LmcXi6T0D42089JwLGc8Z8+bCsAeWe2Qc/wBVde+NrOyCKfDxl/eaJF7nomYOmhH5rR1qKZ0\n2G2cOmMKxzw9sgxCJKV/UAva6a8ef77hCM83HGF2lZuppfkcbD7JMY8v7e8rjCMQVMlz6dfX7DYb\nc6YX09UzyNHWzGSLxOSZOmjrOdIGOGtu+Eztba836/L3hLVpI21XBoK2RlEUPjV/Girw6J/2Zex9\nRfb1+0P0D+q7cqB6xhRAlhKaiamDdkCHHdGizZlWTKk7j7++eUwqKkVC2px2viszhWia02aVMKOi\niIa3j9Pw9vGMvrfInmBInzXa0eZOn4ICvPaO9COzMHfQDupbiGazKSw5bzbdvYM0HmjV5W8K69LS\n464MzmlDeLR9ce1sXA4b99W/KUVpOSAYUlFV/YN2fp6D2VVu3jt6kg/bZbrFDEwdtPU45Ws0rdDj\ndy+8q9vfFNY0XIiWufS4pnxKPv/6pXPw9flZ84u/8ae/fZDxNojM0fac12NjldHOHJoW/Etji+5/\nW+jP1EE7oNNRddGmlhZQVpxH00ddspm+GFffYAiHXdGlEHIigsEQ8+eW0dbRx+v7JTNkZcNBW/++\ndtrMEhx2G//16gcEgzItaHSmDtp6nPI1mqIozJtVQiCo0nhA1myL+PoHQxktQhtNURQu+uQsigqc\nvPGeR9KbFqZtXZuOB0SX086CU8vw9flpkLltwzN10B6e09b3Y8ybXQrAX96QKnIRX79fzUpqPJrL\nYafunJmEQioP/ufbst7WotKZHgc45/RKAH679YAU4RqcqYN2OkbaABUl+VSWFbBr70dyULyISVVV\n+gdDWQ/aAKfPLmF2lZvGA228flD6qxUNDtVPpCM9DlBanMfHT6ugudXHI/+9l1BIHv6MytRBOzKn\nrfPTp6IonHt6JSEV/rRd9ngWY/UPBlFVcOm42cVEKYrCZ2tnU+J28dzrJ3lgyx6aZbMMS0n3SBvg\nMx+fwaxKN//16gc88fz+tL2PmBxTB+10VI9rTj+lhHyXnecbjki6SIzR0+cHslM5HsuUojwu/fRc\nClw2/nv7Yf5t08vc9LO/8tKuo5Iyt4B0zmlr8lx2LvvMXKYUufj9y+/zbpM3be8lJs7UQVs7Zcnl\n0P9j2G02zppTRv9gkL/LhvpilEjQNsBIW1NVVsiSs4u59NNzmDO9mPebT/Kz+jf59o//IishTC6d\n1ePRCvIcXFJ7Cirwo0f/TqdvIK3vJ1Jn6qDdp21ukaYb54LqCgAe//P+yH80QgCcHLqZZXo3tETs\nNoUzTinjCxeexqrLFzC7ys2R4118/1fb5QZsYsNz2ulLj2tmVbk5/2PTae/s5z8e3pn29xOpMXfQ\nHggXAtl0Ol92tIqSfM4+fSotbT42/n+78A2NroQ40dkPQFGBM8stia+40MXnLzyN+XPLeO/oSW75\n+atSWGlSAxkaaWvOm19FZWkB7x3tYP9hSZMbibmD9mAo7enJRR+fzinTwpW5P33qDZkfFACc6OwD\nwJ1v3KAN4ZH3JZ86hfPOquKYp4dv//gvvLSrKdvNEikazMCcdjSbonDhJ2YC8NsXDmTkPUVyTB60\nVfLTHLSdDjufv/A0ZlW62bn3I55vOJLW9xPm4I2MtI2VHo9FURQWnT2Df1h4CqGQys/q35ITwkwm\nE9Xjo82c6mZ2lZu33vPIjnsGYtqgPeAP6n6+bDw2RWHp+XMoLnTy6z++Q9NHXWl/T2FsJ7qMnx4f\nbf7ccq5aegal7jyeefl9tmw7mO0miSRF0uNpWCkzngvOnoHDbuPHTzbKfc8gTBu0fb2DAOQ5MzPS\ncRc4ufATsxgMhPjBAw00HmjlnUPtki7PUSc6+7DbwtW2ZjKlKI8vXHQaRQVOHvnvvbz8+tFsN0kk\nId2bq8RTWVbIkk/Owtfn55af/437n9kdmRoS2WHioB0uCkt3ejzaaTNLOHteBd6uftY99Bq3/WI7\nP/zNTvwBqSzPNSc6+ykusKOkqQgynaYUufjChaeR57Tzs/q3ZI7bBHoHwvc7ZxqWtyYy/9RyPls7\nm2AwxPMNR7jxnm0clVF31pg2aHdrI+0Mr5O96BOzuHzRqXxqfhWzKt38fV8rv9rytmz7l0OCIZWO\n7gGKC42zRjtVFSX5/GNdNQWucOC+49evsfeDEwTklCdDahs6M724yJWV9/9YdQX/+ws1XHjuTLp7\n/dz0s1e4+4nXaWmTnfcyzVy5vSjdkZF2Zj+CoiicNquE02aVEAiG+P2293lhZxNtHb1c8qlTcBc4\nOW1WCRUlBWOu9QeC/PkR6nAAABMKSURBVLnhCCdO9rNgmgR5szrZ3U8opDKlwLxBG2DG1CJ++r2L\n2fz0W7y+vzVSbJTnsjO1JJ8ln5zNF5fMo9DgFfK54KMTvdhtmc0sjqYoCueeUYnDbuPN99p45c1j\nvPbOR6xYdiZlxfl8rLqcmZXucf/GgSYvx9t7mJKhNluRaYO2L0sj7WgOu40rLprHCzubeOs9D2+9\n5wHCO7T9P5//GFdcNG/E79/+qx3sG1rz+IXzy/jM+RlvstDBoWOdAEydYtr/fCLees/DhefOpHpm\nCQdbTtLR1Y8/GKLV28dvX3iXLX85yMrLF4S3SB1n/j4UUvEHjXGAitWoqkqrt5fCPJshpmNqTqvg\nY9XlHGw5ybbGFh57LrxPudNh47ov1LC8rjpmO9872sH/e9+rAKy6ZCq1GW21dZj2rqNtdJLNJ08I\nFyJdcdFptHX04enopXcgwDuHTvDQH97h3SMdnH5KKYvOnkFXzyD7D3txOmz4AyHePCSbXJjVvg9O\nADCnKi/LLdGHoijMrgov79H4A0F2v9/OG++28es/vsPjz+3nf118Ov/r4uGRt6/Pz/bdH/K3t46x\n56CHkBr+77GsOJ/S4jxOmVbMsk/PydbHsozuXj99AwGmlRrndq0o4Z33ZlW6aW7z0dcf4I132/jV\nf77N9j3H+dSCaXzyrEoGBoN88GEn/QMBntw6vN77xTc7+fJy1RAPIWaTVC/YuHEju3fvRlEU1qxZ\nwznnnBN5bceOHdx7773Y7XYWL17M9ddfH/ea48ePs3r1aoLBIJWVldx99924XBObo4nMaRvgyV5R\nFKaVFzKtvBCA+XPL+NP2I7zy1jFeeesYjz23L3L29+WLTuWt9z0c/aibpo+6mDtdEkWxGLHPafYd\n9mKzKcyucNFl0VkOp8POpxZMo+a0Ct451M7bh07wuxff5bkdhyMPoa/vb40cpjO1NJ8ClwOnw05H\ndz8HjvSw/4iXl3Y1sfQTJdSaZFg1ug9Fy1a/a/WGH/AL8oxXglSY7+SsOWUAzJtdwsuvN/P2oXbe\nPtTOI/898ncVBZadP4eefj879hzn0LFOTp9dmoVWm1vCoL1r1y6ampqor6/n0KFDrFmzhvr6+sjr\n69ev5+GHH2batGmsXLmSyy67DK/XG/Oa++67j6985Stcfvnl3HvvvTzzzDN85StfmVDDPxhKURpx\nneyUojxWLDuTto4+TnT2sedgOz19fhadPYNTphUz6A9y9KNuXtx5lK9/8ePZbq7hGLXPQXh0+X7z\nSU6bOYU8pw0G9fjExlWQ52Dhx6Zz7pmV9A0E2LLtIFtfC1eblxbnMX9uGWecUsaUUQVSIVXlWJuP\n/3m9mRfe7KS4dC+rln8Mu824I6tY/W716tWR17PV71qHitAKDRi0oxUXuvji4nl09w7yocfHMU8P\nDoeNqrICnA4bVWWFFBe6IvfuR/+0jx/+6wVZbrX5JAzaDQ0NLF26FIB58+bR2dmJz+fD7XbT3NxM\nSUkJM2bMAGDJkiU0NDTg9XpjXrNz507uuOMOAD772c/ym9/8ZkId+e1D7TQeaGN2hYsStzFTlNGj\n748NHTyiOXXmFFwOha2vHeETZ1biLnSS57ST73KgotI/EKR/MEBBnoMpRS4URSEUUgmG1KH/DREK\nqUSWiCvD/xOdblKU4e+VEb+nEJ2VSvQ7o3+3uy9IR3f/uH9romxK/D4HZK3PBYIhWr29Q1mTEBd9\nYhaQO8teXA47Loeday5fwEnfAE6HneJCZ9x/b5uicMq0Yr508en8/uV3+f22gzS8fZwF1eWUT8mn\nYko+5SX5lLrzsY+zy9eIvsWo31NifjmmTdHfBoIh/IEQx04MUnG8C5fThsthx13ojNnvenvDATNb\n/c7XFx6VAhTlZT+rmIziQhdnzS3nrLnlMV+fO70Yhx0OHPHy+v5WKkryCQRDOOw28lzDZ0nYbAoO\nu83QD3oAPf3BCR/Go93fU5EwaLe3t1NTUxP5vry8HI/Hg9vtxuPxUF5ePuK15uZmOjo6Yl7T19cX\nSRFVVFTg8XhSaizArn0f8cOhk2fOP2v8SkWjsttszJuRx/7mfu749WvZbs7E/OfxtP3p4o6D/O8Y\n/QfISp8DWPtgA3sOtgNw5pxSvrh4Hm+99eaE/paZ2e22mCsj4plS5OLCGjdHvXbeP3qSD9sNVMux\ntS3yZUGenVNDrWP6UGdneFSYjX73fnMHqze/SiCocsYppUwtscZcjN1uY8EpBbx9pM+897/Rtkzs\nfnj5olP5tyvPTemalCsbJrIDWKxrkv07jY2NI763A+u+MjvqJydSbo8RXH3R1Gw3wbAeesjPoUOH\nKCsLz5V1d3ezd+/eyCgnGZPpczC2333p/Hy+dP5wv9MCdqXLWP2v8nQ3RvtvonL+FC4AuKAo200Z\n10MPecf0u1Tp3e9uv3rWqN8w1r/tRP3TBRX8k2TGgcCYf/NEEgbtqqoq2tvbI9+3tbVRWVkZ87XW\n1laqqqpwOp0xryksLKS/v5/8/PzI746n1izVK0JXO3bsoKSkJPLv39vby8UXX4zb7aalpSWtfQ6k\n3+WqeP0O0n+vA+l3IjkJKxvq6urYunUrAHv37qWqqgq3O5yWnj17Nj6fj5aWFgKBANu2baOuri7u\nNRdccEHk5y+88AIXXXRRuj6XMDHpcyIbpN8JM1DUJHI399xzD6+//jqKorB27Vr27dtHcXExy5Yt\n4+9//zv33HMPAJdeeinXXXddzGvmz59PW1sbt9xyCwMDA8ycOZM777wTp9N41d8i+6TPiWyQfieM\nLqmgLYQQQojsM/bCPyGEEEJESNAWQgghTMI4m9mmaLxtLrPtvffe49/+7d+49tprWblyZdwtDZ99\n9lkeffRRbDYbV111FV/+8pfx+/3ceuutfPjhh9jtdu68805OOeWUjLR706ZNNDY2EggE+Nd//VfO\nPvtsU7Q707LZ93bu3Ml3vvMdzjjjDADOPPNMvv71ryf976QnI/bz0W269dZb2bt3L6Wl4e0yr7vu\nOi6++GJT9WEj3+tiMWK/mChD3hNVE9q5c6f6jW98Q1VVVT148KB61VVXZblFw3p6etSVK1eqt99+\nu/r444+rqqqqt956q/rcc8+pqqqqP/7xj9Unn3xS7enpUS+99FK1q6tL7evrU//xH/9R7ejoULds\n2aKuW7dOVVVVffXVV9XvfOc7GWl3Q0OD+vWvf11VVVX1er3qkiVLTNHuTMt233vttdfUG2+8ccTP\nUvl30osR+3msNt1yyy3qyy+/POb3zNKHs93fUmXEfjFRRr0nmjI9Pt42l9nmcrl46KGHRqzL3Llz\nJ//wD/8AhLc0bGhoYPfu3Zx99tkUFxeTn5/PeeedxxtvvEFDQwPLli0D4IILLuCNN97ISLsXLlzI\nz372MwCmTJlCX1+fKdqdaUbse6n8O+nFiP08VptiMVMfNmJ/G48R+8VEGfWeaMqg3d7eHtm1CEZu\nc5ltDoeD/Pz8ET+LtaVhe3v7mG0RR//cZgufnzs4mP5TKex2O4WF4VPKnnnmGRYvXmyKdmeaEfre\nwYMH+eY3v8k///M/s3379pT+nfRixH4eq00ATzzxBNdccw3//u//jtfrNVUfNkJ/S4UR+8VEGfWe\naNo57WiqiVatxWtrqj9Pl5deeolnnnmG3/zmN1x66aUJ22GUdmdLpj/nqaeeyg033MDll19Oc3Mz\n11xzDcFgMGF7Mt1Oo/SXL37xi5SWlrJgwQIefPBBfv7zn/PJT34yq22aDCO2KRVG6RepMNo90ZQj\n7fG2VjUibUtDGN7+MNZn0H6uPUn7/X5UVZ30+c/JevXVV/nVr37FQw89RHFxsWnanUnZ7nvTpk1j\n+fLlKIrCnDlzmDp1Kp2dnUn/O6WTEfvLokWLWLBgAQCXXHIJ7733XtbblIps9zc9GLFfJMuI90RT\nBu3xths0olhbGp577rm8/fbbdHV10dPTwxtvvMGnPvUp6urqeP755wHYtm0bn/70pzPSxu7ubjZt\n2sQDDzwQqbQ1Q7szLdt979lnn+Xhhx8GwidPnThxgi996UtJ/zulkxH7y4033khzczMQnls944wz\nst6mVGS7v+nBiP0iGUa9J5p2R7RYWwcawTvvvMNdd93FsWPHcDgcTJs2jXvuuYdbb711zJaGzz//\nPA8//DCKorBy5UquuOIKgsEgt99+O0eOHMHlcvGjH/0opdOtJqq+vp7NmzdTXV0d+dmPfvQjbr/9\ndkO3Oxuy2fd8Ph8333wzXV1d+P1+brjhBhYsWBBzy8xY/056MWI/j9WmlStX8uCDD1JQUEBhYSF3\n3nknFRUVpurDRr3XxWLEfjFRRr0nmjZoCyGEELnGlOlxIYQQIhdJ0BZCCCFMQoK2EEIIYRIStIUQ\nQgiTkKAthBBCmIQE7SzYtGkTq1at4qqrruLjH/84q1atYtWqVfzhD39I6voHH3yQv/zlL+ltpDC0\nr371q7z00ksjftbf38/ChQs5fvx4zGtWrVrFjh072L9/Pz/84Q/HvN7U1MQll1wy7vv29fXxwgsv\nAPDKK6/wy1/+coKfQOSSyd7zNK+//jotLS1paqVJ6HLsiJiQ5uZm9aKLLsp2M4QJbdmyRb3++utH\n/OzZZ59Vr7vuurjXrFy5Ut2+fXvc148cOaJ+9rOfHfd9X3/9dfWmm25KrbFCDJnsPe/WW29VX3vt\nNR1bZD6W2HvcKjZv3kxLSwsffvght9xyC/39/dxzzz24XC76+/tZu3YtNTU13HrrrdTW1rJo0SK+\n9a1vceGFF7Jnzx56enp44IEHmDZtWrY/ikizz33uc9x11110dHREDpT4wx/+wJe//GVefPFFfv3r\nX+NyuQgGg2zatInZs2dHrt25cyc//elPeeqpp3jjjTdYu3Yt5eXl1NTURH7n0KFDrF27Frvdjs/n\n47vf/S4LFy7k+9//Pl1dXWzatInTTz+dHTt2cM8997B7925+9KMf4XA4UBSFH/zgB5x++umsWrWK\nRYsW8eabb3LkyBFuvPFGXTd5EeY2ODjIunXraG5upqenhyuuuIJrr72WAwcOsG7dOpxOJwMDA9xw\nww309vbywgsvsG/fPr7//e9z/vnnZ7v5WSHpcYNpaWnhscce4+Mf/zgnT55k3bp1PPbYY1xzzTU8\n8MADY37/0KFDfOlLX+LJJ59kwYIF/PnPf85Cq0WmFRQUcOmll/KnP/0JCO93fODAAS655BK6urr4\nyU9+wuOPP86SJUt48skn4/6dTZs2cfPNN/Poo4+O2NO6vb2d73znOzz66KPcfvvt/OQnPyE/P59v\nfOMbXHDBBaxevXrE31m9ejW33XYbjz/+OF/72te44447Iq/19vby0EMPsWHDBn7961/r/P+EMLNH\nHnmEWbNm8fjjj/P000/zxz/+kYMHD1JfX8+ll17K448/zv33309HRwef+9znOPPMM1mzZk3OBmyw\nyClfVnLuueeiKAoAU6dOZdOmTQwMDNDd3U1JScmY3y8rK+OMM84AYObMmZw8eTKj7RXZc+WVV3LH\nHXewcuVKnn32WT7/+c/jcrmYOnUqt9xyC6qq4vF4xpxqFe3dd9+ltrYWgM985jM8/vjjAFRWVrJp\n0yZ+8pOf4Pf7x+1XXV1dnDhxgnPOOQeA888/n+9973uR17Ub7MyZM+ns7Jz05xbWsXPnTtrb23nt\ntdcACAQCNDU1cdlll/H973+f5uZmLr74YsnORJGgbTBOpzPy9erVq7njjjtYtGgR27b93/bu3yW1\nMI7j+JuTBQ5WBE0tkjgWkX9CbYUQHWw6BIogDXFACkRwCSMbGhvDwTEIEUUHuVs6OCg0Bc0uToJD\nmdw7RHK9XZvuJU9+Xtt5nnMengNf+J7znB/fH9zc3HzYf2ZmZmT7p/5KOzXW19d5eXnh6emJfD7P\n1dUV/X4f27a5u7vD6/WSy+V4eHj4dBzDeFtw+73E59nZGTs7O5imyePjI7FYbOzx7xeZ7/6MQZfL\nNbZPptvc3BzHx8dsb29/6CsUCtRqNW5vbykWi1xeXn7BDCePlscnWKfTwe/3MxgMKJfLX1YMXibX\n/v4+19fXuN1u/H4/vV4PwzBYWVnh+fmZarX6adz4fD6azSYA9/f3w/b32AMolUrDMQzD4PX1dWQM\nj8fD8vIyrVYLgFqtxsbGxj89T/meNjc3h4/0BoMB6XSabrdLNpul0+mwtbVFOp0extbf4m/aKGlP\nsGg0yuHhIbFYjL29PdrtNtls9qunJRMkGAxSqVQwTROAxcVFdnd3MU0T27aJRCLU6/Wx7zqcnJxw\nfn5ONBql1+sN28PhMKenp0QiEQKBAAsLC1xcXLC2tkaj0SCRSIyMk8lkyGQyWJZFLpcjlUr9v5OW\nb8OyLGZnZzk4OCAUCrG0tMT8/Dyrq6vYto1lWRwdHRGPx4G30pjJZPLD547TRFW+REREHEJ32iIi\nIg6hpC0iIuIQStoiIiIOoaQtIiLiEEraIiIiDqGkLSIi4hBK2iIiIg6hpC0iIuIQvwBHgFAkM+2e\nPQAAAABJRU5ErkJggg==\n","text/plain":[""]},"metadata":{"tags":[]}}]},{"metadata":{"id":"Hl9cjrD3yFVt","colab_type":"code","colab":{}},"cell_type":"code","source":["TEXT.build_vocab(train_data, max_size=25000, vectors=\"glove.6B.100d\")\n","LABEL.build_vocab(train_data)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"7lma5DcLyH40","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch.nn as nn\n","import torch.nn.functional as F\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"olfUcyORzBWE","colab_type":"code","colab":{}},"cell_type":"code","source":["class SingleSelfAttnELU(nn.Module):\n"," def __init__(self, embedding_dim):\n"," super().__init__()\n"," self.cnn_q = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n"," self.cnn_k = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n"," self.cnn_v = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n","\n"," self.softmax = nn.Softmax(dim=2)\n"," \n"," def forward(self, x):\n"," q = (F.elu(self.cnn_q(x))) \n"," k = (F.elu(self.cnn_k(x)))\n"," v = (F.elu(self.cnn_v(x))) \n"," # q = [batch_Size, emd_dim, seq_len]\n"," # k = [batch_Size, emd_dim, seq_len]\n"," # v = [batch_Size, emd_dim, seq_len]\n","\n"," a = torch.bmm(q.permute(0,2,1), k) # a = [batch, seq_len, seq_len]\n"," a = self.softmax(a/q.size(1)**0.5) # a = [batch, seq_len, seq_len]\n"," \n"," output = torch.bmm(a,v.permute(0,2,1)) # output= [batch, seq_len, emd_dim]\n"," \n"," return output"],"execution_count":0,"outputs":[]},{"metadata":{"id":"JSi7oSIKzDqH","colab_type":"code","colab":{}},"cell_type":"code","source":["class MultiSelfAttnELU(nn.Module):\n"," def __init__(self, embedding_dim, num_heads):\n"," super().__init__()\n"," self.num_heads = num_heads\n"," self.cnn_q = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n"," self.cnn_k = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n"," self.cnn_v = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n","\n"," \n"," self.softmax=nn.Softmax(dim=2)\n"," \n"," def forward(self, x):\n"," q = (F.elu(self.cnn_q(x))) \n"," k = (F.elu(self.cnn_q(x))) \n"," v = (F.elu(self.cnn_q(x))) \n"," # q = [batch_Size, emd_dim, seq_len]\n"," # k = [batch_Size, emd_dim, seq_len]\n"," # v = [batch_Size, emd_dim, seq_len]\n"," \n"," q = torch.chunk(q, self.num_heads, 1)\n"," k = torch.chunk(k, self.num_heads, 1)\n"," v = torch.chunk(v, self.num_heads, 1)\n"," heads = []\n"," for i in range(len(q)):\n"," a = torch.bmm(q[i].permute(0,2,1), k[i])\n"," a = self.softmax(a/q[i].size(1)**0.5)\n"," \n"," h = torch.bmm(a, v[i].permute(0, 2, 1))\n"," heads.append(h)\n"," \n"," output = torch.cat(tuple(heads),dim = -1)\n"," return output # output = [batch_size, seq_len, emd_dim]\n"," "],"execution_count":0,"outputs":[]},{"metadata":{"id":"viWbUPL90xWG","colab_type":"code","colab":{}},"cell_type":"code","source":["class MultiSelfAttnTANH(nn.Module):\n"," def __init__(self, embedding_dim, num_heads):\n"," super().__init__()\n"," self.num_heads = num_heads\n"," self.cnn_q = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n"," self.cnn_k = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n"," self.cnn_v = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n","\n"," \n"," self.softmax=nn.Softmax(dim=2)\n"," \n"," def forward(self, x):\n"," q = (F.elu(self.cnn_q(x))) \n"," k = (F.elu(self.cnn_q(x))) \n"," v = (torch.tanh(self.cnn_q(x))) \n"," # q = [batch_Size, emd_dim, seq_len]\n"," # k = [batch_Size, emd_dim, seq_len]\n"," # v = [batch_Size, emd_dim, seq_len]\n"," \n"," q = torch.chunk(q, self.num_heads, 1)\n"," k = torch.chunk(k, self.num_heads, 1)\n"," v = torch.chunk(v, self.num_heads, 1)\n"," heads = []\n"," for i in range(len(q)):\n"," a = torch.bmm(q[i].permute(0,2,1), k[i])\n"," a = self.softmax(a/q[i].size(1)**0.5)\n"," \n"," h = torch.bmm(a, v[i].permute(0, 2, 1))\n"," heads.append(h)\n"," \n"," output = torch.cat(tuple(heads),dim = -1)\n"," return output # output = [batch, seq_len, emd_dim]\n"," "],"execution_count":0,"outputs":[]},{"metadata":{"id":"yJTf267OzEkv","colab_type":"code","colab":{}},"cell_type":"code","source":["class MultiTgtAttn(nn.Module):\n"," def __init__(self, embedding_dim, num_heads):\n"," super().__init__()\n"," self.num_heads = num_heads\n"," self.target = nn.Parameter(torch.rand(1,embedding_dim))\n"," stdv = 1. / math.sqrt(self.target.size(1))\n"," self.target.data.uniform_(-stdv, stdv)\n"," self.cnn_k = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n"," self.cnn_v = nn.Conv1d(in_channels = embedding_dim, out_channels = embedding_dim, kernel_size=3)\n"," self.softmax=nn.Softmax(dim=1)\n"," \n"," def forward(self, x):\n"," # x = [batch_size, emd_dim, seq_len]\n"," k = self.cnn_k(x)\n"," v = self.cnn_v(x)\n"," t = self.target\n"," \n"," k = torch.chunk(k, self.num_heads, 1)\n"," v = torch.chunk(v, self.num_heads, 1)\n"," t = torch.chunk(t, self.num_heads, 1)\n"," heads = []\n"," for i in range(len(k)):\n"," \n"," a = F.linear(k[i].permute(0,2,1), t[i])\n"," a = self.softmax(a/k[i].size(1)**0.5)\n"," h = torch.bmm(a.permute(0,2,1), v[i].permute(0,2,1)).squeeze(1)\n"," heads.append(h)\n"," output = torch.cat(tuple(heads), dim=-1)\n"," return output #[batch, emd_dim]"],"execution_count":0,"outputs":[]},{"metadata":{"id":"fCCDGS6X8mTt","colab_type":"code","colab":{}},"cell_type":"code","source":["class SenAttn(nn.Module):\n"," def __init__(self, embedding_dim, num_heads, output_dim):\n"," super().__init__()\n"," self.attnelu = MultiSelfAttnELU(embedding_dim, num_heads)\n"," self.attntanh = MultiSelfAttnTANH(embedding_dim, num_heads)\n"," self.mta = MultiTgtAttn(embedding_dim, num_heads)\n"," self.fc = nn.Linear(embedding_dim, output_dim)\n"," \n"," def forward(self, x):\n"," elu = self.attnelu(x) #elu = [batch, seq_len, emd_dim]\n"," tanh = self.attntanh(x)\n"," \n"," inp = elu*tanh #[batch, seq_len, emd_dim]\n"," inp = inp.permute(0,2,1) #[batch, emd_dim, seq_len]\n"," \n"," out = self.mta(inp) #[batch, emd_dim]\n"," \n"," out = self.fc(out)\n"," return out\n"," "],"execution_count":0,"outputs":[]},{"metadata":{"id":"iabAhct91TDx","colab_type":"code","colab":{}},"cell_type":"code","source":["class WordAttn(nn.Module):\n"," def __init__(self, vocab_size, embedding_dim, sentence_len\n"," ,num_heads):\n"," super().__init__()\n"," self.senlen = sentence_len\n"," self.embedding = nn.Embedding(vocab_size, embedding_dim)\n"," self.attnelu = MultiSelfAttnELU(embedding_dim, num_heads)\n"," self.attntanh = MultiSelfAttnTANH(embedding_dim, num_heads)\n"," self.mta = MultiTgtAttn(embedding_dim, num_heads)\n"," \n"," def forward(self, x):\n"," #x = [seq_len, batch_size]\n"," x = x.permute(1,0)\n"," embedded = self.embedding(x)\n"," embedded = embedded.permute(0, 2, 1) \n"," # embedding = [batch, emd_dim, seq_len]\n","\n"," cnn_sen_embedding = []\n"," split_inputs = torch.split(embedded,self.senlen,2)\n"," for inputs in split_inputs:\n"," \n"," elu = self.attnelu(inputs)\n"," tanh = self.attntanh(inputs)\n"," \n"," inp = elu*tanh\n"," inp = inp.permute(0,2,1)\n"," \n"," sen_embedding = self.mta(inp) #[batch, emd_dim]\n"," sen_embedding = sen_embedding.unsqueeze(-1) #[batch, emd_dim, 1]\n"," \n"," cnn_sen_embedding.append(sen_embedding) \n"," \n"," output = torch.cat(tuple(cnn_sen_embedding),dim=-1) \n"," # output = [batch, emd_dim, nos]\n"," \n"," return output"],"execution_count":0,"outputs":[]},{"metadata":{"id":"ikMVPtxqzHEd","colab_type":"code","colab":{}},"cell_type":"code","source":["class Network(nn.Module):\n"," def __init__(self, vocab_size, embedding_dim, sentence_len,\n"," num_heads, output_dim):\n"," super().__init__()\n"," self.word = WordAttn(vocab_size, embedding_dim, sentence_len,\n"," num_heads)\n"," self.sentence = SenAttn(embedding_dim, num_heads, output_dim)\n"," \n"," def forward(self, x):\n"," w = self.word(x)\n"," s = self.sentence(w)\n"," return s\n","\n"," \n"," "],"execution_count":0,"outputs":[]},{"metadata":{"id":"7D8Xksg6C_Ng","colab_type":"code","colab":{}},"cell_type":"code","source":["INPUT_DIM = len(TEXT.vocab)\n","EMBEDDING_DIM = 100\n","OUTPUT_DIM = 1\n","BATCH_SIZE = 16\n","NUM_HEADS = 5\n","SEN_LEN = 50\n","\n","model = Network(INPUT_DIM, EMBEDDING_DIM, SEN_LEN, NUM_HEADS, OUTPUT_DIM)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"OQexAp2uEsrz","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":139},"outputId":"a4609013-7cdd-4241-ad12-69bb74634551","executionInfo":{"status":"ok","timestamp":1546969066760,"user_tz":-330,"elapsed":1118,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}}},"cell_type":"code","source":["pretrained_embeddings = TEXT.vocab.vectors\n","\n","model.word.embedding.weight.data.copy_(pretrained_embeddings)"],"execution_count":43,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([[ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n"," [ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n"," [-0.0382, -0.2449, 0.7281, ..., -0.1459, 0.8278, 0.2706],\n"," ...,\n"," [-0.3971, -0.3226, -0.2672, ..., -0.2575, -0.0519, 0.5823],\n"," [ 0.2112, -0.3741, 0.1258, ..., 0.0768, -0.3270, 0.0697],\n"," [ 0.0152, -0.6117, 0.2832, ..., -0.5245, -0.2323, -0.4019]])"]},"metadata":{"tags":[]},"execution_count":43}]},{"metadata":{"id":"-F5_xaNKHFne","colab_type":"code","colab":{}},"cell_type":"code","source":["'''\n","def binary_accuracy(preds,y):\n"," rounded_preds = torch.round(torch.sigmoid(preds))\n"," correct = (rounded_preds==y).float()\n"," acc = correct.sum()/len(correct)\n"," return acc\n","'''\n","def precision_recall_f1(preds, y):\n"," rounded_preds = torch.round(torch.sigmoid(preds))\n"," tp = y*rounded_preds\n"," fp = (y==0).float()*rounded_preds\n"," tn = (y==0).float()*(rounded_preds==0).float()\n"," fn = y*(rounded_preds==0).float()\n"," \n"," return sum(tp), sum(fp), sum(tn), sum(fn) \n","\n"," \n","def train(model, iterator, optimizer, criterion):\n"," \n"," epoch_loss= 0\n"," epoch_acc = 0\n"," tp = 0\n"," fp = 0\n"," tn = 0\n"," fn = 0\n"," \n"," model.train()\n"," \n"," for batch in iterator: \n"," optimizer.zero_grad()\n"," \n"," predictions = model(batch.text).squeeze(1)\n"," loss = criterion(predictions, batch.label)\n"," #acc = binary_accuracy(predictions, batch.label)\n"," a, b, c, d = precision_recall_f1(predictions, batch.label)\n"," \n"," loss.backward() \n"," optimizer.step()\n"," \n"," epoch_loss += loss.item()\n"," #epoch_acc +=acc.item()\n"," tp += a.item()\n"," fp += b.item()\n"," tn += c.item()\n"," fn += d.item()\n"," \n"," return epoch_loss/len(iterator), tp, fp, tn, fn\n"," \n"," \n","def evaluate(model, iterator, criterion):\n"," \n"," epoch_loss = 0\n"," epoch_acc = 0\n"," tp = 0\n"," fp = 0\n"," tn = 0\n"," fn = 0\n"," \n"," model.eval()\n"," \n"," with torch.no_grad():\n"," \n"," for batch in iterator:\n"," predictions = model(batch.text).squeeze(1)\n"," \n"," loss = criterion(predictions, batch.label)\n"," \n"," #acc = binary_accuracy(predictions, batch.label)\n"," a, b, c, d = precision_recall_f1(predictions, batch.label)\n"," \n"," tp += a.item()\n"," fp += b.item()\n"," tn += c.item()\n"," fn += d.item()\n"," epoch_loss += loss.item()\n"," #epoch_acc += acc.item()\n"," \n"," return epoch_loss / len(iterator), tp, fp, tn, fn"],"execution_count":0,"outputs":[]},{"metadata":{"id":"NDoNGLrwHSLb","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch.optim as optim\n","device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","\n","optimizer = optim.Adam(model.parameters(),lr=0.001)\n","criterion = nn.BCEWithLogitsLoss()\n","model = model.to(device)\n","criterion = criterion.to(device)\n","\n","train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits((train_data,valid_data,test_data), batch_size=BATCH_SIZE,device=device)\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"HpwIDPiZHUh5","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":156},"outputId":"5ff0c8c4-5c1c-4837-ef3f-6c4e0aa2759d","executionInfo":{"status":"ok","timestamp":1546970632686,"user_tz":-330,"elapsed":961743,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}}},"cell_type":"code","source":["N_EPOCHS = 3\n","\n","for epoch in tqdm(range(N_EPOCHS)):\n"," model.word.embedding.weight.requires_grad=True\n"," if epoch>0:\n"," model.word.embedding.weight.requires_grad=False\n"," lr = 0.0001\n"," for param_group in optimizer.param_groups:\n"," param_group['lr'] = lr\n"," train_loss, train_tp, train_fp, train_tn, train_fn = train(model, train_iterator, optimizer, criterion)\n"," valid_loss, valid_tp, valid_fp, valid_tn, valid_fn = evaluate(model, valid_iterator, criterion)\n"," train_acc = (train_tp+train_tn)/(train_tp+train_fp+train_fn+train_tn)\n"," valid_acc = (valid_tp+valid_tn)/(valid_tp+valid_fp+valid_fn+valid_tn)\n"," valid_precision = valid_tp/(valid_tp+valid_fp)\n"," valid_recall = valid_tp/(valid_tp+valid_fn)\n"," valid_f1= (2*valid_precision*valid_recall)/(valid_precision+valid_recall)\n"," print('| Epoch: {} | Train Loss: {:0.2f} | Train Acc: {:0.2f} | Val. Loss: {:0.2f} | Val. Acc: {:0.2f} | Val. F1 {:0.2f} |'.format(epoch+1,train_loss, train_acc*100,valid_loss, valid_acc*100, valid_f1))"],"execution_count":46,"outputs":[{"output_type":"stream","text":["\n"," 0%| | 0/3 [00:00=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2018.11.29)\n","Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (3.0.4)\n","Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (1.22)\n","Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2.6)\n","Installing collected packages: torchtext\n","Successfully installed torchtext-0.3.1\n","Requirement already up-to-date: spacy in /usr/local/lib/python3.6/dist-packages (2.0.18)\n","Requirement already satisfied, skipping upgrade: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.1)\n","Requirement already satisfied, skipping upgrade: thinc<6.13.0,>=6.12.1 in /usr/local/lib/python3.6/dist-packages (from spacy) (6.12.1)\n","Collecting numpy>=1.15.0 (from spacy)\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/ff/7f/9d804d2348471c67a7d8b5f84f9bc59fd1cefa148986f2b74552f8573555/numpy-1.15.4-cp36-cp36m-manylinux1_x86_64.whl (13.9MB)\n","\u001b[K 100% |████████████████████████████████| 13.9MB 3.0MB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.18.4)\n","Requirement already satisfied, skipping upgrade: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.1)\n","Requirement already satisfied, skipping upgrade: dill<0.3,>=0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.2.8.2)\n","Requirement already satisfied, skipping upgrade: ujson>=1.35 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.35)\n","Requirement already satisfied, skipping upgrade: regex==2018.01.10 in /usr/local/lib/python3.6/dist-packages (from spacy) (2018.1.10)\n","Requirement already satisfied, skipping upgrade: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.9.6)\n","Requirement already satisfied, skipping upgrade: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.2)\n","Requirement already satisfied, skipping upgrade: msgpack<0.6.0,>=0.5.6 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.5.6)\n","Requirement already satisfied, skipping upgrade: cytoolz<0.10,>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.9.0.1)\n","Requirement already satisfied, skipping upgrade: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (4.28.1)\n","Requirement already satisfied, skipping upgrade: wrapt<1.11.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (1.10.11)\n","Requirement already satisfied, skipping upgrade: six<2.0.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (1.11.0)\n","Requirement already satisfied, skipping upgrade: msgpack-numpy<0.4.4 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.4.3.2)\n","Requirement already satisfied, skipping upgrade: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.22)\n","Requirement already satisfied, skipping upgrade: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.6)\n","Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n","Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2018.11.29)\n","Requirement already satisfied, skipping upgrade: toolz>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from cytoolz<0.10,>=0.9.0->thinc<6.13.0,>=6.12.1->spacy) (0.9.0)\n","\u001b[31mfeaturetools 0.4.1 has requirement pandas>=0.23.0, but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n","\u001b[31mcufflinks 0.14.6 has requirement plotly>=3.0.0, but you'll have plotly 1.12.12 which is incompatible.\u001b[0m\n","Installing collected packages: numpy\n"," Found existing installation: numpy 1.14.6\n"," Uninstalling numpy-1.14.6:\n"," Successfully uninstalled numpy-1.14.6\n","Successfully installed numpy-1.15.4\n","Requirement already satisfied: en_core_web_sm==2.0.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz#egg=en_core_web_sm==2.0.0 in /usr/local/lib/python3.6/dist-packages (2.0.0)\n","\n","\u001b[93m Linking successful\u001b[0m\n"," /usr/local/lib/python3.6/dist-packages/en_core_web_sm -->\n"," /usr/local/lib/python3.6/dist-packages/spacy/data/en\n","\n"," You can now load the model via spacy.load('en')\n","\n","Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (3.2.5)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.11.0)\n"],"name":"stdout"}]},{"metadata":{"id":"EopWaTjVW2lo","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch\n","from torchtext import data\n","from torchtext import datasets\n","import random\n","import nltk\n","import re\n","import numpy as np\n","from tqdm import tqdm\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","%matplotlib inline"],"execution_count":0,"outputs":[]},{"metadata":{"id":"LHeE87yJXbq_","colab_type":"code","colab":{}},"cell_type":"code","source":["SEED = 10\n","torch.manual_seed(SEED)\n","torch.cuda.manual_seed(SEED)\n","torch.backends.cudnn.deterministic = True #for reproducible results"],"execution_count":0,"outputs":[]},{"metadata":{"id":"7It70WIDXf3X","colab_type":"code","colab":{}},"cell_type":"code","source":["def clean_text(text):\n"," text = text.lower()\n"," text = re.compile('<.*?>').sub(\" \", text)\n"," text = re.sub(r\"i'm\", \"i am\", text)\n"," text = re.sub(r\"he's\", \"he is\", text)\n"," text = re.sub(r\"she's\", \"she is\", text)\n"," text = re.sub(r\"that's\", \"that is\", text)\n"," text = re.sub(r\"what's\", \"what is\", text)\n"," text = re.sub(r\"where's\", \"where is\", text)\n"," text = re.sub(r\"\\'ll\", \" will\", text)\n"," text = re.sub(r\"\\'ve\", \" have\", text)\n"," text = re.sub(r\"\\'re\", \" are\", text)\n"," text = re.sub(r\"\\'d\", \" would\", text)\n"," text = re.sub(r\"won't\", \"will not\", text)\n"," text = re.sub(r\"can't\", \"cannot\", text)\n"," text = re.sub(r\"[-()\\\"#/@;:<>{}+=~|.?,]\", \"\", text)\n"," text = text.split()\n"," return text\n","\n","TEXT = data.Field(tokenize=clean_text,lower=True, fix_length=800)\n","LABEL = data.LabelField(dtype = torch.float)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"hetNKN7xXicX","colab_type":"code","colab":{}},"cell_type":"code","source":["from torchtext import datasets\n","train_data, test_data = datasets.IMDB.splits(text_field=TEXT,label_field=LABEL,)\n","train_data, valid_data = train_data.split(split_ratio= 0.8, random_state=random.seed(SEED))"],"execution_count":0,"outputs":[]},{"metadata":{"id":"36xWKUorYYQw","colab_type":"code","outputId":"e0536fe9-6749-47dd-edef-2f0424f5a461","executionInfo":{"status":"ok","timestamp":1546011931951,"user_tz":-330,"elapsed":19074,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":89}},"cell_type":"code","source":["print('Number of training examples: {}'.format(len(train_data)))\n","print('Number of validation examples: {}'.format(len(valid_data)))\n","print(vars(train_data[0]))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Number of training examples: 20000\n","Number of validation examples: 5000\n","{'text': ['wow', 'this', 'movie', 'bored', 'the', 'pants', 'off', 'me', 'when', 'i', 'saw', 'it', 'bland', 'pointless', 'and', 'unmoving', 'apparently', 'ash', 'and', 'co', 'can', 'travel', 'through', 'time', 'with', 'the', 'help', 'of', 'the', 'spirit', 'of', 'the', 'forest', \"'princess\", \"mononoke'\", 'much', 'there', 'they', 'meet', 'a', 'dorky', 'kid', 'named', 'sam', 'and', 'the', 'plot', 'begins', 'so', 'tom', 'ash', 'and', 'huck', 'sam', 'get', 'high', 'with', 'nature', 'become', 'hippies', 'and', 'try', 'to', 'free', 'celebi', 'the', 'spirit', 'from', 'some', 'weirdo', 'hunter', 'guy', 'i', \"don't\", 'even', 'know', 'what', 'else', 'went', 'on', 'it', 'all', 'went', 'by', 'in', 'a', 'blur', \"ash's\", 'friends', 'were', 'hardly', 'in', 'it', 'and', 'all', 'the', 'fight', 'scenes', 'were', 'boring', 'after', 'saving', 'the', 'day', 'ash', 'and', 'his', 'infamous', 'friends', 'must', 'return', 'to', 'their', 'time', 'while', 'watching', 'sam', 'float', 'away', 'with', 'celebi', 'that', 'scene', 'was', 'just', 'creepy', 'oo', 'then', 'after', 'returning', 'to', 'their', 'time', 'ash', 'learns', 'that', 'his', 'new', 'friend', 'is', 'actually', 'his', \"rival's\", 'grandpa', 'and', 'i', 'think', 'that', 'is', 'it', 'pretty', 'retarded', \"isn't\", 'it', 'if', 'you', 'love', 'your', 'children', 'you', 'will', 'not', 'expose', 'them', 'to', 'this', '1', 'out', 'of', '10'], 'label': 'neg'}\n"],"name":"stdout"}]},{"metadata":{"id":"1GMrHuUxXlg-","colab_type":"code","outputId":"1b6e4687-abd9-4000-e73b-d3f4138a0fed","executionInfo":{"status":"ok","timestamp":1546011932567,"user_tz":-330,"elapsed":19531,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":379}},"cell_type":"code","source":["train_data_dist = [len(train_data[i].text) for i in range(len(train_data))]\n","valid_data_dist = [len(valid_data[i].text) for i in range(len(valid_data))]\n","test_data_dist = [len(test_data[i].text) for i in range(len(test_data))]\n","\n","sns.set(style='whitegrid')\n","f, axes = plt.subplots(1, 3)\n","\n","sns.distplot(a = train_data_dist,ax=axes[0], axlabel='Train')\n","sns.distplot(a = valid_data_dist,ax=axes[1], axlabel='Validation')\n","sns.distplot(a = test_data_dist,ax=axes[2], axlabel='Test')\n"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":19},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAe0AAAFZCAYAAAC173eYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xt8lOWd+P3PPaecJuQAmRDl4IgH\naJS2RFptLKiF0rJd+9vdVtgW+vT52T10sUd4lGb9CX0W8ER5WtG26rK+trXSuJbt9mBFu00rSyJo\npKhBUBBiwiGZYUJgcpzD/fwxuSeTZCYzSeZ03/N9v159lcnkTq7Bi/t7X9f1vb6XoqqqihBCCCGy\nninTDRBCCCFEYiRoCyGEEDohQVsIIYTQCQnaQgghhE5I0BZCCCF0QoK2EEIIoROWRL5p27ZtHD58\nGEVRqKurY+HCheH3Ghsb2bFjB2azmSVLlrBu3bqY1/h8PjZu3EhraytFRUU88sgjlJSUpOaTCSGE\nEAYTN2gfPHiQ1tZW6uvrOXHiBHV1ddTX14ff37JlC7t27aKyspI1a9awYsUKPB5P1GueffZZysrK\n+N73vkd9fT2vvfYan/jEJ2L+7ubm5uR8SmEYNTU1Kf8d0u9EpHT0OZB+J0aK1e/iBu2mpiaWLVsG\nwLx58+ju7sbr9WK322lra6OkpISqqioAli5dSlNTEx6PJ+o1DQ0NfP3rXwdg1apVU2o4hDp5uv5B\nJZu0fXK/N13ifT49//ebrFz9zOlk1PvdaPJZ4v/MWOKuabvdbsrKysKvy8vLcblcALhcLsrLy8e8\nF+ua06dP8/LLL7N27Vq+9a1vceHChUl9ICGEECIXJbSmHWkyVU+1a1RVxel0ctddd/HDH/6Qxx9/\nnHvuuWfca+M96ep5SknaLoQQYiLiBm2Hw4Hb7Q6/7uzspKKiIup7HR0dOBwOrFZr1GtmzJjB4sWL\nAbj55pvZuXNn3AYadbpI2j653yuEELks7vR4bW0te/fuBaClpQWHw4Hdbgdg1qxZeL1e2tvb8fv9\nNDQ0UFtbG/OaJUuWsG/fvvDXnU5nqj6XEEIIYThxR9qLFi2iurqa1atXoygKmzZtYs+ePRQXF7N8\n+XI2b97M+vXrAVi5ciVOpxOn0znmGoC1a9dyzz338Nxzz1FYWMiDDz6Y2k8nhBBCGEhCa9obNmwY\n8Xr+/PnhPy9evHjEFrBY1wAUFBTwyCOPTLSNQgghhEAqogkhhBC6IUFbCCGE0AkJ2kIIIYROSNAW\nQgghdEKCthBCCKETErSFEELHfvL8EX7XdCrTzRBpMuEyptnuhaZT4T9/6qYrMtUMkeO0fih9UKSS\nzx/kP/77XQA+LX0tJ8hIWwghdOp8d1+mmyDSTIK2EELolPuCBO1cI0FbCCF0yiVBO+dI0BZCCJ1y\ndUnQzjUStIUQQqdkpJ17JGgLIYROubp6ATCblAy3RKSLBG0hhNApbaRts8qtPFfIf2khhNCpSz2D\nAPgDaoZbItJFgrYQQuhUUA0F60AgmOGWiHQxXEU0gIs9g+TZzJluhpiCbdu2cfjwYRRFoa6ujoUL\nF4bfa2xsZMeOHZjNZpYsWcK6detiXuPz+di4cSOtra0UFRXxyCOPUFJSkqmPJURSBYOhoB1UQ382\nydq24RlupK2qKj/93dv85LdHMt0UMUkHDx6ktbWV+vp6tm7dytatW0e8v2XLFnbu3Mnu3bvZv38/\nx48fj3nNs88+S1lZGc899xwrV67ktddey8RHEiIlghGz4oGgjLZzgeFG2r6haaJBv3RgvWpqamLZ\nsmUAzJs3j+7ubrxeLwBtbW2UlJRQVVUFwNKlS2lqasLj8US9pqGhga9//esArFq1KgOfRojUCUZE\nbZ8/iNUiM4xGZ7yg7ZNgrXdut5vq6urw6/LyclwuFwAul4vy8vIR77W1tdHV1RX1mtOnT/Pyyy/z\n8MMPM2PGDDZt2kRpaWncNjQ3N0/pe1rfDz1kNNvOx/05epLI34tIH21NGyAQlGS0XGC8oC0jbMNR\n1YnfjLRrVFXF6XRy11138cMf/pDHH3+ce+65J+71NTU1477f3Nw87ve4Bk8N/ZwrEm5ztov3mY0o\n2x9SIkfafklGywmGW9P2+QOZboKYIofDgdvtDr/u7OykoqIi6nsdHR04HI6Y18yYMYPFixcDcPPN\nN3P8+PE0fQohUi8yaAdk21dOMFzQlrVs/autrWXv3r0AtLS04HA4sNvtAMyaNQuv10t7ezt+v5+G\nhgZqa2tjXrNkyRL27dsX/rrT6czMhxIiBSJnoWSknRtkelxknUWLFlFdXc3q1atRFIVNmzaxZ88e\nOjs7qampYfPmzaxfvx6AlStX4nQ6cTqdY64BWLt2Lffccw/PPfcchYWFPPjgg5n8aEIkjaqqI7LH\nJWjnBgMGbZkeN4INGzaMeD1//vzw+uLixYupr6+Pew1AQUEBjzzySGoaKUQGjc47k+nx3GC86XHJ\nHhdC5IDgqKgtI+3cYLigLdPjQohcEBy1q0K2fOUGAwZtmR4XQhifKiPtnGS4oC3Z40KIXDB6pC1B\nOzcYLmjL9LgQIheMXdOW6fFcYLigPegbnh4f3amFEMIoRq9hy/GcucFwQTtypC2n3gghjGrs9LgM\nUnKBoYO2dGIhhFGNLskvg5TcYMCgPTw9LtNFQgijkjXt3GS4imiDMtIWQkzStm3bOHz4MIqiUFdX\nN+K9xsZGduzYgdlsZsmSJaxbty7qNQsXLmTjxo20tLSEj4G98847ueWWW5La1jFBW5Jwc4Lhgras\naQshJuPgwYO0trZSX1/PiRMnqKur4+677w6/v2XLFnbt2kVlZSVr1qxhxYoVeDyeMddoJXa//e1v\nc+utt6asvdqatsWs4A+ocr/LEQYM2oGIP0snFkIkpqmpiWXLlgEwb948uru76e3tBaCtrY2SkhKq\nqqoAWLp0KU1NTXg8njHXeL3etLRXC9pWiwl/ICAziznCcGvakckZUtZPCJEot9tNWVlZ+HV5eTnd\n3d0AuFwuysvLR7zncrmiXuNyuQB4+umn+dKXvsS3vvUtPB5P0turTY9bLWZAcnhyheFG2nK+rBAi\nGdTR6dkTuOazn/0spaWlLFiwgCeeeIJHH32U++67L+712kl2ibzf2e0b+qWh2cVT77fR3Hxhwm3O\nlHifVU/S+VkMF7QjB9dyVJ0QIlEOhwO32x1+3dnZGU4kG/1eR0cHDocDq9U65pqKigqcTmf4a7fd\ndhubN29OqA01NTUx32tubh7x/qmzF+G3HRQV5nOxt4eZMy+jpuaahH5Ppo3+LHqWis8y3kOA4abH\nkZG2EGISamtr2bt3LwAtLS04HA4KCgoAmDVrFl6vl/b2dvx+Pw0NDdTW1ka9xm6387WvfY22tjYA\nDhw4wNVXX5309u7782lguAqkTI/nhoRG2tG2NGiybhuEjLSFEJOwaNEiqqurWb16NYqisGnTJp5/\n/nk8Hg/Lly9n8+bNrF+/HoCVK1fidDpxOp1jrgH44he/yDe/+U0KCgooLCzk/vvvT36DhwYoZpMC\ngF9yeHJC3KAdbRuEtqUBsmsbxOg1KL9sgRBCTMCGDRtGvO7p6QlPfS5evHjEvS/WNQA33ngjv/jF\nL1LTyCFajDabQhOmMtLODXGDdrRtEF6vF7vdnnXbIEY/aEonFkIYlTZIMZuHRtoys5gT4gZtt9tN\ndXV1+LW2pcFut0fdBtHW1kZXV1fUayC0DeKpp55i+vTp/J//839GXB/NRLIpR3fao8feIXCpLd5H\nzBg9Z0/que1CGIE2sWgxh0baksOTGyacPZ7ubRATyaYc9AWg/nT4tdM5j5rrqibc3nTQc/Zkptou\nDwpCDBs70pagnQviZo9H2wZRUVER9T1tG0Ssa2666SYWLFgAhLZBvPPOO0n7ICAF9IUQuUMds6Yt\n97tcEDdox9rSANm3DWLs+bLy5CnS7/Vjnfz3q+9L/xMpFVl7HCTxNlfEnR6Ptg1iz549FBcXZ902\niDGJaNKJRQY89txhOj295OdZ+MzNV2a6OcKghkfaoaAtI+3ckNCa9ugtDfPnzw//OZu2Qcj0uMgG\nM8sL6fT08ud3XPgDwXCikBDJNLymLYloucRQd5PRQVu2fIlMyLOZw3/u6fNlsCXCyCR7PDcZKmhr\nT55KaLZIRtoiIyKPhB30yY1UpIbKyIpoMj2eGwwVtIPhsn5D2ZSypi0yYETQjjjfXYhkCq9py5av\nnGKsoD3UZy1SIUhkkH/ESFuCtkiN4Kja4wGpPZ4TjBW0RyVmyJq2yITI0bUEbZEqw8uBCiaTgvtC\nX4ZbJNLBWEE7OPLJ0ydBW2TAyOlx6YMiNbTpcUVRMCnKmERcYUyGCtqjy/pJYobIBJ9Mj4s00IK2\nSQkNVEYXlxLGZKigra3pWEyyBUJkjm/E9Lj0QZEakedAmEwy0s4VhgrawdEjbenEIgNkpC3SQbvf\nmRQFkyL3u1xhqKA9uoC+jLRFJkQGbZ9s+RIpMmJNW6bHc4ahgnY4EU3WtEUGSSKaSIfIYlIyPZ47\nJnyedjYLypq2YWzbto3Dhw+jKAp1dXUsXLgw/F5jYyM7duzAbDazZMkS1q1bF/OajRs30tLSQmlp\nKQB33nknt9xyS8raHQiqI6YpZXpcpIrWy8IjbQnaOcFYQVuyxw3h4MGDtLa2Ul9fz4kTJ6irqxtx\nKM2WLVvYtWsXlZWVrFmzhhUrVuDxeGJe8+1vf5tbb701LW3XpsMVQjdVSUQTqTJipC1bvnKGoYO2\nnC+rT01NTSxbtgyAefPm0d3djdfrBaCtrY2SkhKqqqoAWLp0KU1NTXg8npjXpJNWDc1mNTPgC8hI\nW6TM8JYvRbZ85RBDrWmrQzE6nIgm64m65Ha7KSsrC78uLy/H5XIB4HK5KC8vH/PeeNc8/fTTfOlL\nX+Jb3/oWHo8npW3X1rC1k75kTVukSlBG2jnJkCNti9TiNRR1EiMI7ZrPfvazlJaWsmDBAp544gke\nffRR7rvvvrjXNzc3T+p7urz+oQaERtjtp8/S3Nw/gZZnt0T+XkR6jM0eD/V7RTvmUBiSsYJ2OHtc\nEtH0zOFw4Ha7w687OzupqKjA4/GMea+jowOHw4HVao16jdPpDH/ttttuY/PmzQm1oaamZtz3m5ub\no35Pe+cl+NU5iosKuNjrpbRsOjU1H07od2a7WJ/ZyLL5IWV09jiEDkmyWiRoG5mhpsfHnHojiWi6\nVFtby969ewFoaWnB4XBgt9sBmDVrFl6vl/b2dvx+Pw0NDdTW1sa85mtf+xptbW0AHDhwgKuvvjql\nbfeF17RD/7QkEU2kSnikjRIO2nJIkvEZcqRtMikoyEhbrxYtWkR1dTWrV69GURQ2bdrEnj176Ozs\npKamhs2bN7N+/XoAVq5cidPpxOl0jrkG4Itf/CLf/OY3KSgooLCwkPvvvz+lbdeCdp5VW9OWRDSR\nGqOzxwH8siRoeMYK2qOmiwKSPa5bGzZsGPF6/vz54anKxYsXj9gCFusagBtvvJFf/OIXqWlkFKOD\ntk8S0USKBEdlj4OMtHOBoabHR08X+WV6XKSZtk/bpo20ZcuXSJHoa9oStI3OUEFbmx4Pj7SlA4s0\n07Z4WSwmTIoEbZE6o7PHARmo5ABjBe3wk2foUHjpwCLdtOlws0nBbDZJIppImWhr2jJQMT5jBe2I\nkbZZ1rRFBoSDttmExWySRDSRMtFH2nLPMzpjBe3IkbZJkYpoIu18Q9PhFpOC2azI9LhIGTV8nvbw\nmrYUlDI+QwVtrYypaWi6SLY/iHTzBbSRtoLFZJIypiJltJ6lKArmoelx2a1gfIYK2oHh+SJJRBMZ\nMbymbcJsVsIjbyGSLkr2uBSUMj5DBe1wcRUFTCbJpBTppwVtk0nBYjYxIIloIkWC0da0JY/H8AwV\ntCMPljApJhlpi7QLr2mbQwUv/IGgnL4kUkKyx3OToYL28EhbwWySkn4i/cJr2kPT4yAZvSI1RmaP\nh/4ss4vGZ6ygPfqouqA6qWMdhZisEfu0h+6kkhwkUiEye1zra/KAaHwGC9pRCujLk6dIo8g1bbPs\nnRUpFG2kLYloxmesoB2Mlk0pN0yRPoGIk+a0PigjbZEK4UEKkYMU6WtGZ6igrY4qrgLSiUV6aQ+J\npqHiKiBBW6RGtIpoUgXS+AwVtEeMtGV6XGSA9pAYSoaUdUaROtFP+ZL7ndEZK2hHGWnLk6dIJ21N\n0Ryxpi0jbf3Ytm0bq1atYvXq1bzxxhsj3mtsbORzn/scq1at4rHHHkvomn379nHttdempK3aSNs0\ndEASyHJgLrBkugHJpMVnBSKSgOTJU6SPVtzCNCJoS1U0PTh48CCtra3U19dz4sQJ6urquPvuu8Pv\nb9myhV27dlFZWcmaNWtYsWIFHo9nzDX19fUADAwM8MQTT1BRUZGS9qoMD1LCfU2CtuEZf6QtnVik\nkTbSNiky0tabpqYmli1bBsC8efPo7u6mt7cXgLa2NkpKSqiqqsJkMrF06VKampqiXuP1egH48Y9/\nzBe+8AVsNltK2htRtVnKmOYQQwXtEafeSDalyAAte1wxgUnWtHXF7XZTVlYWfl1eXk53dzcALpeL\n8vLyEe+5XK6o17hcLk6ePMnRo0f59Kc/nbL2Rt/iKn3N6Aw2PR5tTVuePEX6aDdNsyLZ43o3mcJM\n2jX3338/995774Svb25uTvj9vt4+ANra2/Bc8of+3Haa5uZLE/69mRDvs+pJOj9LQkF727ZtHD58\nGEVRqKurY+HCheH3Ghsb2bFjB2azmSVLlrBu3bq41+zbt4+vfOUrHDt2LKkfJnr2uNwwRfqEp8cl\nEU13HA4Hbrc7/Lqzs5PS0tKo73V0dOBwOLBarWOusdlsvPfee2zYsCH8tTVr1vD000/HbUNNTU3M\n95qbm0e8n/f7/4aLXubOmUve+R44dgLHzJnU1CxI/ENnyOjPomep+CzjPQTEnR6PTM7YunUrW7du\nHfH+li1b2LlzJ7t372b//v0cP3583GtSmZwR7dQbWeMR6eQPBDEpI5OD5MFRH2pra9m7dy8ALS0t\nOBwOCgoKAJg1axZer5f29nb8fj8NDQ3U1tZGvebyyy/n97//Pc8++yzPPvssDocjoYA9UcPZ4xGJ\nt/KAaHhxR9qxEi3sdvuI5AwgnJzh8XhiXqMlZzz88MNJ/zAjKgRJNqXIgGBQxWwOPQtLRTR9WbRo\nEdXV1axevRpFUdi0aRPPP/88Ho+H5cuXs3nzZtavXw/AypUrcTqdOJ3OMdeky8hTDWU5MFfEDdpu\nt5vq6urway3Rwm63R03OaGtro6urK+o1LpeLo0eP8o1vfCM1QTtyTVv2LYoM8AeD4VGPFrwlaOuH\nNqWt6enpCU99Ll68OLyda7xrRvvDH/6QvAZGUFUVBakAmWsmnIiW7uSMiSRmnD17AYCOznNcvBhK\nzDh67B0Cl9om9DvTRc+JGHpueyoFAsMjbZkeF6kUVEMBG2TLVy6JG7SjJWdo69HpSM6YSGLGn0+/\nBW97qZpZBZYeaD+D0zmPmuuq4n3MtNNzIkam2q6HBwV/IIhlKGtcEtFEKqmqylDMlsTbHBI3ES1a\nooXdbgeyLzkjMjEjfCi8rPGINAoE1HDNcQnaIpVUxo60JWgbX9yRdrTkjD179lBcXJx1yRnBiBJB\nsqYtMiEQjBxpS3EVkTqRI22zTI/njITWtEcnWsyfPz/852xKztAS0Uxy6o3IEH9AJc9mBpDiKiKl\nVHV4Wjx8v5MDkgzPUGVMpfa4yLRARPa4bPkSqRSMMtKWvmZ8xgrawYh92lpihqxpizTyB1Qskj0u\n0kCNyB6XoJ07jBm0ZaQtMiQQCIanxbU1bbmRilRQVRWTlj0uQTtnGCpoRzuqTkY5Ip0CQRWLaXRF\nNDlPWyRfMDg80tYGKlLG1PgMFbQj17TNiiSiifQbMdI2Sx8UqaOqavjBEEJT5DLSNj6DBm1kelyk\nXTCoElSJsk9bRtoi+SIT0WAoaAekrxmdsYJ2lDVtGeWIdAkMbbeRNW2RDpGJaBCqdS99zfgMGrQj\nT72RTqxH27ZtY9WqVaxevZo33nhjxHuNjY187nOfY9WqVTz22GMJXbNv3z6uvfbalLZZe0AcnT0u\nN1KRCqFENJkezzUTPjAkmw2XMZWRtp5Fnsd+4sQJ6urqRhTw2bJlC7t27aKyspI1a9awYsUKPB5P\nzGtSeYZ7JO1YxMh92iZFkiFFakSdHpegbXjGGmlHOU9b1rT1J9YZ7sCIM9xNJlP4DPfxrtHOcLfZ\nbCltt9bXtJE2gMVilhupSInIimgQWpaRvmZ8xgraUc7TllGO/rjdbsrKysKvtfPYgahnuLtcrpjX\nnDx5kqNHj/LpT3865e3W+po5IqPXKjdSkSLB4OiRtqxp5wJDTY9HZo+HC+hLRTTdS/cZ7pDYMaCj\nv+dCT+gM9wsXumh9v2foq0G8Pb26OFY0EUb5HEYwZqRtUvAHgkMHiSjjXCn0zFhBOzh2y5c8eepP\nrDPcPR5PWs5wh/HPcYfoZ4qfcXvhv85R6ZjB3DmhUX/BO+9gNim6PTs9kp7PgJ+sbH5ICY4KzpFl\nc60Wc6aaJVLMUNPjwxXRFKn7rGN6OsM9knYsojliTdtqkSlLkXyRAxSN1u+kvxmb4UfaErT1J9YZ\n7p2dndTU1GTVGe6RtKUYS8SatsVsoifgS3tbhLFpS4GjK6KBBG2jM1bQjixjKh1Y16Kd4a5NVWbT\nGe6RwoloY0baUqVKJFe0kbYsCeYGQ02PR275Ck+PSwcWaRKIkj3e2+djYFD6oEguLWiP3vIFErSN\nzlhBO0oZU59Mj4s0CRdXMY+8kQZVNdw3hUiGQMS9TjNcNldmdozMcEFbG+QoioLFLEfVifQJjCpj\nChE3Unl4FEkUXtOOkj0uI21jM1TQHl1AX1EUznf3Z7BFIpcMr2lHmbL0yehHJI/2gDi6jCnIA6LR\nGSpoB0bV4jWZFCmuItJmOHs8ykhbRj8iiaJmj8uWr5xgqKA9uhKQ2aTIKV8ibaKNtC1Dfx6UG6lI\nonFH2j7pa0ZmqKAdDKpEFu8zmxRJABJpM3zK1/A/q+FtODI9LpIncnurxix9LScYL2hHdGKZHhfp\nNHzK18jiKiBTliK5xt3yJWvahmasoK2Oni4ySdAWaeOPUsZUG/0MSiKaSCJt2W/0/Q7kAdHojBW0\ng1HWtAMStEV6RBtpS3KQSIWoI23Z8pUTjBW01dHny8qatkgff5Q17fBIW26kIokCUQ8MkaCdCwwV\ntFVVHfHkaTJJNSqRPkHZpy3SJPpIW2Z1coGhgnYwqEKUJ0/Z9iXSIdpI2yIV0UQKRCtjKjsVcoPh\ngrY8eYpMib6mrSWiSR8UyTO85Wv4a3JIUm4wVNAOjAnassYj0id69rg8OIrk0xJsR1ZEky1fucBw\nQVuJ+ERah/ZLJxZpMN5IW6YsRTJFK65ikQfEnGC4oC0jbZEp42aPy/S4SKLhRLThr0n2eG4wVtAO\nBMdkj4N0YpEe2kjbaolIRNP2aQdkpC2SZ/zztOV+Z2TGCtqji6sM3TBlelykg7aWaDZFme2RkbZI\novHKmA7KUoyhGS5oR+QAYVZkpC3SR8vatUSMtLUHRymuIpIpGKW4ijarIyVzjc0wQVsdKqKiRHny\nlJG2SAdtytIapfa4JKKJZNL6WmT2uJYAOTAofc3ILJluQLIEo3RiWdMW6aT1sxFbviQ5SFe2bdvG\n4cOHURSFurq6Ee81NjayY8cOzGYzS5YsYd26dVGvWbhwIYcOHeKhhx7CYrFgs9l4+OGHKS8vT1o7\nxx9pS18zMsOMtP3jFNCXkbZIB3+0LV8mmbLUi4MHD9La2kp9fT1bt25l69atI97fsmULO3fuZPfu\n3ezfv5/jx4/HvOapp57ioYce4qc//Skf/vCHefbZZ5Pa1mCURDRFUbCYTQz4/En9XSK7GGakrWXu\njig2INmUIo1Od3oBePnQaQryQv+0LDLS1o2mpiaWLVsGwLx58+ju7qa3txeAtrY2SkpKqKqqAmDp\n0qU0NTXh8XjGXOP1ennkkUeA0LJdR0cHNTU1SW2rVpo5cpACkGczy0jb4BIK2tGmfzTZMmUUbbpI\nRtoinaKtM8qDo3643W6qq6vDr8vLy+nu7gbA5XKNuFeVl5fT1tZGV1fXmGtcLhd2u52XX36ZrVu3\ncuWVV3L77bcn1Ibm5uaE3j/xXg8Anq7ztL7vDb+vEODipd64Pycb6KGNiUrnZ4kbtCOnf06cOEFd\nXR319fXh97ds2cKuXbuorKxkzZo1rFixAo/HE/Uabcpo9uzZPProozz77LP84z/+Y1I+SCDK9Lis\naYt0CoaLq0SriCZ9UG9UdeKnA0Zes2TJEj7+8Y+zfft2nnjiiYTudeONyJubm8PvdwVa4ZUuKqbP\nYO6c4YeJ4hPHGfAFkj6yT7bIz6J3qfgs4z0ExF3TjjZl5PWGnuwip4xMJlN4yijWNY888gizZ88O\nTxnNnDkzGZ8PiDHK0bLH5YYp0iD6SFv2zuqFw+HA7XaHX3d2dlJaWhr1vY6ODhwOR9RrKioqeOml\nl4DQOvOKFSuSPhKLVlwFwGY1Sf6EwcUN2m63m7KysvBrbfoHok8ZuVyuca95+eWX+dSnPoXb7U54\nyigR2hR41DKmMj0u0kCrB20alRxkMilSXEUHamtr2bt3LwAtLS04HA4KCgoAmDVrFl6vl/b2dvx+\nPw0NDdTW1ka9xm63s3PnTt5++20ADh8+jNPpTGpbh3fLjPy6zWqWoG1wE05ES/eUUaJrPB5vKGOy\nt7eH1vdbATjvGQTg5MlWmvM8E253qul5TUfPbU+VYFAdMTWusZgUmR7XgUWLFlFdXc3q1atRFIVN\nmzbx/PPP4/F4WL58OZs3b2b9+vUArFy5EqfTidPpHHMNwNatW/nud7+L2WwmPz+fhx56KKltjTXS\nzrOZGfQHQ8cUR+mLQv/iBu1Y0z/R3tOmjKxWa8wpo+XLl4enjHbu3Bm3gYmu8Zx2eeFX5ygutjN3\nzmwAVOtFOH6SmVWXU1NzddzflU56XtPJVNuz/UEhEONGaTabZHpcJzZs2DDidU9PT7ivL168eEQ+\nT6xrAK6//np+/vOfp6aRRE9klaBXAAAgAElEQVS8hdBIG0LLMfk2w2wOEhHiTo/Hmv6B7JoyCowz\nPS7Z4yIdYo1uzCZFypiKpIqWeAuQNxS0pSqaccV9FIs2ZbRnzx6Ki4uzaspovCQgmZoU6RBzetxs\nwifrjCKJohVXgeGgLXu1jSuh+ZPR0z/z588P/zlbpowCgbGd2CQjbZFGsafHFfoGpEqVSJ7hpMeR\nX+/sChWDkeUY4zJMGdPhCkHDX5ORtkinoBp9pG01m2S6UiRVrEQ0re699DfjMlDQjrZPW45FFOkT\nDKpj1hghdFRnIKjKjI9ImmjnaQNYtTO1ZTnGsAwXtCOfPC3hp06ZmhSpFwgGo06PW2T0I5IsECN7\nPDzSlqBtWIbZE6Blj5ujnC8rSRn6o4d696ONl4gGodFPUYE1Jb9b5JZoRxFDxAOiBG3DMuBIe/hr\n0oH1SU9HJEaKteXLYgl9TfqhSJZYa9qRD4jCmIwz0o6yxiMdWJ8SrXcPmT8iURMMqgRVxh1py/S4\nSJZYxVUssqZteMYZaQfGTheZTAomRZERjs7opd59pPDuhfGCtvRDkSSxztOWB0TjM8xIO1Y2pcWs\nSAfWuUwckZhIydTI7xkYypsY6O8P177X9Hj7AHjjrSNccuUn1P5sle2lZHNF7JG29oAoeTxGZZig\nrW2nUUYnZljkqDq9iVXv3uPxpKXePYxf8x7G1l+/1DsI/3EGe1ERc+fMHfG953s7effMWa5wXkXN\ngsqEfn820nO9/MnK1oeUoZgddZACMOCTHTNGZZzp8WD0CkEWswRtvdFLvftI2pntkogm0kHbLRM7\nEU1G2kZlmJF2cJw1HrlZ6kusevednZ3U1NRkTb37SNqZ7bJPW6RDuIzpqGGXJN8an2GCdrSKaBCa\nLvL2yVOn3kSrd69NVWZLvftIWiLkuNnjciMVSSJlTHOXYabH/VEODIGhs4x9gUklMwmRKL+MtEUa\nxTrlyypLMYZnmKA9vOVm5NetUn9cpEEiQVumLEWyxMrhMUtfMzzDBO1YW77MUmxApIEWtM1RDwyR\n0Y9IrpgjbVmKMTzDBO1o52mDTE2K9PD7o+dUQMSNVPqgSJLYgxQZaRudcYJ2MHoikExNinRIaE1b\n+qBIkpinfJkUFEUeEI3MOEE75r5FmZoUqeeLcsqcxmKRI2JFcg1v+Yr+kCj7tI3LOEF7nOIqIEFb\npFYgoUQ0uZGK5AgGoo+0QWpTGJ3xgrZMj4sM8I830pbZHpFk4ZF2tMRHsxySZGTGC9qSiCYyYLxE\nNOmDItliFVcBKd1sdAYK2rEODNG2fMnUpEgdfzD2SFtRFGwWkxziIJImGFSjTo2DBG2jM07QDsia\ntsic8Q4MAcizmemXkbZIkkF/APPoSlJDtOlxqQJpTMYJ2nH2LUrQFqkU3vIVY/hTkGehf0BG2iI5\n+gcCWC2xgrYJVR3uk8JYjBO0Y2TvWiURTaTBeFu+IBS0e/slaIvkGPAFwrOIow0PVCRoG5FxgnbM\nU2+GMndlalKk0HiJaBAK2n0DfpmyFEkxMOiPOdK2Sl0AQzNO0A5Ev2lKLV6RDoP+UP+KNfopyLMQ\nCKr45OAakQT9g+ONtCX51siME7S1U75GDXSsVjMAvf2+dDdJ5BBtJidm0M4PHV3fJ+vaYoq0h7/x\n1rRBBipGZaCgHT0RLc8a+oiynihSScuZ0AqpjFaQJ0FbJIc27R0vaEsejzEZJmgHY1REs1lkpC1S\nTxvVWGLcSCVoi2TpjzOrIyNtYzNM0I6ViDY8PS43S5E64aA9zpo2SD8UU9cfd6QtybdGZpigPXw0\n4sivm00KeTazjLRFSg3GCdqF+VZARtpi6sL5EzI9npMME7RjHQoPUJhnoUdGOCKFhhPR4qxpSz8U\nU9Q/EOpr1hh9TYK2sRkmaAeCKiYlegH9wnyrjLRFSmnba+JOj8tIW0yRNj1uMZujvi+nyhmbcYJ2\nIIgpRi3eogILPX1ysxSpM+DzYzYpUR8aQRLRRPJoiWiy5Ss3GSZo+4NquKjAaIV5VvyBID6/dGKR\nGoO+YMxRNoSWaECCtpg6bctXrDVtsxwFa2iGCdrBgBqz7vOl3kEAGW2LlBkYDMRczwYpriKSJzzS\njvGQKFUgjc0wQdsXiD3SsUlVNJFiA75AeIQTzfCWL+mDYmriTY9bhwpKaQlrwlgsmW5Asly41B/z\nPZtURRMpNuALkGeNnhgEcKDlHCAj7Wy3bds2Dh8+jKIo1NXVjXivsbGRHTt2YDabWbJkCevWrYt6\nzcKFCzl79izf+c538Pv9WCwWHn74YSoqKpLSxvD0eKyR9lAw75cDQwzJOCNtfzDmGo9WFa1HRjki\nRQbHOSoRwDbUNyVoZ6+DBw/S2tpKfX09W7duZevWrSPe37JlCzt37mT37t3s37+f48ePx7zm+9//\nPnfccQdPP/00y5cv56mnnkpaO+OOtIfud9LXjCmhkXa0J0lNtjx9+gPBmGs8Mj0uUik4dIDDeGva\nVgnaWa+pqYlly5YBMG/ePLq7u+nt7QWgra2NkpISqqqqAFi6dClNTU14PJ4x13i9XjZt2kReXh4A\nZWVltLS0JK2d8UrmygOiscUdaevh6TMYVPEH1HHWtGV6XKTOYJybKITqB1gtJnr65MExW7ndbsrK\nysKvy8vL6e7uBsDlclFeXj7iPZfLFfUal8tFYWEhZrOZQCDAM888w1/+5V8mrZ3H2y4AsRPRtH4o\na9rGFHekHe3p0+v1Yrfbs+bpU3vyjDVdpI20ZXpcpEK8uuOaPJsZrwRt3VBVdUrXBAIB7r77bm68\n8UZuuummhK5vbm6O+373xUsAdHScobsrdv3x813dcX9eJmVz2yYqnZ8lbtB2u91UV1eHX2tPkna7\nPerTZ1tbG11dXVGvcTqdAOGnT20qfar64+xbzJNDQ0QKJRy0rWYZaWcxh8OB2+0Ov+7s7KS0tDTq\nex0dHTgcDqxW65hrtCW/73znO8ydO5e77ror4TbU1NTEfK+5uZmamhpsDQ2AjyvmzomZ/Gg/chTF\nbB3352WS9lmMIBWfZbyHgAlnj6f76TORJ0+PNxSMB/r7aH2/dcz3dA29f7K1neZm70SanlJ6ftLU\nc9uTLd5Z2po8q5nz3f0EgrFrCojMqa2tZefOnaxevZqWlhYcDgcFBQUAzJo1C6/XS3t7OzNnzqSh\noYHt27fT1dU15hq73c6vfvUrrFYrX//615PeTp8/VDI31vQ4QH6eWbLHDSpu0I729Kk9Sabj6TOR\nJ89TZy8C5ygrKWbunFljvm/axX72HzlGccl0amo+lNDvTTU9P2lmqu3Z+qAwEOd8Y02ebWiZps/H\ntCJbytslJmbRokVUV1ezevVqFEVh06ZNPP/883g8HpYvX87mzZtZv349ACtXrsTpdOJ0OsdcA/DM\nM88wMDDA2rVrgdAy4ebNm5PSTn8giMmkYBrnwS/fZuFST29Sfp/ILnGDdrSnT7vdDmTP02e8fYvh\nNW2ZmhQpEO+wEI0E7ey3YcOGEa97enrCD6iLFy+mvr4+7jUAP//5z1PTQEIj7fFG2RAq5tM34EdV\n1Zj18IU+xQ3a0Z4+9+zZQ3FxcdY8ffbHOV82vOVLtkCIFBjwaQ+N8abHQ//cvH2DQFGqmyUMyh8I\nxky61RTkWQiqMOgPjlv0R+hPQmvao58k58+fH/5zNjx9DsSpxWsxK5gU6JWRtkiBhEfaQzdPb6/0\nQzF5vgQCcX5e6P3+Ab8EbYMxRBlTLeEi1tOnoihYrWZ6JHtcN/RQ0EczkS1fgGz7ElPi8wcpKrCO\n+z2RR8GW2PPS0SyRJoYoYxpvehxCo5w+2aetC3oo6BNpIIH+ByPXtIWYDFVVx63+qCmwyalyRmWs\nkfZ4tZ+tJjmaUydiFfSB7ConqRn0J77lC2SkLSYvXiEpTf7QSFuqohmPIYJ2IiMdm8WMe0D2yOpB\nrII+EL2cZCoK+iSyvUz7nhPvhSpUeTznaVUvxvz+rkuhh8bjJ9tobo79fdksW7fd5YpEZ3Uip8eF\nsRgiaMc7FB6GM8j7BvzY46wHieySiXKS8fahR+5V/93hA0A3l82sZHZlccxrii/20/i2Vi/ggwm1\nI5voubbAZGXbQ0oi9zoYTkTrkwIrhmOQNe3xy5iCnPSlJ5Mp6DPeNZMpJzkRWoWqxLPHB1PSDmF8\nidzrYHhNu19G2oZjiKAdb8sXyElfelJbW8vevXsBxi3o4/f7aWhooLa2NuY1qSwnqQkEtKA9/rJL\n/lAi2iUJ2mKSErnXARTky/S4URliejzRNW2QzF09iFXQp7Ozk5qamqwo6BPJNxS0zXFupGazicJ8\nC91eCdpichIdaRcXhCruXeqRvmY0hgja8fZpw/DUpDx56kO0gj7a+mI2FPSJFAiE1s/jTY8DlBTl\ncbFnINVNEgYVHmnHCdpvvRdaKnrjhJu/TXmrRDoZYno8sUS00Hsy0hbJ5k9wehxgmt1Gt3dwUsl1\nQiSeiCbT40ZliKA9MBhAgXFPvQkfGiKJaCLJwkE7zugHQiPtQFCVh0cxKQMTTETrkxwewzFE0PYF\ngpjNyrin2WhJQFL3WSSbfyLT4/bQWmO3rDWKSehPcHrcZFLIt5llpG1AxgjavgBmU7wSkqEnT8nc\nFckWPt84gSMQtTrQ3V5Z1xYT15/g2e0wfDynMBZjBG1/cNypcZCRtkgdfyCY0Ho2RIy0JYNcTEIi\nSbeagjwL/YOB8JZEYQzGCNoJ3DTzZI+sSJFQ0E7sn9K0otBIWzLIxWQkuk8bhkuZXpR7nqEYI2j7\ng3Gnx21WM4oihzWI5PMH1ISDdml4elxupGLiEjnRUBMO2tLXDMU4QTvOSNukKBTlW2WkLZLO7098\npN2i7Z9915XKJgmDChdXmcBI+4LkTxiKcYJ2Aid3FRfaZE1bJN1E1rS1G2mvJAiJSfD5Eq8JoJUy\nvXBJgraRGCJo+/2BhIK2vdAqhzWIpAoGVQLBxKfHC/OtKIrkVojJ0WoCxFsOhOFSpuc8PSltk0gv\n3QftQCBIUAVTAp3YXmBl0B8MHyQvxFQN+hPfggOh/bP2AiuXZMZHTIIWtOPtlgEomxbKn2jv8Ka0\nTSK9dB+0B/3aYQ2JTY+DHI0okmfQl9ixnJGKC2309PnCR3oKkajhkXYC97siG2aTQlvnpVQ3S6SR\n7oO2duNLdHockFGOSJrhE+YSW9OG4YfH8919KWmTMC6t+l4CdXwwKQqlxXm0dXgJBqXWvVEYIGgn\nPj2p3SxlPVEky0SnxwGKhx4eO7t6U9ImYVxa9b3xSjZHKp+Wz6AvgOuCPCAahQGCduIj7fbO0NrO\ny4faU9omkTu0kXa8s7QjFReFHh47PXIjFRPjDyS2U0ZTVpwPQFuHTJEbhWGCdiKJGYVDWyB6+mS7\njUiOQZ9WoWri0+MuGWmLCfInULI5klY2t8Mjfc0odB+0w4kZCYx0tDXtnj6ZHhfJMZmR9rShkfZp\nl2zFERPjDwQTOphGoz0gdkrQNgzdB+0JJaIVaGvakogmkmNgEmva04psWC0mTp3tTlWzhEH5A+qE\npse1pZg3jksFPqPQfdDWpicT6chFBaGRttQfF8mi9b+JBG1FUZg+LZ+2Tm/4eiESoSWiJaoo34JJ\nUbjYI/c8o9B90B4eacf/KGaTQlG+hR4J2iJJwlu+JrCmDTC9tIBgUKX+pWOpaJYwqIkmoimKgr1Q\nzlwwEv0H7UDixVUAigpC9cdl36JIhsmMtAFmlISyet3d/UlvkzCuiSaiQWg5pm/AL5UgDUL/QXsC\na9oQ2iMbVFW65eQbkQTajTCRoxIjzSgtAMAt+2fFBPiDE1vTBtmtYDTGCdoJjnTsQ+vabqlGJZIg\nHLQTWJ6JNH1opC1V0cRETGakPVzMR/qaEeg+aPv9iSeiwXA25bnz8tQppm6ya9pWi5kSuw33hX5U\nVZZqRHxBNXSi3ERH2loCbtdFWYoxAt0H7YlOj5fYQyffnHHLyTdi6sIHhkxwehxgRkkBA74A7gty\nMxXxBYfOl5noSLswPxS0PRK0DcE4QTvBkU6pFrSlsIVIgoFJJqLB8Lr2SdmvLRIQGEqenUhxFQht\n+wLouiR5PEag+6A9OIEtXxCaHlcUOOuWoC2mbrLZ4zC8rn3yjARtEd/QRpmE73WawqHpcY/sVDAE\n3QftiU6Pm00K04psnHbJ9LiYuuGR9sRGPxAx0j5zMaltEsYUHmlPcHq8IM+CgkyPG4UBgvbEEtEg\nNEV+sWcQrxQcEFM0nIg28X9K9gIreVYzp2SkLRKgBe2JJqKZFIWCfAtdlyRoG4EBgvbEtnzB8Lp2\nu4y2xRRNZXpcURSml+Rzxt1D/4CcPCfGN9lENAhlkHu6ZaeCEeg+aPsnOD0OUD60lth6VqYlxdQM\n+AKYFGVSN1IITZGrKrSek74oxjfZ6XGAonwrg/4gPf3ycKh3ug/aEy1jCqGtNgCnZC1RTNGgLzCp\n9WzNcDKa9MVssG3bNlatWsXq1at54403RrzX2NjI5z73OVatWsVjjz0W95qf/OQnVFdX09OTnKTX\nyU6PAxQOZZB7pJiP7iUUtLO5I0/kwBBN2bR8FKD5WGdS2iBy18BgYEJLM6MNJ6PJunamHTx4kNbW\nVurr69m6dStbt24d8f6WLVvYuXMnu3fvZv/+/Rw/fjzmNb/85S85f/48Docjae0LTGF6PHyutlRF\n0z1LvG+I7JQnTpygrq6O+vr68Ptbtmxh165dVFZWsmbNGlasWIHH44l6TSo68kSzxwGsFhMlxXmc\n7+5DVVWUCe57FEIz6AtgnURhFU35tHxMJkVG2lmgqamJZcuWATBv3jy6u7vp7Q1VTmxra6OkpISq\nqioAli5dSlNTEx6PZ8w1Xq+XZcuWYbfb+fWvf5209k12nzZAiT0UtGWrq/7FDdrROrLX68Vut2dF\nRw6fpz3BKcoZJfkcvzSAq6sPR3lh0tojcsuALzCpamgai9nE5RV2Tp29SDCoTnptXEyd2+2muro6\n/Lq8vJzu7tAMiMvlory8fMR7bW1tdHV1jbnG5XLhdDon1Ybm5uaY72lB+9Klblrfn9jOl15vaC37\n8Nsnuaywa1JtS7bxPqvepPOzxA3a0Tqyy+XCbrenpSPH+8s47wl1wNOn2ye0tqgEQ9sfGvYf4qrL\n8ifcrmTQc6fVc9uTacAXpCAv7j+jcTkvm0ZbxyU6u3qZOb0oSS0TUzWZTOupZmfX1NTEfO/4b/YD\nUF5Wxtw5lRP6uZWDfv7nSAtBc9G4vyNdmpubs6IdyZCKzzLe/XXCd5t0d+Tx/jKam5spLCoGBnDO\nnTuhUcoAXbx75n0KSmdSUzNv0u2bLD132nS0fdu2bRw+fBhFUairq2PhwoXhjtzY2MiOHTswm80s\nWbKEdevWxbwGQnkUDz74IAcPHqSoKHlBUVVVBn1TW9OG4SWek2e6JWhnkMPhwO12h193dnZSWloa\n9b2Ojg4cDgdWq3XMNRUVFSlp31QS0fJtFuwFVs6dl+lxvYt7t4nWkbVOGasjj3dNsvmGjqqb6LRi\nWXFor/bpTtmrnW2yPSFIo5XQtU4xaM+QDPKsUFtby969ewFoaWnB4XBQUBBKFJw1axZer5f29nb8\nfj8NDQ3U1tZGvcZut6ekfVPZpw2hDPIzrh6CQdmrrWdxR9q1tbXs3LmT1atXj+mUkR155syZNDQ0\nsH37drq6umJek2w+/+QSgUq0oC0FVrJOrDwKyI6EIM1k8ylGmy4Z5Flh0aJFVFdXs3r1ahRFYdOm\nTTz//PN4PB6WL1/O5s2bWb9+PQArV67E6XTidDrHXAPwox/9iMbGRlwuF3/3d3/Hhz70Ie6+++4p\ntW8qI20InXDY2dUnyzA6FzdoR+vIe/bsobi4OCs6ss8fnNRIx2YxU1RglZF2FoqVRwHpSwhKxFRK\nmEYqyrdSkGeRkXYW2LBhw4jXPT094aWgxYsXj9g5E+sagK9+9at89atfTWrbplJcBaCyvJB32y7w\nxnG3BG0dS2hNe3SnnD9/fvjPme7IPn9w0ltuSu15nHZ56R/wkz/FZCKROplICEok0e71Q4cB6O/v\npfX91in9vqI86PD0sr/pVfJt2VvzSBIQM2f4lK/JBe05M4vhMLx+tJNPfnRuElsm0kn3kcoXmELQ\nLg4F7XOeXq6ompbklonJipUT4fF40pYQFC/Rrrm5mauvXQC/6aB0WjFz58ya9O8CaL9wBvdFFyUO\nJ9fNmzGln5Uqek6enKxsekiZ6ki71J5HcaGNP7/rIhAITjmBUmSG7v+reXt9DPqCk7p22lCVoA7J\nqMwq4yX3ZENCkGZgCoeFjFY5VCvgWGt27KEV2WeqQVtRFGZX2unp83FKzl3QLd2PtAPBIGbz5D5G\ncdFQ0Pb0JrNJYopi5VF0dnZSU1OT8TwKzVRO+BqtamiN8chJD38z5Z8mjCg8Pa5Mvr/NnF7EkZMe\n3j7lYd6s0iS1TKST/oN2QJ30Gs80CdpZK1oehTZVmek8Ck2/log2hYpomqICK9OKbLxx3CWV0URU\nUx1pw/DD4dunPHzm5iuT0i6RXrqeHldVlUBQndBhIZEkaIup0M7Ankrt8UgzpxfRPxiQbYgiquAU\nE9EgVIN8WpGNt095ktQqkW66DtpajYDJ7pPNt5kpyDNL0BaToo20kxW0q2ZoU+Tnk/LzhLEkY6St\nKAoLrijH1dWH+4Kc+KVHug7a/sDUig0oikJhvpXTLu+UtwiJ3NM/ODTSTlIWbuS6thCjJSNoAyy4\nIlTnQEbb+qTroD1cIWjyH6O0OA+fP8i58zLaFhPTP5DckXb5tDzyrGbelqAtogiEutuUpscBunsG\nAHih6dTUGiQyQtdB26914imUkbxsRmhb0Jsn3HG+U4iRtJF2MhLRIDTzM3N6IWfP99B1sT8pP1MY\nR0BNzkjbUVaISVHk8BCd0nfQnmItXoDLK0JTkhK0xUQNr2mbk/Yzw+vaMnUpRtGWAydyBHE0FrOJ\nirICXBf6wsmUQj90HbQDgak/eZZPyyffZubN425Z1xYTEs4eT2JlqeF1bUlGEyP5/EODlCTVBVBV\neLftwpR/lkgvXQft8Eh7Cp1YURTmzJzG+e5+3nlfqlGJxCU7exzAUV6IxazIurYYY3ikPfX+NnNG\nqALfkVPycKg3ug7a2kjbMsU1nquHKgPt+/OZKbdJ5I5w9ngSg7bFbOKqWaWcON1Nb78vaT9X6J8v\nSdPjQPiUr6OnZKCiN7oO2v4kFBsAmD3TTlGBlf2HT8sUuUjYQBIrokW6YUElwaDKnw6dTurPFfrm\nD6iYFDApUw/aRfmhCnxHTp7H55/c2Q0iM3QdtLWR9lTXeMwmE4sXVOLu7pdC+iJhfQN+bBZTUm6i\nkZZ9ZA4mk8LvGk/KQ6QI8wVUbFYzSpL629yZxfT2+znynkyR64mug7Y/ScUGAGzW0F/FwSPnpvyz\nRG7oHwyQZ0t++f7pJQV87PoqTp65yJ9eb0/6zxf65B8K2snivKwEkHue3ug6aAemWBEt0uzKYhQF\nXnnzrIxuREL6B/0U5CXvJhrpy5+pxmY1s+vXLeG1c5HbfP7kBu3LKoqwWkz88fV2uefpiK6D9nBx\nlal/jHybhbkzp3G8vVvKSIqE9A+kZqT9QtMpDh3r5Pp507lwaYBH6g8l/XcI/fEHVPKsybtlm00m\n5sws5mLPIO93XErazxWppe+gnYTiKpEWXesA4BcN7ybl5wljG0jhSBvgg1dXYDErHDrmkmQhQb8v\nGN5mmCzOqmkAHGyRKXK90HXQDiQ5aFfNKOLq2aU0H+2k65KUkRSxBYMqg/4g+SkYaWsK8ix8wDkd\nb59P1rYFgWBy9mhHmjtzGgrwyltnk/pzReroOmgns9iAZub0QoJBlSf+882k/UxhPINDfS/PlrqR\nNsCHrqnApCjs+eO7su6YwwJBFVVNftDOz7Mwy2HnnfcvcMYt57jrga6DdrKOqot09ewyTAoca5Wi\nAyK2QV+o7xWkcKQNUFxo46rZpbR1ePnX/3orpb9LZK9B31BNgCQUVhntmrllAPyxWWZz9EDXQduf\npKPqIhXkWZgzcxquC328f072bIvotDXmVI+0Aa6fNx2QQ21y2XDQTv4t+8rLSrCYTfx633sEApI7\nke10HbTDa9pJfvq8Zk7oyfMPr7Ul9ecK4xgcOrwhPy+1I22AyvJCZpTm03r2It3egZT/PpF9BoaC\ndjJ2yoxms5pZcEUZ3j4fTbK2nfV0HbT94X3ayf0YzsumkW8z80LTKbx9Uv9ZjNXvC41IzrpTfyax\noijMn1tOUIU/HZIpzFyUyulxgIVXVQDwzN6jslMhyxkkaCe3I1vMJj58jYOefj+/3f9eUn+2MIbe\ngdCNLT8N0+MAV88uRVFk9idXDQ49JKZiehygtDiP666cTluHl6d+00IwKEmP2UrXQVtbfkn29DjA\ndfOmYzGb+K8/nZAOLMbo04J2GqbHAQrzrcydOY0T7d20Sn38nJPqkTbAjddVcXmFnV/ve4+nX3g7\nZb9HTI2ug3aqpschtM5z9exSLvX6ePO4JACJkXoH0zvSBrh2KMv396++n7bfKbJDKte0NXk2Mytu\nnMu0Ihu/+MO7HGuVypDZSNdBe2Bo7cWWxNJ+kRZcUQ7Az/YeDSe9CQERI+0Ub/mKdEXVNMqK8/hd\n0yk8F6X4Ty5JZfZ4pII8C7fVzEYFHvj3VyXxMQvpOmhrN85kFtGPVDWjiHmzSnj7lIdf/EFKm4ph\n6V7ThtAN+29XzGdgMMAPfn6I3n5JkswVw2vaqZse11zusPORD8zE3d3P/7vrQMp/n5gYfQftwSB5\nVnPSzzOOtORDlzOjJJ+f/u5tXnu7I2W/R+hLute0NWpQZU5lMa8f6+RL393L937WzCtvnZX9tQY3\nkKaRtmbRfAcVpQW8834Xb8sBSllF10G7fzCY8uIWhflWbls8B5Oi8Nhzh2V0I4DQmrZJAZslvf+E\nTCaFlbVX8NHqmeTbzNLa+1oAABQ8SURBVPzx9Xa2PnWQ//tfXuR424W0tkWkz2Aa1rQjmRSFmz90\nGQDPvHg0Lb9TJEbXQbtvMJiW6cmK0gIWXVuB+0If/9/u1yWbXNA3ECTPZkFJ4SxPLGaTiRsWVPLF\nFfP53G1X8wFnOV2XBrj70X3sfeWUPFgaUDqyx0e7bIadWQ47f37HJbOMWUS3QXvAF8AfSE8ZSYAb\nPlDJwqtm8Mpb53jhlVNp+Z0ie/Wm6YFxPIqiUFleyK01s/mLWic2i4lH/+Mwq+99ng0/eJmW985n\ntH0iecLT4ynYKTOej11fhcVs4ns/a6ZVyjpnBd0GbW/vIAB51vSsKZpNJhbNd2C1mHjq1y14LvbL\nOmKOCgRV+gdTeyznRF1RNY2/uuUqPnxNBVXTizj2fhd1P9ov5yQbRKqLq8RSUVbI0g9fjrfPxz2P\n/g+PPXeY8919aW2DGEnHQTs0BZjO0U5RvpUbFlTSPxjg//ruXj5f91t2v3hMpstzTE+fD1WF/LzM\njrRHK7Hn8bGFl/FXt1zFZ5fMw2ox8cBPXuWN465MN01MUe9A6H5nTXMOBcD8K8q5tWYWgUCQF5pO\n8bXtDXKYUgbpNmhf0kbaaZ6i/PA1Fdww30FRgRWrxcQze4/ywE9epcPTm9Z2iMzR9q5m00h7tFkO\nO5/8yFwCQZUt/3ZACmXoXOfQ/aW4yJaR3/8B53T+919Wc/MHL+NSr4/1P3iZh59+jfbOSxlpTy7L\n3rtOHNpBHukO2oqi8NHrqvjodVX0D/p5fv8pmt48y2tvd/DXt17FB6+qoKKsgJnTi6Je33r2IheH\nHjiEPr1/LnSjKivOy3BLxjdnZjGf/OgcXnyllc1PvsK2f6rFeVlJppslJuHc+V7MpvTOLI6mKAof\nvLoCi9nEoXc6efnQaV556xyrl19DWXE+H3CWc1mFfdyfcbTVw1l3D9PS1GYj0m/QHgp8mRzt5Nss\n/K9b5vHu+xc49E4n9S+9Q/1L72AyKaxefi2rl18zIrv4mb1H2f3iMQDW3DqDmkw1XEzJqaHa39NL\nCjLckvjmXV7KrTVB/vu1Nr65449MLy3gUs8gc6um8befvJaa+ZXh733vdDcvHWilqMDK/1o6D3uh\njUAgyKP/8WeCKnzUKTkcmaCqKh2eXgrzTBnZrTBa9ZXT+YCznOPtF2hobucnz4fqlFstJu78y2pW\n1jqjtvOd97v4fx7ZB8Da2+T+N1m6DdqXejMz0h7NpChcO7cM5+XTeONdN/0Dft47080ze4/i6upl\nzacXUD4tn5cPtYcDNsDBY15WfSaDDReTdupsNwDTS/Mz3JLEzL+iHJvVzMEj5+gf8FNUYOWd1i42\nP/kKf1Hr5Muf+QDvvN/Fln87QN9AKEv51//zHquWXUNDc3v4IeXYSSsfXRxM215hEXKp10ffgJ/K\n0uy5XSuKwtWzy7i8wk5bp5e+fj+vH+vkx//5JvvfOMsNCyr58LUVDAwGeO9MN/0Dfn62d3i/90uH\nuvn8SjUrHkL0JqFesG3bNg4fPoyiKNTV1bFw4cLwe42NjezYsQOz2cySJUtYt25dzGvOnj3L3Xff\nTSAQoKKigocffhibbXJrNNr0eH6KSphOlM1i5oYFoVHLB6+p4Nf/8x4vHXyfPx06zZxKOydOd2O1\nmPirpfP406HTvHOml3Pne2JOo+e6bOxzmlNnL1KYZ6IwzdXQpuLKy0u48vLhqXHXhT5eOtjKb/ef\n5NUj5+i6NICqwrLFc7jUO8irb3fw1G+OADD/ijIGBgOcPHORPX88zuc/cU2mPkbKje5DkTLV7zo8\noTPbC/Ky72GpMN/KtXNCB9nMm1XCH15r480Tbt484eap34z8XkWB5R+ZQ0+/j8Y3znLidDdXzSrN\nQKv1LW4vOHjwIK2trdTX17N161a2bt064v0tW7awc+dOdu/ezf79+zl+/HjMax555BG+8IUv8Mwz\nzzB37lyee+65STdcO56wID/7bpzFhTZWLbuWWxbNojDPwvH2bhxlhfzV0nlUlBVy/bzpALzQdCqj\n7cxW2drnADwX+zl3vhdHqVXXo4SK0gLu+MQ1LLxqBq4LfdisZlZ+7AqunVvGDQsq+cInr2X9F2t4\nZP0tfOKGOdx2w2zs+Sae2XvMsJnD2drvtCTXwiwM2pGKC218dsk8vrRyAcsWz2bBFeVcf9UMPrF4\nNp+6aS5rP72Aa+aUUVIUygX5998eyXCL9SluxGtqamLZsmUAzJs3j+7ubrxeL3a7nba2NkpKSqiq\nqgJg6dKlNDU14fF4ol5z4MABvvvd7wJw66238m//9m984QtfmHCjj5w8z4GWc1SVWSm1Z2cykNmk\nUH3ldKqvnI6qjpwGmjerlJcPtfOb/zlJ1Qw704psVJYXkp9nxucLMugPMOgLUpBnocRuQ1EUgkGV\nQFAd+v8gwaCKqu00U4b/L/L3aH9UFIXwVxXQXkXGnPG+Z/T3XeoL0HWxP+bPmgqzSYnZ54CM9blA\nIIjrQh9P/aYFgOvmZP96djwWs4mPf+hyahdehsk08j9giT2P/gE/x1q7gFD+xmc+UsbPXz7Ppidf\n4Su3X0dpcR6KEloiCvWT4f/Xvhb5nskU+bWhr6NgMg1fa1IUVBV8/iC+QICBwdD/UMBqNmG1mPD5\ng/QPBjCbFWwWM1ZL6Os2qxmzSSGohv5tBIMqQVUN/1vR/jzgC+DzBbFaTOTZzOTZzEwrskXtd729\noYCZqX7n7QuNSgGKsmyLYSzFhTaunVvOtXPLo74/d2YxFjMcPeXhtbc7mF6Sjz8QxGIO/ffQzpIw\nmRQsZhNmU3Y/HPf0ByZ9Gtq0ItuEH/7jBm232011dXX4dXl5OS6XC7vdjsvlory8fMR7bW1tdHV1\nRb2mr68vPEU0ffp0XK6J7x89eOQc/zJ08sxHr7XrYrQzuo0Ws4mrL8+npbWPR//jzxlq1RT959mU\n/ejiruP87yj9B8hInwO474km3hg6V/2q2aUsmlfEef+kflTWGR2wY/H2B7jxupm88tY5HvjJqylu\nVXoV5Jm5Itgxpg91d4fyFzLR795t6+LunfvwB1Sunl3KjBJj1IMwm00smF3Am6f6+O6/vpLp5iTH\nnsndDz990xX80+c+OKFrJjy3rKoT7zjRrkn05zQ3N494bQY2f2FWxFf0Warx87XT+XxtpluRnZ58\n0seJEycoKwutlV26dImWlpbwKCcRU+lzMLbf/fVH8vnrj8wa8bUKmz773mRVXBXazvOphbPifKc+\nPfmkZ0y/m6hk97t7V10+6juM0ef+5mPT+ZuPZboV2cA/5r95PHGDtsPhwO12h193dnZSUVER9b2O\njg4cDgdWqzXqNYWFhfT395Ofnx/+3vHU1MimgFzU2NhISUlJ+L9/b28vt9xyC3a7nfb29pT2OZB+\nl6ti9TtI/b0OpN+JxMTNbKitrWXv3r0AtLS04HA4sNtDT9yzZs3C6/XS3t6O3++noaGB2tramNd8\n7GMfC3/9xRdf5OMf/3iqPpfQMelzIhOk3wk9UNQE5m62b9/Oa6+9hqIobNq0iSNHjlBcXMzy5ct5\n9dVX2b59OwCf/OQnufPOO6NeM3/+fDo7O7nnnnsYGBjgsssu4/7778dqtab2Ewpdkj4nMkH6nch2\nCQVtIYQQQmRedm/8E0IIIUSYBG0hhBBCJ7KvnFiCxitzmWnvvPMO//RP/8SXv/xl1qxZE7Ok4a9+\n9Sv+/d//HZPJxB133MHnP/95fD4fGzdu5MyZM5jNZu6//35mz56dlnY/9NBDNDc34/f7+Yd/+Aeu\nv/56XbQ73bK5702WXvvsVOilv+utvxmpL2VlH1F16MCBA+rf//3fq6qqqsePH1fvuOOODLdoWE9P\nj7pmzRr13nvvVX/605+qqqqqGzduVJ9//nlVVVX1e9/7nvqzn/1M7enpUT/5yU+qFy9eVPv6+tS/\n+Iu/ULu6utQ9e/aomzdvVlVVVfft26d+4xvfSEu7m5qa1K985Suqqqqqx+NRly5dqot2p1s2973J\n0mufnQq99He99Tcj9aVs7SO6nB4fr8xlptlsNp588skR+zIPHDjAJz7xCSBU0rCpqYnDhw9z/fXX\nU1xcTH5+PosWLeL111+nqamJ5cuXA/Cxj32M119/PS3tXrx4MT/4wQ8AmDZtGn19fbpod7plc9+b\nLL322anQS3/XW38zUl/K1j6iy6DtdrvDVYtgZJnLTLNYLOTnjzyyMVpJQ7fbPaYs4uivm0yh83MH\nBwdT3m6z2UxhYSEAzz33HEuWLNFFu9Mtm/veZOm1z06FXvq73vqbkfpStvYRXQbt0VQd7VqL1daJ\nfj1Vfv/73/Pcc89x3333JdSObGl3puTC5zTyf3u99Xc9/d1Gk61/r+PJtj6iy6A9XmnVbKSVNITh\n8ofRPoP2de1J2ufzoarqlM9/TtS+ffv48Y9/zJNPPklxcbFu2p1Oeut7k5UL/+310N+N0N+y8e81\nUdnYR3QZtMcrN5iNopU0/OAHP8ibb77JxYsX6enp4fXXX+eGG26gtraWF154AYCGhgY++tGPpqWN\nly5d4qGHHuLxxx+ntLRUN+1ON731vcky+n97vfR3I/S3bPx7TUS29hHdVkSLVjowG7z11ls8+OCD\nnD59GovFQmVlJdu3b2fjxo1jShq+8MIL7Nq1C0VRWLNmDbfffjuBQIB7772XU6dOYbPZeOCBByZ0\nutVk1dfXs3PnTpxOZ/hrDzzwAPfee29WtzsTsrXvTZZe++xU6Km/66m/GakvZWsf0W3QFkIIIXKN\nLqfHhRBCiFwkQVsIIYTQCQnaQgghhE5I0BZCCCF0QoK2EEIIoRMStDPgoYceYu3atdxxxx1cd911\nrF27lrVr1/LLX/4yoeufeOIJ/vjHP6a2kSKrffGLX+T3v//9iK/19/ezePFizp49G/WatWvX0tjY\nyNtvv82//Mu/jHm/tbWV2267bdzf29fXx4svvgjAyy+/zI9+9KNJfgKRS6Z6z9O89tprtLe3p6iV\nOpGUY0fEpLS1takf//jHM90MoUN79uxR161bN+Jrv/rVr9Q777wz5jVr1qxR9+/fH/P9U6dOqbfe\neuu4v/e1115T169fP7HGCjFkqve8jRs3qq+88koSW6Q/uj1P24h27txJe3s7Z86c4Z577qG/v5/t\n27djs9no7+9n06ZNVFdXs3HjRmpqarjpppv46le/ys0338wbb7xBT08Pjz/+OJWVlZn+KCLFPvWp\nT/Hggw/S1dUVPlDil7/8JZ///Od56aWX+Nd//VdsNhuBQICHHnqIWbNmha89cOAA3//+99m9ezev\nv/46mzZtory8nOrq6vD3nDhxgk2bNmE2m/F6vXzzm99k8eLF/PM//zMXL17koYce4qqrrqKxsZHt\n27dz+PBhHnjgASwWC4qicN9993HVVVexdu1abrrpJg4dOsSpU6f42te+xu233572vy+RnQYHB9m8\neTNtbW309PRw++238+Uvf5mjR4+yefNmrFYrAwMD3HXXXfT29vLiiy9y5MgR/vmf/5mPfOQjmW5+\nRsj0eJZpb2///9u7g5Cm2ziA499mkzyUFttlu4iyIFBTB6FdChUNFMH6Y5empBjiIUfaX9YkHDHR\nHdrJQIzU/fVQBDJjaYfw5hZ4KPCgoQfx4KGFazLB3PA9hOP19bXL25tb/j6n7Xme/Xie8bDf/s9/\nex58Ph8FBQVEIhH6+vrw+Xw0NTUxPDx8qP3q6io3b95kcnKSS5cuMTMzcwy9Fr9bVlYW1dXVBAIB\n4Md+x0tLS1RUVBCNRvF6vWiaxrVr15icnDwyjsfjobu7m/Hx8QN7WofDYTo7OxkfH6e3txev18uZ\nM2e4d+8eV69eRVXVA3FUVcXhcKBpGnfv3sXlciXrtre3GRkZwe128/z581/8Toh0Njo6itlsRtM0\nXr16hd/vZ2VlhZcvX1JdXY2maQwNDbG5ucmNGze4ePEijx49OrEJG0CutFPM5cuXOXXqFAAGgwGP\nx8POzg5bW1tkZ2cfan/+/HksFgsAJpOJSCTyW/srjo+iKLhcLu7cucP09DR1dXVkZmZiMBjo6elh\nb2+PL1++UFJScmSM5eVlrFYrAGVlZWiaBoDRaMTj8eD1etnd3f3pvIpGo3z9+pWioiIArly5woMH\nD5L1+x+wJpOJb9++/edxiz/Hhw8fCIfDhEIhAOLxOGtra9TU1OB0OllfX+f69euyOvM3krRTjF6v\nTz5WVRWXy0V5eTlzc3O8ePHiUPuMjIwDz/dkV9oTo6ioiO/fv7O6uorf7+fp06fs7u5it9uZmpoi\nNzeXiYkJFhcXfxpHp/ux4JZIJJJlT548oba2FkVR+Pz5M+3t7Ue+fv9L5r5/zsHTp08fWSdOtszM\nTO7fv09VVdWhujdv3hAMBnn9+jWBQACPx3MMPUw9sjyewsLhMBaLhUQiwezs7LEdBi9S161bt3j2\n7BlZWVlYLBZisRg6nQ6z2czOzg7v37//6bzJz8/n48ePAMzPzyfL9+cewNu3b5MxdDod8Xj8QIyz\nZ89iNBr59OkTAMFgkOLi4l86TvFnKi0tTd7SSyQSuN1uotEoY2NjhMNhKisrcbvdybn1b/PvpJGk\nncLa2tpobm6mvb2dhoYGNjY2GBsbO+5uiRRSX1/Pu3fvUBQFgJycHOrq6lAUBbvdTmtrK6FQ6Mjf\nOjx8+JD+/n7a2tqIxWLJ8paWFlRVpbW1FavVSnZ2NgMDAxQWFrKwsIDD4TgQZ3BwkMHBQWw2GxMT\nEzx+/Pj/G7T4Y9hsNvR6Pbdv36axsZELFy5w7tw58vLysNvt2Gw2Ojo66OrqAn4cjel0Og/93fEk\nkVO+hBBCiDQhV9pCCCFEmpCkLYQQQqQJSdpCCCFEmpCkLYQQQqQJSdpCCCFEmpCkLYQQQqQJSdpC\nCCFEmpCkLYQQQqSJvwDE03d2iIU0YgAAAABJRU5ErkJggg==\n","text/plain":[""]},"metadata":{"tags":[]}}]},{"metadata":{"id":"Zy_iTThEXzjR","colab_type":"code","colab":{}},"cell_type":"code","source":["TEXT.build_vocab(train_data, max_size=25000, vectors=\"glove.6B.100d\")\n","LABEL.build_vocab(train_data)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"mqVeGRU0YSpI","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch.nn as nn\n","import torch.nn.functional as F"],"execution_count":0,"outputs":[]},{"metadata":{"id":"1T8S07AVY-I0","colab_type":"code","colab":{}},"cell_type":"code","source":["\n","class CNN(nn.Module):\n"," def __init__(self,vocab_size, embedding_dim, n_filters, filter_sizes, output_dim, dropout):\n"," super().__init__()\n"," \n"," self.embedding = nn.Embedding(vocab_size, embedding_dim)\n"," self.embedding.weight.requires_grad = False\n"," self.conv_0 = nn.Conv2d(in_channels=1, out_channels=n_filters, kernel_size=(filter_sizes[0],embedding_dim))\n"," self.conv_1 = nn.Conv2d(in_channels=1, out_channels=n_filters, kernel_size=(filter_sizes[1],embedding_dim))\n"," self.conv_2 = nn.Conv2d(in_channels=1, out_channels=n_filters, kernel_size=(filter_sizes[2],embedding_dim))\n"," self.fc = nn.Linear(len(filter_sizes)*n_filters, output_dim)\n"," self.dropout = nn.Dropout(dropout)\n"," \n"," def forward(self, x):\n"," #x = [sen_len, batch_size]\n"," x = x.permute(1,0)\n"," #x = [batch_size, sen_length]\n"," embedded = self.embedding(x)\n"," \n"," # embedded - [batch_size, sent_len, emb_dim]\n"," embedded = embedded.unsqueeze(1)\n"," #embedded = [batch size, 1, sent len, emb dim]\n"," \n"," conved_0 = F.relu(self.conv_0(embedded).squeeze(3))\n"," conved_1 = F.relu(self.conv_1(embedded).squeeze(3))\n"," conved_2 = F.relu(self.conv_2(embedded).squeeze(3))\n"," \n"," #conv_n = [batch size, n_filters, sent len - filter_sizes[n]]\n"," \n"," pooled_0 = F.max_pool1d(conved_0, conved_0.shape[2]).squeeze(2)\n"," pooled_1 = F.max_pool1d(conved_1, conved_1.shape[2]).squeeze(2)\n"," pooled_2 = F.max_pool1d(conved_2, conved_2.shape[2]).squeeze(2)\n"," \n"," #pooled_n = [batch size, n_filters]\n"," \n"," cat = self.dropout(torch.cat((pooled_0, pooled_1, pooled_2), dim=1))\n","\n"," #cat = [batch size, n_filters * len(filter_sizes)]\n"," \n"," return self.fc(cat)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"LqJezieMZSIc","colab_type":"code","colab":{}},"cell_type":"code","source":["INPUT_DIM = len(TEXT.vocab)\n","EMBEDDING_DIM = 100\n","N_FILTERS = 100\n","FILTER_SIZES = [3,4,5]\n","OUTPUT_DIM = 1\n","DROPOUT = 0.5\n","BATCH_SIZE = 128\n","\n","model = CNN(INPUT_DIM, EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"6DkugEsaZru9","colab_type":"code","outputId":"f6a6e01e-d6f8-4ee6-ffec-54f6c1d99843","executionInfo":{"status":"ok","timestamp":1546011934704,"user_tz":-330,"elapsed":20648,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":139}},"cell_type":"code","source":["pretrained_embeddings = TEXT.vocab.vectors\n","model.embedding.weight.data.copy_(pretrained_embeddings)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([[ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n"," [ 0.0000, 0.0000, 0.0000, ..., 0.0000, 0.0000, 0.0000],\n"," [-0.0382, -0.2449, 0.7281, ..., -0.1459, 0.8278, 0.2706],\n"," ...,\n"," [ 0.3581, -0.3348, -0.4465, ..., -0.7128, -0.1006, -0.2799],\n"," [ 0.3896, 0.0446, -0.3506, ..., 0.1727, 0.0145, -0.9884],\n"," [ 0.0994, -0.7076, -0.0949, ..., -0.0898, -1.1580, -0.1628]])"]},"metadata":{"tags":[]},"execution_count":24}]},{"metadata":{"id":"GCoIXAZfZsj6","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch.optim as optim\n","device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","\n","optimizer = optim.Adam(model.parameters(),lr=0.01)\n","criterion = nn.BCEWithLogitsLoss()\n","model = model.to(device)\n","criterion = criterion.to(device)\n","\n","\n","train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits((train_data,valid_data,test_data), batch_size=BATCH_SIZE,device=device)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"QSzKWu6HZ4iP","colab_type":"code","colab":{}},"cell_type":"code","source":["def binary_accuracy(preds,y):\n"," \n"," rounded_preds = torch.round(torch.sigmoid(preds))\n"," correct = (rounded_preds==y).float()\n"," acc = correct.sum()/len(correct)\n"," return acc\n","\n","\n","def precision_recall_f1(preds, y):\n"," rounded_preds = torch.round(torch.sigmoid(preds))\n"," tp = y*rounded_preds\n"," fp = (y==0).float()*rounded_preds\n"," tn = (y==0).float()*(rounded_preds==0).float()\n"," fn = y*(rounded_preds==0).float()\n"," \n"," return sum(tp), sum(fp), sum(tn), sum(fn)\n"," \n"," \n"," \n"," \n"," \n","def train(model, iterator, optimizer, criterion):\n"," \n"," epoch_loss= 0\n"," epoch_acc = 0\n"," tp = 0\n"," fp = 0\n"," tn = 0\n"," fn = 0\n"," \n"," model.train()\n"," \n"," for batch in iterator: \n"," optimizer.zero_grad()\n"," \n"," predictions = model(batch.text).squeeze(1)\n"," loss = criterion(predictions, batch.label)\n"," acc = binary_accuracy(predictions, batch.label)\n"," a, b, c, d = precision_recall_f1(predictions, batch.label)\n"," \n"," loss.backward() \n"," optimizer.step()\n"," \n"," epoch_loss += loss.item()\n"," epoch_acc +=acc.item()\n"," tp += a.item()\n"," fp += b.item()\n"," tn += c.item()\n"," fn += d.item()\n"," \n"," return epoch_loss/len(iterator), epoch_acc/len(iterator), tp, fp, tn, fn\n"," \n"," \n","\n","def evaluate(model, iterator, criterion):\n"," \n"," epoch_loss = 0\n"," epoch_acc = 0\n"," tp = 0\n"," fp = 0\n"," tn = 0\n"," fn = 0\n"," \n"," model.eval()\n"," \n"," with torch.no_grad():\n"," \n"," for batch in iterator:\n"," predictions = model(batch.text).squeeze(1)\n"," \n"," loss = criterion(predictions, batch.label)\n"," \n"," acc = binary_accuracy(predictions, batch.label)\n"," a, b, c, d = precision_recall_f1(predictions, batch.label)\n"," \n"," tp += a.item()\n"," fp += b.item()\n"," tn += c.item()\n"," fn += d.item()\n"," epoch_loss += loss.item()\n"," epoch_acc += acc.item()\n"," \n"," return epoch_loss / len(iterator), epoch_acc / len(iterator), tp, fp, tn, fn"],"execution_count":0,"outputs":[]},{"metadata":{"id":"sPxLvKHKabWc","colab_type":"code","outputId":"2a31e49f-3e4c-41e0-817a-9dfef619726c","executionInfo":{"status":"ok","timestamp":1546012023566,"user_tz":-330,"elapsed":85046,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":176}},"cell_type":"code","source":["N_EPOCHS = 3\n","\n","for epoch in tqdm(range(N_EPOCHS)):\n"," model.embedding.weight.requires_grad=True\n"," if epoch>0:\n"," lr = 0.001\n"," for param_group in optimizer.param_groups:\n"," param_group['lr'] = lr\n"," model.embedding.weight.requires_grad=False\n"," train_loss, train_acc, train_tp, train_fp, train_tn, train_fn = train(model, train_iterator, optimizer, criterion)\n"," valid_loss, valid_acc, valid_tp, valid_fp, valid_tn, valid_fn = evaluate(model, valid_iterator, criterion)\n"," valid_precision = valid_tp/(valid_tp+valid_fp)\n"," valid_recall = valid_tp/(valid_tp+valid_fn)\n"," valid_f1= (2*valid_precision*valid_recall)/(valid_precision+valid_recall)\n"," print('| Epoch: {} | Train Loss: {} | Train Acc: {} | Val. Loss: {} | Val. Acc: {} | Val. F1 {} |'.format(epoch+1,train_loss, train_acc*100,valid_loss, valid_acc*100, valid_f1))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["\n"," 0%| | 0/3 [00:00