├── Question Answering Memory Network.ipynb ├── README.md ├── images └── architecture.png ├── memorynetwork.py ├── model.h5 ├── presentation └── presentation slides.pdf └── trained_models ├── gru_32.h5 ├── gru_32_32.h5 ├── gru_64.h5 ├── lstm_32.h5 ├── lstm_32_32.h5 └── lstm_64.h5 /Question Answering Memory Network.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Question Answering System using End to End Memory Networks" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Imports" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "name": "stderr", 24 | "output_type": "stream", 25 | "text": [ 26 | "Using TensorFlow backend.\n" 27 | ] 28 | } 29 | ], 30 | "source": [ 31 | "import keras\n", 32 | "from keras.models import Sequential, Model\n", 33 | "from keras.layers.embeddings import Embedding\n", 34 | "from keras.layers import Input, Activation, Dense, Permute, Dropout\n", 35 | "from keras.layers import add, dot, concatenate\n", 36 | "from keras.layers import LSTM, GRU\n", 37 | "from keras.utils.data_utils import get_file\n", 38 | "from keras.preprocessing.sequence import pad_sequences\n", 39 | "from keras import backend as K\n", 40 | "\n", 41 | "from functools import reduce\n", 42 | "import tarfile\n", 43 | "import numpy as np\n", 44 | "import re\n", 45 | "\n", 46 | "import IPython\n", 47 | "import matplotlib.pyplot as plt\n", 48 | "import pandas as pd\n", 49 | "%matplotlib inline" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "### Helper functions" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 2, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "def tokenize(sent):\n", 66 | " return [ x.strip() for x in re.split('(\\W+)?', sent) if x.strip()]\n" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 3, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "def parse_stories(lines, only_supporting=False):\n", 76 | " '''Parse stories provided in the bAbi tasks format\n", 77 | " If only_supporting is true, only the sentences\n", 78 | " that support the answer are kept.\n", 79 | " '''\n", 80 | " data = []\n", 81 | " story = []\n", 82 | " for line in lines:\n", 83 | " line = line.decode('utf-8').strip()\n", 84 | " nid, line = line.split(' ', 1)\n", 85 | " nid = int(nid)\n", 86 | " if nid == 1:\n", 87 | " story = []\n", 88 | " if '\\t' in line:\n", 89 | " q, a, supporting = line.split('\\t')\n", 90 | " q = tokenize(q)\n", 91 | " substory = None\n", 92 | " if only_supporting:\n", 93 | " # Only select the related substory\n", 94 | " supporting = map(int, supporting.split())\n", 95 | " substory = [story[i - 1] for i in supporting]\n", 96 | " else:\n", 97 | " # Provide all the substories\n", 98 | " substory = [x for x in story if x]\n", 99 | " data.append((substory, q, a))\n", 100 | " story.append('')\n", 101 | " else:\n", 102 | " sent = tokenize(line)\n", 103 | " story.append(sent)\n", 104 | " return data\n" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 4, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "def get_stories(f, only_supporting=False, max_length=None):\n", 114 | " data = parse_stories(f.readlines(), only_supporting=only_supporting)\n", 115 | " flatten = lambda data: reduce(lambda x, y: x + y, data)\n", 116 | " data = [(flatten(story), q, answer) for story, q, answer in data if not max_length or len(flatten(story)) < max_length]\n", 117 | " return data" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 5, 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):\n", 127 | " X = []\n", 128 | " Xq = []\n", 129 | " Y = []\n", 130 | " for story, query, answer in data:\n", 131 | " x = [word_idx[w] for w in story]\n", 132 | " xq = [word_idx[w] for w in query]\n", 133 | " # let's not forget that index 0 is reserved\n", 134 | " y = np.zeros(len(word_idx) + 1)\n", 135 | " y[word_idx[answer]] = 1\n", 136 | " X.append(x)\n", 137 | " Xq.append(xq)\n", 138 | " Y.append(y)\n", 139 | " return (pad_sequences(X, maxlen=story_maxlen),\n", 140 | " pad_sequences(Xq, maxlen=query_maxlen), np.array(Y))\n" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 6, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "class TrainingVisualizer(keras.callbacks.History):\n", 150 | " def on_epoch_end(self, epoch, logs={}):\n", 151 | " super().on_epoch_end(epoch, logs)\n", 152 | " IPython.display.clear_output(wait=True)\n", 153 | " pd.DataFrame({key: value for key, value in self.history.items() if key.endswith('loss')}).plot()\n", 154 | " axes = pd.DataFrame({key: value for key, value in self.history.items() if key.endswith('acc')}).plot()\n", 155 | " axes.set_ylim([0, 1])\n", 156 | " plt.show()" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "## Downloading the dataset" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": 7, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "try:\n", 173 | " path = get_file('babi-tasks-v1-2.tar.gz', origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz')\n", 174 | "except:\n", 175 | " print('Error downloading dataset, please download it manually:\\n'\n", 176 | " '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\\n'\n", 177 | " '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')\n", 178 | " raise\n", 179 | "tar = tarfile.open(path)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "metadata": {}, 185 | "source": [ 186 | "## Getting train and test stories" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 8, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "name": "stdout", 196 | "output_type": "stream", 197 | "text": [ 198 | "Extracting stories for the challenge: single_supporting_fact_10k\n" 199 | ] 200 | }, 201 | { 202 | "name": "stderr", 203 | "output_type": "stream", 204 | "text": [ 205 | "/anaconda3/lib/python3.6/re.py:212: FutureWarning: split() requires a non-empty pattern match.\n", 206 | " return _compile(pattern, flags).split(string, maxsplit)\n" 207 | ] 208 | } 209 | ], 210 | "source": [ 211 | "challenges = {\n", 212 | " # QA1 with 10,000 samples\n", 213 | " 'single_supporting_fact_10k': 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt',\n", 214 | " # QA2 with 10,000 samples\n", 215 | " 'two_supporting_facts_10k': 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt',\n", 216 | "}\n", 217 | "challenge_type = 'single_supporting_fact_10k'\n", 218 | "challenge = challenges[challenge_type]\n", 219 | "\n", 220 | "print('Extracting stories for the challenge:', challenge_type)\n", 221 | "train_stories = get_stories(tar.extractfile(challenge.format('train')))\n", 222 | "test_stories = get_stories(tar.extractfile(challenge.format('test')))" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 9, 228 | "metadata": {}, 229 | "outputs": [ 230 | { 231 | "data": { 232 | "text/plain": [ 233 | "(10000, 1000)" 234 | ] 235 | }, 236 | "execution_count": 9, 237 | "metadata": {}, 238 | "output_type": "execute_result" 239 | } 240 | ], 241 | "source": [ 242 | "len(train_stories), len(test_stories)" 243 | ] 244 | }, 245 | { 246 | "cell_type": "markdown", 247 | "metadata": {}, 248 | "source": [ 249 | "## Preprocessing the data" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 10, 255 | "metadata": {}, 256 | "outputs": [], 257 | "source": [ 258 | "vocab = set()\n", 259 | "for story, q, answer in train_stories + test_stories:\n", 260 | " vocab |= set(story + q + [answer])\n", 261 | "vocab = sorted(vocab)\n", 262 | "\n", 263 | "# Reserve 0 for masking via pad_sequences\n", 264 | "vocab_size = len(vocab) + 1\n", 265 | "story_maxlen = max(map(len, (x for x, _, _ in train_stories + test_stories)))\n", 266 | "query_maxlen = max(map(len, (x for _, x, _ in train_stories + test_stories)))" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": 11, 272 | "metadata": {}, 273 | "outputs": [ 274 | { 275 | "data": { 276 | "text/plain": [ 277 | "(68, 4)" 278 | ] 279 | }, 280 | "execution_count": 11, 281 | "metadata": {}, 282 | "output_type": "execute_result" 283 | } 284 | ], 285 | "source": [ 286 | "story_maxlen, query_maxlen" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": 12, 292 | "metadata": {}, 293 | "outputs": [ 294 | { 295 | "name": "stdout", 296 | "output_type": "stream", 297 | "text": [ 298 | "-\n", 299 | "Vocab size: 22 unique words\n", 300 | "Story max length: 68 words\n", 301 | "Query max length: 4 words\n", 302 | "Number of training stories: 10000\n", 303 | "Number of test stories: 1000\n", 304 | "-\n", 305 | "Here's what a \"story\" tuple looks like (input, query, answer):\n", 306 | "(['Mary', 'moved', 'to', 'the', 'bathroom', '.', 'John', 'went', 'to', 'the', 'hallway', '.'], ['Where', 'is', 'Mary', '?'], 'bathroom')\n", 307 | "-\n", 308 | "Vectorizing the word sequences...\n" 309 | ] 310 | } 311 | ], 312 | "source": [ 313 | "print('-')\n", 314 | "print('Vocab size:', vocab_size, 'unique words')\n", 315 | "print('Story max length:', story_maxlen, 'words')\n", 316 | "print('Query max length:', query_maxlen, 'words')\n", 317 | "print('Number of training stories:', len(train_stories))\n", 318 | "print('Number of test stories:', len(test_stories))\n", 319 | "print('-')\n", 320 | "print('Here\\'s what a \"story\" tuple looks like (input, query, answer):')\n", 321 | "print(train_stories[0])\n", 322 | "print('-')\n", 323 | "print('Vectorizing the word sequences...')\n", 324 | "\n", 325 | "word_idx = dict((c, i + 1) for i, c in enumerate(vocab))\n", 326 | "idx_word = dict((i+1, c) for i,c in enumerate(vocab))\n", 327 | "inputs_train, queries_train, answers_train = vectorize_stories(train_stories,\n", 328 | " word_idx,\n", 329 | " story_maxlen,\n", 330 | " query_maxlen)\n", 331 | "inputs_test, queries_test, answers_test = vectorize_stories(test_stories,\n", 332 | " word_idx,\n", 333 | " story_maxlen,\n", 334 | " query_maxlen)\n" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 13, 340 | "metadata": {}, 341 | "outputs": [ 342 | { 343 | "data": { 344 | "text/plain": [ 345 | "((10000, 68), (10000, 4), (10000, 22))" 346 | ] 347 | }, 348 | "execution_count": 13, 349 | "metadata": {}, 350 | "output_type": "execute_result" 351 | } 352 | ], 353 | "source": [ 354 | "inputs_train.shape, queries_train.shape, answers_train.shape" 355 | ] 356 | }, 357 | { 358 | "cell_type": "code", 359 | "execution_count": 14, 360 | "metadata": {}, 361 | "outputs": [ 362 | { 363 | "name": "stdout", 364 | "output_type": "stream", 365 | "text": [ 366 | "-\n", 367 | "inputs: integer tensor of shape (samples, max_length)\n", 368 | "inputs_train shape: (10000, 68)\n", 369 | "inputs_test shape: (1000, 68)\n", 370 | "-\n", 371 | "queries: integer tensor of shape (samples, max_length)\n", 372 | "queries_train shape: (10000, 4)\n", 373 | "queries_test shape: (1000, 4)\n", 374 | "-\n", 375 | "answers: binary (1 or 0) tensor of shape (samples, vocab_size)\n", 376 | "answers_train shape: (10000, 22)\n", 377 | "answers_test shape: (1000, 22)\n", 378 | "-\n", 379 | "Compiling...\n" 380 | ] 381 | } 382 | ], 383 | "source": [ 384 | "print('-')\n", 385 | "print('inputs: integer tensor of shape (samples, max_length)')\n", 386 | "print('inputs_train shape:', inputs_train.shape)\n", 387 | "print('inputs_test shape:', inputs_test.shape)\n", 388 | "print('-')\n", 389 | "print('queries: integer tensor of shape (samples, max_length)')\n", 390 | "print('queries_train shape:', queries_train.shape)\n", 391 | "print('queries_test shape:', queries_test.shape)\n", 392 | "print('-')\n", 393 | "print('answers: binary (1 or 0) tensor of shape (samples, vocab_size)')\n", 394 | "print('answers_train shape:', answers_train.shape)\n", 395 | "print('answers_test shape:', answers_test.shape)\n", 396 | "print('-')\n", 397 | "print('Compiling...')\n" 398 | ] 399 | }, 400 | { 401 | "cell_type": "code", 402 | "execution_count": 15, 403 | "metadata": {}, 404 | "outputs": [], 405 | "source": [ 406 | "train_epochs = 100\n", 407 | "batch_size = 32\n", 408 | "lstm_size = 64" 409 | ] 410 | }, 411 | { 412 | "cell_type": "markdown", 413 | "metadata": {}, 414 | "source": [ 415 | "## Building the model" 416 | ] 417 | }, 418 | { 419 | "cell_type": "code", 420 | "execution_count": 16, 421 | "metadata": {}, 422 | "outputs": [ 423 | { 424 | "name": "stderr", 425 | "output_type": "stream", 426 | "text": [ 427 | "WARNING: Logging before flag parsing goes to stderr.\n", 428 | "W0204 16:15:39.949661 4521829824 deprecation_wrapper.py:119] From /anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n", 429 | "\n", 430 | "W0204 16:15:39.962125 4521829824 deprecation_wrapper.py:119] From /anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n", 431 | "\n", 432 | "W0204 16:15:39.966588 4521829824 deprecation_wrapper.py:119] From /anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n", 433 | "\n", 434 | "W0204 16:15:39.977391 4521829824 deprecation_wrapper.py:119] From /anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:133: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n", 435 | "\n", 436 | "W0204 16:15:39.984817 4521829824 deprecation.py:506] From /anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", 437 | "Instructions for updating:\n", 438 | "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n" 439 | ] 440 | }, 441 | { 442 | "name": "stdout", 443 | "output_type": "stream", 444 | "text": [ 445 | "Input sequence: Tensor(\"input_1:0\", shape=(?, 68), dtype=float32)\n", 446 | "Question: Tensor(\"input_2:0\", shape=(?, 4), dtype=float32)\n", 447 | "Input encoded m Tensor(\"sequential_1/dropout_1/cond/Merge:0\", shape=(?, 68, 64), dtype=float32)\n", 448 | "Input encoded c Tensor(\"sequential_2/dropout_2/cond/Merge:0\", shape=(?, 68, 4), dtype=float32)\n", 449 | "Question encoded Tensor(\"sequential_3/dropout_3/cond/Merge:0\", shape=(?, 4, 64), dtype=float32)\n", 450 | "(?, 68, 4)\n", 451 | "Match shape Tensor(\"activation_1/truediv:0\", shape=(?, 68, 4), dtype=float32)\n", 452 | "Response shape Tensor(\"permute_1/transpose:0\", shape=(?, 4, 68), dtype=float32)\n", 453 | "Answer shape Tensor(\"concatenate_1/concat:0\", shape=(?, 4, 132), dtype=float32)\n" 454 | ] 455 | } 456 | ], 457 | "source": [ 458 | "# placeholders\n", 459 | "input_sequence = Input((story_maxlen,))\n", 460 | "question = Input((query_maxlen,))\n", 461 | "\n", 462 | "print('Input sequence:', input_sequence)\n", 463 | "print('Question:', question)\n", 464 | "\n", 465 | "# encoders\n", 466 | "# embed the input sequence into a sequence of vectors\n", 467 | "input_encoder_m = Sequential()\n", 468 | "input_encoder_m.add(Embedding(input_dim=vocab_size,\n", 469 | " output_dim=64))\n", 470 | "input_encoder_m.add(Dropout(0.3))\n", 471 | "# output: (samples, story_maxlen, embedding_dim)\n", 472 | "\n", 473 | "# embed the input into a sequence of vectors of size query_maxlen\n", 474 | "input_encoder_c = Sequential()\n", 475 | "input_encoder_c.add(Embedding(input_dim=vocab_size,\n", 476 | " output_dim=query_maxlen))\n", 477 | "input_encoder_c.add(Dropout(0.3))\n", 478 | "# output: (samples, story_maxlen, query_maxlen)\n", 479 | "\n", 480 | "# embed the question into a sequence of vectors\n", 481 | "question_encoder = Sequential()\n", 482 | "question_encoder.add(Embedding(input_dim=vocab_size,\n", 483 | " output_dim=64,\n", 484 | " input_length=query_maxlen))\n", 485 | "question_encoder.add(Dropout(0.3))\n", 486 | "# output: (samples, query_maxlen, embedding_dim)\n", 487 | "\n", 488 | "# encode input sequence and questions (which are indices)\n", 489 | "# to sequences of dense vectors\n", 490 | "input_encoded_m = input_encoder_m(input_sequence)\n", 491 | "print('Input encoded m', input_encoded_m)\n", 492 | "input_encoded_c = input_encoder_c(input_sequence)\n", 493 | "print('Input encoded c', input_encoded_c)\n", 494 | "question_encoded = question_encoder(question)\n", 495 | "print('Question encoded', question_encoded)\n", 496 | "\n", 497 | "\n", 498 | "# compute a 'match' between the first input vector sequence\n", 499 | "# and the question vector sequence\n", 500 | "# shape: `(samples, story_maxlen, query_maxlen)\n", 501 | "match = dot([input_encoded_m, question_encoded], axes=(2, 2))\n", 502 | "print(match.shape)\n", 503 | "match = Activation('softmax')(match)\n", 504 | "print('Match shape', match)\n", 505 | "\n", 506 | "# add the match matrix with the second input vector sequence\n", 507 | "response = add([match, input_encoded_c]) # (samples, story_maxlen, query_maxlen)\n", 508 | "response = Permute((2, 1))(response) # (samples, query_maxlen, story_maxlen)\n", 509 | "print('Response shape', response)\n", 510 | "\n", 511 | "# concatenate the response vector with the question vector sequence\n", 512 | "answer = concatenate([response, question_encoded])\n", 513 | "print('Answer shape', answer)\n", 514 | "\n", 515 | "#answer = LSTM(lstm_size, return_sequences=True)(answer) # Generate tensors of shape 32\n", 516 | "#answer = Dropout(0.3)(answer)\n", 517 | "answer = LSTM(lstm_size)(answer) # Generate tensors of shape 32\n", 518 | "answer = Dropout(0.3)(answer)\n", 519 | "answer = Dense(vocab_size)(answer) # (samples, vocab_size)\n", 520 | "# we output a probability distribution over the vocabulary\n", 521 | "answer = Activation('softmax')(answer)" 522 | ] 523 | }, 524 | { 525 | "cell_type": "code", 526 | "execution_count": 17, 527 | "metadata": {}, 528 | "outputs": [ 529 | { 530 | "name": "stderr", 531 | "output_type": "stream", 532 | "text": [ 533 | "W0204 16:15:43.860381 4521829824 deprecation_wrapper.py:119] From /anaconda3/lib/python3.6/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n", 534 | "\n", 535 | "W0204 16:15:43.880635 4521829824 deprecation_wrapper.py:119] From /anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.\n", 536 | "\n" 537 | ] 538 | } 539 | ], 540 | "source": [ 541 | "# build the final model\n", 542 | "model = Model([input_sequence, question], answer)\n", 543 | "model.compile(optimizer='rmsprop', loss='categorical_crossentropy',\n", 544 | " metrics=['accuracy'])" 545 | ] 546 | }, 547 | { 548 | "cell_type": "markdown", 549 | "metadata": {}, 550 | "source": [ 551 | "## Model summary" 552 | ] 553 | }, 554 | { 555 | "cell_type": "code", 556 | "execution_count": 18, 557 | "metadata": {}, 558 | "outputs": [ 559 | { 560 | "name": "stdout", 561 | "output_type": "stream", 562 | "text": [ 563 | "__________________________________________________________________________________________________\n", 564 | "Layer (type) Output Shape Param # Connected to \n", 565 | "==================================================================================================\n", 566 | "input_1 (InputLayer) (None, 68) 0 \n", 567 | "__________________________________________________________________________________________________\n", 568 | "input_2 (InputLayer) (None, 4) 0 \n", 569 | "__________________________________________________________________________________________________\n", 570 | "sequential_1 (Sequential) multiple 1408 input_1[0][0] \n", 571 | "__________________________________________________________________________________________________\n", 572 | "sequential_3 (Sequential) (None, 4, 64) 1408 input_2[0][0] \n", 573 | "__________________________________________________________________________________________________\n", 574 | "dot_1 (Dot) (None, 68, 4) 0 sequential_1[1][0] \n", 575 | " sequential_3[1][0] \n", 576 | "__________________________________________________________________________________________________\n", 577 | "activation_1 (Activation) (None, 68, 4) 0 dot_1[0][0] \n", 578 | "__________________________________________________________________________________________________\n", 579 | "sequential_2 (Sequential) multiple 88 input_1[0][0] \n", 580 | "__________________________________________________________________________________________________\n", 581 | "add_1 (Add) (None, 68, 4) 0 activation_1[0][0] \n", 582 | " sequential_2[1][0] \n", 583 | "__________________________________________________________________________________________________\n", 584 | "permute_1 (Permute) (None, 4, 68) 0 add_1[0][0] \n", 585 | "__________________________________________________________________________________________________\n", 586 | "concatenate_1 (Concatenate) (None, 4, 132) 0 permute_1[0][0] \n", 587 | " sequential_3[1][0] \n", 588 | "__________________________________________________________________________________________________\n", 589 | "lstm_1 (LSTM) (None, 64) 50432 concatenate_1[0][0] \n", 590 | "__________________________________________________________________________________________________\n", 591 | "dropout_4 (Dropout) (None, 64) 0 lstm_1[0][0] \n", 592 | "__________________________________________________________________________________________________\n", 593 | "dense_1 (Dense) (None, 22) 1430 dropout_4[0][0] \n", 594 | "__________________________________________________________________________________________________\n", 595 | "activation_2 (Activation) (None, 22) 0 dense_1[0][0] \n", 596 | "==================================================================================================\n", 597 | "Total params: 54,766\n", 598 | "Trainable params: 54,766\n", 599 | "Non-trainable params: 0\n", 600 | "__________________________________________________________________________________________________\n" 601 | ] 602 | } 603 | ], 604 | "source": [ 605 | "model.summary()" 606 | ] 607 | }, 608 | { 609 | "cell_type": "markdown", 610 | "metadata": {}, 611 | "source": [ 612 | "## Training the model and using Keras Callbacks for Visualization" 613 | ] 614 | }, 615 | { 616 | "cell_type": "code", 617 | "execution_count": 19, 618 | "metadata": { 619 | "scrolled": false 620 | }, 621 | "outputs": [ 622 | { 623 | "data": { 624 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd4VFX6wPHvm0klBdITWkgg9E7oiIBSZFUUG4hYUFjrWlbXdd1qWdey9rYWxIr4U1FsICpdBEKTIgkhtFDSIQnpmfP7444YIJAJhMwk836eZ57M3HvunXeG4b33nnPuOWKMQSmllOfwcnUASimlGpYmfqWU8jCa+JVSysNo4ldKKQ+jiV8ppTyMJn6llPIwmviVUsrDaOJXSikPo4lfKaU8jLerA6hJRESEadeunavDUEqpRmPt2rU5xphIZ8q6ZeJv164dycnJrg5DKaUaDRHZ7WxZrepRSikPo4lfKaU8jCZ+pZTyMG5Zx6+U8jwVFRVkZGRQWlrq6lDcmr+/P61bt8bHx+e096GJXynlFjIyMggODqZdu3aIiKvDcUvGGHJzc8nIyCA+Pv6096NVPUopt1BaWkp4eLgm/VMQEcLDw8/4qkgTv1LKbWjSr119fEfumfgry1wdgVJKNVnumfjLi1wdgVJKNVnumfgrtFVfKeXegoKCTrpu165ddO/evQGjqRv3TPyVJa6OQCmlmiz37M5ZUQLGgDb0KOWR/vXFFrbuL6jXfXZtGcI/Lup20vX3338/cXFx3HrrrQD885//RERYunQp+fn5VFRU8MgjjzBhwoQ6vW9paSm33HILycnJeHt78/TTTzNy5Ei2bNnCDTfcQHl5OXa7nU8++YSWLVty5ZVXkpGRQVVVFX/729+46qqrzuhz18Q9E7+9Egr2Q/NWro5EKeUhJk2axF133XU08X/00UfMnz+fu+++m5CQEHJychg0aBAXX3xxnXrWvPTSSwBs2rSJbdu2MWbMGFJTU3n11Ve58847mTJlCuXl5VRVVfH111/TsmVLvvrqKwAOHz5c/x8Ud038AJlbNPEr5aFOdWZ+tvTp04esrCz2799PdnY2oaGhxMbGcvfdd7N06VK8vLzYt28fmZmZxMTEOL3f5cuXc8cddwDQuXNn4uLiSE1NZfDgwTz66KNkZGQwceJEEhMT6dGjB/feey/3338/F154Ieecc85Z+azuWccPkLnZ1REopTzM5Zdfzscff8ycOXOYNGkS77//PtnZ2axdu5YNGzYQHR1d55unjDE1Lr/66quZN28eAQEBjB07lh9++IGOHTuydu1aevTowQMPPMBDDz1UHx/rBO55xm/z1cSvlGpwkyZNYvr06eTk5LBkyRI++ugjoqKi8PHxYdGiReze7fSQ90cNHz6c999/n1GjRpGamsqePXvo1KkT6enpJCQk8Ic//IH09HR+/vlnOnfuTFhYGNdccw1BQUHMmjWr/j8kbpr4K23+VlWPUko1oG7dulFYWEirVq2IjY1lypQpXHTRRSQlJdG7d286d+5c533eeuut3HzzzfTo0QNvb29mzZqFn58fc+bM4b333sPHx4eYmBj+/ve/s2bNGu677z68vLzw8fHhlVdeOQufEuRklyGu1DU+xmy9vhz+sh98/F0djlKqAfzyyy906dLF1WE0CjV9VyKy1hiT5Mz2blnHX2J8wFRB9jZXh6KUUk2OW1b1FFU5xpnO3AIte7s2GKWUOolNmzYxderUY5b5+fmxatUqF0XknFoTv4jMBC4EsowxJ9yDLCL3AVOq7a8LEGmMyRORXUAhUAVUOnsZUlhpw3gHINrAq5RyYz169GDDhg2uDqPOnKnqmQWMO9lKY8yTxpjexpjewAPAEmNMXrUiIx3rnUr6AAYoC+uoPXuUUuosqDXxG2OWAnm1lXOYDMw+o4gcMgMS4eBma+gGpZRS9abeGndFpBnWlcEn1RYb4FsRWSsiM5zeF5AqcVCSB4UH6ytEpZRS1G+vnouAFcdV8ww1xvQFLgBuE5HhJ9tYRGaISLKIJNvEsLYk1lqh/fmVUg3kVEMtNyX1mfgncVw1jzFmv+NvFjAXGHCyjY0xrxljkowxSUH+vvyQH2WtyNxUjyEqpZSql8QvIs2Bc4HPqy0LFJHgX58DYwCnWmv9fWykFnhjD25l1fMrpVQDMsZw33330b17d3r06MGcOXMAOHDgAMOHD6d37950796dZcuWUVVVxfXXX3+07DPPPOPi6GvnTHfO2cAIIEJEMoB/AD4AxphXHcUuBb41xhyptmk0MNcxfKk38IExZr4zQfn72CgDDod0JDRrq5MfRSnVZHzzZzhYz1f7MT3ggv84VfTTTz9lw4YNbNy4kZycHPr378/w4cP54IMPGDt2LA8++CBVVVUUFxezYcMG9u3bx+bN1knqoUOH6jfus6DWxG+MmexEmVlY3T6rL0sHep1OUAGOxL/XJ57QA8ugshy8fU9nV0opVWfLly9n8uTJ2Gw2oqOjOffcc1mzZg39+/dn2rRpVFRUcMkll9C7d28SEhJIT0/njjvu4He/+x1jxoxxdfi1css7d71tQligL1sqW9PTXgm52yG64cfnVkq5iJNn5mfLycYwGz58OEuXLuWrr75i6tSp3HfffVx77bVs3LiRBQsW8NJLL/HRRx8xc+bMBo64btxyrB6AzjHBrCyKtl5kanWPUqrhDB8+nDlz5lBVVUV2djZLly5lwIAB7N69m6ioKKZPn86NN97IunXryMnJwW63c9lll/Hwww+zbt06V4dfK7c84wfoHBPC/61ujvHxRrK2AFe4OiSllIe49NJLWblyJb169UJEeOKJJ4iJieHtt9/mySefxMfHh6CgIN555x327dvHDTfcgN1uB+Cxxx5zcfS1c9/EHxtMYYUX5VGJ+OkZv1KqARQVFQEgIjz55JM8+eSTx6y/7rrruO66607YrjGc5VfntlU9XWJCAMhp1l5v4lJKqXrktok/MToIL4F0iYOCDChx/y5SSinVGLht4vf3sZEQGcS6spbWgqxfXBuQUuqsc8cZAd1NfXxHbpv4AbrEhrA4P8J6kaXVPUo1Zf7+/uTm5mryPwVjDLm5ufj7n9mUtG7buAvQrWUIX2wMwt48BC9t4FWqSWvdujUZGRlkZ2e7OhS35u/vT+vWrc9oH26f+EEoDEmkuQ7doFST5uPjQ3x8vKvD8AhuXdXTrWVzADJ8EqybuPQSUCmlzphbJ/6wQF9im/uzpbI1lB2Ggn2uDkkppRo9t078YFX3rCj8dWx+beBVSqkz5faJv2vL5iw65OjZo4lfKaXOmNsn/m4tQygwgZQHtgRt4FVKqTPWKBI/QFaz9jpKp1JK1QO3T/ytWgTQPMCHNNpCTipUlrk6JKWUatTcPvGLCN1ahrCorBPYK2Djh64OSSmlGrVaE7+IzBSRLBGpcdZzERkhIodFZIPj8fdq68aJSIqIpInIn083yG4tQ5idl4i9VRIseULP+pVS6gw4c8Y/CxhXS5llxpjejsdDACJiA14CLgC6ApNFpOvpBNmtZXPKKw0Zfe6xRupcO+t0dqOUUgonEr8xZimQdxr7HgCkGWPSjTHlwIfAhNPYz9EG3jX0hHbnwNKnoLz4dHallFIer77q+AeLyEYR+UZEfp0VvRWwt1qZDMeyOkuIDMLfx4stBwph5INwJAtWv3amMSullEeqj8S/DogzxvQCXgA+cyyXGsqedLAdEZkhIskiknz86Hw2L6FzTAhb9h+GuMHQ4XxY8SyUFtRD+Eop5VnOOPEbYwqMMUWO518DPiISgXWG36Za0dbA/lPs5zVjTJIxJikyMvKE9d1ahrD1QAGVVXYY9VcoyYfF7j+psVJKuZszTvwiEiMi4ng+wLHPXGANkCgi8SLiC0wC5p3u+wxpH0FhaSVX/G8lad6JMGAG/PQypH57ph9BKaU8ijPdOWcDK4FOIpIhIjeKyM0icrOjyOXAZhHZCDwPTDKWSuB2YAHwC/CRMea0B9sZ3yOG5yf3YWfOEcY/v4w3A27ARHeHz26GggOnu1ullPI44o7TnCUlJZnk5OQa12UVlvLg3M0s3JrJf4b7MWn9VGjVD679HLxsDRypUkq5BxFZa4xJcqas29+5e7yoYH9em9qPgfFhvLrFhhn/JOxaBsuednVoSinVKDS6xA/WMA5XJLVhV24xa1tcAN0vg6VPQN5OV4emlFJur1EmfoALusfQzNfG/63dB2MeAS9v+P5frg5LKaXcXqNN/IF+3ozvEctXmw5Q7B8FQ+6ALXNh72pXh6aUUm6t0SZ+gMv7taaorJIFWw7CkD9AUDQseFAnZVdKqVNo1Il/QLsw2oQF8PHaDPALsoZzyFgNWz+rfWOllPJQjTrxe3kJl/dtw487csnIL4Y+10BUN1j4Dx3OQSmlTqJRJ36AiX1bYQy8tWIXa/cWsLH7/ZhDezAvJlmTtmi1j1JKHaPRJ/42Yc0Y0j6cN5fv5LJXfmTC195MKHuILImEub+HmeMga5urw1RKKbfh7eoA6sN/r+zF2t35BPl5E+zvzRvLYhi2NYHvRmUQt+4JeHM0TPoA4s9xdahKKeVyjf6MHyC2eQAX9mzJiE5R9IsL4z+X9SQqpBnXrOtI4XXfQ3AsvDfR6u6plFIerkkk/uM1D/Dh+cm92X+olAd/OISZNh9a9oH/uwFWPAcVJa4OUSmlXKZJJn6AfnFh3HVeIvM27ufjrUesQdw6jYeFf4f/doKv7oUDG10dplJKNbgmm/gBbh3ZgcEJ4fzt882k5FbCVe9ZB4AOo2HdO/C/4fD1n8Be5epQlVKqwTTpxG/zEp6b3Jtgfx9ueX8tRRV2SBgBl78J96bAwJth9f9gzlSdvF0p5TGadOIHaxjn5yf1YVfOER74dBNH5x8ICIULHodxj0PK1/D2hVCUfeqdKaVUE9DkEz/A4Pbh/HFMJ77YuJ93Vu4+duWgm+GqdyFzC7wyBDZ8AHa7awJVSqkG4BGJH+CWc9szqnMU/5i3hbs+XM/+Q9V69nS5CG78FkLj4LNb4M3zIaPmGcCUUqqxc2bO3ZkikiUim0+yfoqI/Ox4/Cgivaqt2yUim0Rkg4i4NJN6eQkvXt2H20a25+vNBxn138U8/W0K+349AMT2gmnfwiWvwuEMeOM8eGs8bPoYKstdGbpSStWrWufcFZHhQBHwjjGmew3rhwC/GGPyReQC4J/GmIGOdbuAJGNMTl2COtWcu/UhI7+Yx+en8MXG/QDERwQytEM4/eJC6RAZTEKIncBN70DyTMjfBYGRMOpv0O+6sxaTUkqdibrMuevUZOsi0g74sqbEf1y5UGCzMaaV4/Uu3DDx/yotq5AlqTmsSMvhp/Rcist/69bZKTqYWTf0IzZnJSx7BnYvhz5TYfxT4ON/1mNTSqm6qEvir++xem4Evqn22gDfiogB/meMea2e3++MdIgKpkNUMDcOi6eiys6unCPsyC4iLauIF35I419fbOPVqedDwkhY9G9Y9hRkboYr34UWbVwdvlJKnZZ6S/wiMhIr8Q+rtnioMWa/iEQBC0VkmzFm6Um2nwHMAGjbtm19heU0H5sXidHBJEYHA1abwBPzU1i4NZPRXaPhvL9Bq74w92Z4aSAk3QCDb4eQ2AaPVSmlzkS99OoRkZ7AG8AEY0zur8uNMfsdf7OAucCAk+3DGPOaMSbJGJMUGRlZH2GdkennJNApOph/fL6ZI2WV1sLOvyN94lcciD0P89PL8FxP+OIuKD3s2mCVUqoOzjjxi0hb4FNgqjEmtdryQBEJ/vU5MAaosWeQO/KxefHviT3Yf7iUpxemklNUxgOfbuK8WXsZnDqJS7xeYGPkRZj178L7V0D5EVeHrJRSTqm1qkdEZgMjgAgRyQD+AfgAGGNeBf4OhAMviwhApaOBIRqY61jmDXxgjJl/Fj7DWdMvLpQpA9vy1oqdfLRmLyUVVVw/pB1D2kcwe/UeLkkJ5Xe2NryQ8TwyezJc/ZE2/Cql3J5TvXoaWkP16nHG4ZIKLn15BXFhzXjwd13pEBV0dN3u3CNMfyeZQUULeajqBUgcaw0E5+3rwoiVUp6o3rtzNjR3Svy1+eVAARNeXMHfY3/impxnIaQVRHWFiI7WTWFdLgLfZq4OUynVxLmyO6fH6RIbwp/GdeKvX9mJH/QkQyt/guxU2LUMKkvhmxbQdyr0vwlC27k63LMqs6CUjPxi+sWFuToUpdQpaOKvB9OGxrMoJYub1tp4aMIF7A0p5peAQ3Qo3cKdIYvwX/ky/Pgi9LwKRj0ILZzrrvrBqj0sTc3m3rEd6RAVfJY/xZlJzy5i8us/kVVYxmOX9mDSgBM/o91u2LTvMEtTs8kuKmP6OQm0Cfvtaij/SDkPf7WV4rIqnrmqNwG+tqPrsgpKmfb2Gi7q2ZLfn9u+QT6TUk2VVvXUkwOHSxj37DIOl1TgJZAQGcS+/BJahwbw0eS2hG6aCatfA2Ng4AwYdg80q/nM2BjDkwtSeHnxDmxegpdY3UvvGJV4TDJ0F2lZhUx+fRV2u6FTTDA/7sjl4Uu6M3VQHAB7cot5ZckOFmw5SN4Ra9wjX5sXXl5wx6hEpp+TwNLUbB6Yu4lDxeVU2g3DOkTw+rVJ+PvYyD9SzlWvrSQ1s4hWLQJYfv9IHJ0GlFIOWsfvIrtzj3CouIJOMcH4+9j4cUcON7y1hoTIIGZPH0iLiiyqvn8Ur59nU2Xz53Cnqwg69zb8ojse3UdFlZ0/f7KJT9ZlMHlAW+4+P5HH56fwyboMWrUIYHjHSNpHBpIQGYgg7MkrZnduMSUVVVyR1Jq+bUNPGaMxht25xazamUvyrnziIwO5bnA7Av1O7+Iv5WAhU974CRBmTx9I2/Bm3PreOr7flsU9ozuyJ6+Yuev3YfMSxnePYWTnKIZ1iKCs0s5DX2xl/paDRAb7kV1YRueYYJ6+sjeb9x/mTx//zKjOUfz3il5c/9ZqfjlYyCW9W/JRcgbzbh9Kz9YtTitepZoqTfxuZGlqNje9k0xiVBDtI4NYtC2L2PKdzPD+iou9VuCNnZW2vsz3H88GvySKKmBnzhHuPr8jfzivw9Ez21XpuTzzXSrbDhZyqLjimPcI8LHhJXCkvIr+7UKZMbw9/duFEuLvg5eXUFRWyfLtOSzalsXS7dkcOFwKWJPSHy6pICLIlztGJXJRr5asSMvh262ZLE3N5khZJXZjMEB0sD9D2oczuH04idHBrN6Zy+KUbNbsyiMs0JcPpg+ifaTV46m80s7tH6zj262Z+Hl7MWVgHDefm0BUyIldXX/Ylsl/v01lZKco/nBeIr7e1q0l76/azYNzNxPs501xRRWvXtOPpLhQkh79jhnDE7h/XOez+K+mVOOjid/NLNqWxe/fXUuQvzfnd4libLcY2oQ1Y8/unQRtepuu+z8lpCqPfFs4ywLH0LzbWM4dPAiCY6CGKo28I+WkZxcB0Da8GZFBfhSXVzFnzV7eXL7z6FDTIlZyLy6rorzKTrCfN8MSIxjSIYLBCWG0jwxi3Z58npifwqqdeUf3HxHky8hOUUSF+CFY778z9wgrd+QeraoB6BgdxIhOUVw7OI7Wocf2XKqosvPVzwcY0iGcqODTu7dh1oqdPPbNNh6/rCeX9GkFwDVvrGLfoRJ++OO5Wt2jVDWa+N1Q/pFyQgJ8sHnVkKyqKiB1gTUBfNpCMI4ZwHwCIaYH9JkC3S8D38Ba36eyys6ilGz25BVzqLicQ8UVNPO1cW6nSPq3C8PHduLN2sYYlm7PYf2efIZ1iKBP29Aa47TbDSmZhaRlFdEvLpSWLQLq/D3UVUWV/ZiY3/tpN3/9bDPz7zqHzjEhZ/39lWosNPE3ZoWZ1gigeemQuwPSF0H2NvANhh6XQXgH6wDgGwRB0RDZyfrrIWe/WYWlDPz39/xhVCJ3j+5Y+wZKeQjtx9+YBUdbD86zXhsDe1fB2lmw8UPr3oDj+TWH6K7Q5WLoPtGqImqiooL96R8XxoItBzXxK3Wa9Iy/MbFXWYPBlR+B8iIo2GfdLJaTYh0cDm4C8YJ258CAGdD5d03ySmDm8p089OVWFt07gviI2qu/lPIEesbfVHnZwD/EegBEJELCiN/WZ6fC5o9h42yYMwWie8CI+60y+9bCnlVWtVHL3tD+PIju1igPDGO7x/DQl1v5ZvMBbh3RwdXhKNXo6Bl/U1RVaR0AljwBeTuqrRAIaWldKQAEx0JMT2tZSCuI7AidxoPNxyVh18WEF5djgHm3D6u1rFKeQM/4PZ3NG3pNgu6Xw5a5kJsGrftDm/7g3xwO74MdP8CO7yEnDfYlQ7Fj/pzmbWDIHdb8wm48uNyYbjE8uSCFrILSGu8PUEqdnJ7xK0tFKaQvhuXPwN6fICAU/FtAWQGUFlhXBz0ugx5XWFVELrZ532EufGE5T13Ri8v7tXZ1OEq5nJ7xq7rz8YdO46zH7h+tXkT2SusKwS8YMrfCiuetA0N4otW+ENIKmreCzhdBRMPWtXeNDSEiyJelqdma+JWqI0386kRxQ6zH8YqyraqjtIWQv9s6QJQegkX/huH3wdC7GmwSGi8v4ZzESJakZmO3G7xqujFOKVUjTfzKeUGR1siiA2f8tqzgACz4Cyx6FDZ/Yh0AbL5grwAEOpxnXTWcBcM7RjB3/T427z+sg7YpVQdOJX4RmQlcCGQZY7rXsF6A54DxQDFwvTFmnWPddcBfHUUfMca8XR+BKzcREgtXvGU1Jn/1R/jkxmPX+4VA0g0w6NZ6v7HsnMRIwBoITxO/Us5z9ox/FvAi8M5J1l8AJDoeA4FXgIEiEoY1OXsSYIC1IjLPGJN/JkErN9RxrHXjWE4KeHmDl49VDbTqf/DjC/DTK9BumDXkRFh7iOpiVSedQdfRiCA/urUMYWlqDrePSqzHD6NU0+ZU4jfGLBWRdqcoMgF4x1hdhH4SkRYiEguMABYaY/IARGQhMA6YfSZBKzfl2wxa9jl2WdtBkPc3+OlVq7fQ3jVQXmitCwiFzhdaw0y0HQw+dR/0bXjHSF5fmk5BaQUh/u5//4FS7qC+6vhbAXurvc5wLDvZcuVJwhJg/BPWc2PgSDZkJFsNxVvmwvp3QWzWJPUte0ObgdYVRFBUrbsenhjJK4t38GNaLuO6N90xipSqT/WV+GvqUmFOsfzEHYjMAGYAtG3r3Jy0qhESsRJ65/HWo6LEun8gIxn2r4eUr60DAQKt+lnjDfW/8aQNxP3iQgn0tbF0e7YmfqWcVF+JPwNoU+11a2C/Y/mI45YvrmkHxpjXgNfAuoGrnuJS7s4nADpdYD3AuiLI3AIp30DqN/D9v2DlS3D+P6D3NeB17HwCvt5eDG4fztLUbIwxOjmLUk44cVaO0zMPuFYsg4DDxpgDwAJgjIiEikgoMMaxTKmaiUBMdzj3Ppj+A8xYAuHtYd4d8MYo66BwnOEdI8nIL2FnzhEXBKxU4+NU4heR2cBKoJOIZIjIjSJys4jc7CjyNZAOpAGvA7cCOBp1HwbWOB4P/drQq5RTWvaGaQtg4uvWGEPvXAKHM44pMrRDBABrdulPSylnONurZ3It6w1w20nWzQRm1j00pRxEoOeV1jSUb4yG2ZNh2vyjU1G2Cw/E1+ZFup7xK+WU+qrqUersi+oCl8+0pqacezPYrbmJbV5CXHgzdmZr4lfKGZr4VePScQyMfhh+mQdL/nN0cXxEoJ7xK+UkTfyq8Rl8G/SabE00c2gPAPGRgezOPUKVXTuEKVUbTfyq8RGBkX+xnq+zRhFpHxFERZVhX36JCwNTqnHQxK8apxZtIXGMlfirKoiPtBp6d+QUuTgwpdyfJn7VeCVNg6JMSPmG+Agr8WsDr1K108SvGq/E0RDSGpJnEh7oS4i/t97EpZQTNPGrxsvLBv2uh/RFSF468ZFBpGtVj1K10sSvGre+U62RPdfOon1EoFb1KOUETfyqcQuOsUbwXP8e7cN82H+4lOLySldHpZRb08SvGr+kG6Akj0GVqwHYlVPs4oCUcm+a+FXj1244+AYRX7gOQBt4laqFJn7V+Nm8oc0AWmQnA5CerQ28Sp2KJn7VNLQdglf2L3QMqdQzfqVqoYlfNQ1xgwHDmOCdOlibUrXQxK+ahlb9wMuHgbYU0rOLsKaIUErVRBO/ahp8AqBVPzqVbaagtJK8I+Wujkgpt6WJXzUdcYOJKNiKP2Vaz6/UKWjiV01H2yF4mUr6eKWRrnfwKnVSzk62Pk5EUkQkTUT+XMP6Z0Rkg+ORKiKHqq2rqrZuXn0Gr9Qx2g7EIAyybdMGXqVOodbJ1kXEBrwEjAYygDUiMs8Ys/XXMsaYu6uVvwPoU20XJcaY3vUXslIn4d8cienOOdlp/HtXHuWVdny99aJWqeM5879iAJBmjEk3xpQDHwITTlF+MjC7PoJTqs7aDqGHSWHD7mymvrlKG3mVqoEzib8VsLfa6wzHshOISBwQD/xQbbG/iCSLyE8icsnJ3kREZjjKJWdnZzsRllI1iBuMj72UN0f7sH7vISa8tJzUzEJXR6WUW3Em8UsNy07WSXoS8LExpqrasrbGmCTgauBZEWlf04bGmNeMMUnGmKTIyEgnwlKqBm2HAHCufxpzZgyitMLOZS//yO5crfNX6lfOJP4MoE21162B/ScpO4njqnmMMfsdf9OBxRxb/69U/QqOhrD2sHsFfdqG8uktQzDAX+Zu0pu6lHJwJvGvARJFJF5EfLGS+wm9c0SkExAKrKy2LFRE/BzPI4ChwNbjt1WqXiWMgJ3LoLKMNmHNuP+CzqxIy+WTdftcHZlSbqHWxG+MqQRuBxYAvwAfGWO2iMhDInJxtaKTgQ/NsadVXYBkEdkILAL+U703kFJnRcexUHEEdi0HYMqAtiTFhfLIV1vJKSpzcXBKuZ644+VvUlKSSU5OdnUYqrEqL4Yn4q35eC94HIDtmYWMf34Z43vE8twkrW1UTY+IrHW0p9ZKOzmrpse3GcQPh9QF4DixSYwO5tYRHfh8w36+25rp4gCVci1N/KppShwD+TshN+3ooltHtqdrbAh3zdnAtoMFLgxOKdfSxK+apsQx1t/t3x5d5Odt483rkwj0szHtrTVkFpS6KDilXEuPu6z5AAAalUlEQVQTv2qaQuMgsrNV3VNNbPMA3ryuP4dKKrjx7TUcKat0UYBKuY4mftV0JY6B3T9C2bF37nZv1ZyXru7L1v0F3PL+OgpLK1wUoFKuoYlfNV0dx4K9AnYsOmHVyM5RPDaxByvScpjw0grSsnRYB+U5NPGrpqvNQPBrDtsX1Lj6qv5tef+mgRSUVDDhxRXM33yggQNUyjU08aumy+YDHUbB9oVgt9dYZFBCOF/cMYzE6GBufm8dH6/NaOAglWp4mvhV09b5QijKhI0fnLRIbPMA5vx+EEM7hPPApz+zemdeAwaoVMPTxK+atm4TIW4ofPNnOLTnpMX8vG28fHU/2oQ24/fvJutonqpJ08SvmjYvL7jkZTB2+Py2k1b5ADRv5sOb1/fHbuDGt5Mp0N4+qonSxK+avtB2MPZR2LkU1rxxyqLxEYG8ek0/duUcYfTTS3hywTZ26fy9qonRxK88Q7/rocP5sPDvkLvjlEUHtw/nnWkD6NayOa8s3sGIpxZzzRuryMgvbphYlTrLdHRO5TkK9sPLgyC6O1z3pVUNVIuDh0v5ZF0Gry7egc0mPHNVb0Z2imqAYJWqGx2dU6mahLSE0Q/D7hWw4T2nNolp7s9tIzsw745hxIT4M23WGp7+NoUqu/udMCnlLE38yrP0mWr18vn2r1CU5fRm8RGBfHbbUC7v25rnf0jjvo83YtfkrxopTfzKs3h5wYXPQkUJzP9znTb197Hx5BW9uGd0Rz5dt49/frFF5/FVjZImfuV5IjvCOffC5k+su3rr6I5RHfj98ATeWbmbJxeknIUAlTq7nEr8IjJORFJEJE1ETjhNEpHrRSRbRDY4HjdVW3ediGx3PK6rz+CVOm3D7oKIjjD3Zmti9joQEf58QWemDGzLy4t38LfPNrN532E9+1eNRq29ekTEBqQCo4EMYA0wufqk6SJyPZBkjLn9uG3DgGQgCTDAWqCfMSb/VO+pvXpUg8hOhTlTrFm6Rj4Iw+5xqqfPr+x2w4OfbWbOmj3YDbRqEcAF3WP445hOBPjazmLgSp2ovnv1DADSjDHpxphy4ENggpOxjAUWGmPyHMl+ITDOyW2VOrsiO8L0RdD9MvjhYfjgCih2fpweLy/hsYk9WPPg+TxxeU+6xIbw5oqd/PH/NtTY8KtXBMpdOJP4WwF7q73OcCw73mUi8rOIfCwibeq4LSIyQ0SSRSQ5OzvbibCUqgd+QTDxdbjwGevO3pljIX9XnXYRHuTHlUlteOO6JB4c34WvNx3kyW9/q/vPLSrjpreTGfHUYg4cLqnnD6BU3TmT+KWGZcefunwBtDPG9AS+A96uw7bWQmNeM8YkGWOSIiMjnQhLqXoiAknTYOpnVhfPN86HfWtPa1c3Dovn6oFteWXxDj5as5elqdmMe24ZS7dnk11YxrRZyRTpdI/KxZxJ/BlAm2qvWwP7qxcwxuQaY8ocL18H+jm7rVJuo91QuHEh+ATArAsh9dvatzmOiPCvi7txTmIED8zdxLUzV9MiwIfPbxvKK9f0IzWzkNveX0dF1ckHi1PqbHMm8a8BEkUkXkR8gUnAvOoFRCS22suLgV8czxcAY0QkVERCgTGOZUq5p8iOcNP3EJEIH117Wmf+PjYvXprSl4HxYVw3OI4v7hhGl9gQzu0YySOXdGdJajZ//3xzvdb5F5VV6t3Ebm79nnyO1OPV3uGSCg6XnN4Ist61FTDGVIrI7VgJ2wbMNMZsEZGHgGRjzDzgDyJyMVAJ5AHXO7bNE5GHsQ4eAA8ZY3SWC+XegqJgyifwxnnwwSS46TsIjavTLkL8ffhg+qATlk8e0Ja9ecW8vHgHWQVl/PPibrQJa3ZMmdyiMjILysguKiP/SDkjOkXSopnvSd8rq6CU372wnM4xwcy6YQA2r5pqWJUr/bgjh6tfX8WITpG8dX1/RM7s36issopLXlqBzUv45s5z8LHV7ZYsHaRNqZPJToE3R0NwLExbAAEt6mW3drvhzeU7eea7VKrshjtGdaBXmxYs2pbNopQsdh43DPT5XaJ4/dqkGpOF3W64duZqVqbnUmU33H1+R+48P/Ho+syCUp5akMK0YfF0iQ2pc6zGGHKKyskvLicxKuiME1Zj8cuBAl5dsoP7xnaidWiz2jc4hfJKO+OfX8a+/BJKKqr4x0VduWFofI1lK6vsvLpkB33bhjKkQ8RJ9/nqkh3855ttADw8oRtTB7erU3fOWs/4lfJYkZ3gqvfg3Ykw5xrreT0kfy8vYfrwBC7sFcvDX27lqW9TAfD19mJwQjiTB7ShVYtmRIX4sWx7Ds9/v50FWzIZ1z3mhH29unQHy9NyrG6lO/N49vtU+seHMqR9BOnZRUx9czX7DpWwfu8hvrxjGP4+td9fUF5p5/Vl6Xz58wH25B7hSHkVAEM7hPOfiT1PuEKpi7LKKr7YeIDZq/cQFujLFf1aM7JzVJ3PWJ1VVFbJXz7dRFK7UK4d3M6pbQ4VlzP9nWQy8ktYvTOP928aSEJk0GnH8ObynaRlFTHz+iTe+2kPj32zjUEJ4ScciIvLK7n9g/X8sC2LFs18WHj3uUQG+52wv6zCUl74fjvnd4miqKySZ77bzoQ+NXaWPCk941eqNhvnwOe3WqN7Xj4LWverdZO6WLMrj4KSCga3D6eZ77HnYhVVdi5+cQX5R8pZeM9wgv19jq5btyefK15dybhuMbx4dR+Ky6u4+MXlFJRW8p+JPbjv458R4PfnJvDvr7dx07B4/nph11PGsnZ3Hg98uonUzCIGxofRJTaEduHNKK+y8/z3adiN4f5xnenfLozladks257DnrxierZuwYB2ofSNCyXE34dKu6HKbqek3E5BaQUFJRWkZhbx3qrdZBeW0SEqiEPFFeQUlRER5MvFvVoxqnMU/eND8fOun5vfDhWXc91ba9i49xC+Ni8W3jOcuPDAU25jtxtumLWGlTtyefiSbjwxPwURePfGgad1xZSRX8zop5cyvGME/5uaRE5RGeOeXUZoMx++qHYgzjtSzrRZa9iYcYhbR7Tn9aU7Gd01mpem9D1hn3/6eCNz1+/j27vPpai0koteXM4tI9rz5wu6OH3Gr4lfKWfsXQ0fT4PCA3D+P2HQbXW6y/dMbNh7iEtfXsF1g9vxz4u7AZCaWci0WVbT2Vd/OIfmAdYBIeVgIRNeWk5phZ3WoQG8e+NA4iMCeXDuJj5YvYcPpw9iYEL40X1nFZSSmllEWlYh6/YcYt7G/bRs7s/Dl3TnvC7Rx8Sx71AJf/7kZ5Ztzzm6rENUEAkRgWzMOERmQRm1Gd4xkpuGxXNOYgSVdsOSlGw+St7L4pRsyqvsBPjYGNw+nDFdoxnTLYawwJO3bVT3+tJ0VqbnMrZbNGO6xlBRZWfqm6vZmXuEf17UjUe+2srQDhG8fu2p8+IzC1N57vvtPHppd6YMjCMtq4hr3lhFSUUVUwfFEeTvTaCvjWa+3jTztRHga6N5gA+9WrfAq4a2lRnvJLNsew7f/fFcWrUIAGBJajbXzVxN37YtaBPWDAHW7z3EgcOlPD+pD+O6x/DSojSeXJDCq9f0ZVz33/rO/JxxiAkvrWDGOQk8ML4LAPfM2cCXmw6w/dHxmviVqncl+fD57bDtS+g9BS5+AbwaZmiGf3y+mXd+2s0jl3Tnu62ZLErJJtDXxrs3DaRv29Bjyn75837mrtvHYxN7EBXiD8CRskoueG4ZBsP//X4I32/L5KPkDDbuPXR0uxB/b65IasM9ozsS6FdzLbAxhvmbD1JYVsk5iRHENg84ujwjv4R1e/Ipr7TjbRO8vbzw8/aieYAPIQE+hAf5EhXsX+N+i8sr+Sk9lyUp2fyQksXevBJsXsLghHAu7tWS3/WMPWlMv9Z3Nw/w4XBJBd5eQrC/N2WVdl6/NomhHSJ4ZfEOHp+/jbenDeDcjtZ9Qjuyi3jky60YILa5PwE+3sxcsZPL+7Xmyct7Hm3P2JtXzPR3kknJLORk6bJfXCj/vrQHnWKCASgoreC1Jem8uCiN+8d15pYR7Y8p/78lO/hwzV6MMdgNNPO18fAl3enfLgywrvQmvLiCrMIyvrtnOC2a+bInt5g756xnb14xi+4dcfTqb9+hEkY9tZhUTfxKnSXGwOLHYMnj0OMKuORVsJ39prKC0gpGP72EzIIywgN9uX5IO64ZFEeok2fEAKt35nHVayuPJq9O0cFc2rcVPVs1p0NUEJHBfm7ReGuMYcv+Ar7ZfICvNx1kZ84Rgvy8uahXS67q34aerZofPbt+a8VO/vXFVi7q1ZJnruzFLwcK+WrTAbYeKOCu8xOPHhTLKqsY+8xSvLyE+XcOZ3laNnfO3oC3TWgd2owDh0vJKSqjX1wo7980sMa2EGMMpRV2isoqKS6vpLi8iuLySrYdLOSpBSkUllYyY3gCAT423li+k8MlFYzvEcOzV/XB17vuV4db9h9mwosr6BwbTGFpJbtzrak/n76yFxP7tj6m7BPzt3G/VvUodZYt+y98/xB0vQQuewNsPrVvc4Y2ZRwmJbOQC3vGOtVIW5N3Vu4iLauIy/q2pmfr5m6R6E/FGMPa3fnMXr2Xrzbtp7TCTligL0PahxMT4s8by3cytls0L17dt9YG4u9/yeTGt5MZ0j6clem5dI0N4bVrk45WwZRX2vGxyWl9J3lHyvn317/w8doMAM7vEs1d5yfSvVXzun/oal74fjuvLU1nYEIYwzpEcE7HSNrX0NBcWFpBSICvJn6lzrofX4RvH4QOo2Hia9AszNURNWkFpRV8tzWT5Wk5rEjLIbOgjBGdIvnf1H5ONQgbYzXcLk7J5uJeLXn8sp71Porq5n2HsXnJaTUEn6m6dOfUxK/UmUh+C76+z+rrf+UsaFW/PX5UzYwxHCwoJTrYv8ZG1ZPJO1LO+j35jOoc5fZXO3Wlk60r1VCSbrBu7gJ4cyys+h8nbQFU9UZEiG0eUKekDxAW6Mt5XaKbXNKvK038Sp2p1v3g90ugw3nwzZ9g9iQo0qHFlfvSxK9UfWgWBpNmw7jHYccieGUwpOp4hMo9aeJXqr54ecGgm2HGYgiKhg+uhM9ugyM5tW2pVIPSxK9UfYvuag3tPPQu+PlDeKEvrH4d7FWujkwpQBO/UmeHjz+M/hfc8iPE9oKv74VXhsKaN6D0sKujUx5OE79SZ1NkJ7h2Hlz+lnWH71d/hP92toZ+2L/e1dEpD6XDMit1tolA94nQ7VLYv87q+7/5E1j/rtXvv/90a51PzePYKFXf9AYupVyh9DBsmG1V/eRuB59AaD8SOo2HjmMh8OSTcChVE52IRSl359/c6gE08Pewcwls/RxS5lsjf3p5Q/fLYMgdENPD1ZGqJsipM34RGQc8hzXn7hvGmP8ct/4e4CasOXezgWnGmN2OdVXAJkfRPcaYi2t7Pz3jVx7JGDiwETZ+COvegYojkDAS+t8EiWPA2/mROJXnqdexekTEBqQCo4EMrInTJxtjtlYrMxJYZYwpFpFbgBHGmKsc64qMMXWat0wTv/J4JfmwdpY1BEThAWgWbg0D3eMKaNmnweYBUI1HfVf1DADSjDHpjp1/CEwAjiZ+Y8yiauV/Aq5xPlyl1AkCQmHY3TD4DtjxA2x4H5JnwqpXrXUJI6xHREdoEQfBMXowUE5zJvG3AvZWe50BDDxF+RuBb6q99heRZKxqoP8YYz6raSMRmQHMAGjbtq0TYSnlAWze0HGM9SjJh7TvrceOH2DL3N/KeflY7QEJI6xG4jYDwfvEibqVAucSf03D2NVYPyQi1wBJwLnVFrc1xuwXkQTgBxHZZIzZccIOjXkNeA2sqh4n4lLKswSEQo/LrYcxkJcO+bvg0B7r756fYMVzsPxpq/G4/3QYeDMERbo6cuVmnEn8GUCbaq9bA/uPLyQi5wMPAucaY47OumyM2e/4my4ii4E+wAmJXylVByIQ3t56VFdaALuWw8bZ1ixhK1+05gduO8iaMyCkpfXwCXBN3MotOJP41wCJIhIP7AMmAVdXLyAifYD/AeOMMVnVlocCxcaYMhGJAIYCT9RX8Eqp4/iHQOfx1iNnu3UFsO4dSH7z2HLNIqBFGwhtB22HQPxw6y5jDx+n3lM4251zPPAsVnfOmcaYR0XkISDZGDNPRL4DegAHHJvsMcZcLCJDsA4IdqzhIZ41xrxZw1scQ3v1KFWPyoqgYB8U7Ld6CBXsg0N74fBeyE6FAmueWAKjoMuF0PsaaNVXDwKNjE69qJRyjjFW+8CuZVaDcco3UFkKkV2g2yUQ3c16HhavvYbcnN65q5RyjoiV1MPioe+11lASW+bC+vdh8WO/lfP2h7ih1nASiaMhLMF1Maszpmf8SqmalRVBTgpkbYODP0Pad5CbZq0TG0c793l5g38LaxayZhFWdVHfa8E30GWheyKt6lFKnR25O6wDQJGjD4cIVFVY9xgU58Kh3XBwEwSEWeMQdZsIvs2sKwbfQO1NdBZpVY9S6uyoqQvp8fasghXPWlVF1auLxMvqQdR1AnS5CEJiz26s6qT0jF8pdXZkp1iTzVSWQVW51aso5WvI3gYI+IVYVwxeNgiMtA4IPa6EiA7W9qUFVlm/EO1q6gSt6lFKua+sbZDyFRRlg6kCY7cOEruWAwaiukJZodXd9FdhCdZcBQkjwOZjbWOM1esoOMZFH8S9aOJXSjU+Bfth86ew/VsIioKoLlZX0sIDsO0r2LkU7BUnbhfT0xq2us0Aa6gKvxBreIvgGI+6StDEr5RqekoLrN5FYPUkslfB3lWwfaH111QdW94vxLp6iO5qHRxa9rFeN9F5DTTxK6U8S0k+5KRB2WGrmuhIjlV9lLnFepQdtsrZfK1RTNsMssYvat3fOmAUZVnb2LwhopM1nlEju1rQXj1KKc8SEApt+te87te7k/evtx4ZydbYRT+9dPL9+QZbvZeCoqxJcJqFQ0grCI2zxjdqEQd+dZpfyq1o4ldKNW3V707uPtFaVllmTXO5f701b0FglJXkK0qsm9ayUyFvh3UlkLUNinOgovjY/QZGWgeB5q2txubyYmv7sHjoeSXEDQMvrwb/uM7Qqh6llKqNMVZ1Uv5O6+ohb6d1s1r+bjicYfU08gkA7wDrBrbyQghuaQ1vIV6OLq1lVhn/FhDQwrqCiO1lzaJm8znjELWqRyml6pOIY0iKMGjV79Rly4sh9Rv4+SPY+rnVruDtbyX3imIoOQSVJb+Vt/lBZEfwaw4+/lbZqgqrraKs0Noufjh0OM9qm6gq/+2g42WDiESr6qkONPErpVR98m0G3S+zHidTWWYl7gMb4cAGqyG6/AgU51mjo9p8rF5JLdpYB4qVL1p3Q3v51Nyl1Va3nkqa+JVSqqF5O87yIztCzytqL19WaN3gtmelVVUUGgct2oG9EnJSrQePOP32WsevlFJNQF3q+N2zyVkppdRZo4lfKaU8jFOJX0TGiUiKiKSJyJ9rWO8nInMc61eJSLtq6x5wLE8RkbH1F7pSSqnTUWviFxEb8BJwAdAVmCwiXY8rdiOQb4zpADwDPO7YtiswCegGjANeduxPKaWUizhzxj8ASDPGpBtjyoEPgQnHlZkAvO14/jFwnoiIY/mHxpgyY8xOIM2xP6WUUi7iTOJvBVQbGJsMx7IayxhjKoHDQLiT2yqllGpAziT+moaoO74P6MnKOLOttQORGSKSLCLJ2dnZToSllFLqdDiT+DOANtVetwb2n6yMiHgDzYE8J7cFwBjzmjEmyRiTFBkZ6Vz0Siml6qzWG7gciTwVOA/YB6wBrjbGbKlW5jaghzHmZhGZBEw0xlwpIt2AD7Dq9VsC3wOJxhw/Y8IJ71kIpJz+x2pSIoAcVwfhRvT7+I1+F8fy9O8jzhjj1FlzrUM2GGMqReR2YAFgA2YaY7aIyENAsjFmHvAm8K6IpGGd6U9ybLtFRD4CtgKVwG21JX2HFGfvQGvqRCRZv4vf6PfxG/0ujqXfh/PccsgG/Qf8jX4Xx9Lv4zf6XRxLvw/n6Z27SinlYdw18b/m6gDciH4Xx9Lv4zf6XRxLvw8nuWVVj1JKqbPHXc/4lVJKnSVulfhrGwyuqRORNiKySER+EZEtInKnY3mYiCwUke2Ov6GujrWhiIhNRNaLyJeO1/GOgQC3OwYGrNvUQ42YiLQQkY9FZJvjNzLYU38bInK34//IZhGZLSL+nvzbqCu3SfxODgbX1FUCfzTGdAEGAbc5voM/A98bYxKx7oXwpIPincAv1V4/Djzj+C7ysQYI9BTPAfONMZ2BXljfi8f9NkSkFfAHIMkY0x2rm/kkPPu3USduk/hxbjC4Js0Yc8AYs87xvBDrP3Yrjh0E723gEtdE2LBEpDXwO+ANx2sBRmENBAie9V2EAMOx7pnBGFNujDmEh/42sO5BCnDcYNoMOICH/jZOhzslfh3QrRrHnAZ9gFVAtDHmAFgHByDKdZE1qGeBPwF2x+tw4JBjIEDwrN9IApANvOWo+npDRALxwN+GMWYf8BSwByvhHwbW4rm/jTpzp8Tv9IBuTZ2IBAGfAHcZYwpcHY8riMiFQJYxZm31xTUU9ZTfiDfQF3jFGNMHOIIHVOvUxNGOMQGIxxoKJhCrivh4nvLbqDN3SvxOD+jWlImID1bSf98Y86ljcaaIxDrWxwJZroqvAQ0FLhaRXVjVfqOwrgBaOC7vwbN+IxlAhjFmleP1x1gHAk/8bZwP7DTGZBtjKoBPgSF47m+jztwp8a8BEh0t875YjTXzXBxTg3LUYb8J/GKMebraqnnAdY7n1wGfN3RsDc0Y84AxprUxph3Wb+EHY8wUYBFwuaOYR3wXAMaYg8BeEenkWHQe1hhYHvfbwKriGSQizRz/Z379Ljzyt3E63OoGLhEZj3VW9+tgcI+6OKQGJSLDgGXAJn6r1/4LVj3/R0BbrB/9FcaYPJcE6QIiMgK41xhzoYgkYF0BhAHrgWuMMWWujK+hiEhvrIZuXyAduAHr5M3jfhsi8i/gKqyecOuBm7Dq9D3yt1FXbpX4lVJKnX3uVNWjlFKqAWjiV0opD6OJXymlPIwmfqWU8jCa+JVSysNo4ldKKQ+jiV8ppTyMJn6llPIw/w9eLHsG0hbXIAAAAABJRU5ErkJggg==\n", 625 | "text/plain": [ 626 | "
" 627 | ] 628 | }, 629 | "metadata": { 630 | "needs_background": "light" 631 | }, 632 | "output_type": "display_data" 633 | }, 634 | { 635 | "data": { 636 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd81dX9x/HXyc3N3pMskhASRtiEpQioqDhxoOJqHZVStdbROvtz1La21Vq1joqj1l1AaNEqqKAyBCRhJ0gSQiCD7L1ucu89vz++EQgEcgOBm9x8no/HfZDv/Y577pcvb07O95zzVVprhBBCuBY3ZxdACCFEz5NwF0IIFyThLoQQLkjCXQghXJCEuxBCuCAJdyGEcEFdhrtS6i2lVJlSaucx1iul1ItKqVyl1Hal1LieL6YQQojucKTm/jYw6zjrLwSS21/zgFdPvlhCCCFORpfhrrVeDVQdZ5PZwDvasAEIUkpF9VQBhRBCdJ97DxwjBig4bLmw/b0DR26olJqHUbvH19d3/NChQ3vg44UQov/IyMio0FqHd7VdT4S76uS9Tuc00FovABYApKWl6fT09B74eCGE6D+UUvsc2a4nessUAnGHLccCxT1wXCGEECeoJ8J9GfCT9l4zk4FarfVRTTJCCCFOny6bZZRSHwIzgDClVCHwOGAG0Fr/A/gMuAjIBZqAW05VYYUQwuVpbbzcDqt7W1uhoQTqSxw+TJfhrrW+rov1GrjT4U88jra2NgoLC2lpaemJw7kcLy8vYmNjMZvNzi6KEMIRdju0NoClDlrqoLXRWG5rAqsFzD7g4QvunlCyA/Z9Z7zqi8HNDO5e4GaClppuf3RP3FDtMYWFhfj7+5OQkIBSnd2n7b+01lRWVlJYWEhiYqKziyNE/9baBEXpsH8DFGyEllpwcwdlAnsbNFVBcxU0V4O2O35cvwGQcCaEJoOtFawtYGsDvwjwHwD+UfDk+Q4dqleFe0tLiwT7MSilCA0Npby83NlFEcI1NNdAzT4jqJUbKGUEsaUBWuuNmnZdEVTvM7ZrLDe2bW001ms7oCBiOPiFg91mvGfygMhU8AkFnxDwCgTPAPAKAA8/o6bu4Wts19ZkHLOtGUKTIGSQUY4e0KvCHZBgPw45N0Ich7UVGsugodRom67IhrJdUJplvO/uCe7eRjNHXZFR2+6SgoAYCI6HASPbg9nPCOyY8RA3EbyDT/lXOxG9LtyFEOIoWkP5D0YTSGuTUUPWdiPIK7KhfDfU7OeoITYBMRAxDKLHGM0b1hawWyH+TAgaaIS2p/+hm5hKGcsefuDpZzSTuHs45SufLAl3IcTpV5hh1J6TzjFC9Ec2KxRvNppCmiqNV2Uu5K8xmkWOZPKEsGSITYPRc402af8BRht1yKBeW6s+HSTcT4Kfnx8NDQ3OLoYQvVNbi1GbDooDs7fxXvEW+PqPkPOFsezuBYNnQsJUKNwEuSuP6BnS3iySdA4kTjNq3N7Bh9rIzT5GM4s4ioS7EOLE2G1GU0lJ+2zg7p7GqyIb9nwN+9cbzSAAgXHgG2aEu1cQzHzCaLP+4X+QtQx++BR8I2DoxUbYDxhp3JD0CpTwPkG9Ntyf/CSTrOK6Hj3m8OgAHr809ZjrH3zwQeLj47njjjsAeOKJJ1BKsXr1aqqrq2lra+P3v/89s2fP7vKzGhoamD17dqf7vfPOOzz77LMopRg1ahTvvvsupaWlzJ8/n7y8PABeffVVzjjjjB741kL0EFsbFKbD3m9h3zoo2mz02e5M+DBIu9UI6dpCo2mlZj/MeBgm/8IIbTBq4xc8DXWFEBDbceCOOCm9NtydYe7cudxzzz0Hw33hwoUsX76ce++9l4CAACoqKpg8eTKXXXZZlz1XvLy8WLp06VH7ZWVl8Yc//IF169YRFhZGVZUxm/Ldd9/N9OnTWbp0KTabTZp7xOmntdHGXbPfeNUVGb1OGkqhrtiodbc2AMoI7dFzIXYCRI0Bk9mopbe1QEA0BHRj1m83N+PmpuhRvTbcj1fDPlXGjh1LWVkZxcXFlJeXExwcTFRUFPfeey+rV6/Gzc2NoqIiSktLGTBgwHGPpbXmkUceOWq/VatWMWfOHMLCwgAICQkBYNWqVbzzzjsAmEwmAgMDT+2XFf2T3Q4Vu42a9I+vmgIjyGuLwNrccXuTJ/hFgn+kEeaJ0432cZ8Q55RfOKzXhruzzJkzh8WLF1NSUsLcuXN5//33KS8vJyMjA7PZTEJCgkPTIxxrP6219FcXp1/5btj2IWxfZDSB/Mg3wqg1R46AlFlG23hQnPFnYGz7zUu5XvsiCfcjzJ07l9tvv52Kigq+/fZbFi5cSEREBGazma+//pp9+xyaSpna2tpO9zv33HO54ooruPfeewkNDaWqqoqQkBDOPfdcXn31Ve655x5sNhuNjY0EBAScyq8qXIWlwQjukh3GKEivIKN3Sm0RVO2BihzjT2WCwefC2Y9A5HAISTK2Fy5Jwv0Iqamp1NfXExMTQ1RUFDfccAOXXnopaWlpjBkzBkefHnWs/VJTU3n00UeZPn06JpOJsWPH8vbbb/PCCy8wb9483nzzTUwmE6+++ipTpkw5lV9V9FVaG0Pg60tgy7uQ8U9jtKV3SPuEVO2/WZp9jL7ekcNhws9g5Byj/7foF5QxqePp19mTmHbt2sWwYcOcUp6+Qs5RP1KwCb74LZRmGjcdVXuXQEudMcoSjP7ewy6FKXcZQ+HBuKnZ1iRNKi5KKZWhtU7rajupuQvR29QdgK+egO0fGcPfx95gvP9joHsFtr+CYNB0CE7ouL/Zy3iJfk3C/STt2LGDm266qcN7np6ebNy40UklEn1OaaYxmKdslzEoqCLHqHGfdT9Mva/j8Hzhkr7MKsXXw8S4+GC8zJ0P2tJa8+QnWQ4fU8L9JI0cOZKtW7c6uxiir9Ea8r6B7/4Oe1Ya7wXFG5NcpVwA434KITJvf2/T3Grj+ZXZLN1cxEvXj2NiomNdQq02O8+s2E2wrwfzpyd1WLdsWzF3f7gFAA93N8YPDOayMdHMnRDXoWfda6vzePu7fIfLKuEuxKlUsx9++Ax2/w/2bzTayN09jT+bq4w+5Oc+BuNvkb7jp4jWuv2pdSd3/+G73AoeXrqDfZVNBPuYmfduOkt+cQaDwo//m1VTq5W7PtjCqh/KAPD1MHHTlAQA8isaeWTJDsYNDOKucwazfk8la3IqeHjJDtLzq/njlSPwdDfx6fZi/vT5D1wyKoqXHSyvhLsQPam2CPLXwv7vYN96Y8AQQNgQYzi+yWw8Xs3aYozuHHWNEfb9TGF1ExUNrYyJCzolx1+dXc7a3Ap2FtWSWVyH3a45d1gEs0YMYFpKOD4eh6KvssFCZnEdmcV1FNU04edpJtDbjJ+XOxX1Fgqqm9hX2UTGvmriQ3344PZJxAb5cMUr67jl7U0sveNMQnw7nxa4qrGVW9/exPbCGp6anco3u8t5fFkmMcHenDk4jF9+uAWTm+LF68YSG+zDOUMj0Vrz4spc/vZVNvsqG5k3bRD3LdxGWnwwz149mpdvcOwcSG+ZPkbOUS9VXwrf/BE2v2PMM+4ZCAMnQcJZxmRYoUldH6OfaG61ceELq8mvbOKqcbH83yXDCPLxwG7XfLK9mJe/zqWo+tBI2TB/T34+LYmr02Ixm7qee+ad9fk89t9MPExuDI3yJzU6AKtN89WuUqqb2nB3U3i6G8fRQFOr7eC+gd5mmlqttNmMXFQKBgR4ERfsw+SkUO6YkXSwTXzz/mquW7CBETGBvP+zSR3ayg/UNrNiZwlvf5dPcW0Lf79uLBekDqDRYuWa19azt6KR6SnhfL6zhNduGs8FqUePeP90ezH3L9yGxWonIdSHJe3/iTjaW0bCvY+Rc9TLWBpgwyuw9nmwWSDtNhh3k/HoNZnNsFNPf7aL11bnceW4GJZtLSbIx8wtZyaybGsxu0vrGRLpz9TkMH5sREnfV83WghoGhvhw1zmDSYn0P3is2GBvwvwO/ebz4ff7eXjJDmYOi+TlG8bi6X7o78Bqs/P93irW5lbQaj30XNPIAC9SowMYHh1AkI8HWmta2uzUt7QR6GPucIwjfbbjAHe8vxmTmyIq0IvYYG+a2+xsKzCmLU6J9OMPV4xkQsKhJrfSuhaueHkdxbUt3HxGAk9cduypVrYV1PDa6j385oKhJIb5Ao53hZRw72PkHPUSLbXw/QJY/4rRdj7sMmMa235cQy+vtzDv3XQsbXZum5rIpaOj8XDvWNPeXljD5S+v49oJcTx95Siyiut48OPt7CiqZVC4L/fOTOHikVEd2se11ny9u4y/fpFN5hEzxZrcFGcPieDaCXFUN7by4JLtTE8J57Wbxh83lHvS2pwKNu6tpKCqiYLqZuxaM3NYJBekDmBwROft8bllDSzbVsydZyd1u5wS7ifh8ssvp6CggJaWFn71q18xb948li9fziOPPILNZiMsLIyVK1fS0NDAL3/5S9LT01FK8fjjj3PVVVed0rL1lnPUb9lt8N2LsPZvRsAnXwDTHzCeBORCvsgs4Y+f7cLkpgjwNtqgJySEcNW4WAYEHt2HPr+ikZ+89T3l9RZig73JKWsgMsCTn0xJ4Oq0WCL8vWiz2bn072upbmrli3unE+htBowadXZpAymRfrgfp9lFa036vmoaWoz+/hrNxr1VfJxRREWDBYCpg8N446dpx+xO6Ar6frh//pAxV0ZPGjASLvxTl5v9ON9Lc3MzEyZMYOXKlaSlpbF69WoSExMPrn/wwQexWCw8//zzAFRXVxMcfGof6yXh7kR1B2DJ7cYj31IuhBkPGc/m7OWqGlvx8TB1GnjVja0Eeps71JRX7ipl/nsZDArzIynCl7pmKxUNFn4oqcdNwYwhEVw0MorEMF/iQrwpqW3hln9uQgNv3TyB0bGBrM6p4PXVeazNrThYuw72MbMoo5DXf5LGecMje+z7tdnsfLO7nB2FNcyfkdThZqkrkhGqJ+HFF19k6dKlABQUFLBgwQKmTZtGYqLR7/jHaXq/+uorPvroo4P7nepgF06U8xUs/bkxrH/2KzDm+lM6tL+oppnvcitYn1eJ1aZJjQ5gREwg4f6epOdX892eCrbsr2F4dADzpg0iLT74YJ/omqZWNuRVsn5PJd/tqSSnrIEgHzM3TornJ2fEE+7nyZqcCl5fk8eanAqGDvDn3vNSOH94JGtyKvjFe5sZOiCA93426WDtGoza+cL0AhZnFB7s1vej2GBv3rl14sFugdNTwpmeEk5uWQOLMgoO1q4vHhXVo8EOYDa5cd7wyB4/bl/Xe8PdgRr2qfDNN9/w1VdfsX79enx8fJgxYwajR49m9+7dR20r0/f2A20tsPJJ46ZpxHC4+m0IH9Jjh2+12vn7qhw+2VbMj79Dt7TZKK0zmhlCfT3wdHdj2bbiDvtFBngyNi6YDXsr+TKrlNFxQYwbGMT3e6vIOlCH1uBtNjExMYTLx8awvbCGl7/JZcHqPGKCvdlb0Ui4vyfzpg3iq6xSfv5uBqnRAewpb2BQuC/v3jaxQ7ADJIT58sCsodx3Xgp7KxopqG6ioKqZ2uY25k6IIyLg6OaawRF+PHzhMH59/hDS86sZHSfPKThdem+4O0ltbS3BwcH4+Pjwww8/sGHDBiwWC99++y179+7t0Cxz/vnn89JLL53WZhlxGpXsgI9vh/JdMOF2OP+pQw967iatNdVNbQT7mA9WCPaUN3DPR1vZUVTL9JRwgn2MMHVzU4yMCWRKUigpEf64uSmqGlvJKq6jpK6FsQODGBTmi1KKplYrH2cU8ubavby/cT/jBgZx78wUzkgKZVRsUIcbmvkVjby1bi+7S+q5Y0YSl42JxtPdxAMXDGHpliJeWJlDfIgv7/9sEkE+nffbBnA3uZEc6U/yYb1WumI2uTElKfSEzp04Mb23zd1JLBYLl19+OUVFRQwZMoTy8nKeeOIJmpubeeSRR7Db7URERPDll1/S0NDAnXfeSUZGBiaTiccff5wrr7zylJavN5wjl2dpMKYFWPucMbPi7FcgeWa3D2O3a7YV1rA8s4QVO0vIr2wiwMud4dEBJIb58p8txXia3fjTlSOZNaIbj6XrhNYaq1071A/8eOXVGD1QRO/V92+oik7JOTqFbFbY8g58/TQ0lkHqlXDRs+Db/RrnhrxKfvdJFlkH6nB3U0xJCmVKUiiF1c1kFteRXVLPxMQQ/jJnFJGdNGcIcSxyQ1WI7shfC5/eZ0wXMHAKzP0A4iZ06xA2u2Z/VRPPrtjN/3YcICbImz9fNZILUgcc1cwh92vEqSbhLvq3pir44v9g63vGs0TnfgBDLnKoJ0xTq5V/fLOHJVuKqG1qo95i9L/2Mrtx78wU5k0bhLdH5/2tJdjFqdbrwl1qNMfmrCY0l7XrU/jkbmMw0tR7YdoD4OHT5W5aa5ZtK+bpz36gpK6Fs4eEEx/qS6C3mSAfMxekDiA66MRuvArRU3pVuHt5eVFZWUloaKgE/BG01lRWVuLlJe2zJ83aajzpaMPLEDUGfvoJRB57fo+Du9nsLM8sYcHqPLYX1jIyJpCXrh9LWoJM1St6n14V7rGxsRQWFlJeXu7sovRKXl5exMbGOrsYfVv1Plh8CxRlwMSfG90bu5hy127XvLthHwtW51FU00xCqA9/mTOKOeNiT3qOcCFOlV4V7maz+eAoUCF6VHO1MR/MxtfA5AFX/wtSL3do10UZBTy+LJO0+GAeu3Q4M4dFSndB0es5FO5KqVnAC4AJeENr/acj1g8E/gUEtW/zkNb6sx4uqxDdp7UxuvTbvxht66OuhXN+C0FxDu3eaLHy1y+yGTcwiEXzp0hzoegzugx3pZQJeBk4DygENimllmmtD39S62+BhVrrV5VSw4HPgIRTUF4huuerJ2Dd85B0Lpz3pDF5XDe8viaPsnoLr944XoJd9CmO1NwnArla6zwApdRHwGzg8HDXQED7z4FAx4kwhHCG9a8YwZ52K1z8XLcn+iqta+G1b/O4eGQU4+NlWgnRtzgyVjkGKDhsubD9vcM9AdyolCrEqLX/srMDKaXmKaXSlVLpctNUnFI7FsOKh2HYpcYo0xOodT/3RTZWu50HZvXcRGFCnC6O1Nw7+1dxZIfr64C3tdZ/VUpNAd5VSo3QWts77KT1AmABGNMPnEiBhehS5lJYOh/ip8KVbzj0uLvC6ibu+mALza02UqMDiA/1ZWFGAbedmUh8qO9pKLQQPcuRcC8EDr/7FMvRzS63AbMAtNbrlVJeQBhQhhCni6UBlj8EW96FmDSY+z6Yux4XkF/RyPWvb6DBYmV8fDBrcytYsqWIYB8zd50z+DQUXIie50i4bwKSlVKJQBEwF7j+iG32A+cCbyulhgFegLS7iNOneCssvhWq8mDqfXD2I2Ayd7lbTmk9N7yxEatd8+G8yaRGG/ONl9W3ABx36lsherMuw11rbVVK3QWswOjm+JbWOlMp9TsgXWu9DLgfeF0pdS9Gk83NWsbKi9PFbocP5wLKGG2aeJZDu+0orOXmf36Pm5vio3mTSTlsfvIIfxkJLPo2h/q5t/dZ/+yI9x477Ocs4MyeLZoQDireAvUH4IoFDgf7/7Yf4P5FWwn19eS9n00iMUza1YVr6VUjVIU4ITkrQLnB4K4fqKG15u+rcnnuy2zGxwfz2k3jCfM7/vQDQvRFEu6i78teAbETjvtQjbK6FlZklfLJ1mK+z6/iynExPH3lSDzdu+5JI0RfJOEu+rb6EjiwFc75v05X1za3cef7m1m3pwKtYVCYL09cOpyfnpEgI06FS5NwF31bzhfGnymzOl39u0+yWJ9Xya/OTebikVEMjvCTUBf9goS76NuyV0BATKfzsX+ZVcrHmwu56+zB3DMzxQmFE8J5TvxR6UI4m9UCed9A8vlHTS9Q3djKw0t2MHSAP3efm+yc8gnhRFJzF33XvnXQ2tBpk8xjyzKpaWrlX7dOwMNd6jCi/5FwF31X9hfg7gWJ0w6+VdFg4fU1eXyyrZj7zks5OOJUiP5Gwl30SbWNrbRtWUa1z1jWfF9KdJAX32aX8/HmIlqtdi4dHc0vZiQ5u5hCOI2Eu+iTNm3ZxMzWIl5uvoB/fmo8WsDT3Y0542O5bWoiSeF+Ti6hEM4l4S76pLY9awB4+K5fcJdvPEU1zcQG+xDiKxN9CQES7qKP8i3bTI0KICgimVClCJUpBIToQLoRiD4prnEnRb6pJ/SEJSH6Awl30edUV5SSSBGNEeOcXRQhei0Jd9HnHMgy2ts9Eyc7uSRC9F4S7qLPad27AZtWRA+XRwgIcSwS7qLP8S3bTK6KJzz02FP8CtHfSbiLvsVuI6YxiwLfEc4uiRC9moS76FOsJZn40Cw3U4XogoS76FOqdq8D5GaqEF2RcBd9imXvBiq1P3FJR8/fLoQ4RMJd9Cm+ZZvZqpMZHOnv7KII0atJuIu+o6mKkJb97PMeIQ+2FqILEu6i7yjcBEBjxFgnF0SI3k/CXfQZLXs3YNVueCVMdHZRhOj1ZFZI0WfYdn/BLj2I5NgIZxdFiF5Pau6ib6jcg2/VTj61TWJYVICzSyNEryfhLvqGnUsAWOtxFhH+Mne7EF2RZhnRN+z8mEz34YQPGISSOdyF6JLU3EXvV5oF5btYZJnEiJhAZ5dGiD5Bwl30fplL0MqNT9smMlLCXQiHSLiL3k1r2LmEkpAJVBAo4S6EgyTcRe92YBtU7WG91zQCvc3EhXg7u0RC9AkS7qJ3y1wCbu4sahrLiJgAuZkqhIMcCnel1Cyl1G6lVK5S6qFjbHONUipLKZWplPqgZ4sp+qXWJti5BNugs0kvU3IzVYhu6DLclVIm4GXgQmA4cJ1SavgR2yQDDwNnaq1TgXtOQVlFf9LWAh9dD7WF7E+6iTablvZ2IbrBkZr7RCBXa52ntW4FPgJmH7HN7cDLWutqAK11Wc8WU/Qr1lZY+BPI+xpmv8x6N2OiMAl3IRznSLjHAAWHLRe2v3e4FCBFKbVOKbVBKTWrswMppeYppdKVUunl5eUnVmLh2mxtsPgWyFkBlzwPY29gR1EtAV7uDAzxcXbphOgzHAn3zu5g6SOW3YFkYAZwHfCGUiroqJ20XqC1TtNap4WHh3e3rMLVWVth0c3ww6dw4V8g7RYAdhbVMiImUG6mCtENjoR7IRB32HIsUNzJNv/VWrdprfcCuzHCXgjHtLXAv280gn3Wn2HSzwFotdrZXVIvTTJCdJMj4b4JSFZKJSqlPIC5wLIjtvkPcDaAUioMo5kmrycLKlxYaxN8OLe9KeZvMHn+wVXZpfW02uzSU0aIbupy4jCttVUpdRewAjABb2mtM5VSvwPStdbL2tedr5TKAmzAb7TWlaey4KKPs9RD/jrYs8oI9ep9MPsVGHtDh812FNUCcjNViO5yaFZIrfVnwGdHvPfYYT9r4L72lxAd2dqMKXtzVkBNAdTsh4ZSQIO7N8SfAbP+BEMuPGrXHUW1+Hu5Ex8qN1OF6A6Z8lf0jOItkPE27F0NUaMhcTrEnwl538B3f4fa/RAQA6FJMHgmBMdD3CTjZfY65mF3FtUyIlpupgrRXRLuovuq8qAiB+qKoLYIcr805oBx94bEs2D/Rshcemj7uElw0TOQfD64OT7jhcVq44cD9dx8ZkLPfwchXJyEu3CcpQFWPgnfv87B3rDKDSJT4aJnYeTV4B1kzORYkQP7v4OwIRA/5YQ+LiO/mlabnUmJIT33HYToJyTcxdG0hqIMaKqCgGjjdWAbfHK30WY+cZ4R5AHR4BcJpiMuI6UgPMV4nYTVORWYTYrJg0JP6jhC9EcS7v2V1vDl/0H2F8aNzOGzYcBI2LUMvnsJijcfvU9oMty6HAZOPi1FXJNTzriBwfh6ymUqRHfJvxpX0VILngFGrflw1lYoy4SoMR3XrfmrcaMzIhXWvwTrngd3L7C2QEgSXPxXiBwJ9cVQVwxuZhj3k+Pe/OxJFQ0WMovr+M0FQ07L5wnhaiTc+7rmGlj1e9j0hlEDv+R58I801lXkwMe3GU0qSecYgR0yCLa8B6ueglHXwuX/gJYa2P05FGyAlFmQcmG3bnyeCutyKwA4KznMqeUQoq+ScO+rtIYdi2DFo9BUAcMuMZpYXplk3Nxsa4bPHwB3Tzjjbkj/J7wyBUZfB5vfMcL+speMEPcJMQYPHTGAyJlWZ1cQ7GMmNVoGLwlxIiTce6O2FmPkZtZ/oXCT0ZVw6MVGIDeUwo7FsP3fUJkD0ePghoUQPRbKs+E/843aOkDCWXDlAuPG56T5sPxByPin0URzzTvg7uHc73kMWmvW5JRz5uAwTG7Sv12IEyHh3k3vbdjHK1/n8vhlqVyQOqBb+2pLA3u2fkPJ9q8x1Rcy+syL8Rl+gdGMYmmA3K+MG5rZK6C1AbyCIHYC7P4fbPsATJ5gsxgHi58KZ90Po64BN5PxXngK3PoFfP+a0UVx4rxD6wJj4Nr3oDADwgaDp38PnpWelV3aQFm9hWnJMnOoECdKwt1BWmue+zKbv6/Kxd/LnfnvZfDABUOZP30QSim01mzcW0Vzq40ZQ8IPjais3oct6xNKNi4ism4Hg7ExSCvq8MHn8+XwORA+FKrzwdpCi0cI2cEzSTn7RrySZ4DJbAzfz18LOV+Ab7jRDTEorvOCmtzRk+8A6HxUZ+z4U3F6etSaHGOu/6nS3i7ECetX4d5gsbIut4KYIG9SIv3xcHdDa82e8kbW51VSXm/h4pFRDBnQsVZrtdl5dOlO/p1ewNwJcfz24mE8+vFmXly+lcLiYsb4VpKfuZHIphxiVAW7fBXJYR6YLbVQvgsTUGuPZ3PQ1QQPm8GIyeexck8Tb378CXfG5XOR/x50wjT+VTOap3YEYq9zI26ZiaevqGVqchgW7caXjUP4T6kfAMNbmhkRXUKonwdZB+rJLKrlh5J6qhpbqWtpo665jYEhPtw6NZE542Px8Tj2X3Ntcxvf763iuz0VbCuowcPdjUBvM4HeZoYOCOCCEQOICfLu8twW1TSzKL2A7/ZUkhDqQ2p0ICNiAhgdG4S7qXs3Z9fkVJAU7ku0A58rhOicMub8Ov3S0tJ0enr6afmsA7XNvL0unw++308v65OdAAAR4klEQVR9ixUAs0kxOMKfygYLZfWWDtuPiQviynExNLfa2Flcx7aCGmxV+/j9sP3M0BtR+9aDth31OW3uvtR6x7G/1obd5EFkSDBLqxNZ1jqe+Vecx5zxsR22f/6rbJ7/KodfzEhiZ1Eta3Iq+NnURM4dFsmjS3eQV9HIWclh7CyqpbqpjahAL3w93ckrb8B+2F9boLeZ4VEBRAR4Euhtxs/TnfV5lWzZX0OQj5mbJsdzx4zBeHuYDu5jsdp4Ylkm/95UgF2Dl9mNUbFBaK2pbW6juqmN8vbzMio2kMtGR3PTlHg83U0dvsPq7HJeX5PH2vbeLanRARTXtFDV2AoYszm+csM44hx8ilJLm40xv/uCuRMG8sRlqQ7tI0R/opTK0FqndbmdK4a73a7JOlDH+j2VrNtTwdqcCuxac+HIKK6fOJDqplZ2FtWRdaCOQG8zZySFMmVQKP5e7izdUsTC9AIaS/cyyW0X07zzmOi2m+i2fcbBw4dB8kzwCgQ3M6WNNlTQQCKS0yAoHtzc2HWgjns+2sru0noSQn149cbxDIsKOKqcWmvuX7SNJZuLcHdT/OGKEVw7YSBghNxLq3L5aNN+Jg0K5Zq0OKa232BsarWy60A91Y2tDI3yJybIu9MmmIx9Vby+ei/LM0tICvflhbljGRETSGWDhfnvZbApv5qfTonnopFRjBkYdFRw55U3sCKzlOU7D7CtsJakcF/+fNUo0hJCqGyw8NSnWfxnazHRgV5cnRbHnPGxxIX4oLWmpK6FNTkVPPVpFm5K8fy1Yzh7aMRx/94qGix8sHE/z32ZzVs3p3HO0MgTvAKEcF39NtwtVhuX/n0t2aUNACSF+3L2kAh+ekaCY7XHyj3ob56GHYtRaPAMhLiJxoRYQy8xZjV0QEubjRWZJZw9NIIAL/Mxt2u12nlhZTbTUyKYeIrmUFmXW8H9C7dR2Wjh9rMGsWxbMeX1Fp69ejSXjo526Bjf7C7j0aU7Ka5t5tJR0azNraC+pY07ZgzmjrOTjvqP4Uf7KhuZ/95mdh2o4+fTBzF/WhLBvod66TS32li6pYj/bi1iU34Vdm3U9hf+fEqH3zSEEIZ+G+7/236AOz/YzIOzhnLluBgiA44YUdlSa0xPW7wFireC3Wp0FfQfAFV7YesHYPIwHvM26hqjpu7kAT09oaaplYeX7ODznSVE+Hvy+k/SGB131GNuj6vRYuWZFbv51/p8RscG8eerRh11f6IzLW02HvvvThamF+JtNjFnfCxzxsey8ocy3l2fT3VTGymRfswaEcWFIwYwdIC/TPErxDH023C/+Z/fs7uknrUPntOxj7TVAt/+Gda9YAQ6QHCCMeS+/oAR+iYPGH+L0cXQ3/WaBIz+4xUMHeBPxJH/6XVDWX0Lob6e3e6DvruknjfW5PHfrcW02uwoBecOjWTetEFMSAiWQBfCAY6Gu0v1limpbWF1djl3zBjcMXiKt8B/7oCyLGOE5qhrjIE8Poc1g7Q2GTdJe3H/75OllGJaysn3HY/wP7H/GIYM8OeZq0fzmwuG8NWuMiYNCiEp3O+kyyOEOJpLhfvHmwuxaw71StEa1j4Hq/4AfhFw/SJIOb/znT3kMW6nS0SAF9dPGujsYgjh0lwm3LXWLM4oZGJCCAlhvkYzzLK7YftHkHolXPIceAc7u5hCCHFauEy4Z+yrZm9FI7+YkQSNlfDvG40nAZ39KEz7zdFT4QohhAtzmXBflF6Ij4eJy7x3wusPQH0pXPUmjJzj7KIJIcRp5xLh3tRqZcv2rSwM/AivReuMJwbd/D+Im+DsogkhhFP0qXCvaLAQ6utxqMtcQznkfsn+1YtZplZjbjbDzCdh8h29djpbIYQ4HfpMuGfsq+Ka1zZw9pAI/jpnBIErfmXMaY4mWAexJeQiJt/8RwiM7fJYQgjh6vpEuGuteerTXfh6mPhmdymr/vYMV1iXszPueh7MTSVl1BSevXYcSh7sIIQQAPSJcfX/23GArQU1/Pbi4Xw7eTNXWJezwHYpl+RcQvyIKTxzzVh5Yo8QQhym19fcLVYbf17+A0MH+DPHfQ1um5+lZdhVbGqez+VeZp65enS35wsXQghX1+vD/d31+yioaubjq4Jw++SXkDgdr6v+wetyw1QIIY6pV1d5a5paeXFlDtNTwhmf/QJ4+MLVb0tPGCGE6EKvDvd/fJtHg8XKU6OrIWcFTL2v42RfQgghOtVrm2UaLVbe37iPC0cMYGDGryEgxphjXQghRJd6bc19UXoB9S1Wfh2TBcWbjTlizPLAZCGEcESvDHebXfPWunwmxvmSuO1ZiEiF0XOdXSwhhOgzemWzzJdZpeyvauLV5CzYlg83LAY3eZ6mEEI4yqGau1JqllJqt1IqVyn10HG2m6OU0kqpLh8BdTxvrd3L4CDF8NzXIH4qDJ55MocTQoh+p8twV0qZgJeBC4HhwHVKqeGdbOcP3A1sPJkCbSuo4fv8Kv4Usw7VWA4zH5e52IUQopscqblPBHK11nla61bgI2B2J9s9BfwFaDmZAr25di/Rni2ML3wHUi6EuIknczghhOiXHAn3GKDgsOXC9vcOUkqNBeK01p8e70BKqXlKqXSlVHp5eflR6+12zZdZpTwduQplqYdzfutA8YQQQhzJkXDvrE1EH1yplBvwN+D+rg6ktV6gtU7TWqeFh4cftb6gugm/tkrOrFhsPEFpwAgHiieEEOJIjoR7IRB32HIsUHzYsj8wAvhGKZUPTAaWnchN1d0l9dzlvhSTtsKMh7u7uxBCiHaOhPsmIFkplaiU8gDmAst+XKm1rtVah2mtE7TWCcAG4DKtdXp3C5NdWs/lpnVYh18BoUnd3V0IIUS7LsNda20F7gJWALuAhVrrTKXU75RSl/VkYfYVlxKomjBHj+zJwwohRL/j0CAmrfVnwGdHvPfYMbadcaKFqSvda/wQEHP8DYUQQhxXr5l+oM1mp626vVOOPAdVCCFOSq8J9/yKRiJ0pbEgNXchhDgpvSbcs0sbiFKVaOUG/lHOLo4QQvRpvSbcd5fWE6MqwS8STL1yPjMhhOgzek24Z5fUk+hRg5L2diGEOGm9J9xL64lxq5L2diGE6AG9Itxb2mzkVzYQaiuXnjJCCNEDekW47ylvIEA3YLZbpOYuhBA9oFeEe3ZpPdGqvRtkoIS7EEKcrF4S7g3EmaqMhQBplhFCiJPVO8K9pJ6R/g3GgtTchRDipPWKcN9dWk+Kdx24mcE3wtnFEUKIPs/p4d5osVJY3Uy8qQoCosDN6UUSQog+z+lJmlNmNMdE6AppbxdCiB7i9HDfV9kIgJ+lVNrbhRCihzg93MvqLCjsuDeWSB93IYToIU4P95K6FuI8GlD2NhmdKoQQPcTp4V5a10Kqb72xIDV3IYToEb0i3JO9ao0FqbkLIUSP6AXhbiHBXG0sSLgLIUSPcGq4a60pqWsxpvp19wbvYGcWRwghXIZTw722uY1Wq51we4XRDVIpZxZHCCFchlPDvaSuBYBga5ncTBVCiB7k3HCvNcLdt6VU2tuFEKIHOTXcy+osmLBhbpaauxBC9CSnN8tEUo3Sdpl6QAghepBTw720roUhPnXGgkwaJoQQPcbp4T7Kq8xYCEl0ZlGEEMKlODncLYw05YOHPwRLuAshRE9xept7si0PBoyUh3QIIUQPclqiaqCqoZnollyIGu2sYgghhEtyWrhbbXYSOIDZ3gJRo5xVDCGEcElOC/c2myZV5RsLUnMXQoge5dSa+wi3fOwmTwhLcVYxhBDCJTkU7kqpWUqp3UqpXKXUQ52sv08plaWU2q6UWqmUiu/qmG02zQi1F1v4cDCZT6TsQgghjqHLcFdKmYCXgQuB4cB1SqnhR2y2BUjTWo8CFgN/6eq4bTY7qW77cI8Z0/1SCyGEOC5Hau4TgVytdZ7WuhX4CJh9+AZa66+11k3tixuALoebapuFQNWIkpupQgjR4xwJ9xig4LDlwvb3juU24PPOViil5iml0pVS6bqlwXhTbqYKIUSPcyTcO3uChu50Q6VuBNKAZzpbr7VeoLVO01qneSkrNtwgItXx0gohhHCII+FeCMQdthwLFB+5kVJqJvAocJnW2tLVQT10CxVeiWD2crSsQgghHORIuG8CkpVSiUopD2AusOzwDZRSY4HXMIK9zJEP9qKV6sAj78sKIYToCV2Gu9baCtwFrAB2AQu11plKqd8ppS5r3+wZwA9YpJTaqpRadozDHeSOFUu4NMkIIcSp4O7IRlrrz4DPjnjvscN+nnkiH+4WLd0ghRDiVHDqVIw+AyXchRDiVHBauFswExke7qyPF0IIl+a0cC8hDD9Ph1qFhBBCdJPzau4mX2d9tBBCuDynhXtEgPRvF0KIU8Vp4R7kLTNBCiHEqSIPLhVCCBck4S6EEC5Iwl0IIVyQhLsQQrggCXchhHBBEu5CCOGCJNyFEMIFSbgLIYQLknAXQggXJOEuhBAuSMJdCCFckIS7EEK4IAl3IYRwQRLuQgjhgiTchRDCBUm4CyGEC5JwF0IIFyThLoQQLkjCXQghXJCEuxBCuCAJdyGEcEES7kII4YIk3IUQwgVJuAshhAuScBdCCBck4S6EEC5Iwl0IIVyQhLsQQrggCXchhHBBDoW7UmqWUmq3UipXKfVQJ+s9lVL/bl+/USmV0NMFFUII4bguw10pZQJeBi4EhgPXKaWGH7HZbUC11now8Dfgzz1dUCGEEI5zpOY+EcjVWudprVuBj4DZR2wzG/hX+8+LgXOVUqrniimEEKI73B3YJgYoOGy5EJh0rG201lalVC0QClQcvpFSah4wr33RopTaeSKFdlFhHHG++jE5Fx3J+eiov5+PeEc2ciTcO6uB6xPYBq31AmABgFIqXWud5sDn9wtyPg6Rc9GRnI+O5Hw4xpFmmUIg7rDlWKD4WNsopdyBQKCqJwoohBCi+xwJ901AslIqUSnlAcwFlh2xzTLgp+0/zwFWaa2PqrkLIYQ4PbpslmlvQ78LWAGYgLe01plKqd8B6VrrZcCbwLtKqVyMGvtcBz57wUmU2xXJ+ThEzkVHcj46kvPhACUVbCGEcD0yQlUIIVyQhLsQQrggp4R7V9MZuDKlVJxS6mul1C6lVKZS6lft74copb5USuW0/xns7LKeTkopk1Jqi1Lq0/blxPapLHLap7bwcHYZTwelVJBSarFS6of2a2RKf742lFL3tv872amU+lAp5dVfr43uOu3h7uB0Bq7MCtyvtR4GTAbubP/+DwErtdbJwMr25f7kV8Cuw5b/DPyt/XxUY0xx0R+8ACzXWg8FRmOck355bSilYoC7gTSt9QiMDh1z6b/XRrc4o+buyHQGLktrfUBrvbn953qMf7wxdJzC4V/A5c4p4emnlIoFLgbeaF9WwDkYU1lAPzkfSqkAYBpG7zO01q1a6xr68bWB0aPPu338jA9wgH54bZwIZ4R7Z9MZxDihHE7XPnvmWGAjEKm1PgDGfwBAhPNKdto9DzwA2NuXQ4EarbW1fbm/XCODgHLgn+1NVG8opXzpp9eG1roIeBbYjxHqtUAG/fPa6DZnhLtDUxW4OqWUH/AxcI/Wus7Z5XEWpdQlQJnWOuPwtzvZtD9cI+7AOOBVrfVYoJF+0gTTmfZ7C7OBRCAa8MVozj1Sf7g2us0Z4e7IdAYuTSllxgj297XWS9rfLlVKRbWvjwLKnFW+0+xM4DKlVD5GE905GDX5oPZfxaH/XCOFQKHWemP78mKMsO+v18ZMYK/Wulxr3QYsAc6gf14b3eaMcHdkOgOX1d6e/CawS2v93GGrDp/C4afAf0932ZxBa/2w1jpWa52AcS2s0lrfAHyNMZUF9JPzobUuAQqUUkPa3zoXyKKfXhsYzTGTlVI+7f9ufjwf/e7aOBFOGaGqlLoIo3b243QGfzjthXASpdRUYA2wg0NtzI9gtLsvBAZiXNRXa6371eRrSqkZwK+11pcopQZh1ORDgC3AjVprizPLdzoopcZg3Fj2APKAWzAqYf3y2lBKPQlci9HLbAvwM4w29n53bXSXTD8ghBAuSEaoCiGEC5JwF0IIFyThLoQQLkjCXQghXJCEuxBCuCAJdyGEcEES7kII4YL+H7VAmO1wQPKfAAAAAElFTkSuQmCC\n", 637 | "text/plain": [ 638 | "
" 639 | ] 640 | }, 641 | "metadata": { 642 | "needs_background": "light" 643 | }, 644 | "output_type": "display_data" 645 | } 646 | ], 647 | "source": [ 648 | "model.fit([inputs_train, queries_train], answers_train, batch_size, train_epochs,callbacks=[TrainingVisualizer()],\n", 649 | " validation_data=([inputs_test, queries_test], answers_test))\n", 650 | "\n", 651 | "model.save('model.h5')" 652 | ] 653 | }, 654 | { 655 | "cell_type": "markdown", 656 | "metadata": {}, 657 | "source": [ 658 | "## Test Result Analysis" 659 | ] 660 | }, 661 | { 662 | "cell_type": "code", 663 | "execution_count": 20, 664 | "metadata": {}, 665 | "outputs": [ 666 | { 667 | "name": "stdout", 668 | "output_type": "stream", 669 | "text": [ 670 | "John travelled to the hallway . Mary journeyed to the bathroom . Where is John ? | Prediction: hallway | Ground Truth: hallway\n", 671 | "-----------------------------------------------------------------------------------------\n", 672 | "John travelled to the hallway . Mary journeyed to the bathroom . Daniel went back to the bathroom . John moved to the bedroom . Where is Mary ? | Prediction: bathroom | Ground Truth: bathroom\n", 673 | "-----------------------------------------------------------------------------------------\n", 674 | "John travelled to the hallway . Mary journeyed to the bathroom . Daniel went back to the bathroom . John moved to the bedroom . John went to the hallway . Sandra journeyed to the kitchen . Where is Sandra ? | Prediction: kitchen | Ground Truth: kitchen\n", 675 | "-----------------------------------------------------------------------------------------\n", 676 | "John travelled to the hallway . Mary journeyed to the bathroom . Daniel went back to the bathroom . John moved to the bedroom . John went to the hallway . Sandra journeyed to the kitchen . Sandra travelled to the hallway . John went to the garden . Where is Sandra ? | Prediction: hallway | Ground Truth: hallway\n", 677 | "-----------------------------------------------------------------------------------------\n", 678 | "John travelled to the hallway . Mary journeyed to the bathroom . Daniel went back to the bathroom . John moved to the bedroom . John went to the hallway . Sandra journeyed to the kitchen . Sandra travelled to the hallway . John went to the garden . Sandra went back to the bathroom . Sandra moved to the kitchen . Where is Sandra ? | Prediction: kitchen | Ground Truth: kitchen\n", 679 | "-----------------------------------------------------------------------------------------\n", 680 | "Sandra travelled to the kitchen . Sandra travelled to the hallway . Where is Sandra ? | Prediction: hallway | Ground Truth: hallway\n", 681 | "-----------------------------------------------------------------------------------------\n", 682 | "Sandra travelled to the kitchen . Sandra travelled to the hallway . Mary went to the bathroom . Sandra moved to the garden . Where is Sandra ? | Prediction: garden | Ground Truth: garden\n", 683 | "-----------------------------------------------------------------------------------------\n", 684 | "Sandra travelled to the kitchen . Sandra travelled to the hallway . Mary went to the bathroom . Sandra moved to the garden . Sandra travelled to the office . Daniel journeyed to the hallway . Where is Daniel ? | Prediction: hallway | Ground Truth: hallway\n", 685 | "-----------------------------------------------------------------------------------------\n", 686 | "Sandra travelled to the kitchen . Sandra travelled to the hallway . Mary went to the bathroom . Sandra moved to the garden . Sandra travelled to the office . Daniel journeyed to the hallway . Daniel journeyed to the office . John moved to the hallway . Where is Sandra ? | Prediction: office | Ground Truth: office\n", 687 | "-----------------------------------------------------------------------------------------\n", 688 | "Sandra travelled to the kitchen . Sandra travelled to the hallway . Mary went to the bathroom . Sandra moved to the garden . Sandra travelled to the office . Daniel journeyed to the hallway . Daniel journeyed to the office . John moved to the hallway . John travelled to the bathroom . John journeyed to the office . Where is Daniel ? | Prediction: office | Ground Truth: office\n", 689 | "-----------------------------------------------------------------------------------------\n" 690 | ] 691 | } 692 | ], 693 | "source": [ 694 | "for i in range(0,10):\n", 695 | " current_inp = test_stories[i]\n", 696 | " current_story, current_query, current_answer = vectorize_stories([current_inp], word_idx, story_maxlen, query_maxlen)\n", 697 | " current_prediction = model.predict([current_story, current_query])\n", 698 | " current_prediction = idx_word[np.argmax(current_prediction)]\n", 699 | " print(' '.join(current_inp[0]), ' '.join(current_inp[1]), '| Prediction:', current_prediction, '| Ground Truth:', current_inp[2])\n", 700 | " print(\"-----------------------------------------------------------------------------------------\")\n" 701 | ] 702 | }, 703 | { 704 | "cell_type": "markdown", 705 | "metadata": { 706 | "collapsed": true 707 | }, 708 | "source": [ 709 | "## Custom Inputs for demo" 710 | ] 711 | }, 712 | { 713 | "cell_type": "code", 714 | "execution_count": null, 715 | "metadata": { 716 | "collapsed": true 717 | }, 718 | "outputs": [], 719 | "source": [ 720 | "print('-------------------------------------------------------------------------------------------')\n", 721 | "print('Custom User Queries (Make sure there are spaces before each word)')\n", 722 | "while 1:\n", 723 | " print('-------------------------------------------------------------------------------------------')\n", 724 | " print('Please input a story')\n", 725 | " user_story_inp = input().split(' ')\n", 726 | " print('Please input a query')\n", 727 | " user_query_inp = input().split(' ')\n", 728 | " user_story, user_query, user_ans = vectorize_stories([[user_story_inp, user_query_inp, '.']], word_idx, story_maxlen, query_maxlen)\n", 729 | " user_prediction = model.predict([user_story, user_query])\n", 730 | " user_prediction = idx_word[np.argmax(user_prediction)]\n", 731 | " print('Result')\n", 732 | " print(' '.join(user_story_inp), ' '.join(user_query_inp), '| Prediction:', user_prediction)" 733 | ] 734 | }, 735 | { 736 | "cell_type": "code", 737 | "execution_count": null, 738 | "metadata": { 739 | "collapsed": true 740 | }, 741 | "outputs": [], 742 | "source": [ 743 | "# Mary went to the bathroom . John moved to the hallway . Mary travelled to the office . # Where is Mary ?\n", 744 | "# Sandra travelled to the office . John journeyed to the garden ." 745 | ] 746 | }, 747 | { 748 | "cell_type": "code", 749 | "execution_count": null, 750 | "metadata": { 751 | "collapsed": true 752 | }, 753 | "outputs": [], 754 | "source": [] 755 | } 756 | ], 757 | "metadata": { 758 | "kernelspec": { 759 | "display_name": "Python 3", 760 | "language": "python", 761 | "name": "python3" 762 | }, 763 | "language_info": { 764 | "codemirror_mode": { 765 | "name": "ipython", 766 | "version": 3 767 | }, 768 | "file_extension": ".py", 769 | "mimetype": "text/x-python", 770 | "name": "python", 771 | "nbconvert_exporter": "python", 772 | "pygments_lexer": "ipython3", 773 | "version": "3.6.8" 774 | } 775 | }, 776 | "nbformat": 4, 777 | "nbformat_minor": 2 778 | } 779 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Question Answering System using End to End Memory Networks 2 | 3 | This project is the implementation of End to End Memory Networks to build Question Answering System. It takes small story and query as an input and predicts a possibe answer to the query. Here we have used Facebook Babi Dataset to train the model. After training, user can give similar stories and query as input and it would give you accurate results. 4 | 5 | ## Dependencies 6 | 7 | * tensorflow 8 | * keras 9 | * functools 10 | * tarfile 11 | * re 12 | 13 | ## Architecture of End to End Memory Networks 14 | ![alt text](https://github.com/prashil2792/Question-Answering-System-Deep-Learning/blob/master/images/architecture.png) 15 | 16 | ## Results/Observations 17 | 18 | |Layers |Dropouts |Batch-size | Epochs |Results | 19 | | ----------------------------- | ----------------- | --------- | --------- | --------- | 20 | |LSTM(32) |(0.3) |32 |100 |94.6% | 21 | |LSTM(64) |(0.3) |32 |100 |96.5% | 22 | |LSTM(32), LSTM(32) |(0.5, 0.5) |32 |100 |92.4% | 23 | |LSTM(32), LSTM(32) |(0.5, 0.5) |32 |200 |96.9% | 24 | |GRU(32) |(0.3) |32 |100 |86.4% | 25 | |GRU(64) |(0.3) |32 |100 |87.4% | 26 | |GRU(32), GRU(32) |(0.5, 0.5) |64 |100 |52.6% | 27 | 28 | * The models with two or more layers required more training since there are more parameters that need to be set, but then have greater accuracies than the other models once trained completely. 29 | * Overall, LSTM based models performed better than GRU based models for this task. 30 | 31 | ## References 32 | 33 | - Jason Weston, Antoine Bordes, Sumit Chopra, Tomas Mikolov, Alexander M. Rush, 34 | "Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks", 35 | (http://arxiv.org/abs/1502.05698) 36 | - Sainbayar Sukhbaatar, Arthur Szlam, Jason Weston, Rob Fergus, 37 | "End-To-End Memory Networks", 38 | (http://arxiv.org/abs/1503.08895) 39 | -------------------------------------------------------------------------------- /images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashil2792/Question-Answering-System-Deep-Learning/fc0657f51a220baa2798ef2a164bbffca421e7d2/images/architecture.png -------------------------------------------------------------------------------- /memorynetwork.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Trains a memory network on the facebook bAbI dataset for Question/Answering System. 3 | ''' 4 | 5 | # import the packages 6 | import keras 7 | from keras.models import Sequential, Model 8 | from keras.layers.embeddings import Embedding 9 | from keras.layers import Input, Activation, Dense, Permute, Dropout 10 | from keras.layers import add, dot, concatenate 11 | from keras.layers import LSTM, GRU 12 | from keras.utils.data_utils import get_file 13 | from keras.preprocessing.sequence import pad_sequences 14 | from keras import backend as K 15 | 16 | from functools import reduce 17 | import tarfile 18 | import numpy as np 19 | import re 20 | 21 | train_epochs = 100 22 | batch_size = 32 23 | lstm_size = 64 24 | 25 | def tokenize(sent): 26 | '''Return the tokens of a sentence including punctuation. 27 | >>> tokenize('Bob dropped the apple. Where is the apple?') 28 | ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?'] 29 | ''' 30 | return [x.strip() for x in re.split('(\W+)?', sent) if x.strip()] 31 | 32 | 33 | def parse_stories(lines, only_supporting=False): 34 | '''Parse stories provided in the bAbi tasks format 35 | If only_supporting is true, only the sentences 36 | that support the answer are kept. 37 | ''' 38 | data = [] 39 | story = [] 40 | for line in lines: 41 | line = line.decode('utf-8').strip() 42 | nid, line = line.split(' ', 1) 43 | nid = int(nid) 44 | if nid == 1: 45 | story = [] 46 | if '\t' in line: 47 | q, a, supporting = line.split('\t') 48 | q = tokenize(q) 49 | substory = None 50 | if only_supporting: 51 | # Only select the related substory 52 | supporting = map(int, supporting.split()) 53 | substory = [story[i - 1] for i in supporting] 54 | else: 55 | # Provide all the substories 56 | substory = [x for x in story if x] 57 | data.append((substory, q, a)) 58 | story.append('') 59 | else: 60 | sent = tokenize(line) 61 | story.append(sent) 62 | return data 63 | 64 | 65 | def get_stories(f, only_supporting=False, max_length=None): 66 | '''Given a file name, read the file, 67 | retrieve the stories, 68 | and then convert the sentences into a single story. 69 | If max_length is supplied, 70 | any stories longer than max_length tokens will be discarded. 71 | ''' 72 | data = parse_stories(f.readlines(), only_supporting=only_supporting) 73 | flatten = lambda data: reduce(lambda x, y: x + y, data) 74 | data = [(flatten(story), q, answer) for story, q, answer in data if not max_length or len(flatten(story)) < max_length] 75 | return data 76 | 77 | 78 | def vectorize_stories(data, word_idx, story_maxlen, query_maxlen): 79 | X = [] 80 | Xq = [] 81 | Y = [] 82 | for story, query, answer in data: 83 | x = [word_idx[w] for w in story] 84 | xq = [word_idx[w] for w in query] 85 | # let's not forget that index 0 is reserved 86 | y = np.zeros(len(word_idx) + 1) 87 | y[word_idx[answer]] = 1 88 | X.append(x) 89 | Xq.append(xq) 90 | Y.append(y) 91 | return (pad_sequences(X, maxlen=story_maxlen), 92 | pad_sequences(Xq, maxlen=query_maxlen), np.array(Y)) 93 | 94 | try: 95 | path = get_file('babi-tasks-v1-2.tar.gz', origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz') 96 | except: 97 | print('Error downloading dataset, please download it manually:\n' 98 | '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n' 99 | '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz') 100 | raise 101 | tar = tarfile.open(path) 102 | 103 | challenges = { 104 | # QA1 with 10,000 samples 105 | 'single_supporting_fact_10k': 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt', 106 | # QA2 with 10,000 samples 107 | 'two_supporting_facts_10k': 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt', 108 | } 109 | challenge_type = 'single_supporting_fact_10k' 110 | challenge = challenges[challenge_type] 111 | 112 | print('Extracting stories for the challenge:', challenge_type) 113 | train_stories = get_stories(tar.extractfile(challenge.format('train'))) 114 | test_stories = get_stories(tar.extractfile(challenge.format('test'))) 115 | 116 | vocab = set() 117 | for story, q, answer in train_stories + test_stories: 118 | vocab |= set(story + q + [answer]) 119 | vocab = sorted(vocab) 120 | 121 | # Reserve 0 for masking via pad_sequences 122 | vocab_size = len(vocab) + 1 123 | story_maxlen = max(map(len, (x for x, _, _ in train_stories + test_stories))) 124 | query_maxlen = max(map(len, (x for _, x, _ in train_stories + test_stories))) 125 | 126 | print('-') 127 | print('Vocab size:', vocab_size, 'unique words') 128 | print('Story max length:', story_maxlen, 'words') 129 | print('Query max length:', query_maxlen, 'words') 130 | print('Number of training stories:', len(train_stories)) 131 | print('Number of test stories:', len(test_stories)) 132 | print('-') 133 | print('Here\'s what a "story" tuple looks like (input, query, answer):') 134 | print(train_stories[0]) 135 | print('-') 136 | print('Vectorizing the word sequences...') 137 | 138 | word_idx = dict((c, i + 1) for i, c in enumerate(vocab)) 139 | idx_word = dict((i+1, c) for i,c in enumerate(vocab)) 140 | inputs_train, queries_train, answers_train = vectorize_stories(train_stories, 141 | word_idx, 142 | story_maxlen, 143 | query_maxlen) 144 | inputs_test, queries_test, answers_test = vectorize_stories(test_stories, 145 | word_idx, 146 | story_maxlen, 147 | query_maxlen) 148 | 149 | print('-') 150 | print('inputs: integer tensor of shape (samples, max_length)') 151 | print('inputs_train shape:', inputs_train.shape) 152 | print('inputs_test shape:', inputs_test.shape) 153 | print('-') 154 | print('queries: integer tensor of shape (samples, max_length)') 155 | print('queries_train shape:', queries_train.shape) 156 | print('queries_test shape:', queries_test.shape) 157 | print('-') 158 | print('answers: binary (1 or 0) tensor of shape (samples, vocab_size)') 159 | print('answers_train shape:', answers_train.shape) 160 | print('answers_test shape:', answers_test.shape) 161 | print('-') 162 | print('Compiling...') 163 | 164 | # placeholders 165 | input_sequence = Input((story_maxlen,)) 166 | question = Input((query_maxlen,)) 167 | 168 | print('Input sequence:', input_sequence) 169 | print('Question:', question) 170 | 171 | # encoders 172 | # embed the input sequence into a sequence of vectors 173 | input_encoder_m = Sequential() 174 | input_encoder_m.add(Embedding(input_dim=vocab_size, 175 | output_dim=64)) 176 | input_encoder_m.add(Dropout(0.3)) 177 | # output: (samples, story_maxlen, embedding_dim) 178 | 179 | # embed the input into a sequence of vectors of size query_maxlen 180 | input_encoder_c = Sequential() 181 | input_encoder_c.add(Embedding(input_dim=vocab_size, 182 | output_dim=query_maxlen)) 183 | input_encoder_c.add(Dropout(0.3)) 184 | # output: (samples, story_maxlen, query_maxlen) 185 | 186 | # embed the question into a sequence of vectors 187 | question_encoder = Sequential() 188 | question_encoder.add(Embedding(input_dim=vocab_size, 189 | output_dim=64, 190 | input_length=query_maxlen)) 191 | question_encoder.add(Dropout(0.3)) 192 | # output: (samples, query_maxlen, embedding_dim) 193 | 194 | # encode input sequence and questions (which are indices) 195 | # to sequences of dense vectors 196 | input_encoded_m = input_encoder_m(input_sequence) 197 | print('Input encoded m', input_encoded_m) 198 | input_encoded_c = input_encoder_c(input_sequence) 199 | print('Input encoded c', input_encoded_c) 200 | question_encoded = question_encoder(question) 201 | print('Question encoded', question_encoded) 202 | 203 | 204 | # compute a 'match' between the first input vector sequence 205 | # and the question vector sequence 206 | # shape: `(samples, story_maxlen, query_maxlen) 207 | match = dot([input_encoded_m, question_encoded], axes=(2, 2)) 208 | print(match.shape) 209 | match = Activation('softmax')(match) 210 | print('Match shape', match) 211 | 212 | # add the match matrix with the second input vector sequence 213 | response = add([match, input_encoded_c]) # (samples, story_maxlen, query_maxlen) 214 | response = Permute((2, 1))(response) # (samples, query_maxlen, story_maxlen) 215 | print('Response shape', response) 216 | 217 | # concatenate the response vector with the question vector sequence 218 | answer = concatenate([response, question_encoded]) 219 | print('Answer shape', answer) 220 | 221 | #answer = LSTM(lstm_size, return_sequences=True)(answer) # Generate tensors of shape 32 222 | #answer = Dropout(0.3)(answer) 223 | answer = LSTM(lstm_size)(answer) # Generate tensors of shape 32 224 | answer = Dropout(0.3)(answer) 225 | answer = Dense(vocab_size)(answer) # (samples, vocab_size) 226 | # we output a probability distribution over the vocabulary 227 | answer = Activation('softmax')(answer) 228 | # build the final model 229 | model = Model([input_sequence, question], answer) 230 | model.compile(optimizer='adam', loss='categorical_crossentropy', 231 | metrics=['accuracy']) 232 | 233 | print("-------------Model Summary------------") 234 | print(model.summary()) 235 | 236 | # train, batch_size = 32 and epochs = 120 237 | print("Trainig the model") 238 | model.fit([inputs_train, queries_train], answers_train, batch_size, train_epochs, 239 | validation_data=([inputs_test, queries_test], answers_test)) 240 | model.save('model.h5') 241 | 242 | print('-------------------------------------------------------------------------------------------') 243 | print('Qualitative Test Result Analysis') 244 | for i in range(0,10): 245 | current_inp = test_stories[i] 246 | current_story, current_query, current_answer = vectorize_stories([current_inp], word_idx, story_maxlen, query_maxlen) 247 | current_prediction = model.predict([current_story, current_query]) 248 | current_prediction = idx_word[np.argmax(current_prediction)] 249 | print(' '.join(current_inp[0]), ' '.join(current_inp[1]), '| Prediction:', current_prediction, '| Ground Truth:', current_inp[2]) 250 | 251 | 252 | -------------------------------------------------------------------------------- /model.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashil2792/Question-Answering-System-Deep-Learning/fc0657f51a220baa2798ef2a164bbffca421e7d2/model.h5 -------------------------------------------------------------------------------- /presentation/presentation slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashil2792/Question-Answering-System-Deep-Learning/fc0657f51a220baa2798ef2a164bbffca421e7d2/presentation/presentation slides.pdf -------------------------------------------------------------------------------- /trained_models/gru_32.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashil2792/Question-Answering-System-Deep-Learning/fc0657f51a220baa2798ef2a164bbffca421e7d2/trained_models/gru_32.h5 -------------------------------------------------------------------------------- /trained_models/gru_32_32.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashil2792/Question-Answering-System-Deep-Learning/fc0657f51a220baa2798ef2a164bbffca421e7d2/trained_models/gru_32_32.h5 -------------------------------------------------------------------------------- /trained_models/gru_64.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashil2792/Question-Answering-System-Deep-Learning/fc0657f51a220baa2798ef2a164bbffca421e7d2/trained_models/gru_64.h5 -------------------------------------------------------------------------------- /trained_models/lstm_32.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashil2792/Question-Answering-System-Deep-Learning/fc0657f51a220baa2798ef2a164bbffca421e7d2/trained_models/lstm_32.h5 -------------------------------------------------------------------------------- /trained_models/lstm_32_32.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashil2792/Question-Answering-System-Deep-Learning/fc0657f51a220baa2798ef2a164bbffca421e7d2/trained_models/lstm_32_32.h5 -------------------------------------------------------------------------------- /trained_models/lstm_64.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prashil2792/Question-Answering-System-Deep-Learning/fc0657f51a220baa2798ef2a164bbffca421e7d2/trained_models/lstm_64.h5 --------------------------------------------------------------------------------