├── README.md ├── cbow.ipynb ├── collaborative-filtering-nn.ipynb ├── environment.yml ├── image-caption-tutorial.ipynb ├── images ├── model.png ├── tiny_training2.csv └── tiny_val2.csv └── intro-to-pytoch.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # pytorch-tutorials 2 | 3 | Here are a few basic deep learning tutorial using Pytorch. 4 | -------------------------------------------------------------------------------- /cbow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "toc": true 7 | }, 8 | "source": [ 9 | "

Table of Contents

\n", 10 | "
" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "# import pytorch libraries\n", 20 | "%matplotlib inline\n", 21 | "import torch \n", 22 | "import torch.autograd as autograd \n", 23 | "import torch.nn as nn \n", 24 | "import torch.nn.functional as F\n", 25 | "import torch.optim as optim\n", 26 | "import numpy as np" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "# Text Classification\n", 34 | "In this part of the tutorial we develop a continuous bag of words (CBOW) model for a text classification task described [here]( https://people.cs.umass.edu/~miyyer/pubs/2015_acl_dan.pdf). The CBOW model was first described [here](https://arxiv.org/pdf/1301.3781.pdf)" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "## Subjectivity Dataset\n", 42 | "The subjectivity dataset has 5000 subjective and 5000 objective processed sentences. To get the data:\n", 43 | "```\n", 44 | "wget http://www.cs.cornell.edu/people/pabo/movie-review-data/rotten_imdb.tar.gz\n", 45 | "```" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 2, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "def unpack_dataset():\n", 55 | " ! wget http://www.cs.cornell.edu/people/pabo/movie-review-data/rotten_imdb.tar.gz\n", 56 | " ! mkdir data\n", 57 | " ! tar -xvf rotten_imdb.tar.gz -C data" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 3, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "#unpack_dataset()" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 4, 72 | "metadata": {}, 73 | "outputs": [ 74 | { 75 | "name": "stdout", 76 | "output_type": "stream", 77 | "text": [ 78 | "plot.tok.gt9.5000 quote.tok.gt9.5000 subjdata.README.1.0\r\n" 79 | ] 80 | } 81 | ], 82 | "source": [ 83 | "!ls data" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 5, 89 | "metadata": {}, 90 | "outputs": [ 91 | { 92 | "name": "stdout", 93 | "output_type": "stream", 94 | "text": [ 95 | "the movie begins in the past where a young boy named sam attempts to save celebi from a hunter . \r\n", 96 | "emerging from the human psyche and showing characteristics of abstract expressionism , minimalism and russian constructivism , graffiti removal has secured its place in the history of modern art while being created by artists who are unconscious of their artistic achievements . \r\n" 97 | ] 98 | } 99 | ], 100 | "source": [ 101 | "! head -2 data/plot.tok.gt9.5000" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 6, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "data": { 111 | "text/plain": [ 112 | "[PosixPath('data/plot.tok.gt9.5000'),\n", 113 | " PosixPath('data/subjdata.README.1.0'),\n", 114 | " PosixPath('data/quote.tok.gt9.5000')]" 115 | ] 116 | }, 117 | "execution_count": 6, 118 | "metadata": {}, 119 | "output_type": "execute_result" 120 | } 121 | ], 122 | "source": [ 123 | "from pathlib import Path\n", 124 | "PATH = Path(\"data\")\n", 125 | "list(PATH.iterdir())" 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": {}, 131 | "source": [ 132 | "## Tokenization\n", 133 | "Tokenization is the task of chopping up text into pieces, called tokens.\n", 134 | "\n", 135 | "spaCy is an open-source software library for advanced Natural Language Processing. Here we will use it for tokenization. " 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "metadata": {}, 141 | "source": [ 142 | "### Simple Tokenization" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": 7, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "# We need each line in the file \n", 152 | "def read_file(path):\n", 153 | " \"\"\" Read file returns a list of lines.\n", 154 | " \"\"\"\n", 155 | " with open(path, encoding = \"ISO-8859-1\") as f:\n", 156 | " content = f.readlines()\n", 157 | " return content" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 8, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "obj_lines = read_file(PATH/\"plot.tok.gt9.5000\")" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 9, 172 | "metadata": {}, 173 | "outputs": [ 174 | { 175 | "data": { 176 | "text/plain": [ 177 | "'the movie begins in the past where a young boy named sam attempts to save celebi from a hunter . \\n'" 178 | ] 179 | }, 180 | "execution_count": 9, 181 | "metadata": {}, 182 | "output_type": "execute_result" 183 | } 184 | ], 185 | "source": [ 186 | "obj_lines[0]" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 10, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "data": { 196 | "text/plain": [ 197 | "array(['the', 'movie', 'begins', 'in', 'the', 'past', 'where', 'a',\n", 198 | " 'young', 'boy', 'named', 'sam', 'attempts', 'to', 'save', 'celebi',\n", 199 | " 'from', 'a', 'hunter', '.'], dtype='\":0, \"UNK\":1} # init with padding and unknown\n", 502 | "words = [\"\", \"UNK\"]\n", 503 | "for word in word_count:\n", 504 | " vocab2index[word] = len(words)\n", 505 | " words.append(word)" 506 | ] 507 | }, 508 | { 509 | "cell_type": "code", 510 | "execution_count": 24, 511 | "metadata": {}, 512 | "outputs": [], 513 | "source": [ 514 | "#vocab2index" 515 | ] 516 | }, 517 | { 518 | "cell_type": "markdown", 519 | "metadata": {}, 520 | "source": [ 521 | "## Sentence encoding\n", 522 | "Here we encode each sentence as a sequence of indices corresponding to each word." 523 | ] 524 | }, 525 | { 526 | "cell_type": "code", 527 | "execution_count": 25, 528 | "metadata": {}, 529 | "outputs": [], 530 | "source": [ 531 | "x_train_len = np.array([len(x.split()) for x in X_train])\n", 532 | "x_val_len = np.array([len(x.split()) for x in X_val])" 533 | ] 534 | }, 535 | { 536 | "cell_type": "code", 537 | "execution_count": 26, 538 | "metadata": {}, 539 | "outputs": [ 540 | { 541 | "data": { 542 | "text/plain": [ 543 | "43.0" 544 | ] 545 | }, 546 | "execution_count": 26, 547 | "metadata": {}, 548 | "output_type": "execute_result" 549 | } 550 | ], 551 | "source": [ 552 | "np.percentile(x_train_len, 95) # let set the max sequence len to N=40" 553 | ] 554 | }, 555 | { 556 | "cell_type": "code", 557 | "execution_count": 27, 558 | "metadata": {}, 559 | "outputs": [ 560 | { 561 | "data": { 562 | "text/plain": [ 563 | "'will god let her fall or give her a new path ?'" 564 | ] 565 | }, 566 | "execution_count": 27, 567 | "metadata": {}, 568 | "output_type": "execute_result" 569 | } 570 | ], 571 | "source": [ 572 | "X_train[0]" 573 | ] 574 | }, 575 | { 576 | "cell_type": "code", 577 | "execution_count": 28, 578 | "metadata": {}, 579 | "outputs": [ 580 | { 581 | "data": { 582 | "text/plain": [ 583 | "8" 584 | ] 585 | }, 586 | "execution_count": 28, 587 | "metadata": {}, 588 | "output_type": "execute_result" 589 | } 590 | ], 591 | "source": [ 592 | "# returns the index of the word or the index of \"UNK\" otherwise\n", 593 | "vocab2index.get(\"?\", vocab2index[\"UNK\"])" 594 | ] 595 | }, 596 | { 597 | "cell_type": "code", 598 | "execution_count": 29, 599 | "metadata": {}, 600 | "outputs": [ 601 | { 602 | "data": { 603 | "text/plain": [ 604 | "array([11, 3, 6, 7, 2, 12, 9, 7, 10, 4, 5, 8])" 605 | ] 606 | }, 607 | "execution_count": 29, 608 | "metadata": {}, 609 | "output_type": "execute_result" 610 | } 611 | ], 612 | "source": [ 613 | "np.array([vocab2index.get(w, vocab2index[\"UNK\"]) for w in X_train[0].split()])" 614 | ] 615 | }, 616 | { 617 | "cell_type": "code", 618 | "execution_count": 30, 619 | "metadata": {}, 620 | "outputs": [], 621 | "source": [ 622 | "def encode_sentence(s, N=40):\n", 623 | " enc = np.zeros(N, dtype=np.int32)\n", 624 | " enc1 = np.array([vocab2index.get(w, vocab2index[\"UNK\"]) for w in s.split()])\n", 625 | " l = min(N, len(enc1))\n", 626 | " enc[:l] = enc1[:l]\n", 627 | " return enc" 628 | ] 629 | }, 630 | { 631 | "cell_type": "code", 632 | "execution_count": 31, 633 | "metadata": {}, 634 | "outputs": [ 635 | { 636 | "data": { 637 | "text/plain": [ 638 | "array([11, 3, 6, 7, 2, 12, 9, 7, 10, 4, 5, 8, 0, 0, 0, 0, 0,\n", 639 | " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", 640 | " 0, 0, 0, 0, 0, 0], dtype=int32)" 641 | ] 642 | }, 643 | "execution_count": 31, 644 | "metadata": {}, 645 | "output_type": "execute_result" 646 | } 647 | ], 648 | "source": [ 649 | "encode_sentence(X_train[0])" 650 | ] 651 | }, 652 | { 653 | "cell_type": "code", 654 | "execution_count": 32, 655 | "metadata": {}, 656 | "outputs": [], 657 | "source": [ 658 | "x_train_len = np.minimum(x_train_len, 40)\n", 659 | "x_val_len = np.minimum(x_val_len, 40)" 660 | ] 661 | }, 662 | { 663 | "cell_type": "code", 664 | "execution_count": 33, 665 | "metadata": {}, 666 | "outputs": [ 667 | { 668 | "data": { 669 | "text/plain": [ 670 | "(8000, 40)" 671 | ] 672 | }, 673 | "execution_count": 33, 674 | "metadata": {}, 675 | "output_type": "execute_result" 676 | } 677 | ], 678 | "source": [ 679 | "x_train = np.vstack([encode_sentence(x) for x in X_train])\n", 680 | "x_train.shape" 681 | ] 682 | }, 683 | { 684 | "cell_type": "code", 685 | "execution_count": 34, 686 | "metadata": {}, 687 | "outputs": [ 688 | { 689 | "data": { 690 | "text/plain": [ 691 | "(2000, 40)" 692 | ] 693 | }, 694 | "execution_count": 34, 695 | "metadata": {}, 696 | "output_type": "execute_result" 697 | } 698 | ], 699 | "source": [ 700 | "x_val = np.vstack([encode_sentence(x) for x in X_val])\n", 701 | "x_val.shape" 702 | ] 703 | }, 704 | { 705 | "cell_type": "markdown", 706 | "metadata": {}, 707 | "source": [ 708 | "## Embedding layer\n", 709 | "Most deep learning models use a dense vectors of real numbers as representation of words (word embeddings), as opposed to a one-hot encoding representations. The module torch.nn.Embedding is used to represent word embeddings. It takes two arguments: the vocabulary size, and the dimensionality of the embeddings. The embeddings are initialized with random vectors. " 710 | ] 711 | }, 712 | { 713 | "cell_type": "code", 714 | "execution_count": 35, 715 | "metadata": {}, 716 | "outputs": [ 717 | { 718 | "data": { 719 | "text/plain": [ 720 | "Parameter containing:\n", 721 | "tensor([[ 0.0000, 0.0000, 0.0000, 0.0000],\n", 722 | " [-0.9722, 0.9138, 0.0743, -0.1021],\n", 723 | " [-0.0091, -0.4712, 1.2977, -1.2585],\n", 724 | " [ 0.1368, 1.4354, -0.0935, 0.1110],\n", 725 | " [ 0.7230, -0.9195, 0.9880, 1.2590],\n", 726 | " [-0.4534, 0.8292, -0.2036, 0.3768],\n", 727 | " [ 1.1694, 1.6533, 1.1898, 1.0617],\n", 728 | " [-1.1252, -0.2761, -0.1112, 0.7598],\n", 729 | " [ 0.9987, 1.0012, -0.3599, 0.5257],\n", 730 | " [ 1.2248, -0.2419, 0.2870, -1.5904]])" 731 | ] 732 | }, 733 | "execution_count": 35, 734 | "metadata": {}, 735 | "output_type": "execute_result" 736 | } 737 | ], 738 | "source": [ 739 | "# an Embedding module containing 10 words with embedding size 4\n", 740 | "# embedding will be initialized at random\n", 741 | "embed = nn.Embedding(10, 4, padding_idx=0)\n", 742 | "embed.weight" 743 | ] 744 | }, 745 | { 746 | "cell_type": "markdown", 747 | "metadata": {}, 748 | "source": [ 749 | "Note that the `padding_idx` has embedding vector 0." 750 | ] 751 | }, 752 | { 753 | "cell_type": "code", 754 | "execution_count": 36, 755 | "metadata": {}, 756 | "outputs": [ 757 | { 758 | "data": { 759 | "text/plain": [ 760 | "tensor([[[-0.9722, 0.9138, 0.0743, -0.1021],\n", 761 | " [ 0.7230, -0.9195, 0.9880, 1.2590],\n", 762 | " [-0.9722, 0.9138, 0.0743, -0.1021],\n", 763 | " [-0.4534, 0.8292, -0.2036, 0.3768],\n", 764 | " [-0.9722, 0.9138, 0.0743, -0.1021],\n", 765 | " [ 0.0000, 0.0000, 0.0000, 0.0000]]])" 766 | ] 767 | }, 768 | "execution_count": 36, 769 | "metadata": {}, 770 | "output_type": "execute_result" 771 | } 772 | ], 773 | "source": [ 774 | "# given a list of ids we can \"look up\" the embedding corresponing to each id\n", 775 | "# can you see that some vectors are the same?\n", 776 | "a = torch.LongTensor([[1,4,1,5,1,0]])\n", 777 | "embed(a)" 778 | ] 779 | }, 780 | { 781 | "cell_type": "markdown", 782 | "metadata": {}, 783 | "source": [ 784 | "This would be the representation of a sentence with words with indices [1,4,1,5,1] and a padding at the end. Bellow we have an example in which we have two sentences. the first sentence has length 3 and the last sentence has length 2. In order to use a tensor we use padding at the end of the second sentence. " 785 | ] 786 | }, 787 | { 788 | "cell_type": "code", 789 | "execution_count": 37, 790 | "metadata": {}, 791 | "outputs": [], 792 | "source": [ 793 | "a = torch.LongTensor([[1,4,1], [1,3,0]])" 794 | ] 795 | }, 796 | { 797 | "cell_type": "markdown", 798 | "metadata": {}, 799 | "source": [ 800 | "Our model takes an average of the word embedding of each word. Here is how we do it." 801 | ] 802 | }, 803 | { 804 | "cell_type": "code", 805 | "execution_count": 38, 806 | "metadata": {}, 807 | "outputs": [], 808 | "source": [ 809 | "s = torch.FloatTensor([3, 2]) # here is the size of the vector" 810 | ] 811 | }, 812 | { 813 | "cell_type": "code", 814 | "execution_count": 39, 815 | "metadata": {}, 816 | "outputs": [ 817 | { 818 | "data": { 819 | "text/plain": [ 820 | "tensor([[[-0.9722, 0.9138, 0.0743, -0.1021],\n", 821 | " [ 0.7230, -0.9195, 0.9880, 1.2590],\n", 822 | " [-0.9722, 0.9138, 0.0743, -0.1021]],\n", 823 | "\n", 824 | " [[-0.9722, 0.9138, 0.0743, -0.1021],\n", 825 | " [ 0.1368, 1.4354, -0.0935, 0.1110],\n", 826 | " [ 0.0000, 0.0000, 0.0000, 0.0000]]])" 827 | ] 828 | }, 829 | "execution_count": 39, 830 | "metadata": {}, 831 | "output_type": "execute_result" 832 | } 833 | ], 834 | "source": [ 835 | "embed(a)" 836 | ] 837 | }, 838 | { 839 | "cell_type": "code", 840 | "execution_count": 40, 841 | "metadata": {}, 842 | "outputs": [ 843 | { 844 | "data": { 845 | "text/plain": [ 846 | "tensor([[-1.2213, 0.9080, 1.1367, 1.0548],\n", 847 | " [-0.8354, 2.3491, -0.0192, 0.0089]])" 848 | ] 849 | }, 850 | "execution_count": 40, 851 | "metadata": {}, 852 | "output_type": "execute_result" 853 | } 854 | ], 855 | "source": [ 856 | "embed(a).sum(dim=1)" 857 | ] 858 | }, 859 | { 860 | "cell_type": "code", 861 | "execution_count": 41, 862 | "metadata": {}, 863 | "outputs": [ 864 | { 865 | "data": { 866 | "text/plain": [ 867 | "tensor([[-0.4071, 0.3027, 0.3789, 0.3516],\n", 868 | " [-0.4177, 1.1746, -0.0096, 0.0044]])" 869 | ] 870 | }, 871 | "execution_count": 41, 872 | "metadata": {}, 873 | "output_type": "execute_result" 874 | } 875 | ], 876 | "source": [ 877 | "sum_embs = embed(a).sum(dim=1) \n", 878 | "sum_embs/ s.view(s.shape[0], 1)" 879 | ] 880 | }, 881 | { 882 | "cell_type": "markdown", 883 | "metadata": {}, 884 | "source": [ 885 | "## Continuous Bag of Words Model" 886 | ] 887 | }, 888 | { 889 | "cell_type": "code", 890 | "execution_count": 42, 891 | "metadata": {}, 892 | "outputs": [], 893 | "source": [ 894 | "class CBOW(nn.Module):\n", 895 | " def __init__(self, vocab_size, emb_size=100):\n", 896 | " super(CBOW, self).__init__()\n", 897 | " self.word_emb = nn.Embedding(vocab_size, emb_size, padding_idx=0)\n", 898 | " self.linear = nn.Linear(emb_size, 1)\n", 899 | " \n", 900 | " def forward(self, x, s):\n", 901 | " x = self.word_emb(x)\n", 902 | " x = x.sum(dim=1)/ s\n", 903 | " x = self.linear(x)\n", 904 | " return x" 905 | ] 906 | }, 907 | { 908 | "cell_type": "code", 909 | "execution_count": 43, 910 | "metadata": {}, 911 | "outputs": [], 912 | "source": [ 913 | "model = CBOW(vocab_size=5, emb_size=3)" 914 | ] 915 | }, 916 | { 917 | "cell_type": "code", 918 | "execution_count": 44, 919 | "metadata": {}, 920 | "outputs": [ 921 | { 922 | "data": { 923 | "text/plain": [ 924 | "Parameter containing:\n", 925 | "tensor([[ 0.0000, 0.0000, 0.0000],\n", 926 | " [ 1.6292, 1.2889, 0.7647],\n", 927 | " [ 2.5952, -0.9427, 0.3432],\n", 928 | " [ 0.5775, -2.7160, -1.4606],\n", 929 | " [ 1.2119, 0.8058, -0.0705]])" 930 | ] 931 | }, 932 | "execution_count": 44, 933 | "metadata": {}, 934 | "output_type": "execute_result" 935 | } 936 | ], 937 | "source": [ 938 | "model.word_emb.weight" 939 | ] 940 | }, 941 | { 942 | "cell_type": "code", 943 | "execution_count": 45, 944 | "metadata": {}, 945 | "outputs": [ 946 | { 947 | "data": { 948 | "text/plain": [ 949 | "tensor([[ 0.1384],\n", 950 | " [ 0.5663]])" 951 | ] 952 | }, 953 | "execution_count": 45, 954 | "metadata": {}, 955 | "output_type": "execute_result" 956 | } 957 | ], 958 | "source": [ 959 | "s = s.view(s.shape[0], 1)\n", 960 | "model(a, s)" 961 | ] 962 | }, 963 | { 964 | "cell_type": "markdown", 965 | "metadata": {}, 966 | "source": [ 967 | "# Training the CBOW model " 968 | ] 969 | }, 970 | { 971 | "cell_type": "code", 972 | "execution_count": 51, 973 | "metadata": {}, 974 | "outputs": [ 975 | { 976 | "name": "stdout", 977 | "output_type": "stream", 978 | "text": [ 979 | "4067\n" 980 | ] 981 | } 982 | ], 983 | "source": [ 984 | "V = len(words)\n", 985 | "model = CBOW(vocab_size=V, emb_size=50)\n", 986 | "print(V)" 987 | ] 988 | }, 989 | { 990 | "cell_type": "code", 991 | "execution_count": 52, 992 | "metadata": {}, 993 | "outputs": [], 994 | "source": [ 995 | "def val_metrics(model):\n", 996 | " model.eval()\n", 997 | " x = torch.LongTensor(x_val) #.cuda()\n", 998 | " y = torch.Tensor(y_val).unsqueeze(1) #).cuda()\n", 999 | " s = torch.Tensor(x_val_len).view(x_val_len.shape[0], 1)\n", 1000 | " y_hat = model(x, s)\n", 1001 | " loss = F.binary_cross_entropy_with_logits(y_hat, y)\n", 1002 | " y_pred = y_hat > 0\n", 1003 | " correct = (y_pred.float() == y).float().sum()\n", 1004 | " accuracy = correct/y_pred.shape[0]\n", 1005 | " return loss.item(), accuracy.item()" 1006 | ] 1007 | }, 1008 | { 1009 | "cell_type": "code", 1010 | "execution_count": 53, 1011 | "metadata": {}, 1012 | "outputs": [ 1013 | { 1014 | "data": { 1015 | "text/plain": [ 1016 | "(0.6892560720443726, 0.5245000123977661)" 1017 | ] 1018 | }, 1019 | "execution_count": 53, 1020 | "metadata": {}, 1021 | "output_type": "execute_result" 1022 | } 1023 | ], 1024 | "source": [ 1025 | "# accuracy of a random model should be around 0.5\n", 1026 | "val_metrics(model)" 1027 | ] 1028 | }, 1029 | { 1030 | "cell_type": "code", 1031 | "execution_count": 54, 1032 | "metadata": {}, 1033 | "outputs": [], 1034 | "source": [ 1035 | "def train_epocs(model, epochs=10, lr=0.01):\n", 1036 | " optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n", 1037 | " for i in range(epochs):\n", 1038 | " model.train()\n", 1039 | " x = torch.LongTensor(x_train) #.cuda()\n", 1040 | " y = torch.Tensor(y_train).unsqueeze(1)\n", 1041 | " s = torch.Tensor(x_train_len).view(x_train_len.shape[0], 1)\n", 1042 | " y_hat = model(x, s)\n", 1043 | " loss = F.binary_cross_entropy_with_logits(y_hat, y)\n", 1044 | " optimizer.zero_grad()\n", 1045 | " loss.backward()\n", 1046 | " optimizer.step()\n", 1047 | " val_loss, val_accuracy = val_metrics(model)\n", 1048 | " print(\"train_loss %.3f val_loss %.3f val_accuracy %.3f\" % (loss.item(), val_loss, val_accuracy))" 1049 | ] 1050 | }, 1051 | { 1052 | "cell_type": "code", 1053 | "execution_count": 55, 1054 | "metadata": {}, 1055 | "outputs": [ 1056 | { 1057 | "name": "stdout", 1058 | "output_type": "stream", 1059 | "text": [ 1060 | "train_loss 0.689 val_loss 0.651 val_accuracy 0.582\n", 1061 | "train_loss 0.639 val_loss 0.559 val_accuracy 0.766\n", 1062 | "train_loss 0.544 val_loss 0.467 val_accuracy 0.825\n", 1063 | "train_loss 0.440 val_loss 0.375 val_accuracy 0.866\n", 1064 | "train_loss 0.337 val_loss 0.321 val_accuracy 0.867\n", 1065 | "train_loss 0.270 val_loss 0.274 val_accuracy 0.893\n", 1066 | "train_loss 0.213 val_loss 0.258 val_accuracy 0.900\n", 1067 | "train_loss 0.183 val_loss 0.253 val_accuracy 0.902\n", 1068 | "train_loss 0.160 val_loss 0.250 val_accuracy 0.909\n", 1069 | "train_loss 0.135 val_loss 0.260 val_accuracy 0.905\n" 1070 | ] 1071 | } 1072 | ], 1073 | "source": [ 1074 | "train_epocs(model, epochs=10, lr=0.1)" 1075 | ] 1076 | }, 1077 | { 1078 | "cell_type": "code", 1079 | "execution_count": 56, 1080 | "metadata": {}, 1081 | "outputs": [ 1082 | { 1083 | "name": "stdout", 1084 | "output_type": "stream", 1085 | "text": [ 1086 | "train_loss 0.120 val_loss 0.257 val_accuracy 0.907\n", 1087 | "train_loss 0.115 val_loss 0.257 val_accuracy 0.909\n", 1088 | "train_loss 0.113 val_loss 0.256 val_accuracy 0.907\n", 1089 | "train_loss 0.110 val_loss 0.256 val_accuracy 0.908\n", 1090 | "train_loss 0.107 val_loss 0.255 val_accuracy 0.908\n", 1091 | "train_loss 0.103 val_loss 0.255 val_accuracy 0.907\n", 1092 | "train_loss 0.100 val_loss 0.255 val_accuracy 0.906\n", 1093 | "train_loss 0.098 val_loss 0.256 val_accuracy 0.908\n", 1094 | "train_loss 0.095 val_loss 0.257 val_accuracy 0.906\n", 1095 | "train_loss 0.092 val_loss 0.257 val_accuracy 0.906\n" 1096 | ] 1097 | } 1098 | ], 1099 | "source": [ 1100 | "train_epocs(model, epochs=10, lr=0.01)" 1101 | ] 1102 | }, 1103 | { 1104 | "cell_type": "markdown", 1105 | "metadata": {}, 1106 | "source": [ 1107 | "# Data loaders for SGD" 1108 | ] 1109 | }, 1110 | { 1111 | "cell_type": "markdown", 1112 | "metadata": {}, 1113 | "source": [ 1114 | "Nearly all of deep learning is powered by one very important algorithm: **stochastic gradient descent (SGD)**. SGD can be seeing as an approximation of **gradient descent** (GD). In GD you have to run through *all* the samples in your training set to do a single itaration. In SGD you use *only one* or *a subset* of training samples to do the update for a parameter in a particular iteration. The subset use in every iteration is called a **batch** or **minibatch**." 1115 | ] 1116 | }, 1117 | { 1118 | "cell_type": "code", 1119 | "execution_count": 57, 1120 | "metadata": {}, 1121 | "outputs": [], 1122 | "source": [ 1123 | "from torch.utils.data import Dataset, DataLoader" 1124 | ] 1125 | }, 1126 | { 1127 | "cell_type": "markdown", 1128 | "metadata": {}, 1129 | "source": [ 1130 | "Next we are going to create a data loader. The data loader provides the following features:\n", 1131 | "* Batching the data\n", 1132 | "* Shuffling the data\n", 1133 | "* Load the data in parallel using multiprocessing workers." 1134 | ] 1135 | }, 1136 | { 1137 | "cell_type": "code", 1138 | "execution_count": 58, 1139 | "metadata": {}, 1140 | "outputs": [], 1141 | "source": [ 1142 | "def encode_sentence2(s, N=40):\n", 1143 | " enc = np.zeros(N, dtype=np.int32)\n", 1144 | " enc1 = np.array([vocab2index.get(w, vocab2index[\"UNK\"]) for w in s.split()])\n", 1145 | " l = min(N, len(enc1))\n", 1146 | " enc[:l] = enc1[:l]\n", 1147 | " return enc, l" 1148 | ] 1149 | }, 1150 | { 1151 | "cell_type": "code", 1152 | "execution_count": 59, 1153 | "metadata": {}, 1154 | "outputs": [ 1155 | { 1156 | "data": { 1157 | "text/plain": [ 1158 | "(array([11, 3, 6, 7, 2, 12, 9, 7, 10, 4, 5, 8, 0, 0, 0, 0, 0,\n", 1159 | " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", 1160 | " 0, 0, 0, 0, 0, 0], dtype=int32), 12)" 1161 | ] 1162 | }, 1163 | "execution_count": 59, 1164 | "metadata": {}, 1165 | "output_type": "execute_result" 1166 | } 1167 | ], 1168 | "source": [ 1169 | "encode_sentence2(X_train[0])" 1170 | ] 1171 | }, 1172 | { 1173 | "cell_type": "code", 1174 | "execution_count": 60, 1175 | "metadata": {}, 1176 | "outputs": [], 1177 | "source": [ 1178 | "class SubjectivityDataset(Dataset):\n", 1179 | " def __init__(self, X, y):\n", 1180 | " self.x = X\n", 1181 | " self.y = y\n", 1182 | " \n", 1183 | " def __len__(self):\n", 1184 | " return len(self.y)\n", 1185 | " \n", 1186 | " def __getitem__(self, idx):\n", 1187 | " x = self.x[idx]\n", 1188 | " x, s = encode_sentence2(x)\n", 1189 | " return x, self.y[idx], s\n", 1190 | " \n", 1191 | "sub_dataset_train = SubjectivityDataset(X_train, y_train)" 1192 | ] 1193 | }, 1194 | { 1195 | "cell_type": "code", 1196 | "execution_count": 61, 1197 | "metadata": {}, 1198 | "outputs": [], 1199 | "source": [ 1200 | "train_loader = DataLoader(sub_dataset_train, batch_size=5, shuffle=True)\n", 1201 | "x, y, s = next(iter(train_loader))" 1202 | ] 1203 | }, 1204 | { 1205 | "cell_type": "code", 1206 | "execution_count": 62, 1207 | "metadata": {}, 1208 | "outputs": [ 1209 | { 1210 | "data": { 1211 | "text/plain": [ 1212 | "(tensor([[ 243, 2146, 1, 384, 57, 1, 57, 1, 1,\n", 1213 | " 37, 559, 1, 1, 2632, 1, 42, 24, 15,\n", 1214 | " 645, 3014, 2936, 88, 1, 37, 1, 2632, 2029,\n", 1215 | " 1, 80, 1, 23, 0, 0, 0, 0, 0,\n", 1216 | " 0, 0, 0, 0],\n", 1217 | " [ 147, 15, 1075, 1910, 362, 42, 3155, 125, 588,\n", 1218 | " 32, 588, 63, 40, 41, 1479, 57, 2537, 24,\n", 1219 | " 15, 1, 57, 1725, 152, 40, 233, 23, 0,\n", 1220 | " 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", 1221 | " 0, 0, 0, 0],\n", 1222 | " [ 15, 1873, 24, 119, 29, 2288, 477, 1575, 24,\n", 1223 | " 1645, 42, 24, 15, 127, 635, 436, 147, 173,\n", 1224 | " 128, 1551, 129, 436, 72, 704, 1, 42, 977,\n", 1225 | " 24, 2144, 42, 29, 738, 1, 434, 104, 23,\n", 1226 | " 0, 0, 0, 0],\n", 1227 | " [ 1, 476, 29, 661, 91, 3319, 42, 24, 1,\n", 1228 | " 476, 29, 661, 91, 3320, 23, 0, 0, 0,\n", 1229 | " 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", 1230 | " 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", 1231 | " 0, 0, 0, 0],\n", 1232 | " [ 261, 15, 3115, 354, 1298, 42, 243, 148, 199,\n", 1233 | " 1013, 60, 24, 60, 24, 60, 24, 60, 0,\n", 1234 | " 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", 1235 | " 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", 1236 | " 0, 0, 0, 0]], dtype=torch.int32),\n", 1237 | " tensor([ 1., 1., 0., 0., 0.], dtype=torch.float64),\n", 1238 | " tensor([ 31, 26, 36, 15, 17]))" 1239 | ] 1240 | }, 1241 | "execution_count": 62, 1242 | "metadata": {}, 1243 | "output_type": "execute_result" 1244 | } 1245 | ], 1246 | "source": [ 1247 | "x, y, s" 1248 | ] 1249 | }, 1250 | { 1251 | "cell_type": "code", 1252 | "execution_count": 63, 1253 | "metadata": {}, 1254 | "outputs": [], 1255 | "source": [ 1256 | "model = CBOW(vocab_size=V, emb_size=50)" 1257 | ] 1258 | }, 1259 | { 1260 | "cell_type": "code", 1261 | "execution_count": 64, 1262 | "metadata": {}, 1263 | "outputs": [], 1264 | "source": [ 1265 | "train_loader = DataLoader(sub_dataset_train, batch_size=500, shuffle=True)" 1266 | ] 1267 | }, 1268 | { 1269 | "cell_type": "code", 1270 | "execution_count": null, 1271 | "metadata": {}, 1272 | "outputs": [], 1273 | "source": [ 1274 | "def train_epocs(model, epochs=10, lr=0.01):\n", 1275 | " optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n", 1276 | " for i in range(epochs):\n", 1277 | " total_loss = 0\n", 1278 | " total = 0\n", 1279 | " model.train()\n", 1280 | " for x, y, s in train_loader:\n", 1281 | " x = x.type(torch.LongTensor) #.cuda()\n", 1282 | " y = y.type(torch.FloatTensor).unsqueeze(1)\n", 1283 | " s = s.type(torch.Tensor).view(s.shape[0], 1)\n", 1284 | " y_hat = model(x, s)\n", 1285 | " loss = F.binary_cross_entropy_with_logits(y_hat, y)\n", 1286 | " optimizer.zero_grad()\n", 1287 | " loss.backward()\n", 1288 | " optimizer.step()\n", 1289 | " total_loss += x.size(0)*loss.item()\n", 1290 | " total += x.size(0)\n", 1291 | " train_loss\n", 1292 | " val_loss, val_accuracy = val_metrics(model)\n", 1293 | " \n", 1294 | " print(\"train_loss %.3f val_loss %.3f val_accuracy %.3f\" % (loss.item(), val_loss, val_accuracy))" 1295 | ] 1296 | }, 1297 | { 1298 | "cell_type": "code", 1299 | "execution_count": null, 1300 | "metadata": {}, 1301 | "outputs": [], 1302 | "source": [ 1303 | "train_epocs(model, epochs=10)" 1304 | ] 1305 | }, 1306 | { 1307 | "cell_type": "code", 1308 | "execution_count": null, 1309 | "metadata": {}, 1310 | "outputs": [], 1311 | "source": [] 1312 | } 1313 | ], 1314 | "metadata": { 1315 | "kernelspec": { 1316 | "display_name": "Python 3", 1317 | "language": "python", 1318 | "name": "python3" 1319 | }, 1320 | "language_info": { 1321 | "codemirror_mode": { 1322 | "name": "ipython", 1323 | "version": 3 1324 | }, 1325 | "file_extension": ".py", 1326 | "mimetype": "text/x-python", 1327 | "name": "python", 1328 | "nbconvert_exporter": "python", 1329 | "pygments_lexer": "ipython3", 1330 | "version": "3.6.6" 1331 | }, 1332 | "nav_menu": {}, 1333 | "toc": { 1334 | "nav_menu": { 1335 | "height": "116px", 1336 | "width": "251px" 1337 | }, 1338 | "number_sections": true, 1339 | "sideBar": true, 1340 | "skip_h1_title": false, 1341 | "toc_cell": true, 1342 | "toc_position": {}, 1343 | "toc_section_display": "block", 1344 | "toc_window_display": false 1345 | }, 1346 | "widgets": { 1347 | "state": {}, 1348 | "version": "1.1.2" 1349 | } 1350 | }, 1351 | "nbformat": 4, 1352 | "nbformat_minor": 1 1353 | } 1354 | -------------------------------------------------------------------------------- /collaborative-filtering-nn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Collaborative Filtering with Neural Networks" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook we will write a matrix factorization model in pytorch to solve a recommendation problem. Then we will write a more general neural model for the same problem." 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "The MovieLens dataset (ml-latest-small) describes 5-star rating and free-text tagging activity from MovieLens, a movie recommendation service. It contains 100004 ratings and 1296 tag applications across 9125 movies. https://grouplens.org/datasets/movielens/. To get the data:\n", 22 | "\n", 23 | "`wget http://files.grouplens.org/datasets/movielens/ml-latest-small.zip`" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "## MovieLens dataset" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 1, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "from pathlib import Path\n", 40 | "import pandas as pd\n", 41 | "import numpy as np" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 3, 47 | "metadata": {}, 48 | "outputs": [ 49 | { 50 | "data": { 51 | "text/plain": [ 52 | "[PosixPath('/data2/yinterian/ml-latest-small/ratings.csv'),\n", 53 | " PosixPath('/data2/yinterian/ml-latest-small/tags.csv'),\n", 54 | " PosixPath('/data2/yinterian/ml-latest-small/tiny_training2.csv'),\n", 55 | " PosixPath('/data2/yinterian/ml-latest-small/links.csv'),\n", 56 | " PosixPath('/data2/yinterian/ml-latest-small/tiny_val2.csv'),\n", 57 | " PosixPath('/data2/yinterian/ml-latest-small/README.txt'),\n", 58 | " PosixPath('/data2/yinterian/ml-latest-small/movies.csv')]" 59 | ] 60 | }, 61 | "execution_count": 3, 62 | "metadata": {}, 63 | "output_type": "execute_result" 64 | } 65 | ], 66 | "source": [ 67 | "PATH = Path(\"/Users/yinterian/teaching/deeplearning/data/ml-latest-small/\")\n", 68 | "PATH = Path(\"/data2/yinterian/ml-latest-small/\")\n", 69 | "list(PATH.iterdir())" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 4, 75 | "metadata": {}, 76 | "outputs": [ 77 | { 78 | "name": "stdout", 79 | "output_type": "stream", 80 | "text": [ 81 | "userId,movieId,rating,timestamp\r", 82 | "\r\n", 83 | "1,31,2.5,1260759144\r", 84 | "\r\n", 85 | "1,1029,3.0,1260759179\r", 86 | "\r\n", 87 | "1,1061,3.0,1260759182\r", 88 | "\r\n", 89 | "1,1129,2.0,1260759185\r", 90 | "\r\n", 91 | "1,1172,4.0,1260759205\r", 92 | "\r\n", 93 | "1,1263,2.0,1260759151\r", 94 | "\r\n", 95 | "1,1287,2.0,1260759187\r", 96 | "\r\n", 97 | "1,1293,2.0,1260759148\r", 98 | "\r\n", 99 | "1,1339,3.5,1260759125\r", 100 | "\r\n" 101 | ] 102 | } 103 | ], 104 | "source": [ 105 | "! head $PATH/ratings.csv" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 5, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "data = pd.read_csv(PATH/\"ratings.csv\")" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 6, 120 | "metadata": {}, 121 | "outputs": [ 122 | { 123 | "data": { 124 | "text/html": [ 125 | "
\n", 126 | "\n", 139 | "\n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | "
userIdmovieIdratingtimestamp
01312.51260759144
1110293.01260759179
2110613.01260759182
3111292.01260759185
4111724.01260759205
\n", 187 | "
" 188 | ], 189 | "text/plain": [ 190 | " userId movieId rating timestamp\n", 191 | "0 1 31 2.5 1260759144\n", 192 | "1 1 1029 3.0 1260759179\n", 193 | "2 1 1061 3.0 1260759182\n", 194 | "3 1 1129 2.0 1260759185\n", 195 | "4 1 1172 4.0 1260759205" 196 | ] 197 | }, 198 | "execution_count": 6, 199 | "metadata": {}, 200 | "output_type": "execute_result" 201 | } 202 | ], 203 | "source": [ 204 | "data.head()" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": {}, 210 | "source": [ 211 | "### Encoding data\n", 212 | "We enconde the data to have contiguous ids for users and movies. You can think about this as a categorical encoding of our two categorical variables userId and movieId." 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": 7, 218 | "metadata": {}, 219 | "outputs": [], 220 | "source": [ 221 | "# split train and validation before encoding\n", 222 | "np.random.seed(3)\n", 223 | "msk = np.random.rand(len(data)) < 0.8\n", 224 | "train = data[msk].copy()\n", 225 | "val = data[~msk].copy()" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 8, 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [ 234 | "# here is a handy function modified from fast.ai\n", 235 | "def proc_col(col, train_col=None):\n", 236 | " \"\"\"Encodes a pandas column with continuous ids. \n", 237 | " \"\"\"\n", 238 | " if train_col is not None:\n", 239 | " uniq = train_col.unique()\n", 240 | " else:\n", 241 | " uniq = col.unique()\n", 242 | " name2idx = {o:i for i,o in enumerate(uniq)}\n", 243 | " return name2idx, np.array([name2idx.get(x, -1) for x in col]), len(uniq)" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": 9, 249 | "metadata": {}, 250 | "outputs": [], 251 | "source": [ 252 | "def encode_data(df, train=None):\n", 253 | " \"\"\" Encodes rating data with continous user and movie ids. \n", 254 | " If train is provided, encodes df with the same encoding as train.\n", 255 | " \"\"\"\n", 256 | " df = df.copy()\n", 257 | " for col_name in [\"userId\", \"movieId\"]:\n", 258 | " train_col = None\n", 259 | " if train is not None:\n", 260 | " train_col = train[col_name]\n", 261 | " _,col,_ = proc_col(df[col_name], train_col)\n", 262 | " df[col_name] = col\n", 263 | " df = df[df[col_name] >= 0]\n", 264 | " return df" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 10, 270 | "metadata": {}, 271 | "outputs": [ 272 | { 273 | "name": "stdout", 274 | "output_type": "stream", 275 | "text": [ 276 | " userId movieId rating\n", 277 | "0 11 1 4\n", 278 | "1 11 23 5\n", 279 | "2 2 23 5\n", 280 | "3 2 4 3\n", 281 | "4 31 1 4\n", 282 | "5 31 23 4\n", 283 | "6 4 1 5\n", 284 | "7 4 3 2\n", 285 | "8 52 1 1\n", 286 | "9 52 3 4\n", 287 | "10 61 3 5\n", 288 | "11 7 23 1\n", 289 | "12 7 3 3\n", 290 | " userId movieId rating\n", 291 | "0 0 0 4\n", 292 | "1 0 1 5\n", 293 | "2 1 1 5\n", 294 | "3 1 2 3\n", 295 | "4 2 0 4\n", 296 | "5 2 1 4\n", 297 | "6 3 0 5\n", 298 | "7 3 3 2\n", 299 | "8 4 0 1\n", 300 | "9 4 3 4\n", 301 | "10 5 3 5\n", 302 | "11 6 1 1\n", 303 | "12 6 3 3\n" 304 | ] 305 | } 306 | ], 307 | "source": [ 308 | "# to check my new implementation\n", 309 | "LOCAL_PATH = Path(\"images/\")\n", 310 | "df_t = pd.read_csv(LOCAL_PATH/\"tiny_training2.csv\")\n", 311 | "df_v = pd.read_csv(LOCAL_PATH/\"tiny_val2.csv\")\n", 312 | "print(df_t)\n", 313 | "df_t_e = encode_data(df_t)\n", 314 | "df_v_e = encode_data(df_v, df_t)\n", 315 | "df_v_e\n", 316 | "print(df_t_e)" 317 | ] 318 | }, 319 | { 320 | "cell_type": "code", 321 | "execution_count": 11, 322 | "metadata": {}, 323 | "outputs": [], 324 | "source": [ 325 | "# encoding the train and validation data\n", 326 | "df_train = encode_data(train)\n", 327 | "df_val = encode_data(val, train)" 328 | ] 329 | }, 330 | { 331 | "cell_type": "markdown", 332 | "metadata": {}, 333 | "source": [ 334 | "## Embedding layer" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 12, 340 | "metadata": {}, 341 | "outputs": [], 342 | "source": [ 343 | "import torch\n", 344 | "import torch.nn as nn\n", 345 | "import torch.nn.functional as F" 346 | ] 347 | }, 348 | { 349 | "cell_type": "code", 350 | "execution_count": 13, 351 | "metadata": {}, 352 | "outputs": [], 353 | "source": [ 354 | "# an Embedding module containing 10 user or item embedding size 3\n", 355 | "# embedding will be initialized at random\n", 356 | "embed = nn.Embedding(10, 3)" 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": 14, 362 | "metadata": {}, 363 | "outputs": [ 364 | { 365 | "data": { 366 | "text/plain": [ 367 | "tensor([[[-0.1301, 0.0691, -1.1678],\n", 368 | " [-0.9865, 0.4514, -1.4770],\n", 369 | " [-1.7121, 0.0701, 0.0481],\n", 370 | " [ 1.4485, 0.1340, 0.0099],\n", 371 | " [-1.4074, -0.8650, -0.1255],\n", 372 | " [-0.1301, 0.0691, -1.1678]]])" 373 | ] 374 | }, 375 | "execution_count": 14, 376 | "metadata": {}, 377 | "output_type": "execute_result" 378 | } 379 | ], 380 | "source": [ 381 | "# given a list of ids we can \"look up\" the embedding corresponing to each id\n", 382 | "a = torch.LongTensor([[1,2,0,4,5,1]])\n", 383 | "embed(a)" 384 | ] 385 | }, 386 | { 387 | "cell_type": "markdown", 388 | "metadata": {}, 389 | "source": [ 390 | "## Matrix factorization model" 391 | ] 392 | }, 393 | { 394 | "cell_type": "code", 395 | "execution_count": 15, 396 | "metadata": {}, 397 | "outputs": [], 398 | "source": [ 399 | "class MF(nn.Module):\n", 400 | " def __init__(self, num_users, num_items, emb_size=100):\n", 401 | " super(MF, self).__init__()\n", 402 | " self.user_emb = nn.Embedding(num_users, emb_size)\n", 403 | " self.item_emb = nn.Embedding(num_items, emb_size)\n", 404 | " self.user_emb.weight.data.uniform_(0, 0.05)\n", 405 | " self.item_emb.weight.data.uniform_(0, 0.05)\n", 406 | " \n", 407 | " def forward(self, u, v):\n", 408 | " u = self.user_emb(u)\n", 409 | " v = self.item_emb(v)\n", 410 | " return (u*v).sum(1) " 411 | ] 412 | }, 413 | { 414 | "cell_type": "markdown", 415 | "metadata": {}, 416 | "source": [ 417 | "## Debugging MF model" 418 | ] 419 | }, 420 | { 421 | "cell_type": "code", 422 | "execution_count": 16, 423 | "metadata": {}, 424 | "outputs": [ 425 | { 426 | "data": { 427 | "text/html": [ 428 | "
\n", 429 | "\n", 442 | "\n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | "
userIdmovieIdrating
0004
1015
2115
3123
4204
5214
6305
7332
8401
9434
10535
11611
12633
\n", 532 | "
" 533 | ], 534 | "text/plain": [ 535 | " userId movieId rating\n", 536 | "0 0 0 4\n", 537 | "1 0 1 5\n", 538 | "2 1 1 5\n", 539 | "3 1 2 3\n", 540 | "4 2 0 4\n", 541 | "5 2 1 4\n", 542 | "6 3 0 5\n", 543 | "7 3 3 2\n", 544 | "8 4 0 1\n", 545 | "9 4 3 4\n", 546 | "10 5 3 5\n", 547 | "11 6 1 1\n", 548 | "12 6 3 3" 549 | ] 550 | }, 551 | "execution_count": 16, 552 | "metadata": {}, 553 | "output_type": "execute_result" 554 | } 555 | ], 556 | "source": [ 557 | "df_t_e" 558 | ] 559 | }, 560 | { 561 | "cell_type": "code", 562 | "execution_count": 16, 563 | "metadata": {}, 564 | "outputs": [], 565 | "source": [ 566 | "num_users = 7\n", 567 | "num_items = 4\n", 568 | "emb_size = 3\n", 569 | "\n", 570 | "user_emb = nn.Embedding(num_users, emb_size)\n", 571 | "item_emb = nn.Embedding(num_items, emb_size)\n", 572 | "users = torch.LongTensor(df_t_e.userId.values)\n", 573 | "items = torch.LongTensor(df_t_e.movieId.values)" 574 | ] 575 | }, 576 | { 577 | "cell_type": "code", 578 | "execution_count": 17, 579 | "metadata": {}, 580 | "outputs": [], 581 | "source": [ 582 | "U = user_emb(users)\n", 583 | "V = item_emb(items)" 584 | ] 585 | }, 586 | { 587 | "cell_type": "code", 588 | "execution_count": 18, 589 | "metadata": {}, 590 | "outputs": [ 591 | { 592 | "data": { 593 | "text/plain": [ 594 | "tensor([[ 0.1547, 0.2277, 0.2442],\n", 595 | " [ 0.1547, 0.2277, 0.2442],\n", 596 | " [ 0.6601, 0.8225, -1.2139],\n", 597 | " [ 0.6601, 0.8225, -1.2139],\n", 598 | " [ 0.1672, -1.2177, 0.1403],\n", 599 | " [ 0.1672, -1.2177, 0.1403],\n", 600 | " [-1.1907, -1.2933, -0.5506],\n", 601 | " [-1.1907, -1.2933, -0.5506],\n", 602 | " [ 0.1938, -0.0683, -0.8493],\n", 603 | " [ 0.1938, -0.0683, -0.8493],\n", 604 | " [ 0.8506, -1.1564, 1.1165],\n", 605 | " [ 0.8639, -2.5148, -0.8391],\n", 606 | " [ 0.8639, -2.5148, -0.8391]])" 607 | ] 608 | }, 609 | "execution_count": 18, 610 | "metadata": {}, 611 | "output_type": "execute_result" 612 | } 613 | ], 614 | "source": [ 615 | "U" 616 | ] 617 | }, 618 | { 619 | "cell_type": "code", 620 | "execution_count": 19, 621 | "metadata": {}, 622 | "outputs": [ 623 | { 624 | "data": { 625 | "text/plain": [ 626 | "tensor([[-0.1766, 0.2957, 0.4409],\n", 627 | " [ 0.1205, 0.1733, 0.1165],\n", 628 | " [ 0.5143, 0.6258, -0.5793],\n", 629 | " [-0.5603, 0.3582, -0.5370],\n", 630 | " [-0.1909, -1.5812, 0.2533],\n", 631 | " [ 0.1303, -0.9266, 0.0670],\n", 632 | " [ 1.3594, -1.6793, -0.9940],\n", 633 | " [-0.2324, 1.4822, 0.5151],\n", 634 | " [-0.2212, -0.0887, -1.5335],\n", 635 | " [ 0.0378, 0.0783, 0.7947],\n", 636 | " [ 0.1660, 1.3253, -1.0447],\n", 637 | " [ 0.6730, -1.9135, -0.4004],\n", 638 | " [ 0.1686, 2.8820, 0.7851]])" 639 | ] 640 | }, 641 | "execution_count": 19, 642 | "metadata": {}, 643 | "output_type": "execute_result" 644 | } 645 | ], 646 | "source": [ 647 | "# element wise multiplication\n", 648 | "U*V " 649 | ] 650 | }, 651 | { 652 | "cell_type": "code", 653 | "execution_count": 20, 654 | "metadata": {}, 655 | "outputs": [ 656 | { 657 | "data": { 658 | "text/plain": [ 659 | "tensor([ 0.5600, 0.4103, 0.5608, -0.7391, -1.5187, -0.7294, -1.3139,\n", 660 | " 1.7649, -1.8434, 0.9108, 0.4466, -1.6409, 3.8357])" 661 | ] 662 | }, 663 | "execution_count": 20, 664 | "metadata": {}, 665 | "output_type": "execute_result" 666 | } 667 | ], 668 | "source": [ 669 | "# what we want is a dot product per row\n", 670 | "(U*V).sum(1) " 671 | ] 672 | }, 673 | { 674 | "cell_type": "markdown", 675 | "metadata": {}, 676 | "source": [ 677 | "## Training MF model" 678 | ] 679 | }, 680 | { 681 | "cell_type": "code", 682 | "execution_count": 21, 683 | "metadata": {}, 684 | "outputs": [ 685 | { 686 | "name": "stdout", 687 | "output_type": "stream", 688 | "text": [ 689 | "671 8442\n" 690 | ] 691 | } 692 | ], 693 | "source": [ 694 | "num_users = len(df_train.userId.unique())\n", 695 | "num_items = len(df_train.movieId.unique())\n", 696 | "print(num_users, num_items) " 697 | ] 698 | }, 699 | { 700 | "cell_type": "code", 701 | "execution_count": 22, 702 | "metadata": {}, 703 | "outputs": [], 704 | "source": [ 705 | "model = MF(num_users, num_items, emb_size=100) # .cuda() if you have a GPU" 706 | ] 707 | }, 708 | { 709 | "cell_type": "code", 710 | "execution_count": 23, 711 | "metadata": {}, 712 | "outputs": [], 713 | "source": [ 714 | "def train_epocs(model, epochs=10, lr=0.01, wd=0.0, unsqueeze=False):\n", 715 | " optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd)\n", 716 | " model.train()\n", 717 | " for i in range(epochs):\n", 718 | " users = torch.LongTensor(df_train.userId.values) # .cuda()\n", 719 | " items = torch.LongTensor(df_train.movieId.values) #.cuda()\n", 720 | " ratings = torch.FloatTensor(df_train.rating.values) #.cuda()\n", 721 | " if unsqueeze:\n", 722 | " ratings = ratings.unsqueeze(1)\n", 723 | " y_hat = model(users, items)\n", 724 | " loss = F.mse_loss(y_hat, ratings)\n", 725 | " optimizer.zero_grad()\n", 726 | " loss.backward()\n", 727 | " optimizer.step()\n", 728 | " print(loss.item()) \n", 729 | " test_loss(model, unsqueeze)" 730 | ] 731 | }, 732 | { 733 | "cell_type": "code", 734 | "execution_count": 24, 735 | "metadata": {}, 736 | "outputs": [ 737 | { 738 | "name": "stdout", 739 | "output_type": "stream", 740 | "text": [ 741 | "torch.Size([79799])\n", 742 | "torch.Size([79799, 1])\n" 743 | ] 744 | } 745 | ], 746 | "source": [ 747 | "# Here is what unsqueeze does\n", 748 | "ratings = torch.FloatTensor(df_train.rating.values)\n", 749 | "print(ratings.shape)\n", 750 | "ratings = ratings.unsqueeze(1) # .cuda()\n", 751 | "print(ratings.shape)" 752 | ] 753 | }, 754 | { 755 | "cell_type": "code", 756 | "execution_count": 25, 757 | "metadata": {}, 758 | "outputs": [], 759 | "source": [ 760 | "def test_loss(model, unsqueeze=False):\n", 761 | " model.eval()\n", 762 | " users = torch.LongTensor(df_val.userId.values) #.cuda()\n", 763 | " items = torch.LongTensor(df_val.movieId.values) #.cuda()\n", 764 | " ratings = torch.FloatTensor(df_val.rating.values) #.cuda()\n", 765 | " if unsqueeze:\n", 766 | " ratings = ratings.unsqueeze(1)\n", 767 | " y_hat = model(users, items)\n", 768 | " loss = F.mse_loss(y_hat, ratings)\n", 769 | " print(\"test loss %.3f \" % loss.item())" 770 | ] 771 | }, 772 | { 773 | "cell_type": "code", 774 | "execution_count": 26, 775 | "metadata": {}, 776 | "outputs": [ 777 | { 778 | "name": "stdout", 779 | "output_type": "stream", 780 | "text": [ 781 | "13.23068904876709\n", 782 | "5.119534015655518\n", 783 | "2.3902299404144287\n", 784 | "3.441521406173706\n", 785 | "0.9096018671989441\n", 786 | "1.8109439611434937\n", 787 | "2.749631643295288\n", 788 | "2.278921604156494\n", 789 | "1.1593214273452759\n", 790 | "0.925656795501709\n", 791 | "test loss 1.947 \n" 792 | ] 793 | } 794 | ], 795 | "source": [ 796 | "train_epocs(model, epochs=10, lr=0.1)" 797 | ] 798 | }, 799 | { 800 | "cell_type": "code", 801 | "execution_count": 27, 802 | "metadata": {}, 803 | "outputs": [ 804 | { 805 | "name": "stdout", 806 | "output_type": "stream", 807 | "text": [ 808 | "1.7027523517608643\n", 809 | "1.0512956380844116\n", 810 | "0.7498359680175781\n", 811 | "0.6950282454490662\n", 812 | "0.7596880197525024\n", 813 | "0.8397833108901978\n", 814 | "0.8818210363388062\n", 815 | "0.8753886818885803\n", 816 | "0.8334189653396606\n", 817 | "0.7767009735107422\n", 818 | "0.7246581315994263\n", 819 | "0.6901594400405884\n", 820 | "0.6771144866943359\n", 821 | "0.6810137033462524\n", 822 | "0.69219970703125\n", 823 | "test loss 0.894 \n" 824 | ] 825 | } 826 | ], 827 | "source": [ 828 | "train_epocs(model, epochs=15, lr=0.01)" 829 | ] 830 | }, 831 | { 832 | "cell_type": "code", 833 | "execution_count": 28, 834 | "metadata": {}, 835 | "outputs": [ 836 | { 837 | "name": "stdout", 838 | "output_type": "stream", 839 | "text": [ 840 | "0.7007282376289368\n", 841 | "0.6625022888183594\n", 842 | "0.6684340834617615\n", 843 | "0.6455244421958923\n", 844 | "0.6380830407142639\n", 845 | "0.6450700759887695\n", 846 | "0.6408411264419556\n", 847 | "0.6256920099258423\n", 848 | "0.6144804358482361\n", 849 | "0.6132143139839172\n", 850 | "0.6140048503875732\n", 851 | "0.6083489060401917\n", 852 | "0.5969548225402832\n", 853 | "0.5860226154327393\n", 854 | "0.5791704058647156\n", 855 | "test loss 0.822 \n" 856 | ] 857 | } 858 | ], 859 | "source": [ 860 | "train_epocs(model, epochs=15, lr=0.01)" 861 | ] 862 | }, 863 | { 864 | "cell_type": "markdown", 865 | "metadata": {}, 866 | "source": [ 867 | "## MF with bias" 868 | ] 869 | }, 870 | { 871 | "cell_type": "code", 872 | "execution_count": 29, 873 | "metadata": {}, 874 | "outputs": [], 875 | "source": [ 876 | "class MF_bias(nn.Module):\n", 877 | " def __init__(self, num_users, num_items, emb_size=100):\n", 878 | " super(MF_bias, self).__init__()\n", 879 | " self.user_emb = nn.Embedding(num_users, emb_size)\n", 880 | " self.user_bias = nn.Embedding(num_users, 1)\n", 881 | " self.item_emb = nn.Embedding(num_items, emb_size)\n", 882 | " self.item_bias = nn.Embedding(num_items, 1)\n", 883 | " self.user_emb.weight.data.uniform_(0,0.05)\n", 884 | " self.item_emb.weight.data.uniform_(0,0.05)\n", 885 | " self.user_bias.weight.data.uniform_(-0.01,0.01)\n", 886 | " self.item_bias.weight.data.uniform_(-0.01,0.01)\n", 887 | " \n", 888 | " def forward(self, u, v):\n", 889 | " U = self.user_emb(u)\n", 890 | " V = self.item_emb(v)\n", 891 | " b_u = self.user_bias(u).squeeze()\n", 892 | " b_v = self.item_bias(v).squeeze()\n", 893 | " return (U*V).sum(1) + b_u + b_v" 894 | ] 895 | }, 896 | { 897 | "cell_type": "code", 898 | "execution_count": 32, 899 | "metadata": {}, 900 | "outputs": [], 901 | "source": [ 902 | "model = MF_bias(num_users, num_items, emb_size=100) #.cuda()" 903 | ] 904 | }, 905 | { 906 | "cell_type": "code", 907 | "execution_count": 33, 908 | "metadata": {}, 909 | "outputs": [ 910 | { 911 | "name": "stdout", 912 | "output_type": "stream", 913 | "text": [ 914 | "13.233644485473633\n", 915 | "9.459980964660645\n", 916 | "4.618295669555664\n", 917 | "1.2266862392425537\n", 918 | "2.4537320137023926\n", 919 | "3.888521432876587\n", 920 | "2.6157896518707275\n", 921 | "1.1573508977890015\n", 922 | "0.8204843997955322\n", 923 | "1.3100122213363647\n", 924 | "test loss 2.126 \n" 925 | ] 926 | } 927 | ], 928 | "source": [ 929 | "train_epocs(model, epochs=10, lr=0.05, wd=1e-5)" 930 | ] 931 | }, 932 | { 933 | "cell_type": "code", 934 | "execution_count": 34, 935 | "metadata": {}, 936 | "outputs": [ 937 | { 938 | "name": "stdout", 939 | "output_type": "stream", 940 | "text": [ 941 | "1.9130752086639404\n", 942 | "1.3447301387786865\n", 943 | "0.9572998285293579\n", 944 | "0.7714419364929199\n", 945 | "0.752704381942749\n", 946 | "0.8091325759887695\n", 947 | "0.8543495535850525\n", 948 | "0.8524782657623291\n", 949 | "0.8114585876464844\n", 950 | "0.7577651739120483\n", 951 | "test loss 0.851 \n" 952 | ] 953 | } 954 | ], 955 | "source": [ 956 | "train_epocs(model, epochs=10, lr=0.01, wd=1e-5)" 957 | ] 958 | }, 959 | { 960 | "cell_type": "code", 961 | "execution_count": 35, 962 | "metadata": {}, 963 | "outputs": [ 964 | { 965 | "name": "stdout", 966 | "output_type": "stream", 967 | "text": [ 968 | "0.7163214087486267\n", 969 | "0.7023102045059204\n", 970 | "0.6904919147491455\n", 971 | "0.6807348728179932\n", 972 | "0.6728458404541016\n", 973 | "0.6666097044944763\n", 974 | "0.6618107557296753\n", 975 | "0.6582220792770386\n", 976 | "0.6556380391120911\n", 977 | "0.6538312435150146\n", 978 | "test loss 0.805 \n" 979 | ] 980 | } 981 | ], 982 | "source": [ 983 | "train_epocs(model, epochs=10, lr=0.001, wd=1e-5)" 984 | ] 985 | }, 986 | { 987 | "cell_type": "markdown", 988 | "metadata": {}, 989 | "source": [ 990 | "Note that these models are susceptible to weight initialization, optimization algorithm and regularization." 991 | ] 992 | }, 993 | { 994 | "cell_type": "markdown", 995 | "metadata": {}, 996 | "source": [ 997 | "## Neural Network Model" 998 | ] 999 | }, 1000 | { 1001 | "cell_type": "code", 1002 | "execution_count": 76, 1003 | "metadata": {}, 1004 | "outputs": [], 1005 | "source": [ 1006 | "# Note here there is no matrix multiplication, we could potentially make the embeddings of different sizes.\n", 1007 | "# Here we could get better results by keep playing with regularization.\n", 1008 | " \n", 1009 | "class CollabFNet(nn.Module):\n", 1010 | " def __init__(self, num_users, num_items, emb_size=100, n_hidden=10):\n", 1011 | " super(CollabFNet, self).__init__()\n", 1012 | " self.user_emb = nn.Embedding(num_users, emb_size)\n", 1013 | " self.item_emb = nn.Embedding(num_items, emb_size)\n", 1014 | " self.lin1 = nn.Linear(emb_size*2, n_hidden)\n", 1015 | " self.lin2 = nn.Linear(n_hidden, 1)\n", 1016 | " self.drop1 = nn.Dropout(0.1)\n", 1017 | " \n", 1018 | " def forward(self, u, v):\n", 1019 | " U = self.user_emb(u)\n", 1020 | " V = self.item_emb(v)\n", 1021 | " x = F.relu(torch.cat([U, V], dim=1))\n", 1022 | " x = self.drop1(x)\n", 1023 | " x = F.relu(self.lin1(x))\n", 1024 | " x = self.lin2(x)\n", 1025 | " return x" 1026 | ] 1027 | }, 1028 | { 1029 | "cell_type": "code", 1030 | "execution_count": 77, 1031 | "metadata": {}, 1032 | "outputs": [], 1033 | "source": [ 1034 | "model = CollabFNet(num_users, num_items, emb_size=100) #.cuda()" 1035 | ] 1036 | }, 1037 | { 1038 | "cell_type": "code", 1039 | "execution_count": 78, 1040 | "metadata": {}, 1041 | "outputs": [ 1042 | { 1043 | "name": "stdout", 1044 | "output_type": "stream", 1045 | "text": [ 1046 | "13.101761817932129\n", 1047 | "1.957230806350708\n", 1048 | "1.2605514526367188\n", 1049 | "1.3381402492523193\n", 1050 | "1.061022162437439\n", 1051 | "1.1385098695755005\n", 1052 | "0.9165319800376892\n", 1053 | "0.9622549414634705\n", 1054 | "0.8723138570785522\n", 1055 | "0.8084518909454346\n", 1056 | "0.8500765562057495\n", 1057 | "0.7535637617111206\n", 1058 | "0.791947603225708\n", 1059 | "0.7653028964996338\n", 1060 | "0.7301635146141052\n", 1061 | "test loss 0.869 \n" 1062 | ] 1063 | } 1064 | ], 1065 | "source": [ 1066 | "train_epocs(model, epochs=15, lr=0.05, wd=1e-6, unsqueeze=True) " 1067 | ] 1068 | }, 1069 | { 1070 | "cell_type": "code", 1071 | "execution_count": 79, 1072 | "metadata": {}, 1073 | "outputs": [ 1074 | { 1075 | "name": "stdout", 1076 | "output_type": "stream", 1077 | "text": [ 1078 | "0.7691234350204468\n", 1079 | "0.9072751402854919\n", 1080 | "0.7757670879364014\n", 1081 | "0.7180655598640442\n", 1082 | "0.7918605208396912\n", 1083 | "0.7724899053573608\n", 1084 | "0.7119362950325012\n", 1085 | "0.7106000185012817\n", 1086 | "0.7403213977813721\n", 1087 | "0.7438958883285522\n", 1088 | "test loss 0.816 \n" 1089 | ] 1090 | } 1091 | ], 1092 | "source": [ 1093 | "train_epocs(model, epochs=10, lr=0.01, wd=1e-6, unsqueeze=True)" 1094 | ] 1095 | }, 1096 | { 1097 | "cell_type": "code", 1098 | "execution_count": 80, 1099 | "metadata": {}, 1100 | "outputs": [ 1101 | { 1102 | "name": "stdout", 1103 | "output_type": "stream", 1104 | "text": [ 1105 | "0.7163267731666565\n", 1106 | "0.7032808065414429\n", 1107 | "0.695513904094696\n", 1108 | "0.6967512369155884\n", 1109 | "0.6998187303543091\n", 1110 | "0.700666606426239\n", 1111 | "0.7004959583282471\n", 1112 | "0.6982167959213257\n", 1113 | "0.6955875158309937\n", 1114 | "0.694402813911438\n", 1115 | "test loss 0.796 \n" 1116 | ] 1117 | } 1118 | ], 1119 | "source": [ 1120 | "train_epocs(model, epochs=10, lr=0.001, wd=1e-6, unsqueeze=True)" 1121 | ] 1122 | }, 1123 | { 1124 | "cell_type": "code", 1125 | "execution_count": 81, 1126 | "metadata": {}, 1127 | "outputs": [ 1128 | { 1129 | "name": "stdout", 1130 | "output_type": "stream", 1131 | "text": [ 1132 | "0.6919353008270264\n", 1133 | "0.6934647560119629\n", 1134 | "0.6922585368156433\n", 1135 | "0.6942275762557983\n", 1136 | "0.6926798224449158\n", 1137 | "0.6916202902793884\n", 1138 | "0.6911264061927795\n", 1139 | "0.6923496127128601\n", 1140 | "0.6922929286956787\n", 1141 | "0.6904215812683105\n", 1142 | "test loss 0.795 \n" 1143 | ] 1144 | } 1145 | ], 1146 | "source": [ 1147 | "train_epocs(model, epochs=10, lr=0.001, wd=1e-6, unsqueeze=True)" 1148 | ] 1149 | }, 1150 | { 1151 | "cell_type": "markdown", 1152 | "metadata": {}, 1153 | "source": [ 1154 | "# References\n", 1155 | "* This notebook is based on [lesson 5 of Jeremy Howard's Deep Learning Course](https://github.com/fastai/fastai/blob/master/courses/dl1/lesson5-movielens.ipynb)" 1156 | ] 1157 | }, 1158 | { 1159 | "cell_type": "code", 1160 | "execution_count": null, 1161 | "metadata": {}, 1162 | "outputs": [], 1163 | "source": [] 1164 | } 1165 | ], 1166 | "metadata": { 1167 | "kernelspec": { 1168 | "display_name": "Python 3", 1169 | "language": "python", 1170 | "name": "python3" 1171 | }, 1172 | "language_info": { 1173 | "codemirror_mode": { 1174 | "name": "ipython", 1175 | "version": 3 1176 | }, 1177 | "file_extension": ".py", 1178 | "mimetype": "text/x-python", 1179 | "name": "python", 1180 | "nbconvert_exporter": "python", 1181 | "pygments_lexer": "ipython3", 1182 | "version": "3.6.5" 1183 | }, 1184 | "toc": { 1185 | "nav_menu": {}, 1186 | "number_sections": true, 1187 | "sideBar": true, 1188 | "skip_h1_title": false, 1189 | "toc_cell": false, 1190 | "toc_position": {}, 1191 | "toc_section_display": "block", 1192 | "toc_window_display": false 1193 | } 1194 | }, 1195 | "nbformat": 4, 1196 | "nbformat_minor": 2 1197 | } 1198 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: pytorch 2 | channels: 3 | - conda-forge 4 | - pytorch 5 | dependencies: 6 | - python==3.6 7 | - ipython 8 | - jupyter 9 | - pytorch 10 | - torchvision 11 | - numpy 12 | - matplotlib 13 | - pandas 14 | - opencv 15 | - spacy -------------------------------------------------------------------------------- /images/model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yanneta/pytorch-tutorials/53c1dc84fc11e4c50877fa141f9b84ad18265047/images/model.png -------------------------------------------------------------------------------- /images/tiny_training2.csv: -------------------------------------------------------------------------------- 1 | userId,movieId,rating 2 | 11,1,4 3 | 11,23,5 4 | 2,23,5 5 | 2,4,3 6 | 31,1,4 7 | 31,23,4 8 | 4,1,5 9 | 4,3,2 10 | 52,1,1 11 | 52,3,4 12 | 61,3,5 13 | 7,23,1 14 | 7,3,3 15 | -------------------------------------------------------------------------------- /images/tiny_val2.csv: -------------------------------------------------------------------------------- 1 | userId,movieId,rating 2 | 2,1,5 3 | 4,23,5 4 | 4,2,3 5 | -------------------------------------------------------------------------------- /intro-to-pytoch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "toc": true 7 | }, 8 | "source": [ 9 | "

Table of Contents

\n", 10 | "
" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 1, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "# import pytorch libraries\n", 20 | "%matplotlib inline\n", 21 | "import torch \n", 22 | "import torch.autograd as autograd \n", 23 | "import torch.nn as nn \n", 24 | "import torch.nn.functional as F\n", 25 | "import torch.optim as optim\n", 26 | "import numpy as np" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "# Intro to Pytorch" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "PyTorch consists of 4 main packages:\n", 41 | "* torch: a general purpose array library similar to Numpy that can do computations on GPU\n", 42 | "* torch.autograd: a package for automatically obtaining gradients\n", 43 | "* torch.nn: a neural net library with common layers and cost functions\n", 44 | "* torch.optim: an optimization package with common optimization algorithms like SGD, Adam, etc" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": {}, 50 | "source": [ 51 | "## Pytorch tensors\n", 52 | "Like Numpy tensors but can utilize GPUs to accelerate its numerical computations. " 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 2, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "# Create random tensor\n", 62 | "N = 5\n", 63 | "x = torch.randn(N, 10).type(torch.FloatTensor)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 3, 69 | "metadata": {}, 70 | "outputs": [ 71 | { 72 | "data": { 73 | "text/plain": [ 74 | "tensor([[-1.8239, 0.7380, 0.0890, -1.1650, 0.3185, 1.8577, 0.1110,\n", 75 | " -0.8694, 1.1761, 1.0106],\n", 76 | " [ 1.8847, -0.3493, -0.3044, 1.3749, 0.4396, 1.5092, -0.7950,\n", 77 | " 0.5705, 0.9309, -0.3835],\n", 78 | " [ 0.4320, 0.1081, -0.8353, 0.5639, 0.1228, 1.4746, -0.5602,\n", 79 | " -1.2526, 0.0964, -0.1116],\n", 80 | " [ 1.8627, -1.1173, 2.0276, 0.6197, -1.0586, 0.6214, -0.1054,\n", 81 | " -0.3784, 0.9780, -1.6672],\n", 82 | " [-0.1745, -1.0696, -0.1319, -0.5890, 0.7507, -0.3775, -1.7948,\n", 83 | " 1.2520, -0.8993, 1.2639]])" 84 | ] 85 | }, 86 | "execution_count": 3, 87 | "metadata": {}, 88 | "output_type": "execute_result" 89 | } 90 | ], 91 | "source": [ 92 | "x" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 4, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "data": { 102 | "text/plain": [ 103 | "tensor([[-1.8239, 0.7380, 0.0890, -1.1650, 0.3185, 1.8577, 0.1110,\n", 104 | " -0.8694, 1.1761, 1.0106, 1.8847, -0.3493, -0.3044, 1.3749,\n", 105 | " 0.4396, 1.5092, -0.7950, 0.5705, 0.9309, -0.3835, 0.4320,\n", 106 | " 0.1081, -0.8353, 0.5639, 0.1228, 1.4746, -0.5602, -1.2526,\n", 107 | " 0.0964, -0.1116, 1.8627, -1.1173, 2.0276, 0.6197, -1.0586,\n", 108 | " 0.6214, -0.1054, -0.3784, 0.9780, -1.6672, -0.1745, -1.0696,\n", 109 | " -0.1319, -0.5890, 0.7507, -0.3775, -1.7948, 1.2520, -0.8993,\n", 110 | " 1.2639]])" 111 | ] 112 | }, 113 | "execution_count": 4, 114 | "metadata": {}, 115 | "output_type": "execute_result" 116 | } 117 | ], 118 | "source": [ 119 | "# reshaping of tensors using .view()\n", 120 | "x.view(1,-1) #-1 makes torch infer the second dim" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": {}, 126 | "source": [ 127 | "## Pytorch Autograd\n", 128 | "The autograd package in PyTorch provides classes and functions implementing automatic differentiation of arbitrary scalar valued function. For example, the gradient of the error with respect to all parameters.\n", 129 | "\n", 130 | "In order for this to happen we need to declare our paramerers as Tensors with the requires_grad=True keyword. Here is an example:" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 5, 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "x = torch.tensor([1., 2., 3., 4., 5., 6.], requires_grad=True)" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 6, 145 | "metadata": {}, 146 | "outputs": [], 147 | "source": [ 148 | "x.grad" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 7, 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "data": { 158 | "text/plain": [ 159 | "tensor(48.)" 160 | ] 161 | }, 162 | "execution_count": 7, 163 | "metadata": {}, 164 | "output_type": "execute_result" 165 | } 166 | ], 167 | "source": [ 168 | "L = (2*x+1).sum()\n", 169 | "L" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 8, 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "L.backward() # computes the grad of L with respect to x" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 9, 184 | "metadata": {}, 185 | "outputs": [ 186 | { 187 | "data": { 188 | "text/plain": [ 189 | "tensor([ 2., 2., 2., 2., 2., 2.])" 190 | ] 191 | }, 192 | "execution_count": 9, 193 | "metadata": {}, 194 | "output_type": "execute_result" 195 | } 196 | ], 197 | "source": [ 198 | "x.grad" 199 | ] 200 | }, 201 | { 202 | "cell_type": "markdown", 203 | "metadata": {}, 204 | "source": [ 205 | "## torch.nn module\n", 206 | "A neural net library with common layers and cost functions" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 10, 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "# linear transformation of a Nx5 matrix into a Nx3 matrix, where N can be anything \n", 216 | "# (number of observations)\n", 217 | "D = 5 # number of input featutes\n", 218 | "M = 3 # neurons in the first hidden layer\n", 219 | "linear_map = nn.Linear(D, M)" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 11, 225 | "metadata": {}, 226 | "outputs": [ 227 | { 228 | "data": { 229 | "text/plain": [ 230 | "[Parameter containing:\n", 231 | " tensor([[-0.3345, 0.1780, 0.1944, 0.3522, -0.2162],\n", 232 | " [ 0.1899, -0.1076, 0.3387, 0.3439, 0.4197],\n", 233 | " [-0.3837, -0.2800, 0.1663, 0.1904, 0.0215]]), Parameter containing:\n", 234 | " tensor([-0.1277, 0.4425, 0.4374])]" 235 | ] 236 | }, 237 | "execution_count": 11, 238 | "metadata": {}, 239 | "output_type": "execute_result" 240 | } 241 | ], 242 | "source": [ 243 | "# parameters are initialized randomly\n", 244 | "[p for p in linear_map.parameters()]" 245 | ] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "metadata": {}, 250 | "source": [ 251 | "# Linear Regression with Pytorch" 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "metadata": {}, 257 | "source": [ 258 | "The goal of linear regression is to fit a line to a set of points." 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": 12, 264 | "metadata": {}, 265 | "outputs": [], 266 | "source": [ 267 | "# Here we generate some fake data\n", 268 | "def lin(a,b,x): return a*x+b\n", 269 | "\n", 270 | "def gen_fake_data(n, a, b):\n", 271 | " x = np.random.uniform(0,1,n) \n", 272 | " y = lin(a,b,x) + 0.1 * np.random.normal(0,3,n)\n", 273 | " return x, y\n", 274 | "\n", 275 | "x, y = gen_fake_data(50, 3., 8.)" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": 13, 281 | "metadata": {}, 282 | "outputs": [ 283 | { 284 | "data": { 285 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFgRJREFUeJzt3Xuw7XV53/H3hwMKBE25bAJBjgdSksGhzW3BqE1SWzS1jgPVmkJaJ9BBGK2XNPYSM+2oY5oUTTpNJmaCJ+KIbYIQktFTY4PUS0wbcc4GJQUZB4oIJ6BsPQSHAZEDT/9Yy5ydxdrnt87e+/f7rcv7NbNnr8t37/3wY+/1nO/zfL/flapCkqRDOaLvACRJs89kIUlqZLKQJDUyWUiSGpksJEmNTBaSpEYmC0lSI5OFJKmRyUKS1OjItr5xkg8ArwQeqqpzRo/9DPBO4GzgvKpa3eBrXw78JrADeH9VXdn080466aTatWvX9gQvSUvilltu+UZVrTSNay1ZAB8E3gt8aN1jtwOvBt630Rcl2QH8NvAyYB+wN8meqvrSoX7Yrl27WF2dmHskSRtI8tVpxrVWhqqqzwL7xx67s6q+3PCl5wF3V9U9VfUd4MPAhS2FKUmawiz2LE4D7l93f9/oMUlST2YxWWTCYxOPxk1yRZLVJKtra2sthyVJy2sWk8U+4PR1958HPDBpYFXtrqpBVQ1WVhr7M5KkTZrFZLEXOCvJGUmeBVwM7Ok5Jklaaq0liyTXAp8DfijJviSXJXlVkn3Ai4A/TnLjaOz3J/k4QFUdAN4E3AjcCVxfVXe0FackqVkW5Z3yBoNBuXRWkg5PkluqatA0bhbLUJK01O5Ze5Tr9t7HPWuP9h3KX2tzU54kLa171h5l7737OXfXCZy5ctxhfd0rf+t/UwUJfOzNP3FYX98Wk4UkbbOtvODvvXc/VfD4k09xzFE72Hvv/plIFpahJKnB4ZaF1r/gVw3vT+vcXSeQwDFH7SAZ3p8Fziwk6RA2M0vYygv+mSvH8bE3/8SmSlhtMllI0iFspiy01Rf8M1eOm5kk8V0mC0k6hM3OEmbxBX8rTBaSdAizWhbqmslCkhos2ixhM1wNJUlqZLKQJDUyWUiSGpksJEmNTBaSpEYmC0maU12eTuvSWUmaQ12fTuvMQpLm0FYOK9wMk4UkzaGuT6e1DCVJc6jrY0hMFpI0p7o8hsQylCSpkclCktTIZCFJatRaskjygSQPJbl93WMnJLkpyV2jz8dv8LVPJfni6GNPWzFKkqbT5szig8DLxx57G/DJqjoL+OTo/iSPV9WPjD4uaDFGSdIUWksWVfVZYHyXyIXANaPb1wD/pK2fL6l/XR5HoXZ1vXT2+6rqQYCqejDJyRuMOzrJKnAAuLKqPtJZhJK2RdfHUahds9rg3llVA+CfA7+R5AcmDUpyRZLVJKtra2vdRijpkLo+jkLt6jpZfD3JqQCjzw9NGlRVD4w+3wN8BvjRDcbtrqpBVQ1WVlbaiVjSpnR9HIXa1XUZag9wCXDl6PNHxweMVkg9VlVPJDkJ+HvAezqNUtKWdX0chdrVWrJIci3wEuCkJPuAdzBMEtcnuQy4D/iZ0dgB8Pqqeh1wNvC+JE8znPlcWVVfaitOSe3p8jiKZXPP2qOdJuLWkkVV/ewGT50/Yewq8LrR7T8H/k5bcUnSvOtj8cCsNrglSRvoY/GAyUKS5kwfiwc8olyS5kwfiwdMFpI0h7pePGAZSpLUyGQh6bB43tNysgwlaWqe97S8nFlImprnPS0vk4WkqXne0/KyDCVpap73tLxMFtKC2+4zhDzvaTmZLKQFZkNa28WehbTAbEhru5gspAVmQ1rbxTKUtMBsSG9d1+8bMatMFtKCsyG9efZ8DrIMJUkbsOdzkMlCkjZgz+cgy1CStAF7PgeZLCTpEOz5DFmGkiQ1MllIM8z3jtCssAwlzSiXbWqWOLOQZpTLNjVLWksWST6Q5KEkt6977IQkNyW5a/T5+A2+9pLRmLuSXNJWjNIsc9mmZkmqqp1vnPwU8Cjwoao6Z/TYe4D9VXVlkrcBx1fVL4593QnAKjAACrgF+PGqevhQP28wGNTq6moL/yVSfzxqQm1LcktVDZrGtdazqKrPJtk19vCFwEtGt68BPgP84tiYfwTcVFX7AZLcBLwcuLalUKWZ5bJNzYquexbfV1UPAow+nzxhzGnA/evu7xs99gxJrkiymmR1bW1t24OVJA3NYoM7Ex6bWCurqt1VNaiqwcrKSsthSdLy6jpZfD3JqQCjzw9NGLMPOH3d/ecBD3QQmyRpA10niz3Ad1c3XQJ8dMKYG4GfTnL8aLXUT48ekyT1pM2ls9cCnwN+KMm+JJcBVwIvS3IX8LLRfZIMkrwfYNTY/mVg7+jjXd9tdkvzou+d133/fC2e1pbOds2ls5oVfe+87vvnb4ZLhPvT+9JZaVmt33l9zFE72Hvv/k5fAPv++YdrHpPbMprF1VDSXOt753XfP/9weazJfHBmIW2zvt8wp+nnz1rJZ96S27KyZyEtkVkt+cxaAlsm9iykJXC4L7Kz2s/wWJPZZ7KQ5tRmZgmWfLRZJgtpBmymDLOZWULf/RTNL5OF1LPN9hE2O0uw5KPNMFlIPdtsH8FZgrpkspB6tpU+grMEdcVkIfXMGYLmgclCmgHOEDTrPO5D6ognwWqeObOQOjCrO6elaTmzkFowPovwsDzNO2cW0jabNItYhp3Tnu+02EwW0jabtG/ionN3LvSKp82U2Uwu88VkIW2zjWYRi7zi6XA3FtrDmT8mC2mbLeO+icMts83q6bfamMlCasEizyImOdwEuQw9nEVjspC0LQ4nQS7j7GvemSwk9WLZZl/zzn0WkqRGvSSLJD+f5PYkdyT51xOef0mSR5J8cfTx9j7ilCQNdV6GSnIOcDlwHvAd4E+S/HFV3TU29M+q6pVdxydJeqY+ZhZnAzdX1WNVdQD4U+BVPcQhzQQPGNQ86KPBfTvwK0lOBB4HXgGsThj3oiS3AQ8A/7aq7hgfkOQK4AqAnTt3thex1BI3p2ledD6zqKo7gXcDNwF/AtwGHBgbdivw/Kr6YeC3gI9s8L12V9WgqgYrKystRi21wwMGNS96aXBX1dVV9WNV9VPAfuCusee/VVWPjm5/HDgqyUk9hCq1ys1pmhe97LNIcnJVPZRkJ/Bq4EVjz58CfL2qKsl5DJPaN3sIVWqVm9M0L/ralPeHo57Fk8Abq+rhJK8HqKqrgNcAb0hygGFf4+Kqqp5i1YxalFNL3ZymedBLsqiqn5zw2FXrbr8XeG+nQWmu2BiWuuUObs2lWW0MuwxWi6pxZpHkTcDvVdXDHcQjTWUWG8OLOtvZrnLfopQNl9U0ZahTgL1JbgU+ANxo/0B9m8XG8CK+R8N2JcBFTaTLpLEMVVX/ETgLuBq4FLgrya8m+YGWY5MO6cyV47jo3J0z86Izi7Odrdquct+slg01vaka3KMlrF8DvsZwA93xwA1Jbqqqf99mgNK8mMXZzlZtVwJcxES6bNJUUUryFuAS4BvA+4GPVNWTSY4A7qqqmZhhDAaDWl2ddGqItP2Wqf5uz2KxJbmlqgZN46aZWZwEvLqqvrr+wap6OomnwmrpLFv9fbv2gbifZL5N07N4+3iiWPfcndsfkjTbrL9rGbnPQjpMm6m/u/9C88734JYO0+E2spetbKXFZLKQNuFw6u+LuP9Cy8cylNQyl41qETizkDawXUs9F3H/hZaPyUKaYLzP8Dv/4sf42re+vekXe5eNat6ZLKQJ1vcZnn3kEVz+oVV2HHGEDWotLXsW0gTr+wzD2UXcV6Gl5sxCmmB9n+GU5x7NG37vVhvUWmomC2kD6/sMNqi17EwW0hRsUGvZ2bOQJDUyWUiSGpksJEmNTBZqjSetSoujlwZ3kp8HLgcC/G5V/cbY8wF+E3gF8BhwaVXd2nmg2jRPWpUWS+cziyTnMEwU5wE/DLwyyVljw/4xcNbo4wrgdzoNUlvmGwRJi6WPMtTZwM1V9VhVHQD+FHjV2JgLgQ/V0M3A30pyateBavM8aVVaLH2UoW4HfiXJicDjDEtNq2NjTgPuX3d/3+ixBzuJUFvmSavSYuk8WVTVnUneDdwEPArcBhwYG5ZJXzr+QJIrGJap2Llz5zZHqu/a7FHdbmSTFkcvDe6quhq4GiDJrzKcOay3Dzh93f3nAQ9M+D67gd0Ag8HgGclEW2ejWhL0tHQ2ycmjzzuBVwPXjg3ZA/xchl4IPFJVlqB6cKhGtUtjpeXR19lQfzjqWTwJvLGqHk7yeoCqugr4OMNext0Ml87+y57iXHobNaqdcUjLpa8y1E9OeOyqdbcLeGOnQWmijRrV62ccxxy1g7337jdZSAvMU2fVaFKj2qWx0nIxWWhTXBorLReThTbNpbHS8vAgQUlSI5PFiMtAJWljlqFwGagkNXFmgSekSlITkwUuA5WkJpahcBmoJDUxWYy4DFSSNmYZSpLUyGQhSWpkslhw7h+RtB3sWSww949I2i7OLBaY+0ckbReTxQJz/4ik7WIZaoFtdf/IPWuPuvdEEmCyWHib3T9iv0PSepahNNHee/fz1NPF408+xVNPl/0OacmZLDTRKc89micOPA3AEwee5pTnHt1zRJL6ZLLQRF/71rd59pHDX49nH3kEX/vWt3uOSFKf7FloonN3ncCOI+JKKkmAyUIb8CReSev1UoZK8gtJ7khye5Jrkxw99vylSdaSfHH08bo+4lx2Z64cx0Xn7jRRSOo+WSQ5DXgLMKiqc4AdwMUThl5XVT8y+nh/p0FKkv6GvhrcRwLHJDkSOBZ4oKc4JElT6DxZVNVfAr8O3Ac8CDxSVZ+YMPSfJvmLJDckOb3TICVJf0MfZajjgQuBM4DvB74nyWvHhv0PYFdV/V3gfwHXbPC9rkiymmR1bW2tzbAlaan1UYZ6KfCVqlqrqieBPwJevH5AVX2zqp4Y3f1d4McnfaOq2l1Vg6oarKystBq0JC2zPpLFfcALkxybJMD5wJ3rByQ5dd3dC8af13LwjZuk2dH5Pouq+nySG4BbgQPAF4DdSd4FrFbVHuAtSS4YPb8fuLTrOD1xtV8eZCjNll425VXVO4B3jD389nXP/xLwS50GtY4vVP1b/8ZNxxy1g7337vf/gdQjz4aaYNI7zG1HSWTWyiqzFs96vnGTNFs87mOC8ReqU5579JZnGrM2W5m1eMZ53Ig0W0wWE4y/UG1HSWTWyiqzFs8km33jJknbz2SxgfEXqq2WRGatrDJr8UiabamqvmPYFoPBoFZXV7f0PQ61Amo7Vke1tcJqs9/XFV+SktxSVYOmcc4sRppq+NtREmmjrLKV3oNlHknTcjXUyKQVUPNgXuOWNF9MFiPzWsOf17glzRd7FuvMaw1/XuOW1D97FpvQZQ1/O1/g7T1IapvJogezviFOksbZs+jA+LEaNqUlzRtnFlvUVE6aNIuwKS1p3pgstmCactKkYzUuOnen5x5Jmismiy2Y5nyljWYRNqUlzROTxRZMU07y9FRJi8BksQXTJgJnEZLmnclii0wEkpaBS2clSY1MFpKkRiaLBrP8PtWS1BV7FodwqH0UHt4naZmYLA5ho30Unu0kadn0UoZK8gtJ7khye5Jrkxw99vyzk1yX5O4kn0+yq484N9pH0cXZTpa/JM2SzmcWSU4D3gK8oKoeT3I9cDHwwXXDLgMerqq/neRi4N3ARV3HutE+irbPdnLmImnW9FWGOhI4JsmTwLHAA2PPXwi8c3T7BuC9SVI9vFPTpH0Ube/KnuYYEUnqUufJoqr+MsmvA/cBjwOfqKpPjA07Dbh/NP5AkkeAE4FvdBrsIbS5Gc9TaSXNmj7KUMcznDmcAfwV8AdJXltV/339sAlf+oxZRZIrgCsAdu7c2UK0/fA8KUmzpo8G90uBr1TVWlU9CfwR8OKxMfuA0wGSHAl8L/CMLnJV7a6qQVUNVlZWWg67W2euHMdF5+40UUiaCX0ki/uAFyY5NkmA84E7x8bsAS4Z3X4N8Kk++hWSpKHOk0VVfZ5h0/pW4P+OYtid5F1JLhgNuxo4McndwFuBt3UdpyTpoCzKP9gHg0Gtrq72HYYkzZUkt1TVoGmcZ0NJkhqZLCRJjUwWkqRGJosWeb6TpEXhqbMt8XwnSYvEmUVLujiZVpK6YrJoiec7SVoklqFa4vlOkhaJyaJFbZ5MK0ldsgwlSWpkspAkNTJZSJIamSwkSY1MFpKkRiYLSVIjk4UkqdHCvPlRkjXgq2MPnwR8o4dwZo3X4SCvxZDX4aBlvxbPr6qVpkELkywmSbI6zTtALTqvw0FeiyGvw0Fei+lYhpIkNTJZSJIaLXqy2N13ADPC63CQ12LI63CQ12IKC92zkCRtj0WfWUiStsFCJIskL0/y5SR3J3nbhOefneS60fOfT7Kr+yjbN8V1eGuSLyX5iySfTPL8PuLsQtO1WDfuNUkqyUKuhpnmOiT5Z6PfizuS/H7XMXZlir+PnUk+neQLo7+RV/QR58yqqrn+AHYA/w84E3gWcBvwgrEx/wq4anT7YuC6vuPu6Tr8A+DY0e03LOJ1mPZajMY9B/gscDMw6Dvunn4nzgK+ABw/un9y33H3eC12A28Y3X4BcG/fcc/SxyLMLM4D7q6qe6rqO8CHgQvHxlwIXDO6fQNwfpJ0GGMXGq9DVX26qh4b3b0ZeF7HMXZlmt8JgF8G3gN8u8vgOjTNdbgc+O2qehigqh7qOMauTHMtCnju6Pb3Ag90GN/MW4RkcRpw/7r7+0aPTRxTVQeAR4ATO4muO9Nch/UuA/5nqxH1p/FaJPlR4PSq+liXgXVsmt+JHwR+MMn/SXJzkpd3Fl23prkW7wRem2Qf8HHgzd2ENh8W4W1VJ80Qxpd4TTNm3k3935jktcAA+PutRtSfQ16LJEcA/xW4tKuAejLN78SRDEtRL2E40/yzJOdU1V+1HFvXprkWPwt8sKr+S5IXAf9tdC2ebj+82bcIM4t9wOnr7j+PZ04f/3pMkiMZTjH3dxJdd6a5DiR5KfAfgAuq6omOYuta07V4DnAO8Jkk9wIvBPYsYJN72r+Nj1bVk1X1FeDLDJPHopnmWlwGXA9QVZ8DjmZ4bpRYjGSxFzgryRlJnsWwgb1nbMwe4JLR7dcAn6pRF2uBNF6HUenlfQwTxaLWpqHhWlTVI1V1UlXtqqpdDPs3F1TVaj/htmaav42PMFz4QJKTGJal7uk0ym5Mcy3uA84HSHI2w2Sx1mmUM2zuk8WoB/Em4EbgTuD6qrojybuSXDAadjVwYpK7gbcCGy6lnFdTXodfA44D/iDJF5OM/7EshCmvxcKb8jrcCHwzyZeATwP/rqq+2U/E7ZnyWvwb4PIktwHXApcu4D8qN80d3JKkRnM/s5Aktc9kIUlqZLKQJDUyWUiSGpksJEmNTBaSpEYmC0lSI5OF1JIk547eF+HoJN8zer+Ic/qOS9oMN+VJLUrynxgeG3EMsK+q/nPPIUmbYrKQWjQ6h2gvw/fMeHFVPdVzSNKmWIaS2nUCw/O4nsNwhiHNJWcWUotGhzV+GDgDOLWq3tRzSNKmLMKbH0kzKcnPAQeq6veT7AD+PMk/rKpP9R2bdLicWUiSGtmzkCQ1MllIkhqZLCRJjUwWkqRGJgtJUiOThSSpkclCktTIZCFJavT/AZcXR0Q04H4rAAAAAElFTkSuQmCC\n", 286 | "text/plain": [ 287 | "
" 288 | ] 289 | }, 290 | "metadata": { 291 | "needs_background": "light" 292 | }, 293 | "output_type": "display_data" 294 | } 295 | ], 296 | "source": [ 297 | "import matplotlib.pyplot as plt\n", 298 | "plt.scatter(x,y, s=8); plt.xlabel(\"x\"); plt.ylabel(\"y\"); " 299 | ] 300 | }, 301 | { 302 | "cell_type": "markdown", 303 | "metadata": {}, 304 | "source": [ 305 | "You want to find **parameters** (weights) $a$ and $b$ such that you minimize the *error* between the points and the line $a\\cdot x + b$. Note that here $a$ and $b$ are unknown. For a regression problem the most common *error function* or *loss function* is the **mean squared error**. " 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 14, 311 | "metadata": {}, 312 | "outputs": [], 313 | "source": [ 314 | "def mse(y_hat, y): return ((y_hat - y) ** 2).mean()" 315 | ] 316 | }, 317 | { 318 | "cell_type": "markdown", 319 | "metadata": {}, 320 | "source": [ 321 | "Suppose we believe $a = 10$ and $b = 5$ then we can compute `y_hat` which is our *prediction* and then compute our error." 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": 15, 327 | "metadata": {}, 328 | "outputs": [ 329 | { 330 | "data": { 331 | "text/plain": [ 332 | "3.7264671933272044" 333 | ] 334 | }, 335 | "execution_count": 15, 336 | "metadata": {}, 337 | "output_type": "execute_result" 338 | } 339 | ], 340 | "source": [ 341 | "y_hat = lin(10,5,x)\n", 342 | "mse(y_hat, y)" 343 | ] 344 | }, 345 | { 346 | "cell_type": "code", 347 | "execution_count": 16, 348 | "metadata": {}, 349 | "outputs": [], 350 | "source": [ 351 | "def mse_loss(a, b, x, y): return mse(lin(a,b,x), y)" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": 17, 357 | "metadata": {}, 358 | "outputs": [ 359 | { 360 | "data": { 361 | "text/plain": [ 362 | "3.7264671933272044" 363 | ] 364 | }, 365 | "execution_count": 17, 366 | "metadata": {}, 367 | "output_type": "execute_result" 368 | } 369 | ], 370 | "source": [ 371 | "mse_loss(10, 5, x, y)" 372 | ] 373 | }, 374 | { 375 | "cell_type": "markdown", 376 | "metadata": {}, 377 | "source": [ 378 | "So far we have specified the *model* (linear regression) and the *evaluation criteria* (or *loss function*). Now we need to handle *optimization*; that is, how do we find the best values for $a$ and $b$? How do we find the best *fitting* linear regression." 379 | ] 380 | }, 381 | { 382 | "cell_type": "markdown", 383 | "metadata": {}, 384 | "source": [ 385 | "## Gradient Descent with Pytorch" 386 | ] 387 | }, 388 | { 389 | "cell_type": "markdown", 390 | "metadata": {}, 391 | "source": [ 392 | "For a fixed dataset $x$ and $y$ `mse_loss(a,b)` is a function of $a$ and $b$. We would like to find the values of $a$ and $b$ that minimize that function.\n", 393 | "\n", 394 | "**Gradient descent** is an algorithm that minimizes functions. Given a function defined by a set of parameters, gradient descent starts with an initial set of parameter values and iteratively moves toward a set of parameter values that minimize the function. This iterative minimization is achieved by taking steps in the negative direction of the function gradient.\n", 395 | "\n", 396 | "Here is gradient descent implemented in [PyTorch](http://pytorch.org/)." 397 | ] 398 | }, 399 | { 400 | "cell_type": "code", 401 | "execution_count": 18, 402 | "metadata": {}, 403 | "outputs": [ 404 | { 405 | "data": { 406 | "text/plain": [ 407 | "((10000,), (10000,))" 408 | ] 409 | }, 410 | "execution_count": 18, 411 | "metadata": {}, 412 | "output_type": "execute_result" 413 | } 414 | ], 415 | "source": [ 416 | "# generate some more data\n", 417 | "x, y = gen_fake_data(10000, 3., 8.)\n", 418 | "x.shape, y.shape" 419 | ] 420 | }, 421 | { 422 | "cell_type": "code", 423 | "execution_count": 19, 424 | "metadata": {}, 425 | "outputs": [], 426 | "source": [ 427 | "# Wrap x and y as tensor \n", 428 | "x = torch.tensor(x)\n", 429 | "y = torch.tensor(y)" 430 | ] 431 | }, 432 | { 433 | "cell_type": "code", 434 | "execution_count": 20, 435 | "metadata": {}, 436 | "outputs": [ 437 | { 438 | "data": { 439 | "text/plain": [ 440 | "(tensor([-0.4017], dtype=torch.float64),\n", 441 | " tensor([-0.9494], dtype=torch.float64))" 442 | ] 443 | }, 444 | "execution_count": 20, 445 | "metadata": {}, 446 | "output_type": "execute_result" 447 | } 448 | ], 449 | "source": [ 450 | "# Create random Tensors for weights, and wrap them in tensors.\n", 451 | "# Setting requires_grad=True indicates that we want to compute gradients with\n", 452 | "# respect to these tensors during the backward pass.\n", 453 | "a, b = np.random.randn(1), np.random.randn(1)\n", 454 | "a = torch.tensor(a, requires_grad=True)\n", 455 | "b = torch.tensor(b, requires_grad=True)\n", 456 | "a,b" 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": 21, 462 | "metadata": {}, 463 | "outputs": [ 464 | { 465 | "name": "stdout", 466 | "output_type": "stream", 467 | "text": [ 468 | "114.33519327885749\n", 469 | "0.8827967584092252\n", 470 | "0.1509190011086734\n", 471 | "0.1336058659739962\n", 472 | "0.1237467641246501\n", 473 | "0.1161985772878354\n", 474 | "0.11040320048670306\n", 475 | "0.10595349916476945\n", 476 | "0.10253700937772219\n", 477 | "0.09991382151883298\n" 478 | ] 479 | } 480 | ], 481 | "source": [ 482 | "learning_rate = 1e-3\n", 483 | "for t in range(10000):\n", 484 | " # Forward pass: compute predicted y using operations on Variables\n", 485 | " loss = mse_loss(a,b,x,y)\n", 486 | " if t % 1000 == 0: print(loss.item())\n", 487 | " \n", 488 | " # Computes the gradient of loss with respect to all Variables with requires_grad=True.\n", 489 | " # After this call a.grad and b.grad will be Variables holding the gradient\n", 490 | " # of the loss with respect to a and b respectively\n", 491 | " loss.backward()\n", 492 | " \n", 493 | " # Update a and b using gradient descent; a.data and b.data are Tensors,\n", 494 | " # a.grad and b.grad are Variables and a.grad.data and b.grad.data are Tensors\n", 495 | " a.data -= learning_rate * a.grad.data\n", 496 | " b.data -= learning_rate * b.grad.data\n", 497 | " \n", 498 | " # Zero the gradients\n", 499 | " a.grad.data.zero_()\n", 500 | " b.grad.data.zero_() " 501 | ] 502 | }, 503 | { 504 | "cell_type": "code", 505 | "execution_count": 22, 506 | "metadata": {}, 507 | "outputs": [ 508 | { 509 | "name": "stdout", 510 | "output_type": "stream", 511 | "text": [ 512 | "tensor([ 3.2942], dtype=torch.float64) tensor([ 7.8449], dtype=torch.float64)\n" 513 | ] 514 | } 515 | ], 516 | "source": [ 517 | "print(a,b)" 518 | ] 519 | }, 520 | { 521 | "cell_type": "markdown", 522 | "metadata": {}, 523 | "source": [ 524 | "## Simplified GD Loop" 525 | ] 526 | }, 527 | { 528 | "cell_type": "code", 529 | "execution_count": 23, 530 | "metadata": {}, 531 | "outputs": [ 532 | { 533 | "data": { 534 | "text/plain": [ 535 | "Linear(in_features=1, out_features=1, bias=True)" 536 | ] 537 | }, 538 | "execution_count": 23, 539 | "metadata": {}, 540 | "output_type": "execute_result" 541 | } 542 | ], 543 | "source": [ 544 | "# linear tranformation with input dimension=1 and output dimension=1\n", 545 | "nn.Linear(1, 1)" 546 | ] 547 | }, 548 | { 549 | "cell_type": "code", 550 | "execution_count": 24, 551 | "metadata": {}, 552 | "outputs": [ 553 | { 554 | "data": { 555 | "text/plain": [ 556 | "Sequential(\n", 557 | " (0): Linear(in_features=1, out_features=1, bias=True)\n", 558 | ")" 559 | ] 560 | }, 561 | "execution_count": 24, 562 | "metadata": {}, 563 | "output_type": "execute_result" 564 | } 565 | ], 566 | "source": [ 567 | "# simple way of specifying a linear regression model\n", 568 | "model = torch.nn.Sequential(\n", 569 | " nn.Linear(1, 1),\n", 570 | ")\n", 571 | "model" 572 | ] 573 | }, 574 | { 575 | "cell_type": "code", 576 | "execution_count": 25, 577 | "metadata": {}, 578 | "outputs": [], 579 | "source": [ 580 | "# equivalent way of specifiying the same model\n", 581 | "class LinearRegression(nn.Module):\n", 582 | " def __init__(self):\n", 583 | " super(LinearRegression, self).__init__()\n", 584 | " self.lin = nn.Linear(1, 1)\n", 585 | " \n", 586 | " def forward(self, x):\n", 587 | " x = self.lin(x)\n", 588 | " return x \n", 589 | "model = LinearRegression()" 590 | ] 591 | }, 592 | { 593 | "cell_type": "code", 594 | "execution_count": 26, 595 | "metadata": {}, 596 | "outputs": [ 597 | { 598 | "name": "stdout", 599 | "output_type": "stream", 600 | "text": [ 601 | "[Parameter containing:\n", 602 | "tensor([[ 0.2523]]), Parameter containing:\n", 603 | "tensor([ 0.5015])]\n" 604 | ] 605 | } 606 | ], 607 | "source": [ 608 | "print([p for p in model.parameters()])" 609 | ] 610 | }, 611 | { 612 | "cell_type": "code", 613 | "execution_count": 27, 614 | "metadata": {}, 615 | "outputs": [ 616 | { 617 | "data": { 618 | "text/plain": [ 619 | "torch.Size([10000])" 620 | ] 621 | }, 622 | "execution_count": 27, 623 | "metadata": {}, 624 | "output_type": "execute_result" 625 | } 626 | ], 627 | "source": [ 628 | "x, y = gen_fake_data(10000, 3., 8.)\n", 629 | "x = torch.tensor(x).float()\n", 630 | "y = torch.tensor(y).float()\n", 631 | "x.shape" 632 | ] 633 | }, 634 | { 635 | "cell_type": "code", 636 | "execution_count": 28, 637 | "metadata": {}, 638 | "outputs": [ 639 | { 640 | "data": { 641 | "text/plain": [ 642 | "torch.Size([10000, 1])" 643 | ] 644 | }, 645 | "execution_count": 28, 646 | "metadata": {}, 647 | "output_type": "execute_result" 648 | } 649 | ], 650 | "source": [ 651 | "# you have to be careful with the dimensions that your model is expecting\n", 652 | "x1 = torch.unsqueeze(x, 1)\n", 653 | "x1.shape" 654 | ] 655 | }, 656 | { 657 | "cell_type": "code", 658 | "execution_count": 29, 659 | "metadata": {}, 660 | "outputs": [ 661 | { 662 | "name": "stdout", 663 | "output_type": "stream", 664 | "text": [ 665 | "tensor([[ 0.6813],\n", 666 | " [ 0.6076],\n", 667 | " [ 0.6617],\n", 668 | " ...,\n", 669 | " [ 0.6750],\n", 670 | " [ 0.5348],\n", 671 | " [ 0.5524]])\n" 672 | ] 673 | } 674 | ], 675 | "source": [ 676 | "y_hat = model(x1)\n", 677 | "print(y_hat)" 678 | ] 679 | }, 680 | { 681 | "cell_type": "code", 682 | "execution_count": 30, 683 | "metadata": {}, 684 | "outputs": [], 685 | "source": [ 686 | "# Use the optim package to define an Optimizer that will update the weights of\n", 687 | "# the model for us. Here we will use Adam\n", 688 | "learning_rate = 0.1\n", 689 | "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)" 690 | ] 691 | }, 692 | { 693 | "cell_type": "code", 694 | "execution_count": 31, 695 | "metadata": {}, 696 | "outputs": [ 697 | { 698 | "name": "stdout", 699 | "output_type": "stream", 700 | "text": [ 701 | "79.40501403808594\n", 702 | "0.08850504457950592\n", 703 | "0.08850347250699997\n", 704 | "0.08850349485874176\n", 705 | "0.08850345760583878\n", 706 | "0.08850356191396713\n", 707 | "0.08850352466106415\n", 708 | "0.08850352466106415\n", 709 | "0.08850352466106415\n", 710 | "0.08850352466106415\n" 711 | ] 712 | } 713 | ], 714 | "source": [ 715 | "for t in range(10000):\n", 716 | " # Forward pass: compute predicted y using operations on Variables\n", 717 | " y_hat = model(x1)\n", 718 | " loss = F.mse_loss(y_hat, y.unsqueeze(1))\n", 719 | " if t % 1000 == 0: print(loss.item())\n", 720 | " \n", 721 | " # Before the backward pass, use the optimizer object to zero all of the\n", 722 | " # gradients for the variables\n", 723 | " optimizer.zero_grad()\n", 724 | " loss.backward()\n", 725 | " \n", 726 | " # Calling the step function on an Optimizer makes an update to its\n", 727 | " # parameters\n", 728 | " optimizer.step()" 729 | ] 730 | }, 731 | { 732 | "cell_type": "code", 733 | "execution_count": 32, 734 | "metadata": {}, 735 | "outputs": [ 736 | { 737 | "name": "stdout", 738 | "output_type": "stream", 739 | "text": [ 740 | "[Parameter containing:\n", 741 | "tensor([[ 3.0035]]), Parameter containing:\n", 742 | "tensor([ 7.9942])]\n" 743 | ] 744 | } 745 | ], 746 | "source": [ 747 | "print([p for p in model.parameters()])" 748 | ] 749 | }, 750 | { 751 | "cell_type": "markdown", 752 | "metadata": {}, 753 | "source": [ 754 | "# Logistic Regression" 755 | ] 756 | }, 757 | { 758 | "cell_type": "code", 759 | "execution_count": 33, 760 | "metadata": {}, 761 | "outputs": [], 762 | "source": [ 763 | "# generating fake data\n", 764 | "# Here we generate some fake data\n", 765 | "def lin(a,b,x): return a*x+b\n", 766 | "\n", 767 | "def gen_logistic_fake_data(n, a, b):\n", 768 | " x = np.random.uniform(-20,20, (n, 2))\n", 769 | " x2_hat = lin(a,b, x[:,0])\n", 770 | " y = x[:,1] > x2_hat\n", 771 | " return x, y.astype(int)\n", 772 | "\n", 773 | "x, y = gen_logistic_fake_data(100, 1., 0.5)" 774 | ] 775 | }, 776 | { 777 | "cell_type": "code", 778 | "execution_count": 34, 779 | "metadata": {}, 780 | "outputs": [ 781 | { 782 | "data": { 783 | "text/plain": [ 784 | "array([0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0,\n", 785 | " 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0,\n", 786 | " 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0,\n", 787 | " 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0,\n", 788 | " 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1])" 789 | ] 790 | }, 791 | "execution_count": 34, 792 | "metadata": {}, 793 | "output_type": "execute_result" 794 | } 795 | ], 796 | "source": [ 797 | "y" 798 | ] 799 | }, 800 | { 801 | "cell_type": "code", 802 | "execution_count": 35, 803 | "metadata": {}, 804 | "outputs": [ 805 | { 806 | "data": { 807 | "text/plain": [ 808 | "[]" 809 | ] 810 | }, 811 | "execution_count": 35, 812 | "metadata": {}, 813 | "output_type": "execute_result" 814 | }, 815 | { 816 | "data": { 817 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd0VFXXx/HvnklPCDU0QZqggAoooI9i7+3FXh4LioJdFCtW7BVs2LDig71hRxSxoihVQEVQEJAWpARSp+z3jzNqVAgpM3NnJvuzVlYmU+79rSFkzz3n3n1EVTHGGGM2x+d1AGOMMYnNCoUxxpgqWaEwxhhTJSsUxhhjqmSFwhhjTJWsUBhjjKmSFQpjjDFVskJhjDGmSlYojDHGVCnN6wDR0KxZM23fvr3XMYwxJqlMmzZttaoWbOl5KVEo2rdvz9SpU72OYYwxSUVEfq3O82zoyRhjTJWsUBhjjKmSFQpjjDFVskJhjDGmSlYojDHGVMkKhTHGmCp5VihEpK2ITBKRH0RkrogMidzfREQ+FJH5ke+NvcpojDHG2yOKIHCpqnYFdgXOF5FuwFXARFXtDEyM/GyMMaYyVfj++7jsyrNCoarLVXV65PYG4AdgK6A/MCbytDHAkd4kNMaYBLVgAey3H/TtC7/9FvPdJcQchYi0B3oBU4AWqrocXDEBmnuXzFsaXkt4zVmECw9GyyZ6HceYmFFVwkU3EV65E+E1A1Et9TpSYgoG4Z57YMcdYdo0GDkSWrWK+W49LxQikge8BlysqkU1eN1gEZkqIlMLCwtjFzDGNPA9Wj4Z1eC/Hyu6AyomQ+gXdN3FaLjYg4TGxEFgGpS+BroRKr6Bkpe9TpSYli+H4cPhgAPcsNPgweCL/Z9xTwuFiKTjisRzqvp65O6VItIq8ngrYNWmXquqo1W1t6r2LijYYk+rhBQufg79/UR03Xno2nP+/QTdCIT+eDYQiGM6Y+LJB/rHbSEBPsMmjrIyePppNyfRti3MmgXjxsFWW8UtgpdnPQnwJPCDqo6s9NBbwIDI7QHAm/HOFjelLwNloCVQ8TmqFX97WBpcBr6WQDrkXYj4GnkS05iYS+8FOSeDNIHMfpBzvNeJEsMXX0CvXjBwIHz5pbuvUycQiWsML7vH7g6cCswWkZmR+64G7gBeFpEzgcXAcR7li72MXSG4CAiCvy0iGX97WNI6IM0/9SSaMfEkIkj+FZB/hddREkNREQwbBg8/DO3awfjx0K8fX709lekTZ7PH0buw457d4hZHVHXLz0pwvXv31mRsM64agtJxoOsg+2jEZ5eMRFu4+BkoGQvpOyENb/lXMTYm4ajCbrvBlClw4YVw662Ql8fMSXO49ojbKS+pIDMnk4e+vYN2XdvUaVciMk1Ve2/peSmxHkWyEvFDzjFex0hZGvgRNowEyiC0EtK3h9zTvI5lzKatXg0NG0J6Otx8M+Tmwn/+8+fDv8z6lXAwDIDPLyz+fmmdC0V12YyRSV1aWmksN4yGN3oax5hNUoUXXoCuXeGuu9x9++//tyIBsNuRfcjMySQnP5vc/Bx67rt93CLaEYVJXek9IfMQKHsT0rZBck/2OpExf7dkCZx7Lrz7LuyyC/Tvv9mntmzfnDELHmTJj8vouOPWZOdlxy2mzVEYk4I0vM5dlyCNIPtIN8xpEsurr7qzmUIhNw9x4YXgj++/k81RGFOP6Zr/QvBXIA2C85D8q72OZP6pfXvYfXd3ZlOHDl6nqZIVClMtqkEofQsoj3xCjd9hr6kZ1QAEf8ZdwRaAiq+8jmQAAgG4+25YsQIeeAB694b33/c6VbVYoTDVouuvhbL3AYWy95Emz3odyWyGSDqauSdUfOsmSrPt4jXPTZsGZ57prqo+7jjXsyktef78Jk9S462Kr4BIo7aKGZ5GMVsmjR6Biinga4ikx+/sGPN3WlJCxVVXk/HwKKR5c3jjDTgy+Rpi2+mxpnqy+4Nkg+RA1gFepzFbIJKGZO5uRcJDFeUBhu9xOaEHH+KT7C6s/+yrpCwSYEcUppok7xLI2B0oh4x+XscxJnGtXw/PPMP0jrszY34Rp3MwG4N5nPn2bI65pJ3X6WrFjihMtYgIkrkLkrknIvZrY8wmvfkmdOsGQ4fSuug3wuEwayULf5qfZls18TpdrdkRhTHG1NXKlXDRRfDyy25RoTffZOvevbko1JDxT31Mn4N6sudx/9nydhKUXXBnjKmXNPQ7UIH467hCnCr06AHz5sH118MVV7h+TUnALrgzxpjNCJe8CUXXAormDsLXYEjNN/Lrr24Z0owMGDUKmjeH7baLetZEYIPNxpj6p/gBoByogOLHa/baUAjuvx+6d3frVwPsuWfKFgmwQmGMqY/8HYF0wAf+1tV/3dy50K8fXHyxKw6nnBKrhAnF6zWznxKRVSIyp9J9w0XkNxGZGfk61MuMxpjUI41GQM6Jrh1NkzHVe9GTT7plSefPh7FjXcfXrbeObdAE4fUcxTPAKOCf/SDuVdV74h/HGFMfiC8fyb+uek9Wdeua9Ozp2m/cdx8UFMQ2YILx9IhCVT8D1niZwRhjNmnjRjfEdO657uedd4bnnqt3RQISd47iAhH5LjI0tcmFpEVksIhMFZGphYWF8c5njEkRGi5Ci59BS8eh6pYa5cMPYYcd3KS13w/hsLchPeb10NOmPALcjOuRfDMwAhj4zyep6mhgNLjrKOIZ0BiTOtzaHYsAP6yagQxfDM88A126wGefwR57eJzQewl3RKGqK1U1pK60Pw709TqTMSY1ubU75gMVQCms/dK14Rg2zLUEtyIBJGChEJHKl0keBczZ3HONMalNg0sJFx5KeGVvwsUvRH37Iunw+05w7wbQLOh8KixcCLfdBllZUd9fsvJ06ElEXgD2BpqJyFLgBmBvEemJG3paBJztWUATMxouQTfeB6EVSN4FSHoXryOZBKQb7oDQL0AYNtyCZh+B+PKitHGFJ55ALn8fykvhpFFIq2Ois+0U42mhUNWTNnH3k3EPYuJON9wSWVo1gFZ8Bc2/RiS+C8ubZOAHJHJbqnpizSxYAIMHw6RJyF57weOPQ+fO/3paRXmAkWc9wpwvf+SIcw7khCuScz2Jukq4oSdTTwQX4saFFXQjaJnXiUwCkvxhkN4dfM0h/9boHE2EQnDooW550sceg48/3mSRAHjv8Y/4/PUprFxUyP9ueoX503+p+/6TUCKe9WTqAcm7EF17DhCG7GMQX67XkUwCEn9LpOmr0dnYnDnuTKaMDHj2WWjbFrbaqsqXlJeUoyF3aqyIUF5SHp0sScaOKIwnJHM3pPlkpOAjfA1v9DqOSWVlZXDNNa79xr33uvt23XWLRQLgsMEH0LFHO9LS09jzuP/QfffUbfxXFTuiMJ5xwwhRmpg0ZlO++ALOOsutFTFggLtdA3mNchk15Y4YhUsedkRhjElNI0a46yDKy+GDD9xFdE2bep0qKVmhMCYJaHgj4fXXEl4zCA1873Wcv9HQCjS0zOsYfwmF3Pd99oEhQ2D2bDjwQG8zJTkbejImCWjRjVD2HhBA10yH5t8kxOnE4eIxsME1eta8Ifjyaja0E1WrV7smfllZ8MQTsNNO7svUmR1RJDkN/kp4zdmE116EhlZ6HcfESmgJEHC3tQR3anH8aLgELXkRLX33r8Z5ABsfwa0UVw7Fj8Y101/hFJ5/Hrp2hZdfdmczqbV/iyY7okgSqooWPwnln0D2Ufhy3BWkuvYsCC0GfOi6QqRp9NscGO9Jg6Ho2kGgQcgZgEh2XPeva0+HwI9uXYbALCT/avdAWgcIFLnb/vZxzQTAb7+5C+feew/69nWLC22/ffxzpDgrFMmi/CMofhC0FAKz0bRtkIweEF6N63YSgtAKr1OaGJGMvtB8CmgZ4msU132rKgRm4S6OBMo/BVyhkMYPoRtGAYrkXRDXXICbj5g+3Z32euGFriW4iTorFMkitAL+POQXCEeGmfIugw23ufsaXOlVOhMHIlkg8W9UJyJoRj8ITAMUso/46zFfE6Th9fENNG8ePPUU3HGHW4p04UJr4BdjViiSRfYRUPKMKxhpHSFzTwB8uSej2f1B/HEfjjD1hzR+1A17SgPI2MWbEIEA3H033HQT5OS4IadOnaxIxIEViiQhvkbQ7EPQdSCNEZFKj9lFaya2RNIh64BqP18DsyG0CjL7IZJZ9wDTpsGZZ7o1Io47Dh54AFq2rPt2TbVYoUgiIj6QJl7HMKZK4ZLXoOgmN/Ht3waavvK3DzY1FgjAsce6C+feeAOOrJ8dXL1khcIYE12lrwOlbuI7+APo2tp9wPnyS+jTxzXxe/116NABGsV3It84nl5HISJPicgqEZlT6b4mIvKhiMyPfG/sZUZjTA1l7gWSDWSAvwVIw5q9fv16OPts6NcPHnrI3derlxUJD3l9wd0zwMH/uO8qYKKqdgYmRn429ZBWTHdtK0pedadompgKl4wjXHgw4XVDUS2t9XYkdxDS8E6kweVI01drdgX5m29Ct27uyurLL3cFw3jO6xXuPhOR9v+4uz9ueVSAMcAngJ33Wc9oaAW65gygFErfdqeFZh/udayUpaFlUHQdUA6hpai/HdJgSK22JSKQ9c/Pf9Vw3XVwyy2w446uYPTuXav9m+hLxDmKFqq6HEBVl4tIc68DGQ+EloD43Dg3pWjwR4TkLRSqFRD8EfxtEV8CjqZqCX8tNRqC8Po47VfdJHVWFhx9tPt+xRWQnh6f/Ztq8XroqdZEZLCITBWRqYWFhV7HMdGWviP4twLJAclDspP3TBfVMnT1/6FrBqCF+6HBBdHZbmg1Gvge1WDdN+bvBDnHAj7wb4XkxWHIZ9EiOPhgOP9893OvXm6BISsSCScRC8VKEWkFEPm+alNPUtXRqtpbVXsXFBTENWCiU1W07GO05DU0XOJ1nFoRyUSavo40eR4pmISkbeN1pNqrmOmupNdi0GK09K06b1IrpqGr90PX/Bddc9rfG/XVgojgy78eafEDvoKPEH+LOmfcrFAI7r8funeHyZPrTYfXZJ5nS8RC8RYwIHJ7APCmh1mSkhY/hK6/BC26CV1zstdxak0kA0nvhvhqeNZMoknbulL7lSwkre7LaWrJWNf3S0sgMAdCi+q8TaBu1ztUx4IFsPvurh343nvD3Ll/HVGkKFXloSFPcXDGiZy+7UWsXrbG60g15vXpsS8AXwHbishSETkTuAM4QETmAwdEfjY1Ufah+yNCKQR/cOPjxjPib400eQayT4L8GyHrkLpvNK07EGnZImngS5KpvMxMKCyE556Dd95xvZpS3JJ5y3j/iYmEQ2FW/LKSl+4c53WkGvP6rKeTNvPQfnENkmqyDoPiRYBAWhdEMlyxKH0NNAQ5x7oGcx5QDUV6BmVARr/Yf4JNEJLRC8noFb3t5Q5EJQOCPyM5JyV2G5cpU+DZZ2HUKLdWxLx5kJaI59HERlZOxp/DTr40P7kNczxOVHOSzONmf+jdu7dOnTo1KtvS8BqQ3D/702ioEF03BELLocEV+LKj8GkwxlQVKiZDeC1kHYBIJuG1F0baQytk9MXX5ElPsoXXXewKBUD28fj+WNfApJ7iYrj2WjcfsdVWbj6ibVuvU9VIcVEJv85dQrtubchtmFvr7XwwZhIv3jGOTj3acemT55GdmxiNDEVkmqpu8TxkKxSVhNddBWVvg2QiTZ5F0rcnvO4SKBsPhIAMpPkUxFf7XxivhFf9B8K/ux8kD1+L6d7kWLEjUOZ+8LXA1/xzT3KYGPvwQ9fdddEiOPdc1xI8P9/rVDWydtV6Bu94KRVlFaRlpPHYzHto1jq1eq1Vt1Ak4mS2JzS0AsreAQKgG9GNj0QeKAf+mIjUSreTTFb/SFuFHDc05ZWMPrix9WzI3MO7HCZ2ystdp9eMDPjsM3j44aQrEgBTx8+krLiMkqJSykvKmfLONK8jeab+DBRuiTRwk4JaAWSCv03k7qvcee/hVZA3FPE18DZnLUmDKyFrPzdH4dV6AoA0fiRypXV6rQuWhlaiawdDaCnkXYAv94wopzQ1pgrvvgsHHugmrMePh44dk3qtiPbbt/3bKa0ddmznYRpv2dBTJVoxHd34EKR1RBpc6tmEb32iqmjJi1DxKWT1r9YcUHj9MCgdx5/DgQWTEL9dS7M5qmG05AUI/oTknIikd43uDpYtc6e4jhvnmvidd150t++h6R99x+S3prLLob3oc3D0TkZIFNUderIjikokYyfEo0neeqt8Emy8w53OW/4VmtYOSe+2hRel8Ve7CbAR1KppyVjYMAIoRcvegoJPonNtiio8+SRcdpkbbrrrLjcvkUJ22n9Hdtp/R69jeM7+hxlvhZa44TBwvZ1CS7f4EmlwCaT3BF8B5F+H+JvGOGSSC8wF/ugGG4bQyuhs96KLYNAg13rju+9ct9d6dNprfWL/qjHi+vlkIGmpf0FRnWQdBsWPQ7gIfK0go98WXyK+JkjT5+MQLjVIzklo+Qfuh7QukNap9hsLBqGsDPLyYOBA1+n1zDPBZ585U5kVihgIF90BJc8Dija4Cl9u8rbRiDXxN4OCj92nXH8rROxXMtokoyc0mwjhFZC2bc3Wh6hs1ixXFHbcEZ56yh1J9Eq9cXvzb/YxIBZKxuKuFSiH4ie8TpPwRDKQtLZWJGJI/E2R9O61e4/LylxX1969YckS1/HV1Cv2PzMW0jpA8GfAB9E+w8SYeJo1C044wbXdGDAARoyApjYnVN9YoYgBafwMWjwaJBvJHeR1HGNqr2nTv66LOOggr9MYj1ihiAHxN0Xyh23xeW5d4qx60xjPxJYGvkc3PgZp7ZC8CxDJqN2G3nsPXnoJnnkG2rSBmTPBfkfrNSsUHlANomsHQcVX4G8LTV9CfKnVQ8bEl2oZuuYU0I1QnolqEMm/omYbKSx060Q8/zx06+Z+bt7cioSxyWxPVEyBwAzcOe3LoOQ1rxOZZBcuirSfASiPzJFVk6pbH6JbN3jlFbjhBpg+3RUJY7BC4Q1f40ornvnBZ5OD4eIXCK/chfDqo1yDRlMzvgLI3A/IdHNjeTW4QrqkBIYNc72Zpk+H4cPdvIQxEQk79CQii4ANuIY+wer0I0kWkt4Nzb8WSl5yDfqy+3sdyVMaXgcbbgUqIFiEbrgbaTSi+q/XMnTtEHeUln0s0uDyejfvIyLQ6D4I/Qa+hltuXhkOu6OI44+H3Fz45BNo1w78tbzGwqS0hC0UEfuo6mqvQ8SCL+d4yDne6xgJqoZ/5Etecgs1UQ4lz0HWARDF1eSShYhAWpstP3HePDjrLPjiCwiF4PTT3dGEMZthQ081pMHFhNecRXjNIDS4xOs4KUF8jSD/OpDGkNYVaXB5DbegkS8iNSb5OyLHRCAAt90GPXrA3LnurKYBA7xOZZJAIh9RKDBBRBR4TFVHex0IQNeeC6EF7va685Bmb3ucKDX4ck6AnBNq9+Ls46H8Mwh8B9lHQXr9O5qolrPOcmtXH3ccPPAAtGzpdSKTJBK5UOyuqstEpDnwoYj8qKqf/fGgiAwGBgNsvXUcG++Ff+fPT6zhlBwVSzriy0GaPOV1jMRUUuIa+eXnw9ChcNRRcOSRXqcySSZhh55UdVnk+yrgDaDvPx4fraq9VbV3QUEcF61pcDWQ4b4aXBO//RpTU5MmuQZ+F13kfu7Rw4qEqZWELBQikisiDf64DRwIzPE2lePL+T+kxTSkxTR82Yd7HceYf1u3zi0gtO++7ucozkN8O34Gd54+io9f+KJazz135yu4+fgRFK8vjloGE3+JOvTUAngjcopjGvC8qo73NtJfRFLvHHPVMrToJgj8ALnn4Mu2vj5JafJkOPZYWLnSLSQ0fDjk5ERl0wtn/8qNx95DeUkFn7/6NflNG9D7wB6bfG7JhlKGH3MPFaUVLJq7mLzGuVzy2DlRyWHiLyELhar+Amz6N9DEhG4cDaVvA+Ww/jI0oyfib+F1rITm1vt+BipmIjknIZm7eh0Jtt4attkG3nrLtQWPoqXzV+Dzu0GIcCjM4h+WbrZQVJRVEA65i0pDgRDrVhVFNYuJr4QcejIeCK8GApEfBHSDl2mSQ+lrsOE+KH8fXXs2Gloe/wyqMGaMO5NJ1TXx++yzqBcJgJ3234FGzfLJzssiJz+bPY/dfGFsVNCQ4y49Ap/fR8OCfAbeelLU85j4ScgjChN/kjsYLf/YFYzsw8Bfh+Uy6wkN/oxboArXOC+0HPyt4hdg0SI4+2yYMAH69XNzE40bx2x3ufk5PDH3Xpb+tJxWHZuTnZdd5fMH3vpfBtx4Aj6/r95dKV9d6wrX8/PMRWzTqwMNm+V7HWezrFD8g4bXgWQhkuV1lLiStDZQ8DkQqH176npGco5HS19xzfjSOkP6DvHZcSgEo0bB1Ve7taofegjOOScu61ZnZGXQccd21X6+P81agmzOqiWrObvHZYTDYfxpfkZ/N4JmrROzi7QNPVUSLroTXbU7umoXtOJbr+PEnYhYkagBSeuAFHyGNHsLafIiIunx2XFJiVtpbu+93RXW550XlyJhomvq+JkEKgKUFJUSKA8wbcIsryNtlv12RWi4BEqeAQKgpeiGe72OZJKA+HKQtPaIxPiTc0WFu5q6vBwaNIApU+Cdd9zktUlK2+zU4c/bqso2vTpU8Wxv2dDTHyQDJCcyiZvuFhSKAq2Yia6/DPAhjUYg8RqeSDKqQbTkOQitRHJPQfytvY6UOKZMgTPPdEcPrVq5ietWcZwLMTHRZedO3PbeNcyYOJudD+xBpx7tvY60WaKa/A3UevfurVOnTq3zdjTwvTuS8LdAGlyF+PLqvM3wqr0gHDkbxt8WX8HEOm8zFYWL7nSdX6kAXzOk4NPYf0pPdMXFcO21cP/9sNVW8OijcNhhXqcyKUREplVnCQc7oqhE0rshTR6P8lZDf938c7Ei8y+BWfx5BlF4DWgxSOKeBRIXAwbAa6+5OYjbb3f9mozxgM1RxJg0HOFWH/M1Rxrd7XWcxJUzAMhyw3+ZeyC+evpHcc0aWLvW3b7hBndNxEMPWZFIEtMnzubkducysOsQFs7+1es4UWNDTyZhaHCJO5pI3wGRmn+GUQ0DofidfRRNqvDqq3DBBXDoofD0014n8sSyn1eQlu6n+dZxbPQZRUc1PZ2Na11fq0492/Po9MT+cFjdoSc7ojAJQ9LaIhk9alckyr9GV/VCV/YgXPx8DNLF0LJlcPTRblnSNm1gyBCvE3lizA0vMWjHSzljuyG89cgHXsepFQ3/9cE7HE6doWYrFCYl6IabQUuBIGy4laQ5Up4wAbp1g/Hj4a673BlOPXt6ncoTr454m4rSCirKArx4+xtex6mVa168hMYtGtGyQ3OuePoCr+NEjU1mm9Tgawr8AoRAGiR+ywhV1/aje3fYay93Ad0223idylNbdW7FormLEZ+Pdt2jc3p6vPU5qCcvL4/2CTHeszkKkxI0tAJdfz3oBiT/WiS9u9eRNi0YhHvvhU8/hbffdsXCALB25Tqev/V10rPSOfmao8ltmOt1pJRnp8eaekX8LZEmCbGs+ubNmuUunJs2Dfr3d9dJ5NX9Wp1U0bhFI85/YKDXMcwmVDlHISL5IvKvNqIismPsIhkTXxqYR7jobrT03djMbZSVwTXXuNbfS5bAyy/DG29YkTBJY7OFQkSOB34EXhORuSLSp9LDz8Q6mIgcLCLzRGSBiFwV6/2Z+knDa9A1J0LJ4+j6q6HsnejvJBCAsWPh5JPh++9dCw4bcjJJpKojiquBnVW1J3AG8D8ROTryWEx/y8X1bngIOAToBpwkIt1iuU9TTwUXV/qhFA1EqYNnURHceKM7mmjQwA07PfMMNG0ane0bE0dVzVH4VXU5gKp+IyL7AO+ISBsg1jPgfYEFkSVREZEXgf7A9zHer6lv0ruCrwWEVwJhJLt/3bf57rtufYjffoM+fdwFdI0a1X27xnikqiOKDZXnJyJFY2/cH+xYn1KyFbCk0s9LI/f9SUQGi8hUEZlaWFgY4zgmVYlkIs3GIY1HI80m1K27b2GhG146/HDXcmPyZFckjElyVRWKcwFf5SEfVd0AHAycFeNcmxra+ttRjKqOVtXeqtq7oCA5L/c3iUEkC8nog/hb1G1DAwbAK6+4Hk3Tp8Oum19T2phkstmhJ1WdBSAic0Tkf8BdQFbke2/gfzHMtRSofMVNG2BZDPdnTO0sXgy5uW7u4Z57IByG7bf3OpUxUVWdFh674P5oTwa+xf3B3j2WoSL76SwiHcStzXki8FaM92lM9YXDrqtr9+5w5ZXuvm7drEiYlFSdC+4CQCmQjTuiWKga24UVVDUoIhcAHwB+4ClVnRvLfRpTbT/+CIMGwRdfwAEHuGskjElh1Tmi+BZXKPoA/XCnqr4a01SAqr6nql1UtZOq3hrr/ZmaCRc/S3jFjoRX7YMGF3odJ35efx169HDLkj7zDHzwAXRI3LWO65NVS1bzxLDnGDfqfULB0JZfYKqtOkcUZ6rqH42UVgD9ReTUGGYyCU7DG2HDnUAAwsvQotsTv31GXYVC4Pe7CeqTToI77oCWLb1OZSLC4TAX/edq1q5cT3pGGqsWFzL4rtO8jpUytnhEUalIVL4vlhPZJtGJn79+dXwg2V6mia2SErjiCjj4YNfxtXVrdyThUZFYtWQ1r458mynvTfdk/14KhUKMf+pjXhnxNhvWbvzbY6Uby1i3aj3hUJjy0grmTv7Jo5SpyZoCmhoTyUYbjoSNd4KvJZJ/rdeRYmPSJDcX8fPP7nt5OWRlRWXTv36/hNFXjCWvUQ7n3XcGDZtteanT0o2lnLvzFZQUlZKW7mfII4PY/5S9opInGTw6dAzvP/kx4WCID5/9hNGzRvz5WG5+Dn0O7sXMSXPQsHLMxYd5mDT1WKEwteLLPgCyD/A6RmwUFcGll8ITT0CnTvDxx7DPPlHdxRUH3MTaFevw+f1sXFvMre9evcXXrFi4ikBZgGBFkGBFkGkTvqtXhWLmx3MoLykHYNGcJYRCIfx+/5+P3zjuCuZ9+zONCvJp1bGO18SYv7EV7oz5J5/PFYfLL4fvvot6kVBV1hduQBVCwRArf61eZ4GturSmcYtGZDdFh0J3AAAfb0lEQVTIIjMng31P3iOquRLdoYP3JzMng+y8LHY9Yue/FQkAn89H1106p0yRCAaC3HfOY5zZ/WLGjXrf0yy2cJExACtXwp13wm23ueGl0lLIjt3cywt3vMGzN7yEz+/nhtcuo+8hvar1upINpcyYOJutOreifZKuAlcXP037mY1ri+mxT/d/FYpU8/YjH/DYZc9SXlpBZk4G9395K516tI/qPmzhImOqQxXGjIGhQ93Edf/+bmnSGBYJgJOuOor/O/dA0jLSyMzOrPbrchpks/uRfWOYLLF12flfy+OkrI3rSwiF3Gm+Ij6K15d4lsWGnkz9tXAhHHQQnHGGu8J61ixXJOIkt2FujYqEqV8OG7Q/bbq0RnxC30N7sX2/7TzLYkcUxhOqFQC4Di1VC5dOgNJXIHM3JOd0JFqL/px5Jnz7rWvFcc45bm7CmASR37QBj383knA4jM/j300rFJVo8Bd07fmgRdBguDuzx0RduHQCrB8KKJp/C76cozb7XA38BOsvA8og8A34W0LWIbXf+dy57hqIpk3h0UfdfMTWW9d+e8bEmNdFAmzo6W90/fUQ+gXChbD+ElStDUBMbLgZqAACsGEL3VnCyyIX+AEagODS2u2zosKtONerF1wbue6jSxcrEsZUgx1R/E3ynwGWFHzNXDFGwdek6udm/Af87SD4M/jyIfv/ar6/KVPcMNPcuW5hoZtuqlVsY+orO6KoRBreBP4O4GsKDUciktqn33lFGo+CjD0hY3ekcdU9okQykaavIQUfIAWTar640Jgx8J//uIvo3n0Xxo4FW+jKmBqxI4pKJK0TUjDe6xgpT/xb1aiJoIgf/K1rtpPycsjMdG3AL77YDTs1aFDDpMYYsCMKk2rWrIHTT3envYbDronfyJFWJEzUlBaX8fp97zJu1PtUlFV4HScu7IjCpAZVePVVuOACVyyuvNK1Bk+AM0ZMarm+/53MnTwPAWZ9MpcbXr3M60gxl3D/i0RkuIj8JiIzI1+Hep3JJLjVq+Hoo+H446FtW5g6FW65BdLTvU5mUtC8bxYQKAtQURbg+8nzvI4TFwlXKCLuVdWeka/3vA5jElxWFvz0E9x9N3z9tVuBzpgY2e+UPcnKzSQrN5ODzohuw8hEZUNPSUC1DCq+AX8bJK2j13ESw4IFroHfww9DXp5rv5Fmv84m9i566Cz2Pakf/nQ/3Xbt4nWcuEjUI4oLROQ7EXlKRBpv6gkiMlhEporI1MLC6rVpTkaqQfT3Y9F1Q9DVR6Lln3sdyVvBoDty2GEHeO01mD3b3W9FwsSJiLDDHl1rXSTmfbuAWZ/OJRwORzlZ7HhSKETkIxGZs4mv/sAjQCegJ7AcGLGpbajqaFXtraq9C1L5vPjQrxBcAloMlKGlr3mdyDuzZrk1q/9YmvSHH6BPH69TGVNtz9/2GpftO5xrj7iDu09/yOs41ebJxzBV3b86zxORx4F3YhwnsflagWSBVgAZkL6L14m8oQrnnw9LlsArr8Axx0C0mgMaEyfvPTGRsmK3St8nL0/mymcv9DhR9STc8bqItFLV5ZEfjwLmeJnHKxpcAIHvIWNXpNnrUPoWpHWAzIOqv43QKgivhrTtEEnUUcYt+OIL2G47aNYMnn0WGjWCJlto+2FMgtphj66sXbkeDYfp1LNDlc8tWrOBX2b9Ssce7chv4u11QAlXKIC7RKQnrvHSIuBsb+PEnwa+Q38/BfCBpLv2FXnn1mwb5V+ia891n7rT+0Djx6PXnrsONLwOXXcJBBdB3kWb7xxbVATDhrnJ6iFD4L77oKNN5JvkNvTxc+i2axfKSio4dNB+m33e6t9+Z3CPywgFQ/jT/Iz+bgTNWnv3ASnhCoWqnup1Bs+Vf4HrrhoG8iDwHWTuXaNNaPFTQJkrtxVfQ3gV1LRPUgzohnuhYgoQhKLr0ay9kH82Bnz3Xbc+xG+/uSJxyy2eZDUm2tIz0jni3C2PCkz9YBYVZQHKS8rJzMlk6viZHDxw3zgk3LQkHY9IcRl9gQwgsvpZWreabyN9ByALEJBs8G3y5DEPlOEKIIC61uGVPfggHH445OfD5MnuSCIvL94hPREKhpj9+Q8snb98y082Ka1Tz/b81c1aIz97J+GOKAxIRm9oOhYCsyFjD8TfvObbyLsAlXwILUZyT6vWSnLxIHkXoxWzILQM8s5x3WBVYcMGVxyOPRbWr4fLL3dN/eoJVWXYIbfy45T5hENhrnz2QvY4ZlevY5k62LiumGeuf5Hy4nJOHX48zds2q/ZrO+/UkVvfuZqpH8yk90E96byTt8Ouopr8azD07t1bp06d6nUMUxuLF8O558LGjTBpUr3tzVT0+wZOaD2IYMAtlrXjXt0YMelGj1OZurjmsNuY/tFswuEwrbdpydM/3O91pH8RkWmq2ntLz6uf/yuN98Jht1Z19+7wySdw1FHuyKKeym2YQ36zfHx+H5k5GWzfbzuvI5k6WjLvN4KBIOFQmJWLkvuiYBt6MvG3dCmcdJI79fWAA+Cxx6BD1acKpjp/mp9RU27n3cc/pHmbZhw0sH70EEplpw0/nnsHP4YqnHDlkV7HqRMbejLxV1wMe+4JF10Ep51mF86lGFXls1e+Yl1hEfufsge5DXO9jhQVsz//gZ9nLmLXI3amZfvqzRuuXbmOQEWwRvMT8VTdoScrFCY+pk6FO+5wS5FmZblhJisQKenp61/k9XvfIRxyY/OjZ41IiGt46uLb8TO48dh7CIeUzOwMxsx/kPymyb8Yls1RJCkNrUY3Po6WvkPlIh4ufZ/wuosJl77tYbpaKClxZzDtsgt89ZXr+gpWJFLYt+/NoKy4nIqyAL9+vzQlVoGb8fFsyksqCJQHCIfDLJq7xOtIcWVzFAnEdYo92rXdIB1CvyF5Z6MVM2D9VUAplE1C/S2QjL5ex92ySZNg0CD4+Wf3/a67XAsOk9IOGLAXi3/8DZ9P6LxzRzKzk/80593+rw9vPTwBn0/IzM70/LqGeLNCkUjC6yH8OxB0X+VfQt7ZroMslT6BBxdFLspLYKpw9dXu9scfwz42ORsNP34zn9v+e7+77uK5IQm5HsJRFx5Kl507sb6wiD6H9PQ6TlRs368rD31zO4vmLKHnvtuTm5/jdaS4sjmKBKKq6JoTIDgfNAQNb8eXfRgaXo+u7g+6FqQh0mzcv9teJIo334TddoOCAtfptWlTyKlf/6li6dRO57Ni4SoAmm/djOcWPeJxIpPMbI4iCYkI0mQs0uh+pOkr+LIPc/f7GiIFE5CmryMFHyZmkVixAo47Do48EkaOdPe1bWtFIsoqL3YTDnm78M3PsxYx5d1pKTEHYapmQ08JRiQDMvfc9P1pnTxItAWqMGYMDB3qJq5vvdVNXpuYGDZ2CLeeeC+qMOy5IZ7lmPTSl4w482F8Ph9ttm3NqCm346unV9XXB1YoTN3cdhtcey306wdPPAHbbut1oqhaOn85ww6+hbUr1jHwtv9y9JDDPM2z/e7b8cKSxzzNAPDB05MoL3FHEovmLGHN8rU026qpx6lMrNhHgCSmGnLrTgRmx3fHoRCsXu1uDxwIjzwCn36ackUCYPTlz7JyUSHlpRU8fuVYitcXex0pIfQ+qAdZuZmkZaTRsCCfRs0beh3JxJAnRxQichwwHOgK9FXVqZUeGwacCYSAi1T1Ay8yJgNdd75ba0IVbXAJvtzTY7/TOXPgrLMgI8P1aGrVyq0dkaIyczLx+YVQUBGf4Evzex0pIRxz8eG02LqAlb8Wst/Je5CWboMTqcyrf905wNHA346hRaQbcCLQHWgNfCQiXVQ1FP+IiU01BOWT+LNnfcnLEMtCUV7uhpluvx0aNoT7768XF82dd+/prF25jlWLVzP4rlPJzs3yOlJCEBFrg16PeFIoVPUHYFOX9fcHXlTVcmChiCwA+gJfxTdh4hPxo2ldILgQ8EPmf2K3swULoH9/+P57+O9/3WJCBQWx218CadyiEfdMHO51DGM8lWjHi1sBX1f6eWnkPrMJ0uQ5KH0VpCFkx7A7ZatWrjC88w4c5u1krjEm/mJWKETkI6DlJh66RlXf3NzLNnHfJq8IFJHBwGCArbfeulYZk5348iF3YGw2PmECjBgB48ZBbq6bjzDG1EsxKxSqun8tXrYUaFvp5zbAss1sfzQwGtyV2bXYl9mUNWvcNRFjxrizmH77DbbZxutUxhgPJdrpsW8BJ4pIpoh0ADoD33icqX5QhVdega5d4bnn4JprYOZMKxLGGM9Ojz0KeBAoAN4VkZmqepCqzhWRl4HvcZ3xzrcznjZNVSE4B/Ah6d2jsUE31NS2rRt26tHjbw+HQiGWzltG09ZNyGuUGgvRGGOqx5oCJgBVRYsfgtL3IesAJG/IFhd6CRfdBSXPuR9yT8fX4JKa7zgchqefhv/7PzdZvXKla+KX9vfPD6FQiMv2Gc786Qvx+32M/OwmOvVoX/P9mZQWqAiwbMEKmrcrsNOIk4Q1BUwmFZ/DxschNB9Kno5cH7EFpS8Dpe6r5MWa73PBAthvP3fx3OjR7r4WLf5VJMC1aFgwYyHlJeWUbCjlrYfH13x/JqWVFpdxdo/LuGCXYZzS4TxWLVntdSQTRVYoEkG46K+L1xTQdVt+TXp3IMN9pe9Q/X0Fg3D33bDDDjBjBjz++F/rRmxG09aN+eOEtMycDNpvXz/PMjObN2vSXFYvW0tZcTkl60uY9MKXXkeKqZINpYy+4n/ce/Zj9aIoJtp1FPVT1gFQ8jwEpkH6tpB1yBZfIo0eRkueAwTJObn6+7ruOrd29ZFHwkMPQevWW3xJo4KG3Pnhdbz9yAds07MD/c8/uPr7M/VCq04tCIfcdKI/PY22223596qmvnl/BreffD/iE2549TJ67P3X3Jyq8vajE5j58RwOHrgvfQ/pFfX9V3bPwIf5+p2phIJhZnw8m2fnj4rp/rxmcxQJRDWESAx6CZWVudNeW7d260Z88QUcc0y9aMFh4ufbD2by0f8+Zaf9d+Sg06O/ouGxLc5kfWERAC3aFTB24cN/Pvbpy5O558yHKSsuJzMng0em3UXbbWN3re7ArkNYMs+due/z+xhf8eIW5xUTkc1RJKGYFInPP4eePeGEE9yZTS1bwrHHWpEwUdfnoJ4MGzskJkUCICMr/c/bmTkZf3ts6fzlVJQFAPeHe8Wiwphk+MPJ1x1LRlY6GdkZHD3k0KQsEjVhQ0+pqqgIhg2Dhx+G9u3dkFOK/zKb1Db89cu5+4yH8Kf5ufLZC//22P6n7Mnr979LeUkFrTq0YMc9u8Y0y37/3YOe+2xPRWkFrTq2iOm+EoENPaWiOXPgkEPcVdVDhsDNN0NentepjImpspJyVv+2hlYdmuO3dvDVUt2hJzuiSCWq7qihQwfo1ctdab2rtYI29UNWTiZtOrfyOkZKsjmKVKDq2m7svjuUlromfm+9ZUXCbJKq8s37Mxj/9CRKNpR6HcckASsUyW7xYjj8cDjlFHel9e+/e53IJLhXR77NzcePYNSFTzJk92uo6fDz0vnLGTHoEZ4d/hIV5YEYpTSJxIaeklU4DI8+Clde6W7fdx9ccAH4bWzWVO2LN76hrLgcgCU/LqN4fUm1+3eFQiEu7nct61cXkZGZzrrCIi56aFAs45oEYEcUyUoV/vc/2G03mDvXTVpbkTDVsPcJu5GVm0lWbibtt29LbsOcar+2vKSCjWs3gkJFWYBfZv0aw6QmUdgRRSUaLoLQSkjrGJtrGuoqEIB774XTT4fmzeG996BRIzvt1dTIURceSvvubVmzfB27HdmnRtcA5DTIZp+T+vH5q24hypOuPjpWMU0CsdNjIzTwI7rmJCAMadsiTZ5HJHHq6NwHn6PxsEtpXbyS0IiR+IfWolusMVGgqiz9aRl5jfNo3Lyh13FMHdiV2TWkJS+CFoOWQvAnCP7odSSnpITS8y5ku4tOJaN4PTdn7MmLpdaUz3hHRGi77VZWJOqRxPnI7LW0LiDZrlCg4NvUct8euOoqsh8Zxfi0bXg02I3iYAYN6kG3SmNM4vDkiEJEjhORuSISFpHele5vLyKlIjIz8vVo3DLlnAh5QyGrP9J4DOJvFq9d/9u6dfBrZJLw6qvRiRP59uhzKc/IJr9pA44deoR32Ywx9Y5XRxRzgKOBxzbx2M+q2jPOeRDxIbkD4r3bfxs3Ds47Dzp3hk8+gZYtkZYtuW7ffdmwdiM5DbKtPYExHpn+0Xd89c40+h7Siz4Hxf3PlGc8OaJQ1R9UdZ4X+05YK1bAccfBUUe5M5pGjvzX2UwNGudZkTBJYfWyNSz/ZaXXMaJq/vRfuP7IOxn3wHvceMzdfP/1T15HiptEnMzuICIzRORTEdnD6zBx8c030K0bvP023HorfPst7Lyz16mMqZUJz37CgG0u4KztL+GJq8bWaVs/z1rEuTtfznl9rmTR3CVRSlg7C2cv/tupxItmL/YwTXzFrFCIyEciMmcTX/2reNlyYGtV7QUMBZ4XkfzNbH+wiEwVkamFhbHtPR8zkRXB2H57OPRQmDnTLUuanl7164xJYM/f8hoVZQEqygK8du+7ddrW8KPvZsGMRcyf9gs3HTciSglrp88hvcjKzSQnP5vMnEz6HraTp3niKWZzFKq6fy1eUw6UR25PE5GfgS7Avy6SUNXRwGhw11HULW2chULwwAMwZgx89RXk5MDYun3yMlVTVb54fQoL5yxm35P60aZL9JfqNE777duyavFqQqEwLTs0r9O2/mg14m6X1TVanTRu3pBnfnqQRXOW8OM387lsn+Fs26cTQx8/h8zsTE+zxVpCDT2JSIFELokWkY5AZ+AXb1NF2Zw5rsvr0KHQpg1s3Oh1onrho7GfcdeAUYy9+VUu2GUYG9cVex0pZV0x5kKOvOhQDjlzP+6eeH2dtnX50+eTk59NbsMcLnvyvCglrL3c/BwaFuTz1DUv8Nv85Xz+2hRev79uR03JwJOznkTkKOBBoAB4V0RmqupBwJ7ATSISBELAOaq6xouMURcIuPmH226Dhg3h+efhxBOt/UaczP7se8pK3KfTcCjM8l9W0nmnjh6nSk05DbIZfNepUdlW30N68ea6Z6OyrWgpKy77c64iHApTvD71W7V7ddbTG6raRlUzVbVFpEigqq+pandV7aGqO6nq217kiwmfDyZMgOOPhx9+gJNOsiIRRwcO2JvMnExyGmTTrE1T2nVvG5XtLpq7hGGH3sotJ45k7ar1UdmmSWyderRn/1P2wOf30aZLK4655HCvI8Wc9XqKpY0b4ZZb3DBT8+ZQUuLmI4wnVixaxbIFK+i227Zk5URnTPnENoNZs3wtPr+PnvvuwB3jr43Kdo2JB1sK1WsTJsDgwe4K6+22cx1frUh4qmX75rRsX7fJ1X9av3oDqhAKhilcYotGmdSUUJPZKeH332HAADjoIMjKgs8/d0XCpKSBt56EP91PRnYGZ99zmtdxjIkJG3qKtrPPhqeecivPXXutKxYmpZVuLMWf5icjK8PrKMbUiLUZj6dly+CXyFm8N90EU6e6uYkYF4k5X/7IiW0Gc3yrs5jx8eyY7itWQqEQq5asJhgIeh2l1rLzsq1ImJRmhaIuwmEYPRq6dnXzEQAtWkCPHnHZ/Z2nPcjvy9ayduV67jj1gbjsM5pKi8s4p9flnLHtRZy2zQWsK7SzhoxJRFYoamvBAthvPzfUtPPO8NimGuHGVuUGgT5/8v1TTh0/k5WLCqkoC7C+sIhPXpzsdSRjEt7YW17lv1ufw63/vY+K8kBc9pl8f10SwSefwA47wIwZ8PjjMHEidOoU9xjXvHAxbbq0onWnllz30tC477+umrcrIBwOA67Q1bXdgzGpbt63C3jxjnEULv2dyW9+y/tPTIzLfu302JooL4fMTOjbFwYOhGuugdbe9QzqvFNHnv4x+Yac/rBtb9cn56P/fcauh+/Mrodbx1xjqlJRFvjzOl0Nh6koi88RhZ31VB1lZXDzzfD66zBtml0PYYzxhKoyctCjfPz8F3TeqQO3vX8NOQ2ya709u+AuWj7/HAYNgnnz4IwzIJi8Z+eYxLBh7UZeunMcIsIJVx5JXqNcryOZJCEiXPrEuVz6xLlx3a8Vis0pK4NLL4WHH4b27d2V1gcc4HUqkwKu738nP05ZAKL8MGU+93w83OtIxlTJJrM3JyMDZs+GSy5xrcGtSJgoWTR3CcFAkGBFiIVz6s8qaSZ5WaGorLDQXQ+xcqXr9jpxolu7OteGBuqirKScVJgLi5ZjLjmcjOwMMrMzOHboEV7HMWaLbOgJQNWtDzFkCBQVuT5NxxxjS5LWUTgc5pYTRvLluG9p0qox9395C83bNvM6ludOufZY9j5+NwBbac8kBTuiWLwYDjsMTjkFOnd210Ycc4zXqVLC/OkL+Xb8TMKhMGuWr+XNUe97HSlhtOnSOiZFIhwO/3ltijHR4kmhEJG7ReRHEflORN4QkUaVHhsmIgtEZJ6IHBTzMDfcAJ99BvffD198Ad27x3yX9UV+kzzCYTfklJ6RRuOWjbbwClMXX477hiPyTuGIBqfy1dsJ0iTTpARPrqMQkQOBj1U1KCJ3AqjqlSLSDXgB6Au0Bj4CuqhqqKrt1ek6ilWr3IJC7dvX7vWmSp++PJlXRr5N1106c/Y9p5GWbqOdsXJcy7NYF1llr0mrxrz022iPE5lEl9DXUajqhEo/fg0cG7ndH3hRVcuBhSKyAFc0vopZmObWNiKW9jp+N/aKjMeb2MprlMP6SGNFuzbDRFMizFEMBP4YvN4KWFLpsaWR+/5FRAaLyFQRmVpYWBjjiMYkvuGvX0633bal++7bMfz1y7yOY1JIzI4oROQjoOUmHrpGVd+MPOcaIAg898fLNvH8TY6NqepoYDS4oac6BzYmybXr1pb7Pr/F6xgmBcWsUKjq/lU9LiIDgMOB/fSviZKlQNtKT2sDLItNQmOMMdXh1VlPBwNXAv+nqiWVHnoLOFFEMkWkA9AZ+MaLjMYYYxyvTkEZBWQCH4rrmfu1qp6jqnNF5GXge9yQ1PlbOuPJGGNMbHl11tM2VTx2K3BrHOMYY4ypQiKc9WSMMSaBWaEwxhhTJSsUxhhjqpQSS6GKSCHwax020QxYHaU40WS5asZy1VyiZrNcNVPbXO1UtWBLT0qJQlFXIjK1Ov1O4s1y1YzlqrlEzWa5aibWuWzoyRhjTJWsUBhjjKmSFQonUfsxW66asVw1l6jZLFfNxDSXzVEYY4ypkh1RGGOMqVK9LRQJtRzr33MdJyJzRSQsIr0r3d9eREpFZGbk69F45qoqW+Qxz96zf+QYLiK/VXqfDvUqSyTPwZH3ZIGIXOVllspEZJGIzI68R56umyoiT4nIKhGZU+m+JiLyoYjMj3xvnCC5PP/9EpG2IjJJRH6I/H8cErk/du+ZqtbLL+BAIC1y+07gzsjtbsAsXNPCDsDPgD+OuboC2wKfAL0r3d8emOPxe7a5bJ6+Z//IOBy4zOvfr0gWf+S96AhkRN6jbl7nimRbBDTzOkcky57ATpV/v4G7gKsit6/64/9nAuTy/PcLaAXsFLndAPgp8n8wZu9ZvT2iUNUJqhqM/Pg1bu0LqLQcq6ouBP5YjjVeuX5Q1Xnx2l9NVJHN0/csgfUFFqjqL6paAbyIe69MJar6GbDmH3f3B8ZEbo8BjoxrKDaby3OqulxVp0dubwB+wK0EGrP3rN4Win+o1XKsHuggIjNE5FMR2cPrMJUk2nt2QWRI8SkvhiwqSbT3pTIFJojINBEZ7HWYTWihqsvB/WEEEmlx+0T5/UJE2gO9gCnE8D3zaj2KuIj1cqyxzLUJy4GtVfV3EdkZGCci3VW1KAGyxfw9+9vOqsgIPALcHNn/zcAI3AcBL8T1famh3VV1mYg0x60L82PkE7SpWsL8folIHvAacLGqFkXW9omJlC4UmqDLsW4p12ZeUw6UR25PE5GfgS5AVCcia5ONOC9hW92MIvI48E6sclRDwi7tq6rLIt9XicgbuGGyRCoUK0WklaouF5FWwCqvAwGo6so/bnv5+yUi6bgi8Zyqvh65O2bvWb0dekq25VhFpEBE/JHbHXG5fvE21Z8S5j2L/Af5w1HAnM09Nw6+BTqLSAcRyQBOxL1XnhKRXBFp8Mdt3IkdXr5Pm/IWMCByewCwuaPZuEqE3y9xhw5PAj+o6shKD8XuPfNy9t7jMwcW4MaPZ0a+Hq302DW4s1XmAYfEOddRuE+i5cBK4IPI/ccAc3FnzkwHjvDgPdtkNq/fs39k/B8wG/gu8h+nlce/Z4fizkr5GTd851mWSpk6Rn6PZkV+pzzNBbyAG1oNRH6/zgSaAhOB+ZHvTRIkl+e/X0A/3NDXd5X+fh0ay/fMrsw2xhhTpXo79GSMMaZ6rFAYY4ypkhUKY4wxVbJCYYwxpkpWKIwxxlTJCoUxMSYi40VknYh4efGfMbVmhcKY2LsbONXrEMbUlhUKY6JERPpEmsVlRa5+nisi26vqRGCD1/mMqa2U7vVkTDyp6rci8hZwC5ANjFXVRGuNYUyNWaEwJrpuwvV4KgMu8jiLMVFhQ0/GRFcTIA+38liWx1mMiQorFMZE12jgOtz6Jnd6nMWYqLChJ2OiREROA4Kq+nykJfxkEdkXuBHYDsgTkaXAmar6gZdZjakJ6x5rjDGmSjb0ZIwxpkpWKIwxxlTJCoUxxpgqWaEwxhhTJSsUxhhjqmSFwhhjTJWsUBhjjKmSFQpjjDFV+n9V910W1/ImTwAAAABJRU5ErkJggg==\n", 818 | "text/plain": [ 819 | "
" 820 | ] 821 | }, 822 | "metadata": { 823 | "needs_background": "light" 824 | }, 825 | "output_type": "display_data" 826 | } 827 | ], 828 | "source": [ 829 | "t = np.arange(-20, 20, 0.2)\n", 830 | "import matplotlib.pyplot as plt\n", 831 | "plt.scatter(x[:,0],x[:,1],c=y, s=8);\n", 832 | "plt.xlabel(\"x1\"); plt.ylabel(\"x2\");\n", 833 | "plt.plot(t, t + 0.5, 'r--')" 834 | ] 835 | }, 836 | { 837 | "cell_type": "code", 838 | "execution_count": 36, 839 | "metadata": {}, 840 | "outputs": [], 841 | "source": [ 842 | "x = torch.tensor(x).float()\n", 843 | "y = torch.tensor(y).float()" 844 | ] 845 | }, 846 | { 847 | "cell_type": "code", 848 | "execution_count": 37, 849 | "metadata": {}, 850 | "outputs": [ 851 | { 852 | "data": { 853 | "text/plain": [ 854 | "Sequential(\n", 855 | " (0): Linear(in_features=2, out_features=1, bias=True)\n", 856 | ")" 857 | ] 858 | }, 859 | "execution_count": 37, 860 | "metadata": {}, 861 | "output_type": "execute_result" 862 | } 863 | ], 864 | "source": [ 865 | "model = torch.nn.Sequential(\n", 866 | " torch.nn.Linear(2, 1),\n", 867 | ")\n", 868 | "model" 869 | ] 870 | }, 871 | { 872 | "cell_type": "code", 873 | "execution_count": 38, 874 | "metadata": {}, 875 | "outputs": [ 876 | { 877 | "data": { 878 | "text/plain": [ 879 | "torch.Size([100, 1])" 880 | ] 881 | }, 882 | "execution_count": 38, 883 | "metadata": {}, 884 | "output_type": "execute_result" 885 | } 886 | ], 887 | "source": [ 888 | "model(x).shape" 889 | ] 890 | }, 891 | { 892 | "cell_type": "code", 893 | "execution_count": 39, 894 | "metadata": {}, 895 | "outputs": [], 896 | "source": [ 897 | "x, y = gen_logistic_fake_data(10000, 1., 0.5)\n", 898 | "x = torch.tensor(x).float()\n", 899 | "y = torch.tensor(y).float()" 900 | ] 901 | }, 902 | { 903 | "cell_type": "code", 904 | "execution_count": 40, 905 | "metadata": {}, 906 | "outputs": [], 907 | "source": [ 908 | "learning_rate = 0.1\n", 909 | "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)" 910 | ] 911 | }, 912 | { 913 | "cell_type": "code", 914 | "execution_count": 41, 915 | "metadata": {}, 916 | "outputs": [ 917 | { 918 | "name": "stdout", 919 | "output_type": "stream", 920 | "text": [ 921 | "0.9893282055854797\n", 922 | "0.012103211134672165\n", 923 | "0.008044046349823475\n", 924 | "0.006133016664534807\n", 925 | "0.004931855481117964\n", 926 | "0.0040746103040874004\n", 927 | "0.003419493557885289\n", 928 | "0.002897445810958743\n", 929 | "0.0024693021550774574\n", 930 | "0.0021104554180055857\n" 931 | ] 932 | } 933 | ], 934 | "source": [ 935 | "for t in range(10000):\n", 936 | " # Forward pass: compute predicted y using operations on Variables\n", 937 | " y_hat = model(x)\n", 938 | " loss = F.binary_cross_entropy(F.sigmoid(y_hat), y.unsqueeze(1))\n", 939 | " if t % 1000 == 0: print(loss.item())\n", 940 | " \n", 941 | " # Before the backward pass, use the optimizer object to zero all of the\n", 942 | " # gradients for the variables\n", 943 | " optimizer.zero_grad()\n", 944 | " loss.backward()\n", 945 | " \n", 946 | " # Calling the step function on an Optimizer makes an update to its\n", 947 | " # parameters\n", 948 | " optimizer.step()" 949 | ] 950 | }, 951 | { 952 | "cell_type": "code", 953 | "execution_count": 42, 954 | "metadata": {}, 955 | "outputs": [ 956 | { 957 | "name": "stdout", 958 | "output_type": "stream", 959 | "text": [ 960 | "[Parameter containing:\n", 961 | "tensor([[-21.5997, 21.6212]]), Parameter containing:\n", 962 | "tensor([-10.7937])]\n" 963 | ] 964 | } 965 | ], 966 | "source": [ 967 | "print([p for p in model.parameters()])" 968 | ] 969 | }, 970 | { 971 | "cell_type": "markdown", 972 | "metadata": {}, 973 | "source": [ 974 | "# Data loaders for SGD" 975 | ] 976 | }, 977 | { 978 | "cell_type": "markdown", 979 | "metadata": {}, 980 | "source": [ 981 | "Nearly all of deep learning is powered by one very important algorithm: **stochastic gradient descent (SGD)**. SGD can be seeing as an approximation of **gradient descent** (GD). In GD you have to run through *all* the samples in your training set to do a single itaration. In SGD you use *only one* or *a subset* of training samples to do the update for a parameter in a particular iteration. The subset use in every iteration is called a **batch** or **minibatch**." 982 | ] 983 | }, 984 | { 985 | "cell_type": "code", 986 | "execution_count": 43, 987 | "metadata": {}, 988 | "outputs": [], 989 | "source": [ 990 | "model2 = torch.nn.Sequential(\n", 991 | " torch.nn.Linear(1, 1),\n", 992 | ")" 993 | ] 994 | }, 995 | { 996 | "cell_type": "code", 997 | "execution_count": 44, 998 | "metadata": {}, 999 | "outputs": [], 1000 | "source": [ 1001 | "from torch.utils.data import Dataset, DataLoader" 1002 | ] 1003 | }, 1004 | { 1005 | "cell_type": "code", 1006 | "execution_count": 45, 1007 | "metadata": {}, 1008 | "outputs": [], 1009 | "source": [ 1010 | "def lin(a,b,x): return a*x+b\n", 1011 | "\n", 1012 | "def gen_fake_data(n, a, b):\n", 1013 | " x = np.random.uniform(0,1,n) \n", 1014 | " y = lin(a,b,x) + 0.1 * np.random.normal(0,3,n)\n", 1015 | " return x.astype(np.float32), y.astype(np.float32)\n", 1016 | "\n", 1017 | "# create a dataset\n", 1018 | "class RegressionDataset(Dataset):\n", 1019 | " def __init__(self, a=3, b=8, n=10000):\n", 1020 | " x, y = gen_fake_data(n, a, b)\n", 1021 | " x = torch.from_numpy(x).unsqueeze(1)\n", 1022 | " y = torch.from_numpy(y)\n", 1023 | " self.x, self.y = x, y\n", 1024 | " \n", 1025 | " def __len__(self):\n", 1026 | " return len(self.y)\n", 1027 | " \n", 1028 | " def __getitem__(self, idx):\n", 1029 | " return self.x[idx], self.y[idx]\n", 1030 | " \n", 1031 | "fake_dataset = RegressionDataset()" 1032 | ] 1033 | }, 1034 | { 1035 | "cell_type": "markdown", 1036 | "metadata": {}, 1037 | "source": [ 1038 | "Next we are going to create a data loader. The data loader provides the following features:\n", 1039 | "* Batching the data\n", 1040 | "* Shuffling the data\n", 1041 | "* Load the data in parallel using multiprocessing workers." 1042 | ] 1043 | }, 1044 | { 1045 | "cell_type": "code", 1046 | "execution_count": 46, 1047 | "metadata": {}, 1048 | "outputs": [], 1049 | "source": [ 1050 | "dataloader = DataLoader(fake_dataset, batch_size=1000, shuffle=True)\n", 1051 | "x, y = next(iter(dataloader))" 1052 | ] 1053 | }, 1054 | { 1055 | "cell_type": "code", 1056 | "execution_count": 47, 1057 | "metadata": {}, 1058 | "outputs": [], 1059 | "source": [ 1060 | "#y.type(torch.FloatTensor)" 1061 | ] 1062 | }, 1063 | { 1064 | "cell_type": "code", 1065 | "execution_count": 48, 1066 | "metadata": {}, 1067 | "outputs": [], 1068 | "source": [ 1069 | "learning_rate = 0.1\n", 1070 | "optimizer = torch.optim.Adam(model2.parameters(), lr=learning_rate)" 1071 | ] 1072 | }, 1073 | { 1074 | "cell_type": "code", 1075 | "execution_count": 49, 1076 | "metadata": {}, 1077 | "outputs": [ 1078 | { 1079 | "name": "stdout", 1080 | "output_type": "stream", 1081 | "text": [ 1082 | "56.18629455566406\n", 1083 | "0.09595121443271637\n", 1084 | "0.09600495547056198\n", 1085 | "0.08732529729604721\n", 1086 | "0.09132660925388336\n", 1087 | "0.08278344571590424\n", 1088 | "0.08562881499528885\n", 1089 | "0.08167734742164612\n", 1090 | "0.0862448588013649\n", 1091 | "0.09074677526950836\n" 1092 | ] 1093 | } 1094 | ], 1095 | "source": [ 1096 | "for t in range(1000):\n", 1097 | " for i, (x, y) in enumerate(dataloader): \n", 1098 | " \n", 1099 | " y_hat = model2(x)\n", 1100 | " loss = F.mse_loss(y_hat, y.unsqueeze(1))\n", 1101 | " \n", 1102 | " optimizer.zero_grad()\n", 1103 | " loss.backward()\n", 1104 | " \n", 1105 | " optimizer.step()\n", 1106 | " if t % 100 == 0: print(loss.item())" 1107 | ] 1108 | }, 1109 | { 1110 | "cell_type": "code", 1111 | "execution_count": 50, 1112 | "metadata": {}, 1113 | "outputs": [ 1114 | { 1115 | "name": "stdout", 1116 | "output_type": "stream", 1117 | "text": [ 1118 | "[Parameter containing:\n", 1119 | "tensor([[ 3.0190]]), Parameter containing:\n", 1120 | "tensor([ 7.9957])]\n" 1121 | ] 1122 | } 1123 | ], 1124 | "source": [ 1125 | "print([p for p in model2.parameters()])" 1126 | ] 1127 | }, 1128 | { 1129 | "cell_type": "markdown", 1130 | "metadata": {}, 1131 | "source": [ 1132 | "# Two layer neural network" 1133 | ] 1134 | }, 1135 | { 1136 | "cell_type": "code", 1137 | "execution_count": 51, 1138 | "metadata": {}, 1139 | "outputs": [], 1140 | "source": [ 1141 | "# generating fake data\n", 1142 | "# Here we generate some fake data\n", 1143 | "def sigmoid(x):\n", 1144 | " return 1/(1 + np.exp(-x))\n", 1145 | "\n", 1146 | "def gen_nn_fake_data(n):\n", 1147 | " x = np.random.uniform(0,10, (n, 2))\n", 1148 | " x1 = x[:,0]\n", 1149 | " x2 = x[:,1]\n", 1150 | " score1 = sigmoid(-x1 - 8* x2 + 50)\n", 1151 | " score2 = sigmoid(-7*x1 - 2* x2 + 50)\n", 1152 | " score3 = 2* score1 + 3*score2 - 0.1\n", 1153 | " y = score3 < 0\n", 1154 | " return x, y.astype(int)\n", 1155 | "\n", 1156 | "x, y = gen_nn_fake_data(500)" 1157 | ] 1158 | }, 1159 | { 1160 | "cell_type": "code", 1161 | "execution_count": 52, 1162 | "metadata": {}, 1163 | "outputs": [ 1164 | { 1165 | "data": { 1166 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzsnXd4FcX3h9/ZW9J7oUPoCEgTBBVRVPwqdhFFsIJiRbGiYPnZUFSaFQuIgghSLICKikiTIh0JvfeQXm7fnd8fGwKBhNyeEO77PDwkubsz56bsmTlzzucIKSUhQoQIEeLcRalsA0KECBEiROUScgQhQoQIcY4TcgQhQoQIcY4TcgQhQoQIcY4TcgQhQoQIcY4TcgQhQoQIcY4TcgQhQoQIcY4TcgQhQoQIcY4TcgQhQoQIcY5jrGwD3CE5OVmmpaVVthkhQoQIcVaxevXqTCllSkXXnRWOIC0tjVWrVlW2GSFChAhxViGE2OvOdaHQUIgQIUKc44QcQYgQIUKc44QcQYgQIUKc44QcQYgQIUKc4wTMEQghJgghMoQQ/530tUQhxB9CiO3F/ycEav4QIUKECOEegdwRTASuOeVrLwDzpZRNgfnFn4cIESJEiEokYI5ASrkIyD7lyzcBXxd//DVwc6DmPxtZ8csaJrw0hW2rd5Z8TXWpHNh2CJvFXomWhQgRojoT7DOCGlLKwwDF/6cGef4qy7/z1vHG7aP4bvgPPHP5qxzdewyHzcGjnYbwcIfn6dfgEY7sySj3/kM7j7Bg6lIyD2YF0eoQZSGdW9Ayb0LLvAnp3FI5Ntj/Rst7BWn/GylVpGsfUtr1j51bkdqpa7QQJ6NZf0U72hEtoxvSmV7Z5gScKntYLIQYKIRYJYRYdezYsco2xyOklGxZuZ1dG8qu5cjJyGNoz+E81P5Z1v61EYBt/+7EaXMAoBgU9qYfYOPizRzedRS7xU5hbhF/TFpY5nh70/fzULtnGT1wHA+0fvqccAbb1+yiX9oj3FZjAMvnrK5sc0ohcx8H12ZwbUbmDgr+/I71yJwnwToVmTMImXkNMvN6ZMZlyOx+yOzbkce6Ix2hIs2ykFJC3hCQ+aAdQea95N04Wi5a3otoOY8jXbv9bKV/CbYjOCqEqAVQ/H+5S1wp5edSyo5Syo4pKRVWSJ+R9X9vYsrwWexcv8encdzlw8e/5LkrX+OJi4cy7d0fT3v9g0e/YM2f69m1fi8v3zgCl9NF11svJCwyjMjYCCJjI2l1cTNqpKWiqRoA5nAT9ZrVLnO+1X9sQHWpWAttaJrGpqVbA/r+qgLvD/iEjH2Z5B3L5+27xla2OaWRtpM+tgZ/fteOkz6RoB4BbCDzwLlBt0lakUVflzeCz0jHSrSC95H2FQGbI6AIw/EPQJi8GkLmPgfWn8D+BzL7Pr+ZFgiC7Qh+Bu4t/vhe4KdAT7hhUTrDrh/O169OY3DXlzi8+2igp+T3iX9jK7Jjtzj46ePfTnv9wNZDqC79Ae9yOFFdKg1a1uOrrR/w2g/PM37TaKLioqjbtBavznyObr0v4sF37+ay2y8uc77WXVugKAqmMBNSkzTr1Dig768qoBhO/OoqStXa2Iq4t0FEg4jWPw42YZeDEgUiCkTkSS+YOKEqEw6mlgGZXjrTkdkPQNHnyJwHkc7/Kr6pCiGEQMR/DIZ6YDwPETfCu4HU/YALkKBV7ahGwLSGhBDfAZcDyUKIA8CrwDvA90KIAcA+oHeg5j/O5uXbcTlUNFVDMSjsWr+XWg1rBHTOhm0asGPtbhRF0KJz01KvHT/8PU7NhqmERYQBkFQrgaRapTNqO/2vHZ3+1+6M8zW7oDGjFr3OpqVb6dCjTcDfX1VgyNeDeOP2Udgtdp4Z/0hlm1MKEXYposaa074upQOc6WCoizAkB25+QxIk/wHqLjA0AsdKpHUGmLuAqRNYJ4OhESLq3jOOI6VEWiaCfRlE9EaJ6OGeAc7NIARIAKF/bmrt1q1SamCdiVT3IyJ6I4z13JvTz4iwixEp830bJPpZyHsK0CD6Mb/YFSiElLKybaiQjh07Sm9F5/am72dQl6EIRWAOM/HlptHEJcf62cLSFOYW8dPHv2GOMHPjI1eXPOhBdwQ3xNyF0+4CoP2VrXn3j1cDao+/yMvM58PHviTnaB4D37+H5h2r785jxS9rmDPud1pfeh63P3sjQgifxpPSgcy6FdSDgEQkTka4+XCsLKT1F2T+i8XhrXBE0kyEqWnF96kZyMzr0VfDBkTybIShpltzaoWfQeHHgANEHCJ1MUKYfXkblYrUCgEHQkmslPmFEKullB0ruu6sUB/1hQYt6/HlptHsWr+Xlhc1IzYpJuBzRsdH0W9YrzJfMxgNDPlmEB8NGk9sUixPfDIw4Pb4i9EDx7F8zhpUl8qQq1/nh6yJPj8gqyKHdh7hjd4jsVsdrP3rPxJS47j63st9G9S5WXcCsggAaZmJiKvajgD1AEin/rEwgHYIqNgRCEMqpPwOzi1gao5QPKgbdawGis9YpAW0LDDU8tj0qoJQoivbBLeo9o4AILVeMqn1ArcV95TLel/MZb3LjvdXZTIP5aC6VABshTZUl4rRVP1+hY4dyCo5g3DaHBzaecT3QQ11KY6VgIgA0/m+jxloIm4Ey0TQCsDQQA8tuYlQ4iHM/etL7ovqi3QsB6GA8TxQqn+YsypQ/f6KQwSMh967h2HXDcdhc3Lv63dUSycA0PKiZqS1rsfO9XuJiArjmv5X+DymMCRB4hSkdRYYWyEiqn4tpTDUhJS/9YNOpSaiJJMmgHOGXQ7Jc0A7CqZ2CFG1EgGqK9X+jOBsQ0rJsp9XkXU4hyvuvISouKjKNqkUqkvF5XSVOveojmiaRuaBLOJT4zCHn70x6hDnNu6eEVRbd6uqKh89MZ7+LQcz7b2AZ6n6je/f+4m37xrLuGe+ZtBFw6hqjtpgNFR7JwB6Smpq/RS/OgGt8DO0o+3RMm9AquVXiYfwDildaAWj0LLvQ9rLLr4Mqj3qIaRt3lnxs662jmD+5MX8NmEB+7ccZPLr00lfvq2yTXKLf35eha3IjsPq4NCOIxTlWSrbpBB+QKpHoPAj/bDYtQNZeHoRnFSPoBV8hLRM19MoQ5SLZv0dLaMrWuaNSNd+AKRlMhRNBMc/ekW1ejho9kjXAaRjFbL4cF269iAzeyLzXtD/V/1wzhRAqmeQFyjKs6Bp+h+TEOKseaBe2e9Sdq7bg6II6rWoQ1RcZMU3hTgLMFByWIwASu80pFSRWb1By0RiAvUQIubJYBt5ViClE/KeBhygHUPm/x8icXxxam6xOKNQgpZxJO1LkTmPFB9wN4PEqWBfAlItticSHP9CxA0Bt8Vbqq0juPq+y/lj0kJ2rN1Nuyta0+GqsyBLA7jxkf/RqE0Dsg/n0Pm6DtUyPbO6I7VsZP6boOUgYp5HmM5DGFKQsS/ruwJjQ0TME6fclA9aNqDq/xwrK8P0swTJCacK+vcMROTdSOvPIAvB1BGMgamcPs0ayxR0CQ/0lFn1AJjbozt8o25rFa8ZqfaHxVLK0MM0RFDRch4G+yLABSIBkbq8wt9BKSUy5z5wrgepQdwIlIhr/WqXdO0CqbpVFFbV0ayzIf9NUBIQCZ8ijA2B4uptLR+UpKD93WuF46HwA8AOIhaRuhAhIpCO9eBcBeaLEabzgmLLqYQKyooJ5C/DoZ1HOLI7g9ZdW/h8qLh45nK+eGEyNRqkMHTKYBJS4/xkZYjjOB1ONixMJ6l2ImmtAihdoB5Gr6pFX+mjoYeGykcIAQnjwblWf4gZG/nVJK3wCyj8UDcp8h6U2Gf9On6wUSJuKDPUIoQZAijfURYiqj8o8Uh1ty6LISL0r5vbgrltUG3xlmq/IwgUa+Zv5JWb3kExKNRMS+WTVSO8zqu3FFi5LXUATrsTg1Gh+51dGfK1f+WLM/ZnMu3dn4hJiKLPC7cQHln9M39ORkrJ05e9ws51e9A0jWfHP8rld1wSmLmOx4xxQfQTKNEPB2QeT9AyLjkhfCaiUGqsrVyDQgSF0I4gwPw24S/sFr1/wOFdRzm4/TANWnq3ylRdakmaqKZJrIW2Cu7wnKcve4Vj+7MwmAwc2Z3BC5NKx6gtBVYMRiXgqaEOu5P1C/4jpV5yYFflp5CXmc+WlTtwOfSV+tzP/wyYIxBhl0CNVSAdVUdiwNgCHLmABKN/Q0PSvgJpmw3mi1AirvPr2CGCQ7VNHw00bS5rSXhkGIpBwWQ2klrf++1oTEI0A4bfSViEmZppqQx8926f7ZNSsnXVTg5sP4ymaWTsy0RTNZw252l9GWaNnUuv5P7cmnQ///z8r89zl4emaTx16cu8cfsoHu/8AotnLg/YXKcSkxhNfGocBpOBsMgw2nVvFdD5hDBXHScAiPixEDUQoh5AJHzut3Glaxcy50Gwfg95Q5G2BX4bO0TwCIWGTsLpcDLuma/Z+u9O7njuJi7tVb5WipSSRdOXsW/LQa66qxu1GpWtibJ11U42L99Gp2vaUaeJ96lsuzbsJS8znzbdWmIwVlzqP+LeD1kyawVSkzz+0QC2rtzBH98s1EMkXz7ClX0vLbn2uqh+OKz67qZO01pM3PqB13aeiWMHsriv2SAcNj3X+sKe7XlrztCAzFUWWYdz+H3i3yTXTeTKfpdWuT4GZyPS9hcy75liMT0jInowIvrsEVKs7oRCQ14wa8xcfpuwAIfVwTv3fEjzTo1JrV92dzQhRLmNYo6Tvmwrz/d4HalJJgybwvhNY0ipm+SxXb+On8/HT05AURRaXdKct38tu3WetdDKwR1HqN2kJvO/XYzUdCc/c/QcPl8/kluevI6I6PDTbIhPieXYfl1oLaVu4ORy41NjiY6PIi+rAJPZSLsrgptSl1QrgTtfvCWoc1Z3pKkdSIEeXDBBuO+ZTlLLAelCGHzrTBjCfUKO4CSyDufgsuurVUURFOQUlesI3GHDos24HCqqSyXSbGT76l1eOYI5434vOY9Y88cGbBb7aYe9mYeyeaTDc9itDqLjo6jZMJWMvccwGA20uLApQgjqt6hT5vjvzHuJcc98Q0R0GI+OuR8Am8XOtlU7qd2kJsm1/eMcTGYTH68awZ/fLKRGWiqX33H2KbCGOAXrVMCJnhmled3WEUA6NyMLx4H9D0BBRj+KEv2onwwNcSZCjuAkeg2+nkXTl5FzNI8u119AozYNfBqv49Vt+faNGZjDTSgGhfO6eHdI1+6K1uzdfBDVqZJaP4mwiNNTVZf+sBJLvlUPu0h48L27ydh7jNikGG56/MyrtHrN6/DWnBdLPrdb7Tzc7llyjuahaZLRi16nSfuGXtl+Ksm1E+nzQmhVXm1QM9AdAYDQc/jdbEJzMlI9iMzuU7rHc+GncI44Aqnl6dXIxkYI03lI5zZwbdJrEAyBl+IOOYKTqNEghe/2f4bd6vBLemWT9g35ZPUItq/eRZvLW5FQI96rcfoP70u9FnXJzcjjmv7dy6yNSGtVD1Ec89Y0jfMubMoND13t1XzbV+8i+2gu1gI9e2n+lMV+cwQhqhciqj/S/jtoORB+lfcZSc5tlM5dEV45lIrQz0RVhKg6jz4prchj14PUs7pkzLNQMFpv94kJUuYFvMNZ1fluVBGEEH7Nsa/XvA71mpcdknEXg8HANfd3P+M1rS9tQbOOjdi8fDvNOzWh4fn1vZ6vdpOaHM8hCIsMo0WnJl6PFaJ6I4z1IWUJSCtC8UEy3dxeb9gjAWxgvhQR+4q/zARAOtOR2feBLEBGD6oyYSfp3AoygxLZjKLJ6JIVEkQ0ODdB2KVnGsJnzilHIKXkm9em8/fUpXS54QIeHHFXtckc+eenVWxfvRuXw8XWf3ewaMZyuvfxLk8+sWYCYxa/wYLvltD0gsZc1vsiP1sbojohhALCt74ZQomH5F/1ympjc0QAxOJkwbvFq26g8CNk5L2+OS+/oVFKO0mJBO34YlQERTPpnHIEa+dvZMbIn7EV2ZkzLovzOjel223+f8htWJTOPz//S4cr23Dhte39Pn55qKouvmW3OFg4/R+vHQFA47ZpNG6b5ifLQgQTKYubxp9lGltCiYOwywM3gZKI/shzASafDrb9iTA2R4o4XSwPA0QOQJgag/M/CLtU724XYM4pR1CUf+IgSkpKpKlVVeX7d39i14Z93PJkT1p2aVbhWLs27EUogoatS4dg9mzaz9Ceb2G3OJgz7nfe/vUlzr808IJTF9/UkYioMJzFOfrLfl6Fy+mqtu0kKxPVpVKUbyE2Mcaj+6TUwDoTqe7VNWmMviUjlIWW/yZYJoOSDIlT9NBNFUAXgysAJdEnByXtC5HWH8HcFSWyl0f3itiXkdIG6mFdFVZUjc5zQomC5J/1NqbSjjC3QhibBFWxtHrERcphxdzVTHxlKrs27AXgohsuoHXXFghF0KR9Q7rf2RWAWaPn8u1bM/l72lKG9HiD/KyCM447YdgUnrh4KIO6vMikN6aXem3Pf/tKwk1Sk+xct8f/b6wMDAYD7bq3LnnwR8VFulV4FsIzDmw/zB21H+SOWg/y2m3ve9RBThZNQOa/AUVfILN66w8lPyJdB8AyDdB0nf7Cj/06vrdojs3Io+2Rxy5CZl7nUdMdKaXe5atoIppjFTJnENjmQsHrSNt8j+wQSgJKwicoyT8gwi5CSg2t4AO0rNvRLDOL57Oj5b+BlnUX0r7Eo/F9QsSCZSpYJiEzb0U6AlfhXxbVdrm4fM5q3uwzCrvFwczRc/lqyxiS6yTx9q8vnSZNvWfT/pI8fZBkH8klNqn81d7Pn84ruf6nj37j7pd7l7zW/srziYgORygCRVG46MYKi/r8xuDPHiI8OpzcjDz6v9X3rAsNeILL6cJhcxIZExHUeWeOnk1+ViFSSlbNW8eeTftP2xWWi3MNUPzwlzZQj4HxhN6S1PKRReN0qejohzzPFFFObmJkAiXBs/sDRf5LlKSYqjuQ9gWI8CvdulVaJkLhGL3Ji4iiZO0qnaDu9s0u21woGg9YwbkFaWqBtP0Blu8BOzJnA6T+HfCMHQBcW/XQkCwCQBZ8gkj6KvDzFlNtHcHmFdtLHtaKQbBv80GS6+ixtlMfkLc82ZMls1agulTO69KM+uedOcuncds00pdtQwBNT0mrjEuOZcKWsezesJf6Let6HD7whZiEaJ6b8FjQ5qsstqzczpAeb2C3Orh18HV+0WZyl9R6yZjCTTisDqQmz7hgOBUR2VdfZQoFjM3BUPr3TOYOAscqQCIdqxHJMzyyTSiJyLhRUPQhGJsiov2nYCu1InAsA0M9hKm5ZzeLU7rseXKwbF98orZAAkoNPa5LBIT7KHCnHaNELlwooGUWS4g7TrqmoPhsIcAYG4G0n/jcuQzpWK9LWQeBausIut3WhVlj5qIYBNHxUbToXH5+c5N2DZmyfxy5GXnUalSjwkyi1398nllj5yKEwq2DT/9ljIqNpHXXymlEcS7w1UtTsRToD4dZY+fS76VeRMUGp6Xnbc/cQF5WATvW7Ob2524iqZZ7q26tYDRYp4O5I0QNRJgv0LNtTsa1nRMr511e2adE9ICIHl7dWx5SOpFZt4KWoa/M48ciws+czlyK+JGQeRPIbAjrgRKma3hJLR9ZMAZkESJ6EMJY9/R7I27VnaNQwJAGiVMR2gFQaiMUH3/mETeD5Vv94W9sCeaLQFrANgdQIbwHGIJzxiKUeCQnL1A10I4AIUfgE43bpvHVljHs23yQFp2bVhhCiIqNPOPDxFpk44NHv2Dvpv3c9Upv7n7ldn+bHOIk9mzaz/I5q2l5UTPadCudPpdcNxGj2YjL4cIcZsIcHrzsD5PZxMPv3+vRPdK5ASwT9ZWt419E2OWIsDIEDaMGQsFI/ePI/r4be6odrn3g+AdM7T1b1av79IdS8cpc2n7yyBEohlSosex0e3KfA8cSQEU6VyNS/jz93ojrkcZG+sM67BKECAfFP3UtQkmE5D/0cIyI1v/PexHdGYeBoUFww6vGJuDaXPxJJIR1C97UQZupEkiuk1QSDvKVyW/MYOH3y3Danbx15xi+3fMJ8SmhLmK+UJBTyL7NB2nUpj4R0Sccdcb+TAZ1GYrT7sRoNvDW3KG0veyEbPSjY+5HapKMfZn0f+tOTObAOgKHzQFCYA478zzlZmlJ18mfcEKSoTRK1H3IsCsADWFMO30YaQPXNjCkIZRYt+0HkOphZNZNxQ3VgaTvECY3pbgNtYFIQAVhAHNXj+YuF3UXJ3ZAh8u9TJhagikwufR6DYQe3pNq9onvD3Zw7QzInOXakviVrrWkFYCIRhZNhqh7g5LdVK0dgT/Jy8graWqC1JvHhByB9xzde4yHOzyHpmpERIfz+YaRJecpu9bvRVFEScOe9H+2lXIEUbGRPD/x8aDY+duEvxj76BcIIRg65Um63tL5tGtcThfDrhvO2vkbadQmjZELXyu9uzS1h/BbwfYjGFtDRJ9y5ysv3VNqhcisG3UpBwyQNMuz1FDnhuIPbIAJHMvBTUcgRAQk/wC2X8DQ0LOw0JmIfgLyhgISogb4Z8xTkFIFV7re/tNQu9RrmvUXsHwDpgsQMU+DoR6EXQL2pSAMegvKICC1ImTuI3rdQMStep9j9QBgRKq7EHFvB9yGap0+6k/6DutFQo04hCL43/3dqdUw8EJQ1ZmlP67EbnFgybdiKbCydv5/Ja+1vKgZ5nATkTERmMNNdLm+g9vjHtxxmEM7j/jNznHPfI3L4cJpd/LpUxPLvObf39axefl2pIT92w4yf/LiUq8LIVDiXkWpsRYlaZJ31ayOlboTkEXFcex5nt1vagsIXcYBox4P9wBhqImI6u8/J4Ded1ikLkSk/IkS85Tfxj2OlBKZ8xAy+27ksWtKNc2Rrj2Q94KeyWWZBNbvEUIg4j9FJP+CSFkStINaafkOHGv0rCHL93ooDhWwg2N9UGwI7QjcpHbjmkw9+DkupyvgoYhzgcZt01AMxSJ5qkaDlicOCmOTYhifPob/lm6hcds0ajRwTwr8u7dnMfnNmSAl97x2B3c8d5PPdsalxGLJtyIEJNYsWzQwJjEarbj3g6IoxCQGoDOZsSGU5N+bwFRx0ePJCENNSJ4N9hVgbqMXLFUBApqaqWXrmU7F4SdpmXjCkWnZ+gG0BHAg1SMIijMKjcFroQoghOGEwIQQYOoGzpV6dlSQdiWV0qFMCPEU8AD6j2EjcL88Q3VNVWxeH8J3ls1exZr5G+l6y4WlQj/ecmvS/RTkFAIQmxzDzIwJPo95YNshPn5yAkaTkUEfP0BqvbJbkk4f+TN/fLOQTte2Z8DwvuVmnknnJmThh3rWS8yzHmW+SMdKpHUOmLugRPT06v2cS0jpRGZcWqwvZIao+1Bini5+TUXmPKQfnitJiKTpurOsFDutyNynwLEOIm9HRD+lh7NEtM/V5+52KAu6IxBC1AGWAC2llFYhxPfAL1LKieXdczY6gn/nrWPUA58SHh3OqzOeDWqj9kAhpeTg9sNExES4nTYZTJ7q9jJbVmwHBC0vbsbIBa9Vtkml0B9MnYs1ZcwQcTNK3JuVbVaVQ0oNtCxQEnyWi5aufUjLZDDUQUT2KzWelFL/WYio01N5qwlVvVWlEYgQQjjR0xEOVZIdAePN20eV5Lq/3/9jPlrxTiVb5DujH/qMv75djARemPQEl956+sFpZfLaj88z9Z0fEEL41Pxm4+LNfPzkBOJSYnnuq8f81qENadP/AeAA1z7/jFuNkNKOzOoLri26XlLSLJ9E14SxPiK27L7YQoiSjKFznaC7QSnlQeB9YB9wGMiTUv5+6nVCiIFCiFVCiFXHjh0Ltpk+c/JOqzLCb/7G6XAyb8Jf2K0OHFYH3w2f6dH9Ukr2bNpP1uGcAFmonzV0ub4j/V66jZgE7+L0mqYx7Lrh7Fy3h3V//cfoB8f5zT6hxEDE7ejKl5GImCf8Nna1wf7PibRSLRNsP1W2RecEQXcEQogE4CagIVAbiBJC3HXqdVLKz6WUHaWUHVNSzr4m1sOmPkVizXjqNKnJs+OrRgMMXzCajCTUjEcoAnO4iUYeSlS/2Wc0j3d+kXubPM6Kuav9bt+B7Ye5t+kgXr7xHQa0HExhbpFX40hN6u0+0R1LQY5345SHEvd/iJRFiJR/EObg6VCdNRhqnMjlF0YIQF+CEKdTGYGxq4DdUspjUkonMAuo1C7m6cu30S/tEfrWf5iNizdXfIMbdO7ZgWmHvmDitg9peL7/5YaDjRCC0YvfoOcDV9L72RsZ9JH7ed+FuUUs/WEldosdu9XBtHf9v8r758eV2K3F6aj5VjYsTPdqHIPRwCOj78NoMhCTGM1jY+/3s6UgDEm+yyNUUaR0IC3fIou+RGpnVvEtC2FqCbFv6umtUYMg7JoAWOk7Ukqkc4terV0NqIwzgn1AFyFEJGAFrgQq9ST47X5jydiXCcBbd45h6oHPKtOcKkuthjUYPO6hUl8rzC1i8pszUB0qfV/qRULq6UV2EdHhRCdEkZ9ZgCncFJD+x03aN8RoMqA6VVRVo37Lukx77yfmfbWAjle35aGR92AwuCfLfdNj13Djo/8DThcoBH338eekhdQ/ry7d+1xSrVVePUXmvQS2XwENrL8ikj0LIQIokTdC5I3+N86PyPz/A+sPgETGvoQSeUdlm+QTQXcEUsoVQogZwBp06b+1wOfBtuNkTv5DDv1Ne8Ybt49iw6JNSE2yadlWPvl3xGnXGIwGxi59kxkjZ5NSL4nez/r/j7zDVW14edrTrF+4ia63dCbvWD6TXpuO3WLn2L5Mml7QiB53X+b2eOU93IvyLQzq/CJF+RbCws1YCqxcP9C/Im9nNY5VQLGKpmtLpZoSUKzfoxd9oUtZn+WOoFJypqSUr0opW0gpW0sp75byZP3V4PPit09SMy2V1PrJDPtucGWactaxN30/LoeK6tI4sK385K86TWrx5KcD6Tu0V8AK8jpfdwED372Hlhc1pyjPgqLoD3NN0yjKtfhljox9mbicLqQmsVnsbFpS8cNO0zT++m4Js8bMJT/b83DJWUVkHyBCl572VSa6KmNIQ19Hm8uV6pBSQ8sfiZZ5K1rRd8G0zmNClcXAeZ2bMmmnBSTbAAAgAElEQVRX1ejmVBlMH/kz8ycv5sKe7bn/zTs9CnX0eeFmvnh+MgC9n6k62/kLerSh7eWtWPHLGtJa1aPHve7vBs5Evea1qd24Jod3H0VTJdf0v6LCe7557XtmjpqD6tKYPW4eEzaPrbbhJCV6IDLsYl2p1FR9D8NF4iRk0QRQosvXJLLNBus3+veiYCfSfD4iiO0nPSHkCLwk82AWgN/UTSuLjYs3883/fY+tyM7BHYdp3K4hl/V2X4fm5sd7ctENndBUjVqNqo7+ksFo4I2fXzitG52vGE1GPlw+nPRl26iRluKW5tTqeeuxFemb3kM7j2IrspVSW61uVNWHnT8RhmRE7PNnvkjLOqE8K4Qua1FFqZ7ldAHmx49+4Z4mg7inySBmjJ5T2eb4RH5WQcmDUlM1/py8iJ8/mYfL6argzhPUaJBSpZzAyQRi5W0ON9Oue2u3hQevGXAFYZFhRESH07prizM6ASklUsvVVTNDeIRUDyHVKlSbGnErGOoCii7656bQn3T+hyz8FOnwf5p1eVSK1pCnVDWJidtSB5CXmQ/ogmOzMoPXW9TfOOxOXrzmTTYu2ozRbERTVYwmI1fd3e20DKEQ3rN11U5yj+bSoUebcs9IpHQhs/vrMsRKcrH+TdV0sIFEfyZpCOFelheAVvglFI7VP4l5GiXK/2m/3qC/F5su5e3O9a6dyMxb0Q/czYjEST6poLorMRHaEXhBnWY1MRgNGIwKtZtUjlCVvzCHmRi54DVmF03G5XChujTsVgcb3TgE9RXVpfLFkEk8c/mrLJvtuaPPPpLDwx2e46a4e/junR8CYKH/aN6xMZ2vu+DMB+WO1eDaALhAy0RaPOtZ7C+kehCp+k/K26O5XfuQx7ohj7ZCyx/u/o1Fn6E/PO1Q6L9q8LLQawi2Is/QTOc4Qgi3nQAAznRAABogT+ojEVhCjsALXvvheXo+eCXXDriSN35+waexCnIK+WTwV3zw2BdkHwmc/EJFhIWb6da7C+HR4YRFhnFTcR69O0gp+eSpr+iVcj+v3DxC7+jlBj9++As/ffQbGxal81af0WTsz/TI5q9f/Z7dG/dhKbAy6fXpHt9f5TAknSQ1bayU3YBW+Imu3X+sB1rRt0GfXxZ+rPdGRgPLd0j1oHs3GhuiH3ka9UbwAUTmPYvMuh157H9oVg/7QlSEuTMIk946UxghzD9JDhUROiz2gviUOJ74+EG/jPV675H8t3gzUko2L9/Op6vf9cu4ZyL7SA7D++pFdAPfu7uk69bQKYNJ/2crUXGRHlVDb1y8mV+/nI+tyM7qPzYw76sF3PBIxY4kY38WDrsu5yAUQd6x/HJlnsvCaDYiFKF3UISS/gZnK8LYBBn3tt5Q3XyhHmMONkVfUFIHUPQpRPXzeigpbWBfBoba7vdIVuIBE1C8mHBzNS0SxiELPtZ770QHrnudlDawzUVfsQNFn0OE+4umihCGVEiZpzekMbUM2mLg7P7LqQbs33IQl1PPwz+4o+KtprvkZxfwfI/XubPeQ8z5/I9Sr3361ET+W7KFw7uOMrzfWKxFuiKmoii07nqex5IYpc+ZJO4eO9086FrikmMxmgy0u6I1jduleTTvvf93O60vaUFcahx3DLmZxJrxzPnsDz5/fpJfv5fBRIm4DiVpCkrM4MqRRjbUBwz4urKWUkVm3YHMexqZ1dvtlbOIfgLCrwZjc0T8KLcb1wglESXuZZTYlxGK+xLpUkq0/HfRMi5CyxmElI6TXtPQir5Cy30a6TgeugzTVVFR9I/d7fvsAUJJ1BvouHYgiyYj1cCLboZ2BJVM32G9+OyZrwG43Q8dtY4z6bXpbFyUjsup8sngr+hyXYeSVFdbkR1N1Vc0UtVKPvaWNt1a8r/7uzN/8mJaXtyca/q7186wVsMaTD3wGUX5lpJ+xZ4QmxRDj3sv44NHvuD7d39i9e/r2bl+Dw6rk98m/MV3+8cRFhHm8bjnMiLhS2ThRyCMiOhB3g+kHQHXDkqa01tnVrhyltKJLHgfXLsh8kFEeBAqtp2rwfqtnutvXwjWGRDZV7fHMhUKRgM2pH0+JP+JMKRA0jRk4XgwJCOiHvBoOimlngyAAqYO5Wa1aZafoOAVPVRY9AmkzPfsrMFDQo6gkrnxkf9x0Q0dUV0qNdNS/Tauy+EqWakLQHWdeNg/NPJe9vy3n5yMPAa83bd0o3VP5nC6MJqMCCF4/IMBPP6B5w3IDUaDV07gOFPf/rFELTR9+TaO9/yzWx3kZuS73eYyhI4wpCDifG/oIzUrJU4AQKn45yAt3+oPYuyQ/xLS3NbnDl0Vo6H/hQDIE8qnAK7twPH+EUJ3boYUhKEOIu4Vr2aTBW+AdZY+V0RfROyQsi90/K07JwBpAPUgBLC9aCg0VAVIqZvkVycAcNcrvWnQqh4R0eH0HdaLGg1SyNifybyJC3A5XEza9TFzCidzyyDPWx5qmsabfUbTM7wv9zZ9nJyjuX613RMat2uAKcyEwaCQVCuRsAgzEdHhNLugESn1zu5iP38hXTvdP3T1E0I7CBxfwSpgqFPmdZplOlrOE3pjeS0TXX6M4gKsIPxemTpB+E364ay5E0T2LnlJRPbVv044GJuD8Tzf57P+BNKiP+Sts8q9TIRfr88rIkFJLA7ZBY5QHcE5QtbhHAa0HIzqUpESRi18jWYXNPZqrM0rtvP8Va9hK7JjMCr0GXIL973Rx88Wu4e1yMa0ET9SlGfhjiE3o7lUMg9m07xTEwxG9/PQqyta/nCwTAUkxL6CctKDLpBIrQiZdZNeXYuCSJp12upe2v5C5j1VvPINh4RxkDdUX3mHXY2IH1PpLSSlVgjaUTCkeVTXUB5a9gBwrNQ/CeuKkvBp+XM7N4FrD4R105saeUFVb1VZpflv6RZG3P0hikFh6HeDad7RuwemtzgdTr8Ls21duQMpZfHD28CaPzd67Qii4yORmr6AMJiMxKZUXru/iKhw7nu9tBNKrR8KB5VgmUzJKrvos1Ir3kAilChInquHVwz1EMrp8uSoe06SYDAgpAVSFgAOhKgaZztCiQbFu253ZY6X8BFYpgMCIm8/87WmVgE5jC6LUGioDN7sM5ojezI4tPMIw+8cE7R5HXYnT132MtdF9OXRTkOwFlr9NnazTvpDPyzCrGfpdPf8F+zYgSy+GDKJFXPX8OjY+2nUtgFX33MZN7qRKloRqktl0Yxl/PPTv2iab4fXoB/KLZ61gh8++IX8rOAqfjrsThZOX8aaPzdUfptSQwP0LCAzGFsEdWohwhCm1mU7AYDwniBigQhQaoD54uICrKrhBAKBEBGIqHsQUXdXqfcZ2hGUgTzpQRTMP+Tls1exY+0epIR9mw+wYOo/9HzgSr+MnVw7kc/Wvc/q39fTonNTGnvYalJKyZOXDCPrUA5Gs5GeD1zJZ2vf94ttAMP7jmHlr2sBuOqubjz56UCfxvv+vZ+Y/MYMVFVj1ti5TNz2gduNaXxlaM+32LpyBwB9XriFfsN6BWXeshCJXyMLPytWyfTte+p3RLQe+3b9B6Y2btcMVEWkaz/SMkmvmYi8CyFOPFqlloPMGwbqUUTsEIT5wkq0tGxCO4IyGDplMEm1E0ipl8QLk4PXYDwmMZrjSfhCCOKS/Rdy2bVhL4tnLKdJh0YeOwEAh81B1qEcNFXDYXXoGToesnvjXvrUHch1Uf2Y89nvpV5b+ds6bEV2bEV2lv70r8djn8rKX9diK7LjtDnJPJhNQXahz2O6g6qqrF+wqeS9/D1taVDmLQ9hSNXz62Oe0sM1VQnrdHBtApxg/x2cvv/cKwMpXcis28HyDRSMQhaMLv16/utg/xtcG5E5D6J36K1ahHYEZdD2slZMPRD8pmntrzifu16+jb++W0KXGzpy8U2d/DLugW2HePLiYbicLgxGA2OWvOlxu8iwiDAuuflC/v1tLZom6f30DR7b8fGTX5F1SJfR+OiJCVx9X3fMYfpZyAVXtWH1H+sBvd+zr1x97+Vs/XcHiqJQr0Ud4pJjfR7zOBn7M/njm4XUaphK9zu7lsoFNxgMtLiwCbv/248Q0OX6C/w2b7VDhHFiLSqBwIZKpJYLjn/1YjWjH7NwZAHIfPRUVBs415d+Xcum5JxGOvV/IjDNmbwl5AiqGHc8fzN3PH+zX8fcsnIHKAKXU0UxKKQv2+ZV3+CXpj3F9jW7iU2M9kp2OiI6HMWgoKkaRqMBw0mSEC9Ne4pF05dhMBnpeqvvW+f/3dedtFb1yDyYTadr2vlNjtphd/JYpyEUZBdiMpvIycij1+DrS13z7vxX+XvqUqIToul6S9ULA0gpQd0PSoxHVbh+J+I2XWjPsQoib/NKZVNKCdYfkK7tiMheiHJy7aWWh8zsqWcoSQ2SpuiHsf5AxEPYJXo2kNQQUaXraUTM88jse0EWQfTjCMW7up1AEkofPQc4diCLB89/uljrXvLJ6nep27RW0O3IPJTN2/3GknM0l0fH9Kfj1d7L61YWR/Zk8ECrp7BbdSmCi27oyOs/lVMUVEXR8oaCdTYgEAkfI8IurWyTvEYrmgIFIwAriGhEysIyUy2lfTEy90mQhYCAqMdRYnyonD51fKmBKx2URIShdhmvS8CFCPJOIJQ+epYipWT6yNmsmreOq++7nKv6dfN5zJS6SYxPH0P6sm0079TYI2E3f5JcO5GRC9yvWt2wKJ0/vvmb1l3P4+p7L/d4Va+6VN4f8Alr/tjAZbdfzCOj7/N5Z5BSL4n6LetyYNshNFWj2+0X8eYdozi6L5PUekk0bt+Q3s/cELC+zL4iNQtYf+B443VZ+NFZ7QhwrgOOZ9epoB6GsnLujc3Qw0+6Qqkw+x5+PBkhFDhDZzb9965q/k7AOewICnOL+O7tWWia5M4XbiE2qfJy4U9myawVTHpNbx2ZvmwbDc6rS9MOvsvqJtVK4NJbO/vBwuBwePdRhvYcjt1i5+9p/xAeFe5RC02ABVOXsmTWCmxFdn4dP59O17an0//a+WSXwaCfsWxclE5q/WTGD53Cstmr0FwaW1ZsZ/mc1WQfzvFKbuM4LqcL1aUGRidJhOkpmzIXMIGxqVu3SfUY0jIZlAREZF+EMPvfNnfscO3R9YiUGETMEETkHUjbPL0S2dCwXKE8YagBSTPAvgCM5yPCPP9bkFquXhdhbI5Qyj5zkq5dYJunK4cGSULaH5yzjuC1Xu/x39ItSAmb/tnKB0vfqmyTAMjYl4nq0ldrikGQeTDbL46gsnDYHEwd8SPZh3O5Y8hNbrd3PLzzKIpBX73brQ72pu8HPHMELqeKVlz4pmkS1emf9o/mMBMX9NDDWlmHctBO0nFy2JxsX73L67HXzN/IKzeNwOV08cA7/bjtKc8P5c+EEAZImoIsHAeGGm5JNkspkdl3gnoIMCBduxBxr/vVLneR2ffolb4YkGoWSuLnumyzehhMrUqnbapHkUVf684r6l6EsTEYvSuilOoRZOYN6DspMyTPQRiST7kmC5nVC6QNMEH8KET4VV6/12ByzqaP7vlvPy6HiupU2Zd+wKsx1vy5gfuaP8GTlwwjY59/pGKvvOtSEmrGYwozUq95HS7o0cYv41YWHz0xgWkjfuSXL/9kcNeX3a7LaHVJc1LqJhERHU5kTARX3NnV47mv6NuVhBp6MZPL4WLTsq0ej1ERA9+9m/DocBBgNBkIizBzxxDvD/vHPTMRu8WO6lT5csi3AaljEcbGKPHvocQ8ixDhbtzh0g+XcQH24nBM4JBSouUNQzvSEi3zBuTJTd+1TPQQjwvUfQAIQ02Euf1puxSZ3RcsX0HhB8j8//PNKPvfIO36GYO0gWPJ6deoxxcAKmALas9hXzkndwTWIhvJdRPJzczHYDRw2zOer7o0TeOVm9/FbrFzaOcRRg38jHd+e8ln2+JT4vhm+0fkZeYTnxqHopzdvnrnuj0l6qC5R3Nx2p2YwysOK4RFhDFu7Xvs3XSAmg1TiY73PAfeHGaiILsIAE3V+PWL+Qx4q6/H45yJ8y89jx+zJ+KwOzmy6ygxidElct/ekFw7kX3pB1BdGtEJUX7LdvIFIUzI8GuLH4YaRN7r9VhSPQooupxzebj+A+scwAWunciibxAxg/XXoh+Hwo/1MWKePX186dB1/JVUUA+gOw0VnBu9thk4RXBO00XoTrumZXGltACpIsKv8W3OIHJ2P2W8ZPLr09mbfgCk3ozluoGe655LTZaEcKQmsRU3d9E0jS9emMzAts8wfeTPXtlnMBpIrJlw1jsBgL5Db8UcYSYswkyPey93ywkcx2Q20aR9wzKdgJSS7Wt2cXj30TOO0axjI0xhRkxhJpp3bsK0937itV7vsWa+jw+GkzAYDUREhdPw/AYIRSEvM9/rsZ6b+DgX33Qh7a5ozYjfX/abjb4i4kYhEr5GJP+EEuldpbRWNB557Erkse5oRd+dYbIISvTEMYA48fNXoh9FpC5GpC47LewipQOZdZu+E8jsAebLi8cKh0jvz2wAhLktIuFTiByASPgCYTpdiVQoUYjkuXpDneQ5PjWdDzbnZProyAGfMO/rv5GaxBxh5suNo7zKi583cQEfDZpAdHwkb855kcZt01gwdSmjHvwUW5Gd8Kgw3v51GK27+kG+9iwm63AORXkW6jWv7bcV7lt9x7B89io0TfLM+Ee4ok/ZoSNroZXZ435HURRMYSa+GDIZu8VOWKSZCZvH+jWDasrwmUx+YwYgePrLh/2S8VWd0I5eoBdfASgpKKnlV11rRZPA8jWY2iDi3nZLl0c61iBz+usyzwAR9yEibwIRU24BmZQu9LROd0JkZx/upo+e/UtOL+j7Ui+SayciFEHPB670ygmAXrQ0u2AS3+3/rES2oSC7sEQ0TQhBfpCkDaoySbUSqN+ijl+LuhZ+/w+2IjsOq4Pp75e/84qIjuD2Z2/itqdv4OjeDOxWvR+voihkHswu9z5v+PatWTjtLpx2J1+/Ms2vY1cLDPUoaYNpSDvjpUrU3Sgpf6LEj3JfnM1QhxM7iQiEuSXC1Kp8J+BYjczoiDzaHi3v/9Cy79bbVWr+/b04GzgnzwhqNazBlH3jUFXV70JkV951Kb98+Se71u/hvC5NufDa9n4dvyqwcfFmNi7ezIXXtne7Qll16VXN/nAGJrORlLpJZB7Mxmg20uJC91Igez5wFb+N/wuHzUmjNg1odoF/s7FS6yVzaMdhFINCnUoo2KvqiITPkYVjAcOJmL8/xzfUgISvkdYZYGqrN5w5A7JgxIndg3VK8VeNyFwHIvGz8u/TLMiiCYAdEXW/232VqzLVPjRUlG8hNyOP2o1rBvXgTXWpZ0VjlMO7j/LPj//SuF0a7bqXXxBznE3/bGXI1a/jtLswh5sYt/Y96jQ580Pv6//7nilvzSQmIYr35r9Kw/N9bz+YeTCLWWPnEp8azy1PXOt2AZfNYifnSC410lL8fgaTse8YX708lfDIMO5/606fWnBWdaR9GTLvaUAg4scizP7RxQomWs4gsM+nRAfoOMZWKMk/nOG+R8G+iOOHxme6trIJVRYD21bv5Nkr/g/VpdGmW0uG/zI0aM4gGE5g4itTmTlmLnWb1WL4L8NISC1H970c8rMKeKTD8zisDhSjwsvTnqbzdWcWSduyYjuqS294LxTBjrV7zugICnOLmPrOD2iqRl5mAV+++C1vzRnqkZ1lkVwniYHv3uPxfeGRYV6HAisitX4KQ772n2xBVUbmPVfcfQxk7rOI1IWVbJHniLjXkXkKaMf0LCDrNMCEiH3xzDc60wFdYgTXjkCbGRQq5YxACBEvhJghhNgihNgshPCsUshNZo6ei7XAhsPqYOPidA5uPxyIac7I1n93cGf9h+iV0p9ls/134H1g+2Gmj5yNrdDG7g17+e7t8vuflsfe9ANIKXE6XNgtDtYu2FThPRf2bI/JbCQyNgKT2USby1qe8Xqj2VgiLmc0GYhL8Z8KaDDIy8wnffm2krOFEMc5aQ0pzs71pFASUBLGoiRNQYl7FVFjDaLGqor7BUTcetIgEUhpK//as4TK+gmOBX6TUt4m9CqQgMjxNWhZl7BIM3aLA6EoxHu4YvYHIx/4lMwD+uHTiLs/5Mfcr/0yrtFkONG7QFFK5Jw9oXG7NCKiw/UCHlXS9eaKt/f1mtdhfPoYdq7Tz0AqkncOjwzjtR+f58sXvqVGWgoPj/Q+Bz3Y7N18gCe6DEUiSawRz7h17xMeWXW6SvkTqVlAFiIMqW5dL+LHIvOeL/54ZCBNOyNSuvQCNyXltJ7Ino2jgfVnpHoAIm5HGOuWe60QJiQKuuy0U1cdDTu7M8SC7giEELFAN+A+ACmlg5J9ln+543n9sGj3xn30euo6r4qSfMVgOhEiUoz+24DVTEtl4Ht3M/WdH0lrXZ87X7zF4zEiYyL4fMNI1i/YRINW9ajfoo5b96XUTSKlrvtFUxf0aFsiyRBo5k9ZzMzRc2jRuSmPjr4Po8n7X/G/py7FUmgFCTnkkb5sGx2uPB+Aven7+W/JFtp2b10pSq7+RDrWInPuB+lCRtyAEvd2hfcIc1tEyrwgWFc+uvRFf3Bt0Avd3JB0kFIirbPAtRkRcRvCpLfvlEXjoPAzwI60TIfUheXrKRkbofdOsIJUi7Ohzm4qY0fQCDgGfCWEaAusBp6UUhb5eyKD0UDfobdWfGEAeeGbQbzZZzR2i4Nnxj/ilzEXzljGh49+gTncTFqrerS6pDnhUd7lQccmxnBpry5nvEZKic1iJzwyrEpUuh5ny8rtbF+9iwt7dqBGgxS9wvvBcTisDvamH6B2oxrc5kUDneM0bpdGWEQYdosdTdOo27QmALv/26fvFKREMSh8tu79gJ07BANZ+PFJ2TMz0aSjOHe/coTl3EbLBucajq8jpWVSxY7AOgPy3wSsSOtMSFmAUOL1vgjHVUxlAWg5YCjnZxp2NcTmgmM5IuJWhNHz3h6n2aUeROY8DOpRiHkeJfI2n8f0hMpwBEagAzBISrlCCDEWeAEoVUYphBgIDASoX9+P3YSCTIOW9fhiwyi/jadpGiPu+RBnsWzDsQNZbFyyGaPJQJ8hnu8KKsJaZOOZy19l57o9NGmXxsiFr1eJ8Mj6vzcx7PrhSAnjh05h4rYPKMgpQlF0R+VyuMg5muvTHF1v6cwzX7rYtHQLV/S7lNT6uizCxkWb0TQNh81JRHQ46cu2ndWOAGMaOJZTsjG3zUMaWyGi+1emVaWQWgE4N4Cx6YnwlRKr/9OyATOYK0yO0ccoka2WupCeEo+IvAvp+BeEAsZWoJQfIhNCICLvgMg7fH1bJcj84bqyKRrkD0Mr+gwi70OJ6ue3Oc5EZRwWHwAOSClXFH8+A90xlEJK+bmUsqOUsmNKyhl0SSqRgpxC0pdvw1porfhiP3LqmtxucbBv88GAzLVk1gr2bzmIpmrs23KQJbNWVHyTH8k9lsfTl71Cn7oD+XXC/JKvr5m/EbvFgcPqQErJrvV7adqhIZ2vuwBFESTXSeSWJ3p6NeeeTfv5bcJfHNmTQfc+l/D4hwNo2aVZyettL2+JYlBKdmGtLilDdyYIHN59lCU/rCD3WJ5P44iYZyH8ak48DrRimepiBU9nuh5DryT07mLXInMHITP/h3TtBPRYvUiaAVEPIWJfQkQ9VuFYIuI2XXZCRIKhQYkMtwjvrstCJIxHJH5dCTvfkx/FEtS9UDCi5L0GmqDvCKSUR4QQ+4UQzaWUW4ErgfRg2+ErR/ce4+EOz6G5NCJj9Vh7TEJ0wOfdsmJ7iSKl0WRAKAqmMCO3Dr4uIPOdesB+8udH9x5DKCKgjW4mDPuO9GXbUF0qHz42notu6Eh8ShwXXtuemaNnIxQFo8lIkw4NURSFl6Y+hcP2GKYwk1d/zDvW7WZw15cRgGJQ+HLT6NPOQxq0rMcnq0aQvmwbbbq1pGaaewes/mT3xr08cfEwFEXBGGZkQvoYr/syCxEOcSP1A1DbbDDURUTeg7QvReY8Aggwd4KEL0DmIgs+AlRE9OOnSTH7A+lYi7T+jDB3QkT01MM2skj/hwFsf0K0LictDLURMU+5/17NbSH5T1AP6j0DTuoYplcgexZ9kFKC7Qekc0dxq0zvZK5F7FBkzgFw7QLs6AfR4kTILsBUVtbQIODb4oyhXcD9lWSH1yz9YSV2ix2n3YVEsnb+RrrdFpAs2FKMf/FbnHa9AEbTNMatfpcaDVKIjIkIyHwdr27LXS/fxqIZy+nW+6ISWeypI35g0mvTkcCDI+7ilkHerb4rwuVwlZJilsX9BVpd3JwPlw1nx7o9XNCjTaniLU+E7U5l/YJNqC4Vl8NFZEwEm5dvI6WMn2u95nWo19y9w/VAsOKXtTjtegObSCLY9M9WLr7R+6IuIQQi/n2kHKH3LAC0vCFAcWqkYzlox5C5z4BTl1eWzvUIPxdTSdcBZPZ96DH8WSDMeo5/yY7EBKbzfZpDGFLgTOqnHiAt30LBe4ANaZ0GKQsRiucLQmGohUj+AallI7Pu0iWtI64DY8VFnv6gUhyBlHId4EZAr+rSqG0DlOL8eE3VaNCy/HQzf5JcNwmj2YjL4SI8Mpz6Lep4XLz235LNvHXnGIQQDP1uMK0vaVHutUII7nj+Zu54vrTG/rQRP5XIS095a1bAHMH9b97JznV7OLIng7tf6U1CjfiS1xqe38DtKuV/561j7ud/0LprC3oNvr7c3ULby1thMBowFGd4tejsnnxFsFBVlYXT/qEwpxCDSQGhN91p0i7NL+MfdwKA/sB1/AvY9VCKEl+suV9ciavuBWDn+j1IKWnSzvdDU9Q9epxeAtiRzs0o4VdB0rdI258IcwdE2MW+z+MvSrXKdIF2BJQmXg8nlEREyi9IKYManjo7K0GqAO26t2bot4NZ8+d6Lu11EQ1aBieFbNBHDxQLpmXRf3g/ryqY377rgxLBtbf7jeXbPdY7k8oAACAASURBVJ96PEadZrXYsWY3CKh/XuBWxil1k/hs3fs+jXF411Fe6/UedouD1fPWE5cUS497ym4j2KR9Qz5cPpzNy7fT/orWldbfuTw+GjSBPyctREpJ/RZ16N6nK52v61BykO1PRPTjSCUOXPsQUfcghBkZ9SgUvKNfEDWQia9MZcao2YDglieuZcBwHw83Te1BSdIjIwhEhB7yFKbWiDP0BPY3UlrBsQYM9RHG8v+2ReTtSNsfuvMypFUopucuwT6jqPZaQ+5SkFPI589NoiCnkP5v9XU7p/5spF/DR8nYq3dUq5GWwuRdn3g8Ru6xPCa9PgODUeGul2+r0ro6Gxal89L1b2MttCEUwZ0v3sL9b9xZ2WZ5xd2NHuPIngxAPyP61T41KPMe2HaIyW/OJLFmPHe91I2IKBPCUJteKf3Jz9KlpaMTovgha6LPc0lpA+dWMNZHKAk+j+f5/A5k5o16S0ypIRK/PKOWklQPF585tKlyKbchrSEPGTngE1bMXYvqUtm8fDvTDn5e2SYFjGHfDebtvmNBwIvfPunVGPEpcQz60LdmH8HivC5NadCqHrs27CX8/9k7z6iori4MP3eGYYaiqGAXe++9d40lsUWjicaWaOyxxW6MvdfYYo9dYzex9957F1BEQESkwzD13u/H4CgfHWYAy7NWVoS5955Dm33OPnu/r72SFj81Tu8ppZhGneuwd/EhEASqNE0bG1NJkhhWfwKhAeHY2NoQGhDGyL9NFTrFKhfi3jlTrYdFUkNEH16ng6mLJIlIYVNBcwSkUN6lwKSovbECgSRJSJGrQHsCVK2ROSRf9yoj8SUQRPPqmT8GvekHH/ImFFEUY6hTRoREMqPLIl4+9uXHCR1o+XOT9Jpqqildszibni9L72kkCb1Oj/stT3IWyI5z7pStDhW2Cv68OA3/FwFky50FpV3690GklJ+mdaZio7Jo1FpqfB2r6toqaDU6Qt+Gm3SptHqTu180E3ePYN+Sw0iSRDsrnRMlFUmSeHzFDdEoUqZOyeSnV7QnIGoP73P+AqACRRwLau0piFwOUhQY3JAUpT5KBdZ3fAkE0fSa+SNTOs5HNIr8MKZdLInijZN2cPvkAwx6A0sHraVai0q45Pn4dcgzGpf+vc7uBQcoUb0o3Sd1ZFi9P/B190OSJOacmEipFB7eymSyj7vpKxpBEKicRjuBdywf8jcymQyjaPKU6PbH+65XO0c7Oo9N3+79d6wbv5V9Sw4D0LRbA4Ys/yVJ973zykD6UOlGBcoGpjMKZfPYN4kBZq0vEMAYEOsSbZSWk1suYKtS0KhzHYx6IzqNPl2kbhLjSyCIpsbXldn5eg16rT7OemxtlM7sPAZg1BvTcnqfBW9eBjCj8yK0UTqeXvdANIr4uvsRFWEqYTy85kSKA8EXUs7FvVfN/tx2jqpEpcrTiyPrTqOJNKnEntxyPkmBYOPkHWyZthsHJ3vmnfqdgq61QHsRlLVN3sNCPGKOqlag3mSq+5cXBVXsdOOENrN5dPkpgiBw5p+L3D3zEIPOQIfhrek9M206hpPKZ2lVGR/2mezibcrpNrEjriXzorSzpdOoduQskDG7nT9mQt+GI0RLROi0erRRWvP2XmWvpET1lJflfSHlVG1eEZWDEpWD0txHkhYY9AYiQpIuQVamTglsVQoUKgUlqiXe2BUZGsm2GXsQjSLhQRGsGbsdWdYVyHLdR5Z1Rawg8OSaO10LD6Cza1/unfNCcD6AkOM6gvPOOD2PH158glatQxOp5ebxe2gitRj0RnbN/zfDyZp/2REkEZc82VhzP37NIEmSOLz2JI8vu9H858YJ1uZ/IW6KVCxIxcZluXHkDo5ZHeg0oi1t+jfnyN+nKVyuAM16NkzvKX6WjFo/iLM7LiFJ0KCT9ZsmwSTzMbz+BNQRGhp3rsvIvwcmmvMft2UIh9acRDSItPwl8TM8hVKB3EaOQW+M9spIWKZ+Vrcl+L8wpYCmd17IDr81IMTfPFazdVWuH76NJEnkyO9iOofUGVA5qLCxzVhvvV/KR1PB/fOPCfYPoWarKpzffZVFfVeaVDodlKx9tCjD1aB/LIQFheOQ2f6jsPr8gnWY2fVPTm+7gCSBrUrBmocLyV0o5hmPXqfn3M4r2KoU1Pm2eoqsR2+dvM+a0ZvJWTA7w1b1TbAM+qeSg/FxM5lbZc2VhR2vVif4bKPRyPXDd7BVKShetTArfttI4Ktgfp7emWKVLeuXHR9fyketzN4lh1g3bisIAoXKulKufmk00ds9QSbg/yLgSyBIIRm5J+ELaUOO/NlRKBWm7nVBiPOAdeK3c7kfXbra7HRDfl3aO9njVG5SjuU3Zifp2jGbBjO980IMeiNjNg1O9Hq5XE7NVu/PU0asHZDs+aUVX84IUsjJzefRRGrRRGh4ev0Z9b+rhX0mO1SOKnIVzPEln/2Fz5aQgFC2ztzDgZXHzYfMyaXrhA7kKpgDQRCQjCIetz1jXXPn1APT32Ck1qI2sPFRolpRNnosY6vXCsrXT9ii9WPjy44ghdRqXQWvR95IokSuwjkpXqUwW178hb9XAAVK50uVM1ZqMBqMvPUNwjlP1nSbw8eOx21PtFE6StcqbvFWf4PeQIBPIC55s6GwTZq9aIBPIAqlDVkSyWEnxFvfQMY0n4af5xs6jWxDj0nxa+lvmrKTQ6tPULZuKUauH5hkG9Sn1z2ICFWzZOAa/F+8QW4jx+uRNwP/TL6vgUFnwNfjtdlTe8u03VRqHFNsrmqzCtw+9QCQqPttjWSPkZ6ktZZQYiT4ThFtK5ldkqRn//f58pIk3bPqzDI4XcZ3oGDZ/IS8CaXhD3UQorev6VkjHBURxaAaY/F/EUBml0wsvzE7VW8enyPbZ+9l89RdCIJAo851Gb6qn8WeHRmmZmDV0bz1DcIpe2b+ujmHzM4Jp8H+nrCNnfP+QxDgt3UDaPxD3RSNvWnKLryfvkI0iuyYs58WPzWOs/LtyTV3dszdjyZSy6V/r3N4zUnaDmyR6PN3zPuXjZN2gADa6BJOg95o7jpOLioHFXaZVESGRKJQKnCNQ/Llj12/cXHfdWxVihgpmPTizI6LrBq5ieyuzkzY8VucfUaSJLF4wGoOrj5BniI5mXd6coboR4o3NSQIQifgCbBbEISHgiB82Da33toTy+gIgkCddtX5ps9XOGS2T+/pAHD9yB0CvAPRRukIDQjj7I7L6T2lj47//jpmLvk7semsRZ999cBNAv2CTT+ft2Gc330l0Xt2zNmPXqtHp9GzceKOFI+tslcik79fgcZXtfJO4hxMkt96rT5Jzz+85gRatdYcBOwcVSjtbWndP45mrCQgt5Gz4OwU6neqTbtBLek3P7aEg43ChgYda1GrddV0X11HRUQxp8dSArwDeXLVgxXD18d5ndcjH45vOoskSrx+/oYdc/en7UTjIaEdwTigiiRJfoIgVAc2CYIwTpKkPcQ2yfpCBiBngezmpjeZXEauQmlvmPKxU7p2iWiJEYkiFtLOeUeuQjnMzagCQpI6nV3yZuO1VwACAi55Uy7A1m1iR3zcXuH12IeuE76LV66jbN2SNPqhDqe2XqBYlcJ8/UvCHsDvqNCwDAE+gYhGifyl8tJ/YU8yZ3NMkky4v1cAm6fsxN7Jnu6TOpkXVoXK5uf3bUk3nbEUkiSxacpOzu68TO02Vfl5epdEA40oSuafrSRKGHSGOK+zc1SZPTVkNvI0MbNKCvGWjwqCcF+SpHIffJwbOABsAHpKkpQ2Qidk3PLRjMip7Rc4teU8NVtVoVXfZil+TkbLYaYVOq2eAyuOoY3S0aZ/MxycLJvqO7XtPGf/uUyttlWTJH43+bt5XNx3DUmUyJTVkV0Ba1NUJpkQj664EfgqmOotK6ZYh8loMHJswxkiQ9W07NU4Wd+37kUH4f/iDTIbOTW+rsSkPaNSNAdLcf3oHaZ8Nw9NpKkUfMymwdRpVz3R+/5bcZQ1o7eQLU9WZhwcFyvQ67R6NBEarhy8yT+z91GscmGGreprVe0rS5SPhguCUOTd+UD0zqAhsA8oY5lpfsHSNP6hborzyACBfsGMaDQRX4/XtPipEcNW9YszIAT7h3B252XyFMlF9ZaVUjPlDIWtUkH7Idax/QRo3LkejTvXS/L1ga+CzSvIyDA1eq3eom8ch9acYNngdRj0RrLkyMxmz+VJPsT+ELmNnJa9UibE+OblW0RRQtQZ8HqUPO9tSZJYNmQdR9adpkiFAsw4NC7VwVsdqoZ3v/MSSe5ubt2vOa37xZ0Kc7/1nBGNJ6HT6ClZoyhyGzm2KgWChYN6SkloFv0BmSAI5jopSZLCgRZA8gt2v/BRsGfhAfye+yOJEqe2XeTZ3RexrtFp9fSvMorVozYxpeN8Dq89GftBX7AIPSZ3Qmlni0KpoM2A5hZfPR79+zQ6jR7RKBLkF8KGif9Y9PlJ4bvhrbBVKbBVKeg28bvEb/gAt5vPOfr3abRqLW43n7Nv6eFUz6dW22qUqFYEQRAoUrEgDb9PvSPalmm7UYdFYdAZeHD+CZ73X3Jyy3n2LTmU6mdbgnh3BJIk3QUQBOGBIAibgDmAKvr/VYFNaTLDL6QpjlkdkNvIMRpEJEmK0wv5rU8gESGRZqvKqwdvpXg1+IWEqfJVBf55tYqoSK1VqkvK1CnJo8tu5o/9vd5afIzE6D2rK637N8dWpYhhRZoUbBRys6e1IAgokljqmhC2SgXzTk6KJUWfGnIVyo6tSmH+mwEwGIyEvg23yPNTS1IKzWsAs4FLQCZgC1DHmpP6QvrRYVgrvJ++4ul1DzqNbEueIrliXZMjvwvZ87kQ+CoIURRp3CXlqagvJI6Dk4PFzyre0XvWj9w7+xCP254oVLZ0GtHGKuMkRkpFHItUKEiXce35d/kxSlYvSpsBiZe6JoQoilz57yZ6rT7FshVx8dO0zhh0RlMJryRx5+R9RIPI4ytu6HX6FKXjLEmiWkOCyXttOvAV4Aj8LklS2vjjRZORD4u9n/ry1/ANqByUDPzz5xSbp3xsqMOjuHH0DrkK5aB4lcSVHpNDWGA4C375izcvA/llTtdYjURfsCySJOHvFYCTSybsHGPvAD8nlg5ey9H1Z0CSqNSkHFP2jbb4GIfWnGTZkLXoovSoHFWMWj+Ieu2t0xCX1MPipIS765gse6oBdYHOgiDsSuX8PhnGtpzOjSO3ubT3GjO6LIrzmpdPfPG47YkkSdw8fpfv8/xC10IDeHrjfZ/emX8u0qvMUKZ0nI86PIprh2+zduwWHl91t+h8fT38OLDyOM/veaXqOfaZ7Kj/XS2LBwGA5UP/5srBW7jfes6ENrPRJbGWPbnsXnSANpm78XOpIWYf4IyEJElEhqmxtjBkREgkt0/e58k1D6uO8zFwad91NBEak3T0sbtWGcPByf79TkOScMyS/n1ISUkN9ZIk6d1y/DXQVhCEblac00dFyJswJAmMRpG3vkGxXt+39BBrRm9BkAk07daAczsvm82+5/RcytoHCwn2D2HuT8vQafT4PfNnXq/lXDt0C61ax94lh1lxey75iuVO9Vz9vQLoX3kURqOIIMDCc1PTTAUxOYQGhJmNfwx6AwadIckyB0klLDCctWO3oNca8HX3Y83YLelSsx4fOq2eEY0n4Xbdg9xFcrH40nSr1JwbDUYGVh1NkH8oggAD/vwplg1rVKQGvUZv7oK+9O915v60DIWtgkl7R1K6ZnGLzyu9qNm6Csc3mhoJKzQqa5Ux6nWowdPrHlw9eJOGP9ShopXGSQ6J7gg+CAIffu7LQXE0vWf/iI3CBluVgr7zYnc/7l54EG2UqVP1yNpTfFiJ+a4sUxv13iLPaDDi/yLA/EYok8vweuhtkbk+ueYBAuiidIgGkXtn427/N+gNHFh5nJ3z/k2WMYil6D27K5myOUbbInaM88A6tcjkMt71RQoyweKBJrVcP3ybF/dfYjSIvPEK4OSW81YZJ/BVEIGvQ9CqTeJtF/dei/H6rRP36JijF9/n7cO68VsBmNNjKRHBkQT7hzC/13KrzCu9GLSkF6M3/sqwVf2YtHuEVcaQyWT0mdONtQ8X0W1CxwzRr/PJqpJJksTpbRd48/ItTbs3sJqeR7uBLWneoyEyuSzO0r4S1YsS+CoYURTJXzov/Rf0jF5N2TBm068A5CqYgzYDmrPnz0Nkz+fML7O7MvHbOchtjEiiiIOFto7vRNRUDqZ5VmoSd+598cA1nNpyHtEocnr7BZbfmGOR8ZNKkQoF2R2wDtEoWs2TwDGLA8PX9OPvcdvIWSg7v8zuapVxUkrWXFkQ33WgymVky5W8apqk4pwnG9nzORP4KghJgvodYxrPrB231bxQ+WfOfrpN7BhDnsJWZWuVeaUXMpnsoxOwswSfrDHNP3P2sXnqLvRaA07ZM7PlxXK8n75iYrs5qMPUDFnR12oHNB+ijdKyd8lhNBEavh38dbxWmBCzm/f09ovM/WkZeq3eZHTzcCE58qfeHvPNywDunnlEiepFyR+HkBdAzxK/4uv+GjC9CR3Vp31t+RfgwMpjHNtwhuotK/Hj799ZbeUYGabm0v7r5C6Ug7J1S8V4bdoPC7i47zoGnYHMzpnY9WYtDy89ZX7vv7BVKRi3dSgFSuWzyrw+V9ThURzbcAbHLA407lI3VZVLn70xze2T981G1uHBEYS+DWdR35X4PfcHYFbXP6kTsdni7fr/j9JOyQ+j2iXp2g//0H3d/TDoTXolMrkMr8e+FgkEjlkdeXbvBbdP3afbHx3j1Ltp1bcZ6//YjiCTZQhVR0sQEhDK9cN3KFAmn1UOuK1Bq77NUiUTklQcMtvzVbcGcb42dEVf7DPZE/wmhJ+ndUYQBMrWKcnfj/+0+rw+V0Y2nsSLh94IMhluN58xYOFPVh/zkw0EX//SlAcXnyCTyyhQ2pWsOZ2QyWUIAkiSKS+cEXJz8VG3fQ12zN2PIBOwz2xP6VqWOZBb0GcFl/Zdw6A3cvvUfba9XBnrmu+Gt6Z8g9JEhWsoV78UwW9CGddyOj5ur+gwrBU9p/xgkbmkFVERUfQpP4KoiCgkSWLSnlFUbVYhvaf1UeCYxYHhqy0nxf2FhBFFEfdbnuZKsRtHrVO59P98soGg/ne1yF8qH299gyjfoDQymYzhq/sxpeMCIkMiGbKij9UCwYOLT9i/9AjFqxamw7BWKdp1FCzjyt9PF/PysQ8lqhW12IHpy0c+ZqnhwFfBGI1G5PLYefgPV83bZu7B874XRoPIznn/0rRbA4tUMaUVXo980EZpzTvES/uvfQkEX8iQyGQyqn9diXtnHyFJEs17NkyTcT/ZQACmN9OCZVzNH7uWyMvqe/OtOmawfwhjm09Do9Zy+b/rKJQK2g1qmaJnOefOavEGtW4TOzGrq2lb37p/8ziDwP9jq3wnjmWSuLZRfFym8q4l86JQKlDaGUEQqNioLHsWHUShtKFFr8bp3tX5hS98yOS9o7hx7C4OTvaUrVMyTcb8pANBehD4Ktjs1qCL0uH1yOf9a37BKJQ26WrOXq99Dcq/XIEmUpvktv7OY7/l+T0vPO+/pPPYb8lV8OPyOXDIbM/KO/O48t8NCpbNz5qxW3h6zQNBJnD/whPGbRmS3lP8whfMPL7ixvGNZyletTClaxW3+jkmpGMgEARBDtwAfCVJapVe87A0hcrlp1jlwrjdfI7cRkab/qbDvvV/bDe5EQkCYzb+Sv3vaiXyJOvh5JIZJ5f4Xw96HcyxDWfJns+Zxl3q4uDkwIxD49NuglbAOXdWvunzFQBu15+ZnbcenH+cntP6wmfC0xvPeOsTSNXmFRJUkA16HcyYFtPRqrVcOXATW5VtijMKySE9dwRDgMdA/PWUHyFyGznzTk/C192PbLmzmt2Wts/ah9FgahJbP2F7ugaChDAajQyqMZag1yEobG147elPzkI5WDJwDY5ZHJj67xgKl0/cdSo90Gn1XPnvBpmdMxEeHMH8Xn+htFcyZf9oSlR9f+bRqHMdzu68DJJE85/fm8No1FoW9VvJ87tedBnXnobfZ1xtxaDXwRzfeI7srs40ivbM/kL64Pfcn1WjNqFQmppK/z+de2zjGRYPWINMJpCvRB6WXp0Z7yo/8FUwgiy60VSt5eVjnzivszTpEggEQcgHfINJzG54eszBmshkMlxLxKzRd8mbjTfeb5HbyMlXIo9VxxdFEaPBmKLcd3hQBMH+oRj1Rox6I7dO3ufRZTcMOgPqsCj+HLCaPy9Ms8KsU8+YZlNxv+0JkoRBb8SgMxAZqmZB779YeWee+brf1vTnq+4NUCgVMeQRts/cw7mdV9Br9cz9aRll65YkPCiC60fuULZuSUrXKpEeX1YsjAYjA6uPJcQ/BBuFDQHeb/k+iSXKX7A8476ejq/7a2QyAb/nrxmxdgCuJfOa3+yPbziLVm0qVHhx/yWhAWHxym0XrlCAYpUK4XbzGTY2Nin2fE4u6WWPswgYxbvTx8+AuScn0uiHurTs1YRR6wdZbZyn1z1o7/wTrRy7snla8rUBnVwyU6JaEbP5eMtejZHbmH5NBEHAVpUxD1b1Oj0PLjw2C4aJRtOvliCArV3M7ldBEKjQoEwsjZzQwHDzrk0QBF489GZw7fGs+30bo5pOSZEAoCRJFhfNC30bRmhAKAa9EY1ay83j9yz6/C8kj0C/YCRJwmgUeXLNg4HVxzLx2znmEtBqLSuhclBiq1KQLU9WMrvEf0Yol5syCituzWWr9woKlc2fJl9Dmu8IBEFoBbyRJOlmtPVlfNf1AfoA5M+fNt8Ma5K7cE7Gbh5s9XFWjdxEZKgagM1TdvHd8Nao7JPuaiUIAnNPTuTO6Ye45MlKoXIFcMruxJKBq8mULVO8NeU+7n7cPnmf0rWKU6RCQUt8KclCYaugWOXCeD32QRAE6rSvwZMr7tg5qui/sAd3zz6kWOXCCZbhdh7zLdcO3SbAJ5CvejQkIjgSQSZg1BuRCQKPL7tRqkaxGEYoCRH6NoyhdX/H1+M11ZpXZMr+0RaRzMiaMwtFKhbC65E3olHk695pYwp06+R95v60FFuVLRN2DKdoxUJpMq41SM2uGeDqwZusHbeV3IVz0mV8Bzb88Q9GgxFJlNBGB+fAV0G45HWm42+tyVMkJwHegTTuUjfRSj2ZTEa+4tbNGvw/aS4xIQjCTKAbYMDkeJYZ2CNJUrxiLxnZjyC9+Hf5EVaN2oyTSyZmHB5vbvOf1nkhF/Zcxag3onJQ0mdON5zzZqNW66pWyyP7ewXwS7nh6a5qGhURxentl8js7EiddtURBAFfDz8GVDVpyttnsmP1/QU4ZknY5OVdb0XQ62B6lRmGaBAxGkUcs9gTHhwBEkiYUkxNusTvP7x99j42/PEPBr0BO0cVk/aOIiIkgrk9l2Fja8PkvaMoX790vPcnhE6r5+7pB7jkzUahcmlzZtPe5SfCgyIAU1HEqrvWLcW2Fk+uuTO62VQ0ai09Jn9Pl7Htk3W/OjyKjjl7odPosVHIadqtAX3ndWfrjN38u+woOo0Oeyd7/vFdZVVj+qRgST8CiyJJ0lhJkvJJklQQ+AE4lVAQSAsOrz3JN/Zd6JirN+63nlvkmWFB4fzWaCIdc/dmz+KDybo30C8Y91vPMRqNcb6uUWv5a9h6tGotAd5v+WvYevNrg5f1pk7bapSuXRynHE6sHLGRmT/+yfbZ+5I8viRJbJ+9l9HNpnB2x6VEr0+qqqm1sXO04+veTaj7bQ1z0Lu49xraKB3qsCjUYeoYczPoDbx84osmOn/7jncrNscsDsw5/gcj/h5IkQoFCPILRhelR6fRo9fo+bPfqgTn4+SSCXl0z4UoimR2dmR+r7/QRGqJCI5kYd/YXd1JxVapoFqLSmkWBP6fj0GjLD5WjdyEOiwK0SCyceIOtFHaxG/6AL1WbxYENBpEwoMjcMziwM/Tu/DjhA4069mIheempnsQSA6ffR+B0Whk8YDVGPRGdJpQlgxcw+LLM1L93C3TdvPo0lMMeiNrx2yhTtvqSarbv3P6Ab+3noUgmJRL556YGGslL5PLTDLKeiOCTGZWEwXInC0TE3b8RmSYmvbOP5lz5Zf/vU7nMd8mae6nt11gy9TdaNRaHlx4Qv7S+RLMVSZV1TQ9KFq5MDYKuenw2yiRv7Rp56RRaxlYbQxvXgagtLNl+Y3ZMbScfNxeMbjWOLRROsrUKYlDFgcEQYjxBmjnqEpw7GY9G+L1yIe7Zx7yTd+vKFqxEEo7JeqwKAQh8fszGr9vH8acnqbUkDXPuaxN1pxOyKN/J2xVCmwUyXsbdHLJTOex37Jl2m6y5nTi5+ldALBR2NB5TPJ2FxmFdA0EkiSdAc6k5xzeGV4b9EZkMgF7J8tIPosGkQ8XTaKYtHPxvYsPmSsMHl9xx98rIFYDl61SwYQdv/HX8PW45M3GoCW9Yj3HPpMdRSsX4uVjX5AkGndOuq+wv1eAeZWs0+i5cuBmgoEgez5nVt+bz92zjyhZvWisiqn0pHKTckz4Zzj3zj2izrc1zNIYd08/IMDnLZpILQadgVPbLvLD6PeVN/uXHSE8JBIkU4PP+O3DiAiOJMAnEAcnO2yVtgxd2SfBseVyOf3m94jxuSn7R7Gwz0ps7WwZvSHlb6aWNFZPKpWblme7T8K7oI+BIX/1QRQlgvyC+WVOtxSd23Sf2Iluf2QMLwFL8NnvCGQyGVP/HcOSX9fi5BL/YWhy+XFCBx5fdcPHzY/vR7Uld6HYKp9xUaJaEW4ev4s2SoetUkGWHE5xXlezVZUElUEFQWDBmclc+e8mWXNlSVYuukzdkqbu6OhA5n4z8XRZjvzZ41WwTE/un3/M5f9uUL5+6RhVQrkK5zTvlmxsbchXPKZ2Ut5iuVHaKdGqtUiSRNGKBVl8aXqSxnx0+SlXDt6kYsOyVG5aPsZrJasXi1HKmlwk2Vd89gAAIABJREFUSWLxgNUcXH2CHPldWHB2CjlcE+gO/EIsMjtnYuKu1JvOfCpBAD5hP4KPFaPRyH9/HcPH7RVtBrSI1zPAmkRFRNElfz8iQ9UoVLb0ndedNmlUz2wJHlx8gtdDbwqVL8CoplPQqrUo7ZWM2zqE2m2qma+7dvg2xzacoVKTsnzdu2mMP2yj0ci2GXt4ev0Z3w75hspJTHd5PfZhYLXRaNU6lHa2zDr6eyyN/9Tg9diHgVVHo43SIZPLaN2/GYMWx94RJhVJkgj2DyGzc6Zkp0j+H6PRiMctT5yyZ06SDMn5PVdxv/WMRt/XSbezjk+dz96P4GNFLpenSUt5Qtg52rHs+myObzyLa4k8NEpGWim9ufzfDaZ3XgiSaaX/rktTr9Xjef9ljEBQvWUlqresFOdz5HI5XSd0TPb4nve8om0wQRQl3G95WjQQ2DmqzOcUcoWcTNlS7mOs1+kZ2WQybjeekSlbJpZdm4lLXucUPUuSJP5oO4d75x4hGUXGbR1K7bbV4r3+3K7LzOm5DK1ay77Fh9ngviTeJqtPiZdPfHnz8i2lahbD7cYzsudzjlUq+qFBVVrxJRAkA0mSCAsMxzGLg8UsFI0GIyc2nyMqXEOzng2t4s+bEvIUyUWPyd+n9zTixKA38PDSU5xzZ431R3T14E20apO1osxGhmMWBwTBlAJs+H1tq8+tYuOyKO1sEQST34WljX1yuLowfE1/ts/aS5EKBVPVUXz//BOe3/VCrzUQGhDK0fVn+HF8hxQ9Kzw4glvH72KI9trevfBAgoHg0RV381kYgoCvu98nHwiuHLjJtB8WIMgEJBFkMgHRKDJu21Bqt6mGJEnM6bGUk1vPU6C0K/NPTyKzc9oIVH5WgcDXw48j606Tr3humvVomKyoazQYGdtyOvfPPyZTNkeWXplhEcewpb+u5fimc0iiyMkt51hyZWay7g9+E8qpLefJ7upMvQ41P6m8ZVxIksSor6bicfs5okFk9KbBMSxH67SrzolN5xBkAnaOdqy4NQd/rwByF85JpqwpXz0nlSzZnfj7yWLcbz2nULn8ZMke9xlPaqjUuCy3T9xDEATUYepkNQx+iEvebB+ckyhSddbgkNmeTNkyERoQikKpoGTNYgle37hzHQ6uPIZMJsMpR2aKpkPfSVpzeM1J8yLlQw6sPE7tNtV4fMWNC3uvIokSPk99+Xf5kRTtSlPCZxMIoiI1/FpjLBGhapQqWyJCIukwNOmip4+vuvPkqjsGnYHQgDAOrDxuLhtLDbdO3jevjNyScCh7+9R95vf6C5WjinFbhzCh9SyC/UOQ28h5/SKATiPapHpOGZlg/xCeXHUzm+scWHE0RiCo1qIS889O4eUjH6p/XSlaaTVtdQ0dszhQqbH1SmgntJnNs9ueIMCLh97JXjy8I3/JvIze+CsHVh2nYsMyNOkaf3NcYsht5Cy5MoP9yw6T3dUl0TOl4lWKsO7xn/i6+1GietEUB7OPgac3nhHiH0K5+qW4dfIeOo0OSTS5JCqUCio0MBVy2GWyQ4ruT5DZyHHMYv2Fyzs+m0AQ5BeMTqtHEiU0ai0PLz5NViDIliuLefWkUNpYZDcA0LxnQ7bN3IsgCFT5KnHXrCnfzSciJBKAOT2WEOwfgl5rQK81cPPYnRQHAlEUiQiOJFM2xwy9q8jsnIlMWR0JCQhDoVRQrl7s/HuJqkViqI1+arx+8QZj9O+in+ebVD2rXoea1OtQ0xLTImeB7PSZ0z3J12fP50z2fCk7k0gpDy895dL+a1RsVJZqLeI+H7Ikh9acYPnQ9chkAkUqFmT46n74PX9DxcZluXnsLrkK5qBpt/q89Q0kyC+YHpM7cWjtKcrULsE3fZtafX7v+GwCQa5COShYxpWXT3wRjSKt+n6VrPvzFMnFmM2D2bfkMGXqlqRl78aJ3/QBBr2BbTP34uvxmk4j2pilnLuM60CFhmVRh0dRuWniq8h3bwBg8l4uVK4A3k98EUWR5j81Stac3hEREsmvNcfx2tOfAmVcWXRhWoZdodkobFhydSZH1p4ku6sLdo527Ft6mKZd6ycqHWFp3ni/5eLeaxQs62rVHcD/031iJ1aO2GD696ROaTbux473U19GN5uCVq1j/7IjTD84jgoNylh1zCPrTpl3/I8uuzHtwFizNP27cubn97wYUud3ZHKTP/maBwvM16QVn00gkMvlLDw/lSdXPcju6pwil62639ag7rc1Er8wDjZM2sGuBf9h0Bo4tfU8C89NoUxtkw1dmdpJlzceu3kw83otR2WvZOS6geQvlZdbJ+7jki9bikXAzu64RID3Wwx6I74er7l64CYNOln/YDWl5HB1ofuk79k0ZSc75u7HaBT5d/lR1j5cmGa7mcgwNf0rjyQqQoNMLmfMpl+T9Ltx5/QD/J77U6dd9RQfBLYd2IL6HU1+Flnj6TNJj8qT1HD37EO8HvpQu101XPJks8oYLx54myu6jAaRZ7dfJCsQvPO9Tk6qsUqzinjef4nRIJIjv0usYhBtlJb/VhxFE6kxf+7xFfc099T+bAIBmBQq40olpAWe97wwROe1JVFieudFbPVakezn1Gpdld1v1sX4XGorU5zzZDOXWUqihHOe1Pska6O0rBy5CR+3V3Sb0NEq3/drh2+bDel93f3QRGqwc0ybqisfNz/0OkP0WYWBG0fvxggEGrUW95vPyVc8t7ka5p1BCUhsnrqL9W6LU6x+GV8AADiw8hjLhvyNfSY7Zhwen+o02dpxWzi46gTFKhdm0p4RFv8eX9x3jZldFyNJEhsn72Cjx1KrVM9VaFQGOweVuaKrVptEy+vNPL7qzuivpqDXGWjWoyHDVvZN0n3dJ3Ykf8m8BL8OoWn3+jGCc0hAKP0qjTQJ+UU3cIpGkQLRMihpSXr5EXx2fD+qndnLGEwrkoxCjW8q031SJ8rVK0W/+d0tUve+fsI/HF13itsn7jOu5XTU4VEWmGlMvureAJW9EjtHFaVrFUflkHbaPQVK58MxiwN2mUy+De9W6GBqyOtTbji/t5pJj2K/4vngJWB6w9OqtWjVOkLfhhPgHWjxeRmNRpYNXodBZyAsMJylv65N1fPcbj5j35LDhAdFcP/8I/YtPWKhmb7n6qFbaNVadFE6tFE6fNxeWXwMMOlwrXvyJ1P2j2aD+xJyF0682/+tbyDBb0LZPGUnUREaDDoDxzacISQgNEljCoJAox/q0H7oN7G8yi/tv0FEiBqdRo9CqaB2m2osujAtzc9N4DPbEaQn5eqVYtKekcz7eTkymYzRG39N7ymZEQSBjr+1oeNvlqs4ev3CH53GZMhiNIqow9QWX+W16d+copUKEfImlKrNK6ZpKkRlr2TV3fncPH6P/KXyxtBienzVg9DAcFPwE0ypt0Jl81P/u1rcOnYPBNOKPkd+y0tDyGQyFKpo7Sy5jExZU3duYhLa+/DjVE4wDuq1r8GxDacx6kU0ERrcbj6neBXrHPY7ZLZPcjpo64zdbJ66GwQoU6cECqUCvVaPwtbGIoKBriXy8E7HRSaT8f2Yduni5QFfJCY+SSRJYv3Efzi+4QyVGpdj+Op+FmuASyoedzwZ0WgSWrWWr7o3YPjq/mk6fnryxvstvUoPRRNpkrYYu3kwddpVB0zyF37P/anVuqrVDrfvnXvE0sHrcHLJxOgNg1LcLfyOdeO3cmDlcYpXKczE3ZZPDQG0ztTVnOZzyZuNbd4JS3QHvg7m8WU3CpZxtZqJSyvHruaD3hyuLtRqU5VXz17T9Y+OsdztUsrZnZe5tP8adb+tYbHqrQ9JqsTEJx0Inl73YNOUneQqlIPes7pm2EqYpGDQm7b6WXNmSXTle/XgTSa0mY0kSchkMob89Qtf/5J2pWjv0Ov0REVoYm2JPwee3njG6e0XKVOruFX+wD81epcbjvcTXwRBoHTt4iw4MyXea8/vucrUTvORRAmZjYzhq/rRvGfKKuYSoleZofi4+SGTCVRoWIZZRydYfAxr89lrDek0OkY1nYI6PAqFUgEScco1W218rZ7tM/fwxjuQTiPbkr9kXm6duMfmqbvIXyov/Rf2TLJxRYBPIIOqjyE8OIKS1Ysx58QfCQqEXf7vplmPRhRFPO68sMSXlGwUtgoU2TKmx7G1+dR7GSzNt7+2ZNOUnbjkc2bCjt8SvHbz1J3mxivRILJn0UGrBIJZRyew4Y/tKO2V9JiSdnIrep2ee2cfkS1XljQT4/tkA0FUhAadxtTOrdfq8fV4nabjrxq5kcNrT6LXGLj87w3WPlrIH21no43S8eS6Bw6Z7fllTrckPevIulOEBIQhGkU8bnvy8OJTKjSMP89ZqmYxDq89iWgUEWQCDTrVivfajMapbee5dug2Db+vY3Gdnk+Bc7sus2nKLsIDw8lXMg+/remfZInzpCJJEn/2X8XR9WcoWMaV2ccnWHVX9+KhN38NX49WrSMyVM3d0w9o+H2deK8vWMaV53e9zB+XSkTOIqVkz+fMiHUDrfLs+JAkidFfTcXjtieiKDJ0ZV+a/ljf6uN+slVDPm5+5kMuG4WcbhPTRrPjHZ73X6KL0puF6u6efWRepRu0et74JL1iJEd+F9OuBtMKP7Hyzqbd6tN2UAsKlM5Hn7ndrN40YylunbjHgl9WcnLLeab9sACP257pPaUMhb9XALN7LOXFg5cE+gVz9/RDpna0vG+w283nnNxyHoPOwIsHL9m3+LDFx/iQAO+3ZpMdvc7A6xcBCV4/dGVfWvdvTqHy+ekyrj0DF/9s1fmlJeFBETy+4kZUhAatWseh1SfSZNxPdkewbvxWjAaTEqIoShQq65om44qiyKJ+q2KlY1b+toEmP9bj2Iaz2Ge2S5bK41fdG/DWJ5C7Zx/Run/zRA/H5HI5Axb+lJLppyveT18hRTu5yWQyfNxeUbRSyprkkkKwfwizeywl0DeIfgt6xCvx8eDCY976BlGzddV0PWcKD4pAJot5PhQaGGHxcZR2tuZFi0wuQ+Vo3a+5QsMy5CuRB6+H3jhmcaBpt4RXwHYOKgYv623VOaUXDlnsyZY7K4F+wdgobKjYqGyajPvJHhbP7rmUs9svotcZsM9kx57Av9OkcubivmvM6rbYXAHxDkGAibtHUqlJOZT2tmaD9C+8J9AvmH6VRqJRa8nikpkVt+fg4GQ92YgpHedzaf81jAYRpZ0te4PXx2rwOrj6OH8N24BMJpC7SE7+ujknzS0i3yGKItM7L+L87itIooRCqWDi7hHU+LqyxcfavegA+5YcpkT1ooz6eyC2KluLj/Ehoijy1jeIrDmdYvwMdBodoih91IUeiSFJElERGuwcTc1uQa+DObbhLM55stLkx3qp+n377A+LB/35E3K5jADvQH6a9kOalU+KYtyBVZJgZtfFTP13dJrq0nxMOOfOyqbny/B77k/eorms/uYTGao2N/YZDUaMBhHF/w15cst5cwnhy8e+hAdFpLma6TtkMhkT/hmORq3FxlaOIAhWW1B0GNoqWaKM8WHQG9g0ZRcvH3nTaVQ7StWIO58vk8nI4eqCJEl4PfJGaa/k+V0vpndeiGgU6b/op4/KJS+pREVEMbTeBF489KZIhYIsODuFbLmyxvDPTgs+2UDg4OTAiLUD0nzc2m2rUqtNNa4eNHkF+3u+MZt1iEYRz/svvwSCBFDZK2M0Z1mTfvO7M7r5NMIDw+k9O+7y4tptq+N+8zmSKJHd1TlVjmCW4mNaHW+ZvptdC/5DF6XjxvF7bPdZicLWBm2ULk5/iGVD1nFk3SkkCVQOSnNT4upRmz7JQHB25xV83V8jGkS8n/hyad81GndJuRx4SvlkA0F6IZfLGbdliPnjR5efMqnDPDSRGhS2NtRtnzLRui8kzraZe9g2ay+5C+dkxqHxOOdO+FC9ULkC7Hi1OkGBtg5Dv8G1eG7e+gbRoFPtdEsLfax4P32FLspUvScaRa4fucP8Xssx6Ax8O+Qb+vxf5dyBFcfey69IEjK5DEkUccqePruwpGA0GAkPNu0Uk9vdnjWnU4xu7Szp5NL2yZ4RZCSiIjW8fORDvuK5rZrz/px54/2WnsUHo9fqkdvIaPFzE4au6JPe0/rscbv5jBGNJ2HUG6nWshIhb0J5ePEpAHIbGftCNqKyVxL6Nox75x7z1/C/CfQNQiaXU7VZBeQKOdooHf0X9CR/ybzp/NXEJtAvmEE1xhLyJpSiFQvy84wulKpZPMm7NkmS2DF3Pxf2XqNBx1p8N7y1Ref32Z8RJIbHHU9ObjlPyWpFrS65bOegokS1olYdI72IDFOjclCm++G3TC4zi/oJgoDC9rP91c5QFK9ShH98VxEeFEF2Vxfm9FjK0+vPMOgMqBxUKJQ2hAWF07vMMCLD1eg1BuQ2MrK7OjN26xDsE5CzkCSJkDehOGRxwFaZPo2Lx9afIfh1CEaDkSfXPJjQZhbZcmVl5d152CVBBFEQBL4f1S5V3tOW4LP8awl6Hcywen+gidSgjI7cqQkGRoMRz/svye7qnG4HiWmNKIpM7biAy/9dxym7E4svTSdnAcu4tqUElzzZGLCwJ5un7sK1ZF66TUpd34goihZLAx35+xQbJ+4gb7Fc/P7P8Az1O3L3zEOObThD+Qalk+3jnVTsHO3M+kSDlvZCobThrW8QPaf+gFwu58lVD3RaPXqNSabdaBAJePkWg84Q7zMlSWLyd/O4dugWSnsli85PpUDptCkR/xCXfNmwsbUxl6qblGXDeHrNI81KPy3BZ5nwfOXx2lyPrVVreXLdI8XPMhqMDK03gWH1J9C18EDcbj6z1DQzLFGRGtxuPOPGsTsYDSIh/iHsWvBfvNK8f0/YxrfZejK84UQiQyNTPO61w7dZM2Yzjy6bUgvq8Chm91jCkLrjuXP6Aa36NmO7zyrmnpiY4k5YdXgUA6uNpoXiB8Z9M8P8B54cIkMjeXjpKZFhakICQlk8YA0BPoHcv/CEv3/fluC9b18FMbrZVPpUHMHdsw9T9DX8P74eflz+70as7/2rZ68Z32oGxzacYemvazm/+4pFxksIh8z2DF/dnxmHxpsVRguXz48kSu9z5QIo7ZUJyoq/fOLLjaN30GsNRIZEsmvBf1afe1w0+bEeP4xuS8GyrsgVpl2xaBTJVzx3uswnpXyWgaBYlcJky50Vu0x2qBxVNEnFKb3ng5e8ePASTaQWTYSGg6uOp+g5RoORSe3n0lLVmVFfTTHLY2Q0Vo3cyLdZezDqq6kYot8kZQo5B1cdp7NrPxb88leM6z3ve7F74QEiQiJ5fMWNXQsPpGjcu2ceMqXjPP6Zs59RX03Fx+0Vq0Zt5Ow/l3h0yY3fW88k6gOXp5RyYtM5XjzyRpIkHpx/zPUjd5J1f6BfMD2KDWbc19PpWfxXAv1CzK9JooROq0/w/kV9V3Ln9AM873nxe6uZiGLqfCseXXGjb8WRzOq6mN7lfiMq4r0vxKtn/mbHLl2UDq/HPqka6//xuO3JwVXH8fdKuFPYJa8zS6/NotesruQo4IJMLsOgNbDwl/iNm5xc3gd6W5VtkrwFrIFMJqPrhI6svreA8VuH0nFEGxacnZJqxde05pMPBHEdhivtlKy4PYcZB8eywW2xuXtVG6XF18MPozHpq8Acri7m7bTSXkmRFNpFXjlwk5vH72LQGXh02Y1T2y6m6DlxERESyeX/buDr4Zeq54QFhbN3yWGMBpGo8CgKlnaldO0SuOTJhl5rwKAzcHzTOYL937/5CR+kVwRIcerB/dZzczWJTC7g+cCbwFfB6HXv0wnv6v1Tg4OTvTklJIkSDk7J8469evAWGrUGdVgUmkgtHree02NyJ+wcVbiWzMtPUzvHe6+P2yuCX4cgRvtS67UG879TyoU9V9CqtajDo1CHqXG/9V62o1y9kmTP54ydowr7zPY07lw3wWddP3qH7bP24uOe+O/Ro8tPGVrvd/4atp6+FUcQ9Do4wevzl8zL9yPbEhoQjmgQ0Wp03Dp5P97rs2R3Ytp/Y6navALtfm1Jp5FtE52TtanXoSZ95nSjWOXC6T2VZPPJnhH4uL1iZJPJBPuH0GV8B7pPjGnyrbRTxnDiev3iDQOrj0Gr1pKveB4WX5qepIamzM6ZmHd6EgdXHadIxUK07tcsRfO1tXs/liCAyt4yzVRhQeF0ztcXnUaPTC5j/pnJlK1TMkXPUtrZYqOQY9AZUChtKF6lMMNW9WPZkHUcXH0CvUaPjcIGuw8MaAqWceWHMd+yb/FhilQsmOKqiNptq7Fpyk5sVQpU9koqNCxNniI5eXDhCeqwKNoNakGW7PHbNyaVRp3r4HbDgxvH7vJV9wbJttgsXP59D4QkSRQql5/iVYokehi4bMg6Dq85iSiKqByUGHQG+i3omaDKbFIoV680/y4/hkFn2onkL/W+8sa0IJqL95NX5CyYPUHD9Ev7rzPjxz8xaPVsm7WXjR5LEzzruH3qATqNHkmUkCvkuN/ypMbX8ZfzRoap8XvmT63WVbhy8BZIEo1+iF94DqBio7IfVR4+I/PJlo9O6TifC3uuIkkSNrY2bPNekeAbxeZpu9g0eSeiUcQuk4pJu0dSuWn51E49yUiSxOrRmzm38zLVWlZi0JKfLVKJs278NrbN3GP+uHbbakzeOyrFz7t37hF//76NXIVyMGhJLxwy26NRa1kzZjOvPF7TdcJ3lK5VItXzjouQgFC8HvpQtHIh85uWKIrotfokS3qnBTeO3eXqoVvU/KZyvPpFHyJJEi1sfzCv/vOXysuaBwstdnB78/hd3G95Urd9DfIVS1nuesVv69m98CAA9pntmH5gbIKWpk+uuTOi8SQEQcDG1oZ1j/+M12f5jfdb+lcehV6nJ3O2TPSZ2w3HLA5UalIuTV3nPkU++/JRxywOyG1kJss+mWBW74yPPEVyYatSoInUIhpFq9gIJoQgCPSZ0y1Wg01KkSQJvVZPjvwxc5WFy6dO37x8/dIsPDc1xudU9koGLba+14OTS2byFs+Nrer9z1Imk2WoIABQtVkFqjZLPAC8QxAEchbMjv+LAOQ2MgqVL2DRN8AqX1VIUkBKiPrf1eLAyhPI5TIcnOwpUrFggteXrF6MJVdm4n7zOZWblos3CAAcXHkcdbgag86UkhVksjgXYQ8vPeW15xtqtq6CQ2Z79Do9d888wjl32un2f6qk+Y5AEARXYCOQCxCBVZIk/ZnQPSnZEYQHRzCnx1L8nvvz84wu1G5TLcHrJUli/9LD3D37iJa9miAaRdb/sZ18xXMzbFW/BLfNGY0An0CG1v2dtz6B1G1fE4PRwO3j96nYuCyT9476KFdZRoORUV9N4fEVd+wz27H06kxyFcyR3tNKMf5eATy89JRSNYuRu1BOAnwC2TpjN5myOtJ5XPsk1aCnNa+eveblY1/K1StpscbIf+bsY/2E7WYZFqW9LUuuzIwlM3Jy63kW9lmBIAg4587KmocLGdlkMs/uvEAURYav7kcOVxcW9l2JXSY7xm4eTN6iH1fljjXIsFaVgiDkBnJLknRLEIRMwE2gnSRJj+K7x5qdxWd3Xub4xjNUa1GRNgNaIAgCESGRfJ+nDzqNDhtbG77p0zRNVrwpITJMzd3TD8lXIo+583LVqI3sXngQ0WjKN88/M9lqZuBpxYOLTxjXcjpRERpkMoFOo9rSa8aP6T2tFPH6xRv6VhhhLmRYfnNOilM2HzvtnX8iPNgkpa1QKpj67+g4dy8T2sziyoGbgOmsauH5qQyuPd7ca1C+QWk873kRHhyJIAiUqlmMPy9OT7svJIOS1ECQ5lVDkiT5SZJ0K/rf4cBjIF16xz1uezL3p6VcPXiLNWO2cPk/U7DRRumQpGhVSr2RkIAwq84jOVVKH6JRa+lbYQSzui9hQNXR5rrzLDmyYBPdWSuKUpziXh8bznmymnPoCpWC3IVzpfOMUs69s48QRZGoCA2iUeTOqQfpPSWLog6PSrRK6B2upfIiV8ixUcgpUb1IvCms2u2qoXJQorS3xSlHZvKXzkfWnE7IFXKU9koqNymHTmsKCpIkxZKB/0LCpOsZgSAIBYFKwNU4XusD9AHIn986apT+XgHmUkGjwcjr528Akxxyu1+/ZvfCAzi5ZKbHpE4JPSbFhAdHMLzBRLwevqTGN1WYtHdksg6IvR56Ex4UQVS4qTb8zD+XqNCgDN8ObslrT3+eXPPgu99ap1uNdUp4+yqIf5cdIUtOJ9r0b26umsldKCcTdv7G/qWHKVOnJC1+trxHbUox6A2c23UFuVxG3Q41Ev0ZvrNWVCgVIAiUqWOdw/X04NaJe/zRbjaiQaRV/2aJGiRN3T+aLdN3IwgCXca3j/e6lj83IWeBHLx+7k/d9jVQqmxZenUmxzeewyVvNhp3qUve4rlZ0HsFSnslQ1f2tfSX9kmTblVDgiA4AmeB6ZIk7UnoWmulhjRqLb/WHIvfM38cszry183ZZP1A/c9oMCKTy6yWU985/z/+Hr8Vvc6AnaOKibtHJOtQLyIkkm5FBqKJ0CBX2DB6wyDqdahplbmmBZIk0bXwAN76BGFja8PXvZsw8M+Mb0M4+bt53Dhqajxr0Kl2kuTPXzz05s7pBwS+CuLU1gsUqViQsZsHm6UYPlYG1x7P4ytugEn/6d+wjbEO828ev8uK3zbgnCcbozcMivE39wXLkqGrhgRBUAC7gS2JBQFrorI31VG/9QkiW+4ssdyprG1mkymbIzIbOegMiKKEYzJTOI5ZHFh+Yzbnd12hULn8VGtRKcbrURFRKJSKVNeiWwudVs+eRQcJDQil/dBWZHZ25K13IKIooYvSmaUkMjo3j901pyKuHryVpHsKlnHFVqXgl/K/oYvSEewfwo65/9Jj8vfWnKrVyVs8Nx63n6PXGnDIbB+rWk+v0zOx3Ry0UTq8n/iyZNBa/tj5W6rHDfYP4dFlN4pVLkSO/OnOUfgSAAAgAElEQVSnefWxkubvEIJpeb0WeCxJ0oK0Hv//kcvl6SaW9lX3+jy/+4Lbp+5TvFoR9i09TP0ONanVOtEAbiZ3oZxxdlWuHLGBvYsPo3JQMufEHxnysHjZ4HWc2HQWg97IuV1X2Oy5nNrtqnPj6B0kUaK9BRyy0oLq31Tm6oFbIJj6NJKKXqt/J5iK0SDi6/4Ko9GY7kquqeHXJb2wd1SZpDYmdYol3Gc0iGZpEqNBNB8Ug6n58dDqEzg4OVCwTD4CfIKo2aoK9tENihf2XuXwmpNUalKODsNamXfqgX7B9C47zNR5Lkksuz4L1xLJO3Z8Jwd9+9QDvu7dhPrf1UrNtyFRIkIiCXwVRL4SeTLEzzs9qobqAueB+5jKRwHGSZJ0KL57PnY/gsS4d+4R476egVatRWlvy/zTk1MlWx0WFM73uX8xl+RVa1GRGYfGW2q6FqN/lZF43H4BgCATOKjegtxGztPrz8iUzfGjqaQxGoxc2n8dmVxGrTZVk6VaumrkRvb8eRDRKKG0t6VSk3JM2TfairNNf3YvOsCa0ZtxzOrI7GMTzL0tfSr8hvfTV4BJuM1WZUt2V2dW35+Pz9NX0Z3/OlT2Soav6W/uPD7zz0UW/LKCqAgNCqWCX2Z35dvBXydrTqe2nmdh35VoIk1/g0uvzqJgGeuomXrc8WR4gz+QRIkiFQsy//Rkq2UfMnLV0AVJkgRJkspLklQx+r94g8DngPcTX5OpMaYGqXd/DClFaWeLTXSaS2FrQ3ZX6zXHud96zqYpO7l9Km5dGB93P05sPseDi49jVUd1HNHGJBnhoKTRD3VR2CqQyWSUqlHsowkCYEoh1utQkzrtqscbBMKDI/C444leF1N0rs/c7hSuUNBc6XL1wE20UZ92xUuHoa04pNnGztdrzEFAFEVePPTGoDNpVolGEU2kBv8Xbwh8Fcwb70CzQJ5eZ8D/xRvz84pVKYwkmaQsZHKB0rWKJ3tO/l5v0UdXHcnkMt76BlngK42bAyuOERWuQROp5dmdF3g+eGm1sZJKxkwef2bUbluNDRN3IJPLcMjiQPWvKyV+UwIo7ZTMPDyOvydsJ1ehnPSd191CM43Jq2evGd7gD7RqHbZ2CmYcGk/5+qXNr98//5jRzaaaUiAClK5dkgVnJ5vfLBt3rkepmsWJDFVTpEJBq8wxI/DioTdD6oxHEiVyF87JkqszsVUqzPlyj9ueyGQCgsxkyJIUjauPnf8vwJDJZNTrUJNrh2+begMkCZmNnKw5nXDOnZUsOZxwLZGXFw9e4pjFgabd6pvvzVs0N39enM6No3coW69UinbTX/VowL6lhwkPiqBAqXyUb1A68ZtSSIEyrijtlWaRRJe82aw2VlL5EggyAFlzZmHjs6W88nhN3mK5LCKZULZuKeafnmyB2cXPs7teCDIZkiRh1Bt5ev1ZjEBwatsF9NGyy5IEbjc88HX3i5G/zV3o4yltTSlH159GHWYq8fXz9MftxjPK1inJ+V1XuH/+sUmYzdaG2m2qMmhJr3Tt/A4LDGdMi2l4PfLhm1+a0n9hzzSbz/htQ7l//jEOTvYE+4fi/yKA+t/VRG4jR24jZ8mVGQS+CiZLjsyxCjsKly+QKvkUlzzZ2PJiOSEBYWTLlcWq3tRtB7ZANIi43/akzYDmFhFLTC1fAkEGQWWvTLUOUEJoo7Q8vPiUXIVykKeIZZqxKjQojdJOgSCYDttqtqoc4/WKjcpyZO1J81mFja0N2RIxlP8UKVy+gHkFKIkSuQqaihM+VJy1Uchp2q2BxUopfdz9uH3yPmVql0jW79WePw/iec8Lg97IoTUnadazIUVTKK2eXGQyGRUalEnw9ez5rKfzb6OwwSWPZVfnGrVJu8z+A0VemUxGh2EZqxDiSyD4CAgPjmBR35W8fhFA71k/UqlxuWTdb9AbGFh9LG9eBiAaRab9N9Yi8r2ZnTOx9tEinl7zoFD5ArjkyYaPux9atZbC5QvQoGMtlHa2HFx1HJWjki5j28ep2fTioTebp+0iez5nekz+X3v3GRhVsTZw/D8pu9k0eouRIgjSpCkliiJIEwFBEQtdioKiIFxRpFvoqEjRCwiI3oigAq9UKYqggPSEGggJBAhJKGmbrfN+2BAJkJCyJdmd3xdIsmdmDmR3zpkzz/P0zHPh7+Li6V5PYDKYOL7nNB0HtM4qWhLW9VE6DGjN7rX7ePy5pjR/told+ouPSWBok/9gsVgRQvDZzilZNTcAEuOS2P6/XYTUqEhY10ezXfFrdJrMGhIWkPKeyRpdwWwys2/jIYJKBeSaAdXVdv60h097fY60SIbM7sNzwzq6ekg5cts01M5gsVhITkyhRLlgh95KTu/3Jdv/9ydmkwWtv5bVCYvztXwUfTSG4WFjs/a6P937Cd5b9pbdx7l2wSa+GrUcIQTt+j7J8HmD7nmMxWKhR4WBpF5LxUfjS9s+TzLCTaNCrVYrFrPljmWNvLhZ8jI5KYUhM/vkWvzk95W7mTVoIfoUPb5aHwZ+2ovu73QCwJhh5NWqQzP/vX0YMrMPzw5pR1zUJXaE7yakRgV+X/kXp/afofvbnQpcP8KR3u/4EZG7TtoCEMf3oGcRKEpzN72rD+NytO2htl+AlnUpK5w+hiK7a6g4uXr5GgtHLWP5pJVk3Fb9Ku1GGgPrjuTVKm8w+OF3SU/R59BK4V1PuJG1vGIxmbN2N+RV+Srl8NH44OUl0AZos63j29OqWesw6o0Y0g2sX7Q1T8cY9UbSk9OR0ravPvaYfcslFhUn90XRvUx/ng3sxfefrM738TMHzGf3mr0c3hHJe+2m3LXy3k11wmrZihsFaPH28aZRm3/v/hIuJKFP1WM2WchIM/DP5sOkXk9j2KNjWD5pJbMHLqTl8835PmZhkZwELBYL+zcfQZ9q23WzZdkO14zDbOFKbAJmU87vxXKhZfDOzExQ1KOn1USQi1GtJ/HLF+sJn/oLM/vPy/aznav3kHghybaVLSaBv9Y67o7ltU9etUUhe3vx0vvdCCyZcwrg+JgEtofvylYnNiDYn3l7p9JrfA9GLxlGhwGtHTLOmk0ewFfri7ePN/fXCsnTMbpAHe37t0bj54tWp6H3hB4OGZujJSelMKr1RHqGDGLtgk13/Pzr0d+SdiMdq9nK8ok/3nFhcS9JF69mlepMuZZK1MHoHF9bLrQMs3+fTEj1ipQoF5xtO3LFquWpWK0CuiA/tP4a2vVtxcUzl7FarbYtm+kGju7MMRGwy1nMVh5oUAWNzrbt+JEODZ0+hrTkdAbWG0H/2u/Qr9Zwkq+m3PV1Y8NH8PjzzWjWqTGfbCh6cTy3cvtnBKnX07h6+TqhNSvle/nmYtQlLGYrlswn/Lcqd38ZboaFSpn5tYNUb1CV1QlLsJgtuaaLuHQ2niENR4EAgWDhwRlZCedCqlek93jHfsiOXjqM1XN+JT1Fzwsj8/4wbMRXQ3h1bHf8g/1zneSKsuWTVhKx6wQWk4WF7y4jrMsj2QqYl6poy5RpMVnQ+Pniq8nfW+/1Wf14r93kzOhZGP30JFbFL87x92Htgk3EHr+A2WRhap+5NGpTn6BSgXj7ePPlnk85tC2CClXLUa1eZYwZRkpXLMVVrmG1SFq/0rJQ/xaOsnHJNj5/42uEtxft+rbi4Za1aXWPcpaOsOf/9pMYdxWj3sj1+BvsXPU3nQa3veN1ZSqV4sPwkU4fX0G49URwav8ZRj01EatVUrvZg0zbMi5fk0GnIW3Z9M12pBV6/if7OmSTtg0YNK0Xf/60hyd7hjlsueUmIcQ9cwYd3BaB1WrNir48vCPS7plHr5xPZPcv+6hWvzINWmXf4aHVaXnlg5wzSOamuOeHsZgsWUGBYEv/favh8wZhtVhJjLvG4Bm98x1J2qBVXeqE1eLoH8cByEgzYNAbc/ydSE/WY7lZ+F7KrLz9YNuhduuDaY2fhgUHphPx5wlCH6xUZLPVLhix1LZEarKwf9Nh3lkw2CXjqFC1fNZ/tfASVKxWfAsk3eTWE8EvczegT80A4Pie05w/eZEqtUPzfPybX7zGs0Pa4eevveubo8vQDnQZ2sFu4y2sumE1gX/LctYuQIRlbtJupPFG49FkpBoQ3oIPvn/nnpXfPEXvCT04sS+Ki1GXeWlMN8rfFs0dXCaI8T+OKlQffSf1ZGynT7FarLTv1yrXqnkDPnqZE3tOkxh3lZff737PNWpdgB+Ptnf+Mkt+BJcJQp+qz1xzd93e+7phtXhn4SB2/LCbsK6PFroMaFHg1ruGfpy5lmUTV2JIN+AX6Mf3MQvcokhLbqKPxnB4xzEatKpj9zqux/ecZky7KVkPxjsPbc/wLwfatQ93JqXk6uXrBJb0L3DQYHqKHn1qBmU8MB4j5vgFvnxrMRo/DW/PH1js7iJ/++4Pfpj2Cw82foC3FwxySq3tIp2G2lm6j+iEVVqJPhJLt+HPuP0kAFCtfhWHFfKuWu9+/IN1SCmxWq083q2ZQ/pxR1JKPn31c/78eQ++Wl9mbpuY6xbQnPgH6bIFJxUHV84nsnHxVsrdX5b2/Z8q8FbrKrVDmfHbBDuPzjniYxKYM/grjHojcVGXCalegV7jis7GCLeeCLy9vek5+jlXDyOL1WolMe4qJcuXQFMEA3XuRRfgx1eHZ3JgyxGq1Al12ITjjq7EJrLrl72YDLbtv+FTf2bcysLn4c8ri8UC0vE1Nm4y6A2smbeRjDQD6xZsIjkpFV+tD4lxVx2+aaEoSk/RczNuz2KykJxk22mUkW4gfOrP3EhM5qX3urksJb5bTwS5Sbp0DV2g312vrgx6A+ciLxD6YEUCSthnF4vJaOLdpyZy5mA0/sE65u2dWuxubQGCSwfRqqfzd2oUd4GlArKyZ2r8fAmp4byay/9sPszE7tOxmK2M+HoI7fq0KnBbJqMJQ7rxnru7pvaey571B5AWWxCdlGBIN3J05/EC912cVa17P21ebcmGxduoUKVsVg2Rz9/4mt9//AuL0cze9QdZET3fJbmmPDKOYN7bS+j9wFBerDSQA1uzp09OS07ntTojGN1mIr2rv5ltP35hRO46ybmjsRgzTCQnpbJxyTa7tKvkTJ+WwajWE3k2sBezBy3INQgrvxIvXmVkq/H0qzWcvRsO3vP1AcH+TN00jmadGtP1zY70nuCYOthgW4b6a90//PzFeq4n3GDusP9iSDdiNpqZOWA+iXFJ2V6/cuZaelcfxrQ+c+9Ik32rk/+c4YXyr/FChdeY+9biXMdwcm8UpgyTbZePEOgCbXELnV9vZ5dzLG6EEIz46nU2GsNZHjUva2vx2cMxmDJMWK2ShPNJWG/u9HIyj5sI9GkZrFuwCZPBjCHdyPKJP2T7+aFtESQnpaBPyUCfmsEfq/62S79lQ8tk/Sf7an2p6AFZN11t4+JtHP/7FIZ0A9t/2M2R3+0XKDV32CIid50k7vQlJr0wM9cI05vqhtXio3XvM3h6b4cuDa6dv5FPXvmM/763gjeavEdgqX+v3qVV8uXwJVlfRx2MZvnElVyOvsIfq//m169/y7Hd5RN/sG1LNVlY/9/fuJGYnONruwxrj9Zfg1+AlmeHtGXKujEsPDCjWNfUtofbn4+88kH3rGDKjgPbOG3p7nYetzSk8fPFP0hHyrU0fLU+hNbMHgEbWisEq9X2ge3t403VevapUhT6YCXGho9g3cLNPPxEnWz51BXH8PH15ubCrNloZt+mg9R9rJZdajjrUzOQmb8nVrPFNskXkcc+ezcczMorlZyUwuhvhjKm3UdIKRFCZEtRkpGWkbV2bbVYs7Zb302FKuXw1fpgMpjx8fXBLyDnXS8vvdeNph0bYzKYqPlIdZem1i7KnnwxjLqP1SI9JSPP0fiO4NbbR3MSfTSG5ZNWUjakNAM+eQVdYPbnBAe3HeX3lbtp1OZhnuzh2NqliuMYDSam953Lrp/3YrVKfDTePPXS44xaPLTQbZ+LPM8Hz3xMclIqQ2b0pvMb7e0wYvvYHr6LWQMX4OUlqFC1HAsPzmDj4m18PfpbSpYP5uP1Y7MqwEkpmd5/Htu//5MHGlRh+pbxOa7/69My+GrUMi6eiafvxJ7UDavlzNNSCiCv20c9ciLIyb5Nh/h69LeUr1yW/ywdRomywQ7vU3G8jtqXs5ZuKlQtx4qz8108Isc7tf8M8TGJPNK+AboAP1cPR3ERlX00nwx6A5O6z+BcRCwHthxm/jvfFKo9fVoGv369hR0/7MJqtXJ4RyQ/ffYrl6Lj7TTi4inqUDSzBy+kc2AvupXux+EdkfluIzkphXdafkj3sv35Yfov93x9yxea4xfoh1+Alvb9nkJKyZXYBPRpOS+DFHc1m1SnZfdm+ZoE9qw/wOzBC/lrXdFL+a44lrojyJSWnM4L5QZgNlkQAh7p0IhPfv2gwO291eIDoo/EILwEjds+zP7NR7CaLWh0GpZHfUlwmSA7jr54OH3gLCOeGIch3Zj1vUrVK7D89Jf5amfByKWsmbfRlsBNp2FRxOxcS15arVYObYtA66+ldvMH+fDZqRzeEYGv1pfZv092aGW44uLkvijefWoChnQjWn8N0zaPz3Hp5+azBnsxGU1YzFa3K0hUFKg7gnwKCPanz8QX8fbxomT5Egya1qvAbVmtVk7ujcKgN5KRZuDoH8cwpBswGc1IKYlxQc59KSVxUZe4duWG0/u+6ejO43dsj9MFFnLZIg8XMl5eXjR++mHqhtXi/Ik4jvxxDGOGibQb6fz0+a+F699NxBy7kO3DPSby/B2vsZgtjOsylfa+PRnWdIxdanAc2h5Bt9L9ea5kX8Kn/VzgdqIORvPt5B858NuRQo/JE6mJ4BYvv9+dDYZwVl5aRLV6lQvcjpeXF4+0b5C1HPF4t2Zo/bXognT4BfjxQAPnX4HOGfIVQxqMole1ofz9f/sL3d6ls/F88MzHvN/xI+KiLuXpmMZPP4y3jzdanQbhJXigQRXG/m9Evvt+dezzPNS0BsFlgugzqWeudwO3K1m+xM3s4Wj9NYTWrJTv/t1R02ca4R+kQxekQxeoo0WXOy8i/9l8mMM7IpFWybmIWDbboSjMV6OXY0g3YDFbWDou3BYBnU+XouMZ8cQ4lk9ayfjnpnFoe0Shx+VpPG776L0U5JbXYrGwfOJKju48TtdhHXmyRwsmr3mPvesPElDCnwat6vL8yM7ERJ6nYet6uWaNdARjhpFN32zPuhpfMWVVoevjTuw+g+ijsQCM7zqNxZGf3fOYqnXvZ+HBGZzcG0W9lrXvyNCZV8Flgvhs50cFOrZE2WA+2TCWVbPXUb1hVXq826VA7RQFFouFDYu2kXgxiWcHt81W/yC/SpYrwdLTczl/Io77a4XcsZMOIKCEf1ZQnpeXFwElCv97XDakNNFHYrGYLfgH+xcoD1H00VjbcRLMRgsn952xS01uT+LRE8Hlc1dY+O4yhBC8MadfgT+YNi7exuo5/4ch3cipf85QvWFVQh+sRFjXf1M0V6kdmq8U2Pbkq/XFV+uLId0AAirVKHww2/UrN7I+FG4k5BxYdLv7alTivhquvQqv37I29Vvar+i5lJL4mAT8g3ROffbzzYfh/DJ3PSaDmU1LtvNdzIJC1c7WBfhRs0n1HH9e77GHeGXs82xetoMmbR+m9SuPF7ivm0YtGcrctxZzIyGZITP7FOhCrN7jD6H114KwBcw171y4ixxP5NETwbgu04g5dh4hBPExCczfN61A7STEJWUF6Xh5eXE9/nrWPu2i4OKZy1jNmbfcEvwD/bBarcwevJCdq/bw8JN1GLdyZL6iXYd+3p9pfeYiJbzxWX8Hjbx4+GLof23LJEIwfuVImnVyzgdRxJ/Hsx68X4u/gT5Fb7fcWDl5eUw3Xh7TzW7tlSgbzIcFWB68VXDpIBYfm8OJvVFUq1+ZsiGl7TQ6z+HRzwiSLl5FWmVm5airBW7n2cFtKV2xJN4+XpSqUCLXiEtX0PjZ1uQBfLU+BJUKZP/mw/y+8i/Sk9M5uPUIW1f8ka82n+wRxpoby1lzYzltimhpQ2cwZhhZv2grxgwTRr2RbyevclrfnV9vh4+vN74aHxq0qoO/k5ccnSUtOZ3da/Zx7th5tn63k28n/3jHNuygUoE82r6hwyaBiD+PEz7tF6KPxjikfVfz6Ilg0PRe+Gh88NX6MHh6wXcJlb2vDKOWDMXb14dLZ+MZ8cR4rsTaJ1mdPZQLLcPw+YMIqV6Rph0b8crY5xGZa6o3iQIsKfhqfItlOm178tH4EFwmECFsleHslZIkL3b+tAchBBKJ1WrfLZ1FhUFv4PWGo5na5wtebzSKWYMWsGLKKt5s+j4GvSHH424kJrNyxho2Ld2elTKmoCJ3n2RMh49YOi6c4WFj3TIWyKOWhs4cPse0vnNBwn+WvknHAW14skcYQnDXh2P5cXLfGcxGM1KCl7cX0RHnHZZmeuOSbSyb+AOhD4YwbuXIPK1Lt+/3FO37PZX1dZO2D9Ou35P8/sNuGraux9O9PPOqfuG7y1jz5UZCalRg+m8T8l35y8vLizl/TOG7j1dTumJJh2YVvV3krhOYMmsRn/rnjNP6daaYYxe4kZSMPsUW/Gcx2T7UDXoDiXFX7/q8SUrJ8LCxXIlNwNvHh9gTcQyaWvALvWO7T2IxWbCYLWh0vkQdPJevnWrFgUfdEUzpMYvoI7FEH41lco+ZgK3iU2EnAYAWnZvYEtoF69DqtNSxc73gm67FX+eLYYtIvHCVo38eZ+m48AK1I4TgrbkDWXVlCR+Gj7RLIrbiJvZEHOsWbsZsMhN3+hI/FHAfe2jNEN5b9haDpvV2alBUhwGt8QvQ4hegpU0v90xieF+Nivj4eGfeufvio/FBF+RH5dqhORaNN+iNXD4bj9lowZBu4NC2wm0nbdqpMb5a23vbV+tLvccfYv+Ww8wcMI/t4bsK1fbtpJSsmbeByS/OYt+mQ3ZtOzce9e43ZPwb0apPNZCRbrDbG7da/SosjpxD9NFYareo6bCymJZbArKkVWIy5Jw/Ps9tmi0uS3/rSlqdJisgzcvbyy4XBM404ONXaNH5EawWK3Ufe8jVw7GrqEPR7Pp5L7Wb12T+/unsXPU3VevdT0iNilyJTaTuYw/h7f3v76zJaGLVrHUkXbrGCyM706hNfY79dQqr1UqnwU8XaixVaoeyKGI2UYfOUadFTa5fSWZCt+kY0o3sWPkXgaUCeLR9w8KeMmBLGLhozHdkpBnYu/4AXx2a6ZRddi65IxBCdBBCnBRCRAkhxjir39HfvElQ6UB8fL1JuZbKi5UGcWq//W6py1cuR7NOTQgunfcthPExCexes4/rCXmL+C0bUpreE3qgC/Sjcu376Du5Z0GHi8Vs4YNOn9BR+xJDGo4i7UZagdvKD6vVStSh6DsKpDhbhSrleHPua7ZnJ8805qUxRaesaV4IIajTohb1Hq9dJJ4PmE1mrl6+VugCQFdiExjRchwrpqxico+ZnD95kR6jupB6PZ3Vn/2Kxk9zx7Ophe8uY8WUVaybv4m3w8YyZd0YJqwexWc7P+KZgYWbCMD23g7r8igly5XgwqmLWdXmrGYLsXbMFBB3+hLGDNvFnZe3N/HnnPOs0el3BEIIb2Ae0Ba4AOwTQqyVUtqvakgOGrepz8ID0+n/0NsYM0zoTXrCp/7M+B9HObrruzoXeZ63mn+Al7fAR+PD4sg5lCxX4p7H2WsL3/4tR4jYeRwp4fzJi2xe9jvdhj9T6HZzI6VkwnPTObQ9AmmVjF81iqYdGzm0z9x0fK0NHV9r47L+3UV8TAJvNn+ftOtp1G5ek+lbxhf4LjPm2IWseAhjhimr2tmsgQswpBvYvHQH/z06K9s6/al/zmZ9gF5PSMZsstCkbYPCn9hdNGpT35aZWNpidFq+YL9iO0/3foJf5m7AoDcSUr0C9R53zp2eK+4ImgJRUsqzUkojEA50dVbngaUC/60dq9NwX03XFYPYu/4AJqOJ9GQ9ZoOZiD9POLX/wFIB/0aKensRVNoxy1m3upGYzD+bD5ORZsCgN/LjrLUO71NxvA2Lt5KcmILJYOb0/rMc++tUgduq06Im/sE6/IN1+PlreaxbU84eicGUubTr5S2IO3052zEvju6KVqdB66/liR4tHJp6OyDYn8WRc5j9x2S+PTuvwIGod1OpWgW+i1nAwgPTmbd3Kho/jd3azo0rnhHcB9ya0eoC0Oz2FwkhBgODASpXLnjen9v5B+mYtmU84VN/pvJD99F7fA+7tZ1fdcJq2apoSdtySY1G1Zzbf/Oa9JvyEpuWbKexnSJF7yWwZAABwTqSk1LR+PlS65GcI1mV4uNm9TJDuhGr1UrpSiUL3FZAiQAWRc7h5N4oKtcJpWxIaXx8vflx1lqklJQqX5K6j2XPjNqyezNqPfo5qdfSqFbffp8XOdH4aajR0DHvVz9/7R2VEx3N6WmohRA9gPZSyoGZX/cGmkop38rpGGcVpnGFiD+PE7HrJM06NS5UorviJC7qEj9/sZ4KVcrR/e1OHvmg2t1YrVa+/3g1EbtO0nVYB1p0vmfm43xLTkrh4pnLVKtfGa2uaAVtFlVFtkKZEKIFMFFK2T7z6/cBpJSf5nSMO08EiqIojlKU6xHsAx4UQlQTQmiAlwC1UKwoiuIiTn9GIKU0CyHeBDYB3sASKWX+6xUqiqIoduGSgDIp5XpgvSv6VhRFUbLzqBQTiqIoyp3URKAoiuLh1ESgKIri4dREoCiK4uGcHkdQEEKIBKCgpYHKAol2HE5xoM7ZM6hz9gyFOecqUsp7FkYpFhNBYQgh/slLQIU7UefsGdQ5ewZnnLNaGlIURfFwaiJQFEXxcJ4wEXzt6gG4gDpnz6DO2TM4/Jzd/hmBoiiKkjtPuCNQFEVRcuHWE4GraiO7ihDifiHEdiHEcSFEpBDibVePyRmEEN5CiINCiP9z9RqyWEEAAAOgSURBVFicQQhRUgixSghxIvP/uoWrx+RoQogRmb/TEUKI/wkhHFeCzEWEEEuEEFeEEBG3fK+0EGKLEOJ05p+lHNG3204Et9RG7gjUAV4WQtRx7agczgy8K6WsDTQHhnnAOQO8DRx39SCc6HNgo5TyIaABbn7uQoj7gOHAI1LKetiyFr/k2lE5xFKgw23fGwNslVI+CGzN/Nru3HYiwMW1kV1BSnlJSnkg8+8p2D4g7nPtqBxLCBEKdAIWuXosziCECAaeABYDSCmNUsrrrh2VU/gAOiGED+APXHTxeOxOSvkHcPW2b3cFlmX+fRnwnCP6dueJ4G61kd36Q/FWQoiqQCNgj2tH4nCfAf8BrK4eiJM8ACQA32Quhy0SQgS4elCOJKWMA2YCscAl4IaUcrNrR+U0FaSUl8B2oQeUd0Qn7jwRiLt8zyO2SAkhAoHVwDtSymRXj8dRhBDPAleklPtdPRYn8gEaAwuklI2ANBy0XFBUZK6LdwWqASFAgBCil2tH5V7ceSK4ANx/y9ehuOHt5O2EEL7YJoHvpJQ/uXo8DvYY0EUIcQ7b0l9rIcQK1w7J4S4AF6SUN+/0VmGbGNzZ00C0lDJBSmkCfgLCXDwmZ4kXQlQCyPzziiM6ceeJwONqIwshBLa14+NSytmuHo+jSSnfl1KGSimrYvv/3SaldOsrRSnlZeC8EKJW5rfaAMdcOCRniAWaCyH8M3/H2+DmD8hvsRbom/n3vsAaR3TiklKVzuChtZEfA3oDR4UQhzK/90FmaVDFfbwFfJd5gXMW6O/i8TiUlHKPEGIVcADbzriDuGGEsRDif0AroKwQ4gIwAZgKrBRCvIZtQuzhkL5VZLGiKIpnc+elIUVRFCUP1ESgKIri4dREoCiK4uHURKAoiuLh1ESgKIri4dREoCiFIITYKIS47imZTxX3pCYCRSmcGdhiNxSl2FITgaLkgRDiUSHEESGEnxAiIDM3fj0p5VYgxdXjU5TCcNvIYkWxJynlPiHEWuAjQAeskFJG3OMwRSkW1ESgKHk3GVsOqwxshVIUxS2opSFFybvSQCAQBLhdqUTFc6mJQFHy7mtgHPAdMM3FY1EUu1FLQ4qSB0KIPoBZSvl9Zj3s3UKI1sAk4CEgMDNj5GtSyk2uHKui5JfKPqooiuLh1NKQoiiKh1MTgaIoiodTE4GiKIqHUxOBoiiKh1MTgaIoiodTE4GiKIqHUxOBoiiKh1MTgaIoiof7f+SMdxsa/De7AAAAAElFTkSuQmCC\n", 1167 | "text/plain": [ 1168 | "
" 1169 | ] 1170 | }, 1171 | "metadata": { 1172 | "needs_background": "light" 1173 | }, 1174 | "output_type": "display_data" 1175 | } 1176 | ], 1177 | "source": [ 1178 | "import matplotlib.pyplot as plt\n", 1179 | "plt.scatter(x[:,0],x[:,1],c=y, s=8);\n", 1180 | "plt.xlabel(\"x1\"); plt.ylabel(\"x2\");" 1181 | ] 1182 | }, 1183 | { 1184 | "cell_type": "code", 1185 | "execution_count": 53, 1186 | "metadata": {}, 1187 | "outputs": [ 1188 | { 1189 | "data": { 1190 | "text/plain": [ 1191 | "Sequential(\n", 1192 | " (0): Linear(in_features=2, out_features=2, bias=True)\n", 1193 | " (1): Sigmoid()\n", 1194 | " (2): Linear(in_features=2, out_features=1, bias=True)\n", 1195 | ")" 1196 | ] 1197 | }, 1198 | "execution_count": 53, 1199 | "metadata": {}, 1200 | "output_type": "execute_result" 1201 | } 1202 | ], 1203 | "source": [ 1204 | "model = torch.nn.Sequential(\n", 1205 | " torch.nn.Linear(2, 2),\n", 1206 | " torch.nn.Sigmoid(),\n", 1207 | " torch.nn.Linear(2, 1)\n", 1208 | ")\n", 1209 | "model" 1210 | ] 1211 | }, 1212 | { 1213 | "cell_type": "code", 1214 | "execution_count": 54, 1215 | "metadata": {}, 1216 | "outputs": [], 1217 | "source": [ 1218 | "x, y = gen_nn_fake_data(10000)\n", 1219 | "x = torch.tensor(x).float()\n", 1220 | "y = torch.tensor(y).float()" 1221 | ] 1222 | }, 1223 | { 1224 | "cell_type": "code", 1225 | "execution_count": 55, 1226 | "metadata": {}, 1227 | "outputs": [], 1228 | "source": [ 1229 | "learning_rate = 0.01\n", 1230 | "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)" 1231 | ] 1232 | }, 1233 | { 1234 | "cell_type": "code", 1235 | "execution_count": 56, 1236 | "metadata": {}, 1237 | "outputs": [ 1238 | { 1239 | "name": "stdout", 1240 | "output_type": "stream", 1241 | "text": [ 1242 | "0.5998386144638062\n", 1243 | "0.08418773114681244\n", 1244 | "0.04379989951848984\n", 1245 | "0.02649916149675846\n", 1246 | "0.0179321076720953\n", 1247 | "0.013030633330345154\n", 1248 | "0.009794511832296848\n", 1249 | "0.007520528510212898\n", 1250 | "0.0058765956200659275\n", 1251 | "0.0046782889403402805\n" 1252 | ] 1253 | } 1254 | ], 1255 | "source": [ 1256 | "for t in range(10000):\n", 1257 | " # Forward pass: compute predicted y using operations on Variables\n", 1258 | " y_hat = model(x)\n", 1259 | " loss = F.binary_cross_entropy(F.sigmoid(y_hat), y.unsqueeze(1))\n", 1260 | " if t % 1000 == 0: print(loss.item())\n", 1261 | " \n", 1262 | " # Before the backward pass, use the optimizer object to zero all of the\n", 1263 | " # gradients for the variables\n", 1264 | " optimizer.zero_grad()\n", 1265 | " loss.backward()\n", 1266 | " \n", 1267 | " # Calling the step function on an Optimizer makes an update to its\n", 1268 | " # parameters\n", 1269 | " optimizer.step()" 1270 | ] 1271 | }, 1272 | { 1273 | "cell_type": "code", 1274 | "execution_count": 57, 1275 | "metadata": {}, 1276 | "outputs": [ 1277 | { 1278 | "name": "stdout", 1279 | "output_type": "stream", 1280 | "text": [ 1281 | "[Parameter containing:\n", 1282 | "tensor([[-4.4651, -1.2217],\n", 1283 | " [ 0.7295, 6.5536]]), Parameter containing:\n", 1284 | "tensor([ 33.2324, -42.4598]), Parameter containing:\n", 1285 | "tensor([[-25.9705, 23.0225]]), Parameter containing:\n", 1286 | "tensor([-12.2211])]\n" 1287 | ] 1288 | } 1289 | ], 1290 | "source": [ 1291 | "print([p for p in model.parameters()])" 1292 | ] 1293 | }, 1294 | { 1295 | "cell_type": "code", 1296 | "execution_count": 58, 1297 | "metadata": {}, 1298 | "outputs": [ 1299 | { 1300 | "data": { 1301 | "text/plain": [ 1302 | "array([ 72.7843, -134.4691, -45.7142])" 1303 | ] 1304 | }, 1305 | "execution_count": 58, 1306 | "metadata": {}, 1307 | "output_type": "execute_result" 1308 | } 1309 | ], 1310 | "source": [ 1311 | " np.array([72.7843, -134.4691, -45.7142])" 1312 | ] 1313 | }, 1314 | { 1315 | "cell_type": "markdown", 1316 | "metadata": { 1317 | "collapsed": true 1318 | }, 1319 | "source": [ 1320 | "# References\n", 1321 | "* https://pytorch.org/docs/stable/index.html\n", 1322 | "* http://pytorch.org/tutorials/beginner/pytorch_with_examples.html\n", 1323 | "* https://hsaghir.github.io/data_science/pytorch_starter/" 1324 | ] 1325 | }, 1326 | { 1327 | "cell_type": "code", 1328 | "execution_count": null, 1329 | "metadata": {}, 1330 | "outputs": [], 1331 | "source": [] 1332 | } 1333 | ], 1334 | "metadata": { 1335 | "kernelspec": { 1336 | "display_name": "Python 3", 1337 | "language": "python", 1338 | "name": "python3" 1339 | }, 1340 | "language_info": { 1341 | "codemirror_mode": { 1342 | "name": "ipython", 1343 | "version": 3 1344 | }, 1345 | "file_extension": ".py", 1346 | "mimetype": "text/x-python", 1347 | "name": "python", 1348 | "nbconvert_exporter": "python", 1349 | "pygments_lexer": "ipython3", 1350 | "version": "3.6.6" 1351 | }, 1352 | "nav_menu": {}, 1353 | "toc": { 1354 | "nav_menu": { 1355 | "height": "116px", 1356 | "width": "251px" 1357 | }, 1358 | "number_sections": true, 1359 | "sideBar": true, 1360 | "skip_h1_title": false, 1361 | "toc_cell": true, 1362 | "toc_position": {}, 1363 | "toc_section_display": "block", 1364 | "toc_window_display": false 1365 | }, 1366 | "widgets": { 1367 | "state": {}, 1368 | "version": "1.1.2" 1369 | } 1370 | }, 1371 | "nbformat": 4, 1372 | "nbformat_minor": 1 1373 | } 1374 | --------------------------------------------------------------------------------