├── README.md ├── edge_list_short.txt ├── hetero-intro.ipynb ├── hetero-attention.ipynb ├── basics-of-graphs.ipynb └── GCN_hetero.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # graphs 2 | This is the repo for the post "deep learning on graphs" 3 | -------------------------------------------------------------------------------- /edge_list_short.txt: -------------------------------------------------------------------------------- 1 | 0 1 2 | 0 2 3 | 1 2 4 | 1 3 5 | 1 4 6 | 2 4 7 | 2 0 8 | 2 1 9 | 3 4 10 | 3 1 11 | 4 3 12 | 4 0 -------------------------------------------------------------------------------- /hetero-intro.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### introduction to Heterogenous Graphs\n", 8 | "this note book provide the details of the heterograph of the acm dataset that is discussed in the blog." 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 77, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "%matplotlib inline\n", 18 | "import dgl\n", 19 | "import numpy as np\n", 20 | "import scipy.sparse as sp\n", 21 | "# Creating from networkx graph\n", 22 | "import networkx as nx\n", 23 | "import scipy.io\n", 24 | "import urllib.request\n" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "name": "stdout", 34 | "output_type": "stream", 35 | "text": [ 36 | "['__header__', '__version__', '__globals__', 'TvsP', 'PvsA', 'PvsV', 'AvsF', 'VvsC', 'PvsL', 'PvsC', 'A', 'C', 'F', 'L', 'P', 'T', 'V', 'PvsT', 'CNormPvsA', 'RNormPvsA', 'CNormPvsC', 'RNormPvsC', 'CNormPvsT', 'RNormPvsT', 'CNormPvsV', 'RNormPvsV', 'CNormVvsC', 'RNormVvsC', 'CNormAvsF', 'RNormAvsF', 'CNormPvsL', 'RNormPvsL', 'stopwords', 'nPvsT', 'nT', 'CNormnPvsT', 'RNormnPvsT', 'nnPvsT', 'nnT', 'CNormnnPvsT', 'RNormnnPvsT', 'PvsP', 'CNormPvsP', 'RNormPvsP']\n" 37 | ] 38 | } 39 | ], 40 | "source": [ 41 | "data_url = 'https://data.dgl.ai/dataset/ACM.mat'\n", 42 | "data_file_path = 'ACM.mat'\n", 43 | "\n", 44 | "urllib.request.urlretrieve(data_url, data_file_path)\n", 45 | "data = scipy.io.loadmat(data_file_path)\n", 46 | "print(list(data.keys()))" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 78, 52 | "metadata": {}, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "text/plain": [ 57 | "Graph(num_nodes={'author': 17431, 'paper': 12499, 'subject': 73},\n", 58 | " num_edges={('paper', 'written-by', 'author'): 37055, ('author', 'writing', 'paper'): 37055, ('paper', 'citing', 'paper'): 30789, ('paper', 'cited', 'paper'): 30789, ('paper', 'is-about', 'subject'): 12499, ('subject', 'has', 'paper'): 12499},\n", 59 | " metagraph=[('author', 'paper'), ('paper', 'author'), ('paper', 'paper'), ('paper', 'paper'), ('paper', 'subject'), ('subject', 'paper')])" 60 | ] 61 | }, 62 | "execution_count": 78, 63 | "metadata": {}, 64 | "output_type": "execute_result" 65 | } 66 | ], 67 | "source": [ 68 | "G = dgl.heterograph({\n", 69 | " ('paper', 'written-by', 'author') : data['PvsA'],\n", 70 | " ('author', 'writing', 'paper') : data['PvsA'].transpose(),\n", 71 | " ('paper', 'citing', 'paper') : data['PvsP'],\n", 72 | " ('paper', 'cited', 'paper') : data['PvsP'].transpose(),\n", 73 | " ('paper', 'is-about', 'subject') : data['PvsL'],\n", 74 | " ('subject', 'has', 'paper') : data['PvsL'].transpose(),\n", 75 | " })\n", 76 | "G" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 11, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "data": { 86 | "text/plain": [ 87 | "['author', 'paper', 'subject']" 88 | ] 89 | }, 90 | "execution_count": 11, 91 | "metadata": {}, 92 | "output_type": "execute_result" 93 | } 94 | ], 95 | "source": [ 96 | "G.ntypes" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 17, 102 | "metadata": {}, 103 | "outputs": [ 104 | { 105 | "data": { 106 | "text/plain": [ 107 | "tensor([ 0, 1, 2, ..., 17428, 17429, 17430])" 108 | ] 109 | }, 110 | "execution_count": 17, 111 | "metadata": {}, 112 | "output_type": "execute_result" 113 | } 114 | ], 115 | "source": [ 116 | "G.nodes('author')" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": 112, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "def author(number):\n", 126 | " return data['A'][number][0][0]\n", 127 | "def find_author_by_name(name):\n", 128 | " for i in range(17430):\n", 129 | " if author(i).find(name) >= 0:\n", 130 | " print(i, author(i))" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 113, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "name": "stdout", 140 | "output_type": "stream", 141 | "text": [ 142 | "5100 Jack J. Dongarra\n" 143 | ] 144 | } 145 | ], 146 | "source": [ 147 | "find_author_by_name(\"Dongarra\")" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 104, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "def institution(number):\n", 157 | " return data['F'][number][0][0]\n", 158 | "def find_institution_by_name(name):\n", 159 | " for i in range(len(data['F'])):\n", 160 | " if institution(i).find(name) >= 0:\n", 161 | " print(i, institution(i))" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 115, 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "def subject(number):\n", 171 | " return data['L'][number][0][0]" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 105, 177 | "metadata": {}, 178 | "outputs": [ 179 | { 180 | "name": "stdout", 181 | "output_type": "stream", 182 | "text": [ 183 | "231 Indiana University\n", 184 | "232 Indiana University Purdue University Indianapolis\n" 185 | ] 186 | } 187 | ], 188 | "source": [ 189 | "find_institution_by_name(\"Indiana\")" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 107, 195 | "metadata": {}, 196 | "outputs": [ 197 | { 198 | "name": "stdout", 199 | "output_type": "stream", 200 | "text": [ 201 | "5100 Jack J. Dongarra\n" 202 | ] 203 | } 204 | ], 205 | "source": [] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": 108, 210 | "metadata": {}, 211 | "outputs": [ 212 | { 213 | "name": "stdout", 214 | "output_type": "stream", 215 | "text": [ 216 | "6313 David S. Wise\n", 217 | "9342 Charles Wiseman\n", 218 | "11618 G. Bowden Wise\n" 219 | ] 220 | } 221 | ], 222 | "source": [ 223 | "find_author_by_name(\"Wise\")" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": 35, 229 | "metadata": {}, 230 | "outputs": [ 231 | { 232 | "data": { 233 | "text/plain": [ 234 | "Graph(num_nodes={'author': 17431, 'paper': 12499},\n", 235 | " num_edges={('author', 'writing', 'paper'): 37055},\n", 236 | " metagraph=[('author', 'paper')])" 237 | ] 238 | }, 239 | "execution_count": 35, 240 | "metadata": {}, 241 | "output_type": "execute_result" 242 | } 243 | ], 244 | "source": [ 245 | "G['writing']" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": 57, 251 | "metadata": {}, 252 | "outputs": [ 253 | { 254 | "data": { 255 | "text/plain": [ 256 | "(tensor([5100]), tensor([9626]))" 257 | ] 258 | }, 259 | "execution_count": 57, 260 | "metadata": {}, 261 | "output_type": "execute_result" 262 | } 263 | ], 264 | "source": [ 265 | "G['writing'].out_edges(5100)" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": 58, 271 | "metadata": {}, 272 | "outputs": [ 273 | { 274 | "data": { 275 | "text/plain": [ 276 | "array([array([\"'Bi-objective scheduling algorithms for optimizing makespan and reliability on heterogeneous systems We tackle the problem of scheduling task graphs onto a heterogeneous set of machines, where each processor has a probability of failure governed by an exponential law. The goal is to design algorithms that optimize both makespan and reliability. First, we provide an optimal scheduling algorithm for independent unitary tasks where the objective is to maximize the reliability subject to makespan minimization. For the bi-criteria case, we provide an algorithm that approximates the Pareto-curve. Next, for independent non-unitary tasks, we show that the product { failure rate }x { unitary instruction execution time } is crucial to distinguish processors in this context. Based on these results we are able to let the user choose a trade-off between reliability maximization and makespan minimization. For general task graphs we provide a method for converting scheduling heuristics on heterogeneous cluster into heuristics that take reliability into account. Here again, we show how we can help the user to select a trade-off between makespan and reliability. '\"],\n", 277 | " dtype='\n", 512 | "\n", 525 | "\n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | " \n", 564 | " \n", 565 | "
0123
sosp80.02.016.02.0
soda0.096.02.01.0
sigcom7.04.086.02.0
vldb0.01.02.097.0
\n", 566 | "" 567 | ], 568 | "text/plain": [ 569 | " 0 1 2 3\n", 570 | "sosp 80.0 2.0 16.0 2.0\n", 571 | "soda 0.0 96.0 2.0 1.0\n", 572 | "sigcom 7.0 4.0 86.0 2.0\n", 573 | "vldb 0.0 1.0 2.0 97.0" 574 | ] 575 | }, 576 | "execution_count": 46, 577 | "metadata": {}, 578 | "output_type": "execute_result" 579 | } 580 | ], 581 | "source": [ 582 | "import pandas as pd\n", 583 | "df = pd.DataFrame(mat, index =['sosp', 'soda', 'sigcom','vldb'])\n", 584 | "df" 585 | ] 586 | }, 587 | { 588 | "cell_type": "code", 589 | "execution_count": null, 590 | "metadata": {}, 591 | "outputs": [], 592 | "source": [] 593 | } 594 | ], 595 | "metadata": { 596 | "kernelspec": { 597 | "display_name": "Python 3", 598 | "language": "python", 599 | "name": "python3" 600 | }, 601 | "language_info": { 602 | "codemirror_mode": { 603 | "name": "ipython", 604 | "version": 3 605 | }, 606 | "file_extension": ".py", 607 | "mimetype": "text/x-python", 608 | "name": "python", 609 | "nbconvert_exporter": "python", 610 | "pygments_lexer": "ipython3", 611 | "version": "3.7.4" 612 | } 613 | }, 614 | "nbformat": 4, 615 | "nbformat_minor": 2 616 | } 617 | -------------------------------------------------------------------------------- /basics-of-graphs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stderr", 10 | "output_type": "stream", 11 | "text": [ 12 | "Using backend: pytorch\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "import dgl\n", 18 | "import dgl.function as fn\n", 19 | "import torch as th\n", 20 | "import torch.nn as nn\n", 21 | "import torch.nn.functional as F\n", 22 | "from dgl import DGLGraph\n", 23 | "import numpy as np\n", 24 | "import networkx as nx\n", 25 | "#from res.plot_lib import set_default\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "%matplotlib inline\n", 28 | "from gensim.models import Word2Vec\n", 29 | "import random\n" 30 | ] 31 | }, 32 | { 33 | "cell_type": "raw", 34 | "metadata": {}, 35 | "source": [ 36 | "contents of the file \"edge_list_short.txt\"\n", 37 | "0 1\n", 38 | "0 2\n", 39 | "1 2\n", 40 | "1 3\n", 41 | "1 4\n", 42 | "2 4\n", 43 | "2 0\n", 44 | "2 1\n", 45 | "3 4\n", 46 | "3 1\n", 47 | "4 3\n", 48 | "4 0" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 20, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "data": { 58 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb4AAAEuCAYAAADx63eqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeVhV5frG8S+TguKM84SoqSiKivOcghMgw3LKKbXMY1aaWVmdBsufnUwbLFMbTdPMxaDiBJIDOc+goDiggjMqIjJu9v790XEfSXAE1t7s53NdXdcJcO+bTnGz1nre97UyGAwGhBBCCAthrXUAIYQQojhJ8QkhhLAoUnxCCCEsihSfEEIIiyLFJ4QQwqJI8QkhhLAoUnxCCCEsihSfEEIIiyLFJ4QQwqJI8QkhhLAoUnxCCCEsihSfEEIIiyLFJ4QQwqJI8QkhhLAoUnxCCCEsihSfEEIIiyLFJ4QQwqJI8QkhhLAoUnxCCCEsihSfEEIIiyLFJ4QQwqJI8QkhhLAotloHEEIIYX6S07JQDyRx/HIqqZk6ytvb0rRGeQa3rUMVx9Jax3sgK4PBYNA6hBBCCPNwJDGFb7eeYlv8NQCydHrj5+xtrTEAPZtUZVKPRrSqW1GjlA8mxSeEEOKRLNt9llnrj5Opy+VBzWFlBfa2Nrw7oCkjOzoXW75HJbc6hRBCPNTfpRdHRo7+oV9rMEBGTi6z1scBmFz5yRWfEEKIBzqSmMKw73eTkZOb5+O5Gbe5vv4rMs8ewtqhPJV6jKFs8555vsbBzoaVEzrSso7p3PaUqU4hhBAP9O3WU2Tqcu/7+I3w77CysaPOK8tw8nmD6+ELyL52Ls/XZOpyWbD1VHFFfSRSfEIIIQqUnJbFtvhr9z3T02dnkn5iJxW7j8S6lAP2dZtTplEH7hzbkufrDAbYcuIa19OyijH1g0nxCSGEKJB6ICnfj+tuXMDK2hq7yrWNH7Or1oCcf1zxAVgB6sH8X0cLUnxCCCGYNWsWL774IlFRUej1/xtgOX45Nc+Shbv0ORlYlS6T52PWpcugz86472szdXqOX7pd+KGfkEx1CiGEIDY2luXLl/P7779jMBho2rQp3bt3Z5eNG9hWu+/rre0cMGTlLTlDVjrWpRzyff3UzJwiyf0kpPiEEMLCJCcnExUVxd69e4mJieHMmTMkJCQAkJaWBsCBAwcoVaoUFb3acSWfx3O2lWtj0OeSc+OC8XZn9tUE7KrWz/c9y9vbFc038wSk+IQQogQ6f/48UVFR7N+/n2PHjnH27FmuXr3K7du30ev1lCpVikqVKlG7dm1atGhB2bJl2b9/PwBWVlZUqlQJAP9e7flic/x9tzutS9lTpkknUqJ+o0r/V8m+eob0U3uoMXLOfVnsba1pWrNc0X/Tj0iKTwghzJBeryc+Pp6//vqLAwcOEBcXx/nz57l27Rp37tzBYDBgb29PlSpVqFOnDh07dsTd3Z0uXbrQtm1bbG3z/vhXVZXBgwcDYDAYuHnzJs888wxK2zp8sTk+3wyVvSZxff1XJM0fgbVDeap4TaJUPld8BkBpU6fQ/xk8KVnALoQQJkqv13P48GF27NjBwYMHOXHiBElJSSQnJ5ORkYGVlRVlypTBycmJunXr0rRpU9q0aUPXrl1p3rw51tYPnl+8dOkSISEhqKrKwYMH0ev13L59GwcHB9577z3eeecdACYs3U9E3JUHblNWECsr6OtanYUjPZ7kH0GRkOITQggNZWdns3fvXnbu3Mnhw4eJj4/n4sWL3Lhxg6ysLKytrSlbtizVqlWjfv36uLq64uHhQdeuXWnYsOFjv19SUhLBwcGoqkpMTAze3t4oioKXlxfffvst06dPZ/bs2bz99tvGP1PQzi2PwhR3bpHiE0KIIpaWlsaOHTvYvXs30dHRnD59mosXL5KSkkJOTg42NjaUK1eO6tWr06BBA1q0aIGHhwfdunWjVq1aT/3+586dIygoCFVVOXHiBL6+viiKQp8+fShd+n9HCKWkpLB9+3Z8fX3ve41lu8/yybo4MvNZ2lAQBztr3h3QTPbqFEKIkuifk5IJCQlcvnyZ1NRUdDodtra2VKhQgZo1a9KwYUNatGhB+/bt6dq1K5UrVy70PKdPnzaWXUJCAn5+fiiKQq9evShVqtQTvabfG3OJtmmEwcbWrE9nkOITQohH9CiTkhUrVqR27do0atSIVq1a0bFjRzp16kSZMmUe/gZPKT4+HlVVUVWVCxcuEBAQgKIo9OjR475hlscVExND7969WRW5hxVHrrPlxDWsIM8V4N3z+Ho1qcqkno1M6vbmvaT4hBDiv/R6PSdPniQqKooDBw5w/Phxzp07d9+kZOXKlalTpw7PPPMM7u7udO7cmbZt2z7xldTTiI2NNZZdcnIygYGBKIpC165dsbGxKZT30Ov1dO/enZEjRzJx4kQArqdloR5M4vil26Rm5lDe3o6mNcuhtJET2IUQwqT8c1IyPj6exMTEPJOSDg4OODk5Ua9ePZo0aWKclGzRosVDJyWLmsFgICYmxlh2t2/fRlEUFEWhU6dORZLvl19+4dtvv2X37t2FVqZakuITQpQ4/5yUPHnyJBcuXDBOSlpZWeHo6EjVqlWpX78+zZo1w8PDg+7duz/RpGRRMxgMHDp0yFh2OTk5xrJr165dkZbxzZs3adasGWFhYXh4mM6ShKchxSeEMEv3TkrGxMRw6tSpAiclnZ2dad68Oe3bty+0ScmiZjAY2Ldvn7HsrK2tGTx4MIqi0KZNG6ysrIolx6RJkwBYsGBBsbxfcZDiE0KYrHsnJY8ePcqZM2fynZSsUaOGcVKyQ4cORTYpWdT0ej27d+9GVVWCgoJwcHAwll3Lli2Lrezu2rdvHz4+PsTFxRm3MCsJpPiEEJpKTExk+/bt7N+/n9jYWBISEvKdlKxVq9Z9k5Jly5bVOv5Ty83NZceOHcayq1SpEoqiMHjwYFxdXYu97O7N1bFjR15++WWef/55TTIUFdmrUwhRpB5nUrJ27dq0b9+e1q1b06lTJzw8PDSZlCxqOp2O7du3o6oqwcHB1KhRA0VR2Lx5M82aNdM6HgDff/899vb2jB49WusohU6u+IQQT+3eSclDhw5x4sQJEhMTuX79Ounp6QCUKVOGKlWqUK9ePZo2bUrr1q3p0qULLVu21HxSsjjk5OSwZcsWVFUlNDSU+vXrExgYSGBgII0bN9Y6Xh5Xr16lRYsWREZG4ubmpnWcQifFJ4R4JPdOSh45coT4+HguXLjAzZs3yczMzHdSsm3btnTr1o2GDRtaRLn9U3Z2Nps3b0ZVVdasWUPjxo1RFIWAgAAaNGigdbwCjR07lkqVKjFv3jytoxQJKT4hhNHdSck9e/YQHR1tnJS8desW2dnZ2NjY4OjoaNxTsnnz5sY9JevUMZ1jZ7SUmZlJeHg4qqoSFhZG8+bNjWVXt25dreM91I4dOxg6dCixsbGUL19e6zhFQopPCAtz/fr1+07f/uekZPny5alZsyYuLi7GSclu3bqZ5aRkcUhPT2fjxo2oqsr69etxd3dn8ODB+Pv7m8XSibt0Oh1t2rTh3XffZejQoVrHKTJSfEKUQElJSURFRbFv3z6OHTv20ElJNzc3OnXqRKdOnXB0dNQ6vllIS0tj/fr1qKrKpk2baNeuHYqi4O/vT/Xq1bWO90S++OIL1q9fT3h4uGbTpMVBik8IM6TX6zl16pRxUjIuLi7fSclKlSoZ95Rs1aoVnTt3pl27diVyUrI4pKamEhYWhqqqREZG0qlTJxRFYdCgQVStWlXreE/l4sWLtGzZkh07dtCkSROt4xQpKT4hTJReryc6Opq//vrLePr2gyYln3nmGeOekpYyKVkcUlJSWLNmDaqqsnXrVrp3746iKPj6+paoW7/Dhw/HxcWFWbNmaR2lyEnxCaGh7Oxs9u3bl2dPyaSkpDyTkndP365Xr55xUvLunpJSbkXj+vXrrF69GlVV+euvv3j22WdRFAUfHx8qVKigdbxCFxkZyfjx44mNjS2W45O0JsUnRBG7c+cOO3fuZNeuXfedvp2dnY21tXWe07ddXV1p166dTEoWs6tXrxIaGoqqquzZswdPT08URWHgwIGUK1dO63hFJisri1atWvHZZ5/le/J6SSTFJ0QhuHHjBlFRUezZs8e4p+SlS5fynZRs0KABbm5uxj0lq1SponV8i3Xp0iVCQkJQVZWDBw/Sr18/FEWhf//+JWI7tEcxe/Zsdu7cydq1a7WOUmyk+IR4RHcnJe+evp2QkMCVK1dIS0sjNzc3z6Rkw4YNadmypUxKmqCkpCSCg4NRVZWYmBgGDhyIoij07dsXBwcHreMVq7Nnz9K2bVv2799v0gvqC5sUn3ig5LQs1ANJHL+cSmqmjvL2tjStUZ7BbU3/lOXHpdfrOX36NNu3b+fgwYN5JiXT0tLyTErWrl07z+nbMilp2s6dO0dQUBCqqnL8+HF8fX1RFAVPT09Kly5Z/x4/Dj8/Pzw8PHjvvfe0jlKspPhEvo4kpvDt1lNsi78GQJZOb/ycva01BqBnk6pM6tGIVnUrapTy8d2dlLx7+vaJEyc4f/58vpOSdevWNZ6+3aVLF1q1aiXDJGbkzJkzxrPszpw5g5+fH4qi8Oyzz8ovKUBYWBivv/46MTExFlf+UnziPst2n2XW+uNk6nJ50L8dVlZgb2vDuwOaMrKjc7Hle5i7k5K7du3i8OHDxj0lb9y4kWdSMr89JRs1aiTlZsbi4+ONZXfhwgX8/f1RFIUePXpgZ2endTyTkZGRQfPmzVm4cCFeXl5axyl2Unwij79LL46MHP3Dv/i/HOyseXdAs2Itv/T09DyTkveevn3vpGS1atWMe0rKpGTJFBsbayy7a9euERgYiKIodOvWDRsbG63jmaT333+f48eP88cff2gdRRNSfMLoSGIKw77fTUZOrvFjBl0O18MXkHn2MPrMNGwr1qRSj9E4NPTI82cd7GxYOaEjLesU3m3Pmzdv5pmUPH36NJcvX+bWrVt5JiVr1KiBi4uLcVKyS5cuODk5FVoOYVoMBgNHjx5FVVVWrVpFamoqiqKgKAqdOnWSsnuIkydP0qlTJw4fPmyxvwRK8QmjCUv3ExF3Jc/tTX12Jql7gnB064NNhapknN5P8po51Br3DbYV/7cfoZUV9HWtzsKR/yvE5cuX8+GHH3Lo0KECR8MvXLhg3FMyNjaWM2fO3DcpWaFCBeOeki1btjSWm0xKWg6DwcChQ4eMV3bZ2dnGsmvfvr3cnn5EBoOBfv364enpyRtvvKF1HM3ICewC+Ht6c1v8tfue6VmXsqditxHGvy/TqD22FaqTdflUnuIzGGDLiWtcT8uCrDTGjh1LZGQker2eLVu2cPXq1Xz3lNTr9XkmJT08PHB3d6dTp060b99ehhAsmMFgYN++fcays7KyYvDgwSxfvpy2bduW6E2Ui0pQUBAXLlzgtdde0zqKpqT4BADqgaRH+rrcOzfJuXGBUlXr3fc5vV7PkLfnsW3hv8nN/d/tUh8fnzyTkj169KBNmzZ07twZd3d3+W1dGOn1enbv3o2qqgQFBWFvb8/gwYMJDg6mVatWUnZP4fbt20ydOpXffvvN4gd9pPgEAMcvp+ZZspAfQ66O5DWf4+jWG7sq9x+omaOHc7f+fvZ2l16vZ/bs2bz11luFnlmUDLm5uezYscNYdpUqVUJRFNatW0fz5s2l7ArJzJkz6dWrF927d9c6iuak+AQAqZm6B37eYNCTHDYXbGyp7DmxwK/r0acfPy79N8eOHWPFihUsWbKEO3fuFHZcYeZ0Oh1RUVGsWrWK4OBgatSogaIobN68mWbNmmkdr8Q5duwYv/zyC0ePHtU6ikmQ4RYBwJSVhwg9fDHfzxkMBq6v/wrdrStUG/wh1nYFL3b1d6/NF0PdiyqmMGM5OTls3boVVVUJCQmhXr16KIpCYGAgjRs31jpeiWUwGOjZsydDhgzh5Zdf1jqOSZArPgFA0xrlKW17Od/bnTc2fUvO9USqD/vkgaVnb2tN05oldxd78fiys7OJjIxEVVVWr15No0aNUBSFPXv2WNTekFpatmwZaWlpTJxY8J0aSyNXfAL4e6qzy3/+vK/4dLeucuG7cWBjh5X1/9ZHVe73Mo7Ne+X52tK21ux869kSt4eneDyZmZlERESgqipr167F1dUVRVEICAigXr37h6JE0UlJScHV1ZWQkBA6dOigdRyTIcUnjPJbx/eorIC+zfOu4xOWIyMjg40bN6KqKuvXr6dVq1YoioK/vz+1a9fWOp7FeuWVV8jOzmbRokVaRzEpUnzC6ND5GygL/iLX6vF3vjDostBtnMOLgX1xc3PDxcUFFxeXEnlatfhbWloaGzZsQFVVNm7ciIeHh7HsatSooXU8i3fw4EH69+9PbGysnPn4D1J8Avj7lPBRo0Zx1rYO6U36k/mQpQ33crCzZkK7qkwb1N54dI+NjQ0ZGRkkJSVRs2bNIkwuilNqairr1q1DVVUiIiLo1KkTiqLg5+dH1apVtY4n/kuv19O5c2cmTJjAuHHjtI5jcmTlsODixYv06NGDcuXKsWvpHN4b2AwHOxsetnzKyurvPTrfHdCMqT4eLFy4EFtbWzIzM0lPT8fHx0dKrwRISUlh6dKlDBo0iDp16rBs2TK8vb1JSEhg06ZNvPjii1J6JubHH3/E2tqa559/XusoJkmu+CzckSNH8PX1ZcKECbzzzjvGxcLRSSks2HqKLSeuYQV5rgDvnsfXq0lVJvVsZNyYOjc3l2eeeYYzZ85gZWWFnZ0dixcvZsyYMRp8Z+Jp3Lhxg9WrV6OqKlFRUfTq1QtFUfDx8aFiRfM5f9ESJScn4+rqSnh4OO7usrQoP1J8FmzdunWMHTuW+fPnM3To0Hy/5npaFurBJI5fuk1qZg7l7e1oWrMcSpv8T2CPjIykT58+hIWFER4ezjfffEPHjh3ZsGED5cuXL+pvSTyFa9euERoaiqqq7Nq1C09PTxRFYeDAgfL/nRl54YUXKFu2LF999ZXWUUyWFJ+Fmj9/Pv/3f/9HcHAwnTp1KtTXjo+P55lnngEgOjqavn37kpKSwk8//cTw4cML9b3E07l8+TIhISGoqsr+/fvp168fiqLQv39/Of3CDO3atQtFUYiNjZXBsgeQ4rMwOp2OqVOn8ueffxIWFlYsi4j1ej2TJk1i8eLF9OjRg3Xr1lGmTJkif1+RvwsXLhAcHIyqqhw5coSBAweiKAp9+/aV/1/MmE6no127dkyfPp3nnntO6zgmTYrPgqSmpjJs2DB0Oh1//PFHsT+r2b9/PwMGDCAtLY2lS5cSGBhYrO9vyc6fP09QUBCqqhIXF4ePjw+KouDp6Ym9vb3W8UQh+PrrrwkNDSUyMlI29n4IKT4Lcf78eby9vencuTPz58/X7FgSvV7P+PHjWbJkCZ6enqxevVp+8BaRM2fOGMvu9OnTDBo0CEVR6N27t5xzWMJcunSJli1bsn37dtnk+xFI8VmAffv24efnx7Rp05g6dapJ/Da4a9cuvL29ycrK4vfff8fb21vrSCVCfHy8sewSExPx9/dHURR69uxp8WewlWQjR46kTp06fPrpp1pHMQtSfCVccHAwL730Et9//z1+fn5ax8lDr9czatQoVqxYwYABAwgODpYrkScQFxdnPKX86tWrBAQEoCgK3bp1y3M2oiiZtmzZwvPPP09sbCxly5bVOo5ZkOIroQwGA3PmzGH+/PmsXr2aNm3aaB2pQNu2bWPQoEHo9XqCgoLw9PTUOpJJMxgMHD161Fh2t27dIjAwEEVR6Ny5MzY2j7/lnDBP2dnZuLu7M2vWLPz9/bWOYzak+EqgnJwcJk2axL59+wgLC6NOnTpaR3oonU7HsGHDCA4Oxs/Pjz/++EOuVu5hMBg4fPiwsewyMzNRFAVFUejQoQPW1rIJkyX67LPP2Lp1K+vWrTOJRxjmQoqvhLl58yaKolCmTBlWrFhhdmuxNm/eTEBAANbW1oSGhtKzZ0+tI2nGYDCwf/9+Y9kBxrLz8PCQH3QW7vz587Rp04Y9e/bQsGFDreOYFSm+EuT06dN4e3vTr18/Pv/8c7O95ZWdnY2iKISFhTF06FCWLl1qMVd/er2ePXv2GMuudOnSDB48GEVRcHd3l7ITRoGBgbRs2ZIPPvhA6yhmR4qvhNixYweKovDvf/+bSZMmaR2nUKxfv54hQ4ZQqlQp1q5dS5cuXbSOVCRyc3PZuXMnqqoSFBREhQoVjFd2LVq0kLIT99mwYQOvvPIKR48eleVAT0CKrwRYvnw5U6ZM4ddff6Vfv35axylUmZmZ+Pn5ER4ezqhRo/j5559LxPMsnU5HVFQUqqoSHBxMtWrVUBSFwMBAXF1dtY4nTFhmZiYtWrRg/vz59O/fX+s4ZkmKz4wZDAZmzpzJzz//zNq1a3Fzc9M6UpEJCQlhxIgRlC1blvXr19OuXTutIz22nJwctm7diqqqhISEULduXWPZ3d3bVIiH+eijj4iOjiYoKEjrKGZLis9MZWVlMX78eOLj41mzZo1FnHidnp6Ot7c3W7du5YUXXmDhwoUmf/WXnZ1NZGQkqqqyevVqGjZsaCw7FxcXreMJM3P69Gk6dOjAoUOHqFu3rtZxzJYUnxlKTk7G39+fGjVqsGTJEovbWHjlypU8//zzVKhQgfDwcFq2bKl1pDwyMzOJiIhAVVXWrl1Ls2bNCAwMJDAwkPr162sdT5gpg8HAwIED6dGjB2+99ZbWccyaaf+6LO5z/PhxOnbsSNeuXVm5cqXFlR7A0KFDuXbtGo0bN8bd3Z1XXnkFvV7/8D9YhDIyMoy3Y2vUqMHnn3+Oh4cH0dHR7Nixg9dff11KTzyV0NBQzp49y9SpU7WOYvbkis+M/PnnnwwfPpzZs2czbtw4reOYhCVLlvDSSy/h5OTEpk2baN68ebG99507d1i/fj2qqrJx40batm2Loij4+/tTs2bNYsshSr47d+7g6urKkiVLLHpta2GR4jMTP/30EzNmzOD333+nV69eWscxKSkpKfTr1499+/YxZcoU5s6dW2Tvdfv2bcLCwlBVlYiICDp27IiiKPj5+VGtWrUie19h2d5++20SExP57bfftI5SIkjxmTi9Xs8777yDqqqsW7eOJk2aaB3JZC1evJjJkydTs2ZNNm/eTOPGjQvldVNSUli7di2qqrJlyxa6du2KoigMGjSIKlWqFMp7CFGQuLg4unfvTnR0tNxJKCRSfCYsPT2d0aNHc+XKFUJCQnByctI6ksm7ceMGnp6eHD58mLfffptZs2Y98eusXr0aVVWJioqiZ8+eKIqCj48PlSpVKuTUQuTPYDDQu3dv/Pz8ePXVV7WOU2JI8Zmoy5cv4+vrS5MmTfjhhx8oXbq01pHMyjfffMPUqVOpV68emzdvpkGDBg/9M9euXSM0NBRVVdm1axeenp4oisLAgQMpX758MaQWIq/ly5czZ84c9u3bZzHb9hUHKT4TFBMTg4+PD+PGjePf//63bFn1hK5evUqfPn2IjY3l/fff5/3337/vay5fvkxISAiqqrJ//3769u2LoigMGDDA7Db4FiXLrVu3cHV1RVVVOnXqpHWcEkWKz8Rs3LiR0aNH8+WXX/Lcc89pHadEmDt3Lm+//TYuLi5s3rwZGxsbgoODWbVqFUeOHGHAgAEoikK/fv0scnmIME1TpkwhLS2NH374QesoJY4UnwlZsGABH3/8MaqqltgNmbWyb98+vL29uXr1Kg4ODsZNoL28vGSTX2FyDh8+jJeXF7GxsfJsvwhI8ZmA3Nxcpk2bxqZNmwgLC5OztQpJQkICQUFBqKrKyZMnjae8L126lKZNmxIZGWkRW70J86LX6+natStjx47lxRdf1DpOiSQ7t2js9u3b+Pn5ERMTw86dO6X0ntLJkyf59NNP8fDwoH379sTHxzNz5kwuX77MTz/9xC+//MKZM2fIycmhbt26fPnll1pHFiKPX375hdzcXMaPH691lBJLrvg0lJSUhLe3Nx4eHnz33XfY2dlpHcksHT9+3Hhw6+XLlwkICEBRFLp37/7ASbj333+fWbNm4ebmRkREBFWrVi3G1ELc7/r167i6urJhwwbatGmjdZwSS4pPIwcOHDCuzXnjjTdkcvMxGAwGjh07Ziy7mzdvEhgYiKIodOnS5bFOnj99+jR9+vThwoULfPXVV/zrX/8qwuRCPNhLL71EqVKlmD9/vtZRSjQpPg2Ehoby4osvsmjRIgICArSOYxYMBgNHjhwxll16erpxQKVjx45PfTzRW2+9xeeff07r1q0JDw+ncuXKhZRciEezZ88e/P39iY2NpWLFilrHKdGk+IqRwWBg3rx5zJs3j9DQULM8TLU4GQwGDhw4YCw7vV5vLLt27doV+lXy8ePH8fT05OrVqyxYsECesYhik5ubS/v27ZkyZQqjRo3SOk6JJ8VXTHJycnjllVfYuXMnYWFh1KtXT+tIJkmv17N3715j2dnZ2TF48GAURaF169bFckt46tSpfP3113To0IGNGzfKri2iyH377bf88ccfbN26VR57FAMpvmKQkpLCkCFDsLW1ZeXKlZQrV07rSCZFr9ezc+dOVFUlKCgIR0dHY9m5ublp8oPg6NGjeHl5cePGDX788UdGjBhR7BmEZbhy5QotWrRg69atxXqsliWT4itiCQkJeHt78+yzz/LFF1/Ifnv/lZubS1RUFKqqEhwcjJOTk/E2pqurq9bxgL8LefLkySxcuJBu3bqxbt062cZMFLrRo0dTvXp15syZo3UUiyHFV4R27dpFYGAgM2bM4JVXXtE6juZ0Oh1bt25FVVVCQkKoXbs2iqIQGBho0sctHTx4kH79+pGWlsaSJUsYPHiw1pFECbF9+3ZGjBhBXFyc/FJVjKT4isjKlSuZPHkyv/zyCwMHDtQ6jmays7P5888/UVWV1atX06BBA2PZmdNifb1ez4svvsjPP//Ms88+S1hYmGx1Jp5KTk4OrVu35sMPP0RRFK3jWBQpvkJmMBiYNWsW33//PWvWrKFVq1ZaRyp2WVlZREREoKoqa9eupUmTJiiKQkBAAM7OzlrHe9pbL+oAACAASURBVCp79uxh4MCBZGRksGLFCnx9fbWOJMzU3LlzCQ8PZ+PGjTLQUsyk+ApRVlYWEyZM4NixY6xdu9aiTkvOyMhg06ZNxpPi3dzcjGVXp04dreMVKr1ez5gxY/jtt9/o168foaGhlCpVSutYwowkJSXh7u7Orl27aNy4sdZxLI4UXyG5fv06AQEBVKlShaVLl1K2bFmtIxW5O3fusGHDBlRVZePGjbRp0wZFUfD397eI0v/rr7/w8fFBp9Pxxx9/0L9/f60jCTMxZMgQmjZtysyZM7WOYpGk+ApBfHw8AwcOJCAggNmzZz/1LiKm7Pbt26xfv55Vq1YRERFBhw4dUBQFPz8/qlWrpnW8YqfT6XjuuedQVRVfX19UVZXJXfFA4eHhTJw4kWPHjuHg4KB1HIskxfeUtm3bxtChQ/n4449L7BEit27dYu3ataiqypYtW+jSpQuKojBo0CCqVKmidTyTEBkZadx+LiQkhGeffVbjRMIUZWVl4ebmxrx58/D29tY6jsWS4nsKS5YsYfr06Sxfvpw+ffpoHadQ3bhxgzVr1qCqKtu3b6dnz54oioKPjw+VKlXSOp5JysnJYfDgwaxZswZFUVi+fLlc/Yk8PvnkE/bv309oaKjWUSyaFN8T0Ov1vP/++6xYsYKwsDCaNWumdaRCkZycTGhoKKqqsnPnTvr06YOiKHh7e8u2XY9hw4YNDBkyBDs7O9asWUPXrl21jiRMQEJCAu3atePAgQPUr19f6zgWTYrvMWVkZPD888+TlJREaGio2Z/hduXKFUJCQlBVlX379tG3b18URWHAgAGyoPYpZGdn4+fnx8aNG3nuuef49ddfS/SzX/FwPj4+dO7cmRkzZmgdxeJJ8T2GK1euMGjQIFxcXPjpp5/MdgHzxYsXCQ4ORlVVDh8+zIABA1AUhX79+lGmTBmt45Uoa9asYfjw4Tg4OLB+/Xrat2+vdSShgTVr1vDmm28SHR0tS19MgBTfIzp27Bje3t6MHj2aDz/80OwWnCYmJhrL7ujRo/j4+KAoCl5eXmZb4OYiPT0dX19f/vzzT8aOHcv3338vV38WJD09HVdXV3788Ud69+6tdRyBFN8jCQ8PZ+TIkcybN4+RI0dqHeeRnT17lqCgIFRVJT4+nkGDBqEoCr1796Z06dJax7M4f/zxB2PGjKF8+fJs3LiR1q1bax1JFIN3332XM2fOsGLFCq2jiP+S4nuIRYsW8cEHH7Bq1Sq6deumdZyHOnXqlLHszp49i7+/P4qi0KtXL+zs7LSOZ/HS0tIYOHAgUVFR/Otf/2L+/Ply9VeCnThxgi5duhAdHU2tWrW0jiP+S4qvALm5ubz55puEhYWxbt06GjVqpHWkAp04ccJ4cOulS5cICAhAURS6d+8u4/QmatmyZbzwwgtUrlyZ8PBwWrRooXUkUcgMBgNeXl4MGDCAqVOnah1H3EOKLx9paWmMGDGC1NRUgoKCqFy5staR8jAYDMTGxhrL7vr16wQGBqIoCl27dsXGxkbriOIRpKam0q9fP/bs2cOrr77KF198oXUkUYhWrlzJ//3f/3HgwAH5BdTESPH9w4ULF/Dx8cHd3Z2FCxeazASWwWAgOjraWHZpaWnGg1s7deokt8vM2I8//sikSZOoXr06ERERJn02oXg0t2/fplmzZqxcuZIuXbpoHUf8gxTfPQ4dOoSvry8vv/wyb731luaTmwaDgYMHDxrLTqfTGcuuXbt2UnYlyI0bN/Dy8uLQoUO8+eabzJ49W+tI4ilMmzaNGzdu8PPPP2sdReRDiu+/1q5dy7hx4/juu+80PRTSYDCwd+9eY9nZ2toay65Nmzaal7EoWt999x2vvfYatWvXJjIyEhcXF60jiccUExND7969OXbsmNlvcFFSWXzxGQwGvvrqK+bMmUNISIgmC4z1ej27du1CVVWCgoIoW7YsgwcPRlEU3NzcpOwszNWrV/Hy8iImJoZ///vffPjhh1pHEo/IYDDQvXt3RowYwcSJE7WOIwpg0cWn0+l49dVX2b59O+vWrSvW/fNyc3P566+/UFWV4OBgKleubLyya968ebHlEKbryy+/ZPr06TRo0IDNmzdTr149rSOJh1iyZAnffPMNu3fvliEzE2axxXfr1i2GDh0K/D19VaFChSJ/T51Ox7Zt21BVlZCQEGrWrImiKAQGBtK0adMif39hfi5dukSfPn04ceIEH3/8sezzaMJu3rxJs2bNCAsLw8PDQ+s44gEssvjOnj2Lt7c33bt35+uvvy7SUeOcnBz+/PNPVFUlNDQUZ2dnY9mZ8tpAYVo+/fRT3nvvPZ555hk2b94si6FN0KRJkwBYsGCBxknEw1hc8e3Zswd/f3/efPNNXnvttSJ5fpaVlcXmzZtRVZU1a9bwzDPPGMvO2dm50N9PWIbz58/j6enJmTNn+PTTT5k2bZrWkcR/7d+/Hx8fH2JjY+W8SjNgUcW3atUqJk2axE8//YSPj0+hvnZmZiabNm1CVVXCwsJo0aIFiqIQEBBA3bp1C/W9hGWbOXMmM2fOxNXVlc2bN1OtWjWtI1m03NxcOnbsyOTJkxkzZozWccQjsIjiMxgMfPrppyxYsIA1a9YU2ubA6enpbNiwAVVV2bBhA61bt0ZRFPz9/eVWlChSCQkJ9OnTh/Pnz/Pll1/y8ssvax3JYi1cuJDffvuN7du3ywS2mSjxxZednc3EiRM5fPgwa9eupXbt2k/1emlpaaxfv55Vq1YRHh5O+/btURQFPz8/qlevXkiphXg077zzDv/5z39wd3cnIiLC5LbXK+muXr1KixYtiIyMxM3NTes44hGVyOIbMmQIzs7OvP322wQGBlK+fHmWL19O2bJln+j1UlNTCQsLQ1VVIiMj6dy5M4qiMGjQIJycnAo5vRCPJz4+Hk9PTy5dusSCBQt44YUXtI5kMcaOHUvlypWZO3eu1lHEYyhxxZeYmEjjxo2xsrKifPnyjBw5ks8+++yx19TcvHmTtWvXoqoqW7dupUePHiiKgq+vrzy8FiZp2rRpfPnll7Rr146NGzdSsWJFrSOVaDt27GDo0KHExcVRrlw5reOIx2A2W4Ynp2WhHkji+OVUUjN1lLe3pWmN8gxuW4cqjv87VPXrr79Gr9eTk5NDbm4u3t7ej1x6169fZ/Xq1aiqyo4dO3j22WcZOnQoS5cuLZZ1fkI8jblz5zJu3Di8vLyoUaMGixcvZvTo0VrHKpF0Oh3/+te/mDt3rpSeGTL5K74jiSl8u/UU2+KvAZCl0xs/Z29rjQHo2aQqk3o0oklVeypWrEhWVhZWVlaULl2a2rVrc+rUqQJf/+rVq4SGhqKqKnv27MHLywtFURgwYID8Cy3Mkl6v57XXXuPbb7+lS5cubNiwAUdHR61jlShffPEF69evJzw8XAZazJBJF9+y3WeZtf44mbpcHpTSygrsbW3o7HCJn94ZR/369fH19cXT05Nu3brdd8vn0qVLhISEoKoqBw8epH///iiKQr9+/Z74OaAQpubw4cP069ePW7du8csvvxh3KhJP5+LFi7Rs2ZIdO3bIEVJmymSL7+/SiyMjR//wL/4vBztrXu/VgBd73b/9V1JSEsHBwaiqSkxMDN7e3iiKgpeXFw4ODoUZXQiTodfrmThxIj/88AM9e/YkLCyMMmXKaB3LrA0fPhwXFxdmzZqldRTxhEyy+I4kpjDs+91k5OTm+/mcGxe4+ONkyjbtgpPPG3k+52Bnw8oJHWlZpyLnzp0jKCgIVVU5ceIEvr6+KIpCnz59KF26dL6vLURJtG/fPgYMGEB6ejrLli3D399f60hmKTIykvHjxxMbGyu/QJgxkyy+CUv3ExF3pcDbm1d+/zcGXRa2FardV3xWQAO7W6Rt/JKEhAT8/PxQFIVevXqZzGnqQmhBr9czduxYli5dipeXF6Ghodjb22sdy2xkZWXRqlUrPvvsM3x9fbWOI56CyRVfcloWXf7zZ54hlnvdid1Gevwu7KrURZdy6b7iA7A25DKvRxm8PXsW6QbUQpijHTt24OPjQ3Z2NitXrmTgwIFaRzILs2fPZufOnaxdu1brKOIpWWsd4J/UA0kFfk6flU5K1G9Uenb8A1+jlJ0dV8o2kNITIh9dunTh2rVreHt74+PjYyxBUbBz584xd+5cvv76a62jiEJgcsV3/HJqgVd7KduX4tjKC9vyVR/4Gpk6Pccv3Qb+Xpu3aNEi/Pz8yMzMLPS8QpgjGxsbfv/9d/7880+ioqJwcnJi8+bNWscyWa+99hpTpkyhQYMGWkcRhcDkLolSM3X5fjz7yhkyzx2h5tivHul1DsTE0u7zF4mJicHGxoaMjAysrU2u54XQVM+ePUlOTmbIkCF4eXkREBDA77//LndL7rFu3TpiY2NZuXKl1lFEITG5Z3xTVh4i9PDF+z6eum81Kdt/xarU30sPDNmZYNBjV6VuvmWYEbeVq6s/N/69ra0tI0aMoEGDBri4uNCgQQMaNGhAzZo1pRCFADZt2sTgwYOxtrZm9erV9OjRQ+tImsvIyKB58+YsXLgQLy8vreOIQmJyv9Y1rVGe0raX77vd6ejel7LNuhv/PnVvMLpbV6jc9/7jWOxtrZk2eRzRVVP47bffyMjIoF69enTv3p2EhAQ2bdrEmTNnSEhIICUlhXr16uUpw3vLUfblFJaib9++JCcnExAQQK9evRg+fDhLly616F8MZ8+ejYeHh5ReCWNyV3wPm+q8KyXqtwKnOkvbWrPzrWep4liaLVu2MGzYMHr37s3y5cvv+9qMjAzOnj1rLMJ7/zpz5gxWVlb3leHdv5ydnWXxuyiR1q5dy/DhwyldujTr1q2jY8eOWkcqdidPnqRTp04cPnyYOnXqaB1HFCKTKz54+Dq+B7Gygr6u1Vk40sP4sfT0dHJych57o2mDwcDNmzfzFOG9xXju3DkqV66c75VigwYNqFOnzmOfCiGEqcjMzGTQoEFEREQwZswYfvzxR4u5+jMYDPTv358+ffrwxhv3/3ItzJtJFt/Ddm55kHt3bilqer2eixcv3neVePd/X7t2jTp16uQpxnvL0cnJSTa4FSYvKCiIUaNG4ejoyIYNG2jbtq3WkYqcqqp8+OGHHDp0CDs7O63jiEJmksUHT7ZXZylreN+nOSM7OhddsMeQlZXFuXPn8r2FmpCQQHZ2Ns7OzvfdQr37l+yoL0xFeno6AwcOZNu2bUyYMIEFCxaU2Ku/tLQ0mjVrxm+//Ub37t0f/geE2THZ4oPHO53BKlfHtfBFuDmk8NFHH9G7d2+T/w8zNTU131uod/9ydHQs8PlivXr15DdRUeyWL1/O+PHjqVixIuHh4bi5uWkdqdC9+eabXL58mV9//VXrKKKImHTxAUQnpbBg6ym2nLiGFX8vTr/r7nl8vZpUZXirKvRs6QJAmTJlKF++PHPmzGHkyJHaBH9KBoOBK1euFPh88eLFi9SsWbPA54s1atSQ26iiSNy+fZv+/fuza9cuJk+ezJdfflli/l07duwYPXv25OjRo1SvXl3rOKKImHzx3XU9LQv1YBLHL90mNTOH8vZ2NK1ZDqXN/05gd3Z25ty5cwDY2dnx+eef8+qrr2oZu8jk5OSQmJhY4PPF27dv4+zsXODzRTlRXjytJUuWMGHCBKpVq0Z4eDjNmjXTOtJTMRgM9OzZkyFDhvDyy/cvkxIlh9kU36OYPHky3377LQC1atUiMTHR5G93FpU7d+7ke/v0bjna2dnlewv17jINObZJPIqUlBS8vLw4cOAA06ZN47PPPtM60hNbtmwZX3zxBXv37pVp7BKuRBVfREQEAQEB/PLLL4waNYru3buzceNGrWOZHIPBwPXr1wt8vpiYmIiTk1OBzxdr1aolPxhEHosXL2by5MnUqlWLzZs306hRI60jPZaUlBRcXV0JCQmhQ4cOWscRRaxEFR/8PX1WpkwZDh48SPv27XnppZeMV4Hi0eTm5nLhwoUCny9ev36devXqFfh8sXLlyiXmmY94dMnJyXh6ehIdHc27777LzJkztY70yF555RWys7NZtGiR1lFEMShxxXev1atX4+/vz+eff87rr7+udZwSIyMj475lGveWY25ubr5Xii4uLjg7O8vJ1SXc119/zbRp06hfvz6RkZHUr19f60gPdPDgQQYMGMCxY8eoUqWK1nFEMSjRxQcwb9483njjDUJCQhg0aJDWcSxCSkpKvleKZ86c4dy5c1SoUKHA26h169aVkwFKgCtXrtCnTx/i4uL48MMPee+997SOlC+9Xk/nzp2ZMGEC48aN0zqOKCYlvvgAJk2axOLFi9m7dy9t2rTROo5F0+v1XL58ucDbqJcvX6Z27doF3katVq2a3EY1I3PmzGHGjBk0atSIzZs3m9yel99//z0///wzf/31l8UOwlkiiyg++Hvn+aioKE6dOkWtWrW0jiMKkJ2dzfnz5wu8jZqenp7vLjd3y7FcuXJafwviH5KSkvD09OTkyZPMnj2b6dOnax0J+PuZZPPmzdm0aRPu7u5axxHFyGKKT6/X07x5c65du8b58+flOZOZun37doFLNBISEnBwcChwmUb9+vUpVaqU1t+CxZo1axYffPABTZs2ZfPmzdSoUUPTPC+88AKOjo58+eWXmuYQxc9iig/+3m2+Xr16VKpUibi4OLm1UcIYDAauXbtW4BFTFy5coHr16gU+X5RDiYveuXPn6N27N+fOnWPu3LmabTCxa9cuFEUhNjZWNnOwQBZVfACXL1/GxcWFzp07s3nzZq3jiGKk0+lISkoqcP1iSkoK9evXL/D5ohxKXHjee+89Zs+eTcuWLYmIiMDJyanY3lun09GuXTumT5/Oc889V2zvK0yHxRUfQHR0NG3btmXs2LEsXrxY6zjCRKSnp3P27NkCB2/uHkpc0DINe3t7rb8Fs3Lq1Cn69OnDxYsXmT9/Pi+99FKxvO/XX39NaGgokZGRMihloSyy+ADCwsLw9fXlP//5j8k8bBem6+6hxPkt0UhISOD8+fN5DiX+ZznKocQFmz59OvPmzaNt27aEh4dTsWLhn6UZFRXFSy+9xMcff8zEiRPZvn272e8tKp6cxRYfwPz583nttddYtWoVgYGBWscRZuzuocQFPV9MTk6+71Die8vR0g8ljouLw9PTk2vXrrFo0SKef/75Qn39u1uq6fV6XFxc2L59u+bDNUI7Fl18AK+++ioLFixg165dtGvXTus4ooS691Di/MoxOzv7gcs0ypYtq/W3UOQMBgNTpkxh/vz5dOrUiQ0bNlC+fPlCee1PPvmE999/H4PBgI2NDeXKlePKlSsy5WuhLL74AAYMGMCWLVs4efKkyS2wFZbh1q1bBS7ROHv2rPFQ4vyWaZS0Q4mjo6Pp27cvN2/e5KeffuK5554jJycHVVUZNmzYE10ZT5w4kUWLFlGqVCkqVarEr7/+ipeXVxGkF+ZAio+/b1O1bNmSS5cukZiYKGv8hEkxGAx5drv5ZzleunQpz6HE/yxHczyUWK/XG3dc6t69O+3atePzzz9nzZo1+Pj45PtnktOyUA8kcfxyKqmZOsrb29K0RnkGt61Dz04eHD16lClTpjBr1iz5b9zCSfH9V2ZmJs7Ozjg6OhIfHy/ruYTZuHsocUGDN2lpafcdSnxvOZryOrYDBw7Qp08fUlJSAKhTpw5nzpzJc4V7JDGFb7eeYlv8NQCydHrj5+xtrTEAdW1TGdKiIhMC+xZrfmGapPjucfXqVVxcXPDw8GDr1q1axxGiUKSlpRmXaeRXjqVKlSpwmUb9+vU1PZQ4KyuLZs2akZCQAICNjQ3/93//x5tvvgnAst1nmbX+OJm6XB70k8zKCuxtbXh3QFNGdnQuhuTClEnx/cPRo0dp3bo1o0aN4qefftI6jhBFymAwkJycXODzxcTERKpWrarZocQnT56kZ8+eXLlyBTs7OzIzMwHYvXs3Jw3VmbU+jowc/UNe5X8c7Kx5d0AzKT8LJ8WXjw0bNuDt7c0nn3zCjBkztI4jhGZyc3ONu93kV4w3btwolkOJdTodFy5c4PTp08ydO5e9p69Qwf8DdPzvkUTqgbXciYkk+9pZyjbrgZP31Hxfy8HOhpUTOtKyTuGvFxTmQYqvAAsWLGDy5MmsWLGCoUOHah1HCJOUkZGRZ7ebf5bj3XVz+S3TeJpDiUcu2s5fCbfA6n/Fl35iJ1hZkZFwEENOdoHFZ2UFfV2rs3CkxxO9tzB/cuJnASZNmsSpU6cYMWIEzs7OdOjQQetIQpgcBwcHmjVrVuAuKDdv3sxThnFxcaxfv964TKNixYoFLtMo6FDi5LQs9iXdyVN6AGWadAYg6/IpcnOSC8xsMMCWE9e4npZFFUftnl8K7cgV30MMGjSITZs2ceLECerXr691HCFKDL1ez6VLlwrcG/XKlSt5DiW+W44xuuqoJzLJzs3/R9fN7UvJTU0u8IoP/p72nOr5DC91b1hU354wYVJ8D6HX62ndujXnz58nMTERR0dHrSMJYRGysrLyPZT4SBl3smq2KvDPPUrxAfi71+aLoXIArSWSW50PYW1tzb59+3B2dqZly5acOnVK1vgJUQxKly5N48aNady4cZ6Pj1uyjz+PX33q10/NzHnq1xDmSX6CP4JSpUoRHR3NtWvX6N69u9ZxhLBo5e0L5/f18vYlZ5s38Xik+B6Rk5MTe/fuZe/evYwePVrrOEJYrKY1ylPa9v4fXQZ9LgZdNuhzwaDHoMvGoM/N9zXsba1pWrNcUUcVJkpudT6GZs2asW7dOvr160ejRo14//33tY4khMUJbFObzzcdv+/jt3b8zq0dK4x/f+fYFip0GU7FbiPu+1oDoLSRDektlQy3PIHFixczceJEli5dyogR9/9HJYQoHIcOHSIuLo6MjAwyMjLYtWsXBw8eJKvdaAy13TDw+IvjZR2fkCu+JzBhwgROnTrF6NGjqV+/Pl27dtU6khAl0ocffsiGDRuwsbExblfWrVs3vpj5IiN/2k9GTv63Mh/E3taGST0bFXZUYUbkGd8T+uyzzxg0aBC9e/c2bqArhChc48ePR6fTkZmZiZWVFW3atGHr1q20dXbi3QFNcbB7vB9hf+/V2VS2K7NwcqvzKbVt25bTp09z/vz5QjstWghLl5yczIgRI4iIiKBcuXKkpaVRpkwZ4uLi8hwWLacziCchV3xPac+ePTg6OuLm5oZOp9M6jhBmLTs7m7Fjx1KjRg2OHTvGxo0biYmJwdbWlkWLFuUpPYCRHZ1ZOaEjfV2rU9rWGvt/THva21pT2taavq7VWTmho5SeAOSKr1DcuHEDZ2dnmjdvzq5du7SOI4TZ0ev1fPjhh3z22WeULl2aefPmMX78eOPnb9y4QeXKlR/4GtfTslAPJnH80m1SM3Mob29H05rlUNrUkT05RR5SfIXkxIkTuLm5ERgYyIoVKx7+B4QQAPz4449MmzaNzMxMpk+fzkcffSS7I4kiJVOdhaRJkyZs3LgRT09PGjduzMyZM7WOJIRJi4iIYOzYsVy6dIlRo0axcOFC7O3ttY4lLIAUXyF69tln+eGHHxg/fjwNGzZkzJgxWkcSwuQcPXqU4cOHc+zYMTw9PTl8+DBOTk5axxIWRIqvkI0dO5ZTp04xbtw4nJ2d6dGjh9aRhDAJly9fZvjw4Wzbtg13d3fi4uJo0qSJ1rGEBZIb6UVg1qxZBAYG4uXlxcmTJ7WOI4Sm0tPTGT58OLVr1yYhIYEtW7Zw8OBBKT2hGRluKULt27fnxIkTnDt3jooVZcGssCx6vZ63336bL7/8EkdHR+bPny9b/AmTIMVXhHQ6HS4uLhgMBhISErC1lTvLwjIsWLCAt956C51Ox3vvvceMGTNkUlOYDCm+IpaSkkL9+vV55pln2Ldvn9ZxhChSYWFhvPjii1y7do0XXniBb775Rn7hEyZHfgUrYhUrVuTgwYNER0czePBgreMIUSQOHjxI06ZN8fX1xcPDg+TkZBYuXCilJ0ySFF8xaNiwIREREQQHBzNjxgyt4whRaBITE+nSpQseHh5UqFCB06dPs3btWnmmLUyaFF8x6d69Oz///DP/+c9/+PHHH7WOI8RTSUtLIyAggPr163PlyhV27NjBnj17aNCggdbRhHgouQ9RjEaPHs2pU6eYMGECzs7O9O7dW+tIQjwWnU7H66+/znfffUelSpVYtWoVgYGBWscS4rHIcIsGRowYwapVq4iJiZG1TMJszJ07l/fffx+Ajz/+mNdff13jREI8GSk+jXTq1ImjR49y7ty5h+46L4SWVFXlX//6FykpKUyaNIm5c+fK0Iowa1J8GtHpdDRq1IicnBwSEhIoVaqU1pGEyGPXrl2MGjWKhIQE/P39+eWXX3B0dNQ6lhBPTYZbNGJra0t0dDR37tyhY8eOWscRwighIYH27dvTpUsXatasyblz51BVVUpPlBhSfBoqX748hw4dIjY2Fn9/f63jCAuXkpKCt7c3DRs25Pbt2+zfv5+oqKj7Tj0XwtxJ8WmsQYMGREZGsmbNGqZPn651HGGBsrOzmTBhAk5OThw4cIA1a9YQFxdHmzZttI4mRJGQ4jMBXbp0YenSpcydO5dFixZpHUdYCL1ezyeffEKFChVYsWIFX3/9NZcuXcLb21vraEIUKRnNMhHPPfccp0+fZtKkSTg7O9O3b1+tI4kSbNmyZbz66qukpaUxdepUZs+eLZtIC4shU50mZsyYMSxfvpzDhw/TvHlzreOIEmbr1q2MGTOGpKQkhg0bxg8//ICDg4PWsYQoVlJ8Jqhbt24cOnSIs2fP4uTkpHUcUQKcOHGCYcOGceTIEXr27Mny5cupUaOG1rGE0ITc2zBB27Zto3r16ri5uZGdna11HGHGkpOT8fLyolmzZuTm5hITE8Off/4ppScsmhSfCbK2tubIkSNkZWXh4eGBXq/XOpIwYwDvmgAADO5JREFUM5mZmYwZM4bq1asTFxfHpk2biI6OltvnQiDFZ7IcHR05fPgw8fHxDBo0SOs4wkzo9XreffddKlasyOrVq/nhhx9ITEzE09NT62hCmAwpPhNWr149tm3bxoYNG5g6darWcYSJ++GHH6hUqRJz587l7bff5saNG4wdO1brWEKYHFnOYOI6dOjA8uXLGTZsGA0bNmTy5MlaRxImZtOmTYwbN44rV64watQoFi1aJHu/CvEAUnxmYMiQIZw+fZrXXnsNFxcXBgwYoHUkYQKOHj3KsGHDiI2NxcvLi+joaKpUqaJ1LCFMntzqNBMzZsxg7NixDBo0iJiYGK3jCA1dvHiRnj170rJlS0qXLs2JEyfYuHGjlJ4Qj0iKz4z88MMPdO3alY4dO3L16lWt44hilp6ezrBhw6hbty7nzp1j+/btHDhwgMaNG2sdTQizIsVnZiIjI6lVqxZubm5kZmZqHUcUA71ezxtvvEHFihWJiIhg2bJlJCQk0LVrV62jCWGWpPjMzN01frm5ubRt21bW+JVw33zzDRUqVGDBggV89NFHXL9+neHDh2sdSwizJsVnhsqUKcPhw4c5c+YMAwcO1DqOKAJr1qyhRo0aTJkyhZEjR5KamsqMGTO0jiVEiSDFZ6bq1KlDVFQUERERvPzyy1rHEYVk//79NGnSBD8/Pzp06EBycjLfffcdtrYygC1EYZHiM2MeHh788ccffPfdd3z11VdaxxFP4fz583Tu3Jn27dtTqVIlEhISWL16NRUrVtQ6mhAljhSfmQsICOCzzz5j6tSprFmzRus44jGlpqbi7++Ps7MzycnJ7Nq1i927d1O/fn2towlRYknxlQBvvPEGEyZMICAggMOHD2sdRzwCnU7Hyy+/TJUqVdixYweqqhIfH0+HDh20jiZEiSfn8ZUgffr0YefOnZw6dYpatWppHUcUYM6cOXzwwQdYW1vzySefMGXKFK0jCWFRpPhKEL1ej6urK9evXycxMRF7e3utI4l7/PHHH0yaNIlbt24xefJk5s6di7W13HQRorhJ8ZUwGRkZ1KtXjypVqhAbGys/WE3Ajh07GD16NGfPniUgIICff/4ZR0dHrWMJYbHkp2IJ4+DgwJEjRzh//jx9+/bVOo5FO336NB4eHnTr1o3atWtz7tw5Vq1aJaUnhMak+EqgWrVqsXPnTrZs2cLEiRO1jmNxUlJSGDhwII0bN+bOnTscPHiQ7du3U6dOHa2jCSGQ4iux3N3dCQ4OZvHixcyZM0frOBYhOzubF198EScnJw4dOkRYWBhxcXG4u7trHU0IcQ8pvhLM19eXefPm8dZbbxEUFKR1nBJLr9czc+ZMKlSo8P/t3VtMU3kCBvCvpYWCihQVdYAoyi0SLmExSrzgQLyAD4aMS4A1xSg6iaAkyIOLMRIicWMMmhCBqNHEBRXjg2hAJ7qoBCNocRxqhAqCFEFQ6gWQS6VlHzYy43QWpFwOcL7fG5zTno+k6cf5n///HBQUFCArKwstLS18biLRJMXJLSKQmJiI3NxclJeXIygoSOg408qFCxeQlJSE7u5uJCcnIyMjgxOKiCY5Fp9IhIeH4969e6itreW1pjFQUlKC7du3o7m5GTExMTh79iyXjxBNESw+kTCZTPD19UVrayuamppgZ2cndKQpqbq6GtHR0dBoNAgNDcXFixfh5OQkdCwiGgGOyYiEVCpFZWUlZDIZ/P39+Ry/EXr79i3Wr18PHx8fDAwMQKPR4M6dOyw9oimIxSciCoUCGo0GLS0tCAsLEzrOlNDb2wuVSoWFCxdCq9Xi9u3bqKqqgo+Pj9DRiMhCLD6RcXJyQnl5OcrKyhAfHy90nEnLZDIhNTUVs2fPxo0bN3Du3DnodDr+w0A0DbD4RMjX1xeFhYU4f/48jh49KnScSef06dNQKpXIzMxEamoq9Ho94uLihI5FRGOEk1tE7NSpU9i7dy8uX76MqKgooeMI7ubNm4iPj0dbWxvi4uKQk5MDa2troWMR0RiTCR2AhJOQkIC6ujrExsZi0aJFon0WXFVVFWJiYlBdXY1NmzYhLy8Pjo6OQscionHCoU6RO3HiBMLDwxESEgKdTid0nAnV0tKCkJAQBAQEwNbWFlqtFsXFxSw9ommOxUcoLCyEp6cnAgIC0NXVJXSccdfd3Y2oqCi4urpCp9OhtLQUarUaHh4eQkcjognA4iNIpVKo1WrY2NhM6zV+JpMJycnJcHBwQElJCfLz89HQ0IDVq1cLHY2IJhCLjwAA1tbW0Gg0aGtrQ0hIiNBxxlxWVhbs7e2Rm5uL9PR0tLe3Izo6WuhYRCQAFh8Nmjt3Lh49eoTy8vJpM32/sLAQ8+fPR3JyMlQqFTo6OnDgwAGhYxGRgFh89I1ly5ahqKgIeXl5SE9PFzqOxR4/fgxPT09ERkYiODgYer0e2dnZkMk4kZlI7Fh8ZGbDhg3IyclBWloa8vPzhY4zIo2NjQgODsaKFSswZ84cNDQ04Nq1a7C3txc6GhFNEvz3l/7S7t27UVtbC5VKhcWLF2PVqlVCRxpSR0cHVCoVrl+/Dnd3d1RUVGD58uVCxyKiSYh3bqEhRUZGori4GDU1NXBzcxM6jpn+/n7s27cPZ86cgaOjI3JzcxEZGSl0LCKaxFh8NKzAwEDU19dDp9NNqiHDY8eOIS0tDVKpFBkZGUhKShI6EhFNASw+GpbBYICbmxvkcjnq6uoEnyBSUFCAhIQEdHR0IDExEcePH4dUysvVRPR9+G1Bw/q6xk+v12Pt2rUAgObmZtTX109ojgcPHmDJkiWIjY1FaGgoPnz4gMzMTJYeEY0IvzHouzg6OkKtVkOtViM8PBw+Pj7YtWvXhBz75cuXCAoKwpo1a+Di4gKdTocrV65gxowZE3J8IppeWHz03by8vHD48GHcunULnz59wsOHD9Hf3z9ux3v//j0iIiLg4eGB7u5uPHnyBKWlpXB2dh63YxLR9Mfio+9WWlqKQ4cODf48MDCAysrKUb/vny8zGwwG7Ny5E05OTnj69CmKiorw/PlzBAQEjPpYRERWaWlpaUKHoKlBqVTC2toaz549g9FoRF9fH0wmE7Zs2TK4T3tXHy48bEReRSOuVL7GXe1bvNJ3w23uDNhZm0+KMRgMCAoKgkKhgJ+fH9LT07F582ZotVqcPHkSly5d4lMTiGhMcVYnjZjRaERxcTF27NiBzs5OtLS0oOmzFKfu1eH+i3cAgL7+35/woJBJMQBgndc87Alxh7+rw+C2lJQUZGVlQSaTQS6Xo6enB/v378eRI0c4aYWIxgWLj0YlJSUFd171oXdZBPr6TRjq0ySRAAqZFQ5GeGPbysUoKytDWFgYDAYDAMDPzw8VFRVQKBQTlJ6IxIjFR6Ny4WED0q5VwST9/rV9tnIpdgY64sDWVTAajYO/t7GxQWtrKxwcHIZ4NRHR6PBenWSx35o+4uhNrVnptd84jt5Xv8H0pRdWM5SwX/kTZvlvHNze88WErLJmKJcGINjrBzg7O2PmzJlQKpWQy+UT/WcQkcjwjI8stvvfatyubjMb3jS8a4Rc+QMkMjm+6JvQevGfcPp7GmwWuA/uIwGw0Wc+crcFTWxoIhI9zh4gi7R39eH+i3d/eU3Pet4iSGRfz9wkkECC/g9vvtlnAMBd7Tvou/rGPSsR0R9xqJMscrXy9ZDb9b9k47PmPxjo74P1/KWwXWp+ZicBcPXJa/y8duk4pSQiMsfiI4vUtHZ8s2Thz+Zs3APH9T+jr7kGvToNJFbm1+56+02oedM5njGJiMxwqJMs0tE7/K3KJFIrKFx9YOxsR+evxf/nfb6MdTQioiGx+Mgi9ooRDBaYTGbX+H5/H87iJKKJxeIji3gvsIeNzPzjY/z8EZ+f34fJ0IMBkxE99ZX4XH0fikX+ZvsqZFJ4L5w1EXGJiAbxGh9ZZOvfXHDizgvzDRIJOn+9Cf0v2cCACbLZTlCG7YKd50qzXQcAbA10Gf+wRER/wOIji8ydaYMQz3lm6/is7GZjwT/+NezrJRLgR695mDPTZhxTEhGZ41AnWSxhnTsUMiuLXquQWWHPOvfhdyQiGmMsPrKYv6sDDkZ4w1Y+so+RrVyKgxHe8HPhPTmJaOJxqJNGZdvKxQCAjOIa9PYbR/R0BiIiIfBenTQmql5/RPa9OtzVvoME/1uc/tXX5/H96DUPe9a580yPiATF4qMxpe/qw9Unr1HzphMdvV9gr5DDe+EsbA104UQWIpoUWHxERCQqnNxCRESiwuIjIiJRYfEREZGosPiIiEhUWHxERCQqLD4iIhIVFh8REYkKi4+IiESFxUdERKLC4iMiIlFh8RERkaiw+IiISFRYfEREJCosPiIiEhUWHxERiQqLj4iIRIXFR0REosLiIyIiUWHxERGRqLD4iIhIVFh8REQkKv8Fhpck9/m8gPcAAAAASUVORK5CYII=\n", 59 | "text/plain": [ 60 | "
" 61 | ] 62 | }, 63 | "metadata": {}, 64 | "output_type": "display_data" 65 | } 66 | ], 67 | "source": [ 68 | "G = nx.read_edgelist('edge_list_short.txt', nodetype=int, create_using=nx.DiGraph())\n", 69 | "labels = {0:'0', 1:'1', 2:'2', 3:'3', 4:'4'}\n", 70 | "nx.draw(G, labels=labels)" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 21, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "data": { 80 | "text/plain": [ 81 | "" 82 | ] 83 | }, 84 | "execution_count": 21, 85 | "metadata": {}, 86 | "output_type": "execute_result" 87 | } 88 | ], 89 | "source": [ 90 | "G" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 22, 96 | "metadata": {}, 97 | "outputs": [ 98 | { 99 | "data": { 100 | "text/plain": [ 101 | "[3, 4, 0]" 102 | ] 103 | }, 104 | "execution_count": 22, 105 | "metadata": {}, 106 | "output_type": "execute_result" 107 | } 108 | ], 109 | "source": [ 110 | "nx.shortest_path(G, 3, 0)" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 23, 116 | "metadata": {}, 117 | "outputs": [ 118 | { 119 | "name": "stdout", 120 | "output_type": "stream", 121 | "text": [ 122 | "[0, 2, 1, 4]\n", 123 | "[0, 2, 1, 3, 4]\n", 124 | "[0, 2]\n", 125 | "[0, 2, 4]\n", 126 | "[0, 1, 4]\n", 127 | "[0, 1, 3, 4]\n", 128 | "[0, 1, 2]\n", 129 | "[0, 1, 2, 4]\n", 130 | "[1, 4, 3]\n", 131 | "[1, 3]\n", 132 | "[1, 2]\n", 133 | "[1, 2, 4, 3]\n", 134 | "[3, 4]\n" 135 | ] 136 | } 137 | ], 138 | "source": [ 139 | "for cycle in nx.simple_cycles(G):\n", 140 | " print(cycle)" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 24, 146 | "metadata": {}, 147 | "outputs": [ 148 | { 149 | "data": { 150 | "text/plain": [ 151 | "[2, 3, 4]" 152 | ] 153 | }, 154 | "execution_count": 24, 155 | "metadata": {}, 156 | "output_type": "execute_result" 157 | } 158 | ], 159 | "source": [ 160 | "list(G.neighbors(1))" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 31, 166 | "metadata": {}, 167 | "outputs": [ 168 | { 169 | "data": { 170 | "text/plain": [ 171 | "OutEdgeView([(0, 1), (0, 2), (1, 2), (1, 3), (1, 4), (2, 4), (2, 0), (2, 1), (3, 4), (3, 1), (4, 3), (4, 0)])" 172 | ] 173 | }, 174 | "execution_count": 31, 175 | "metadata": {}, 176 | "output_type": "execute_result" 177 | } 178 | ], 179 | "source": [ 180 | "G.edges()" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 8, 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "c = 0\n", 190 | "for edge in G.edges():\n", 191 | " G[edge[0]][edge[1]]['weight'] = c\n", 192 | " c = c+1\n" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 9, 198 | "metadata": {}, 199 | "outputs": [], 200 | "source": [ 201 | "for i in range(5):\n", 202 | " G.nodes[i]['value'] = 3.14+i" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 16, 208 | "metadata": {}, 209 | "outputs": [], 210 | "source": [ 211 | "dG = dgl.DGLGraph()\n", 212 | "dG.from_networkx(G, node_attrs=['value'], edge_attrs=['weight'])" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": 17, 218 | "metadata": {}, 219 | "outputs": [ 220 | { 221 | "data": { 222 | "text/plain": [ 223 | "DGLGraph(num_nodes=5, num_edges=12,\n", 224 | " ndata_schemes={'value': Scheme(shape=(), dtype=torch.float32)}\n", 225 | " edata_schemes={'weight': Scheme(shape=(), dtype=torch.int64)})" 226 | ] 227 | }, 228 | "execution_count": 17, 229 | "metadata": {}, 230 | "output_type": "execute_result" 231 | } 232 | ], 233 | "source": [ 234 | "dG" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": 18, 240 | "metadata": {}, 241 | "outputs": [ 242 | { 243 | "data": { 244 | "text/plain": [ 245 | "tensor([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])" 246 | ] 247 | }, 248 | "execution_count": 18, 249 | "metadata": {}, 250 | "output_type": "execute_result" 251 | } 252 | ], 253 | "source": [ 254 | "dG.edata['weight']" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 19, 260 | "metadata": {}, 261 | "outputs": [ 262 | { 263 | "data": { 264 | "text/plain": [ 265 | "tensor([3.1400, 4.1400, 5.1400, 6.1400, 7.1400])" 266 | ] 267 | }, 268 | "execution_count": 19, 269 | "metadata": {}, 270 | "output_type": "execute_result" 271 | } 272 | ], 273 | "source": [ 274 | "dG.ndata['value']" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": 33, 280 | "metadata": {}, 281 | "outputs": [ 282 | { 283 | "data": { 284 | "text/plain": [ 285 | "tensor([2, 3, 3, 2, 2])" 286 | ] 287 | }, 288 | "execution_count": 33, 289 | "metadata": {}, 290 | "output_type": "execute_result" 291 | } 292 | ], 293 | "source": [ 294 | "dG.out_degrees()" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 45, 300 | "metadata": {}, 301 | "outputs": [ 302 | { 303 | "data": { 304 | "text/plain": [ 305 | "tensor([2, 3, 2, 2, 3])" 306 | ] 307 | }, 308 | "execution_count": 45, 309 | "metadata": {}, 310 | "output_type": "execute_result" 311 | } 312 | ], 313 | "source": [ 314 | "dG.in_degrees()" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": 65, 320 | "metadata": {}, 321 | "outputs": [], 322 | "source": [ 323 | "dG.ndata['one'] = th.torch.ones(5)\n", 324 | "dG.ndata['deg'] = dG.in_degrees().float()" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": 66, 330 | "metadata": {}, 331 | "outputs": [ 332 | { 333 | "data": { 334 | "text/plain": [ 335 | "tensor([2., 3., 2., 2., 3.])" 336 | ] 337 | }, 338 | "execution_count": 66, 339 | "metadata": {}, 340 | "output_type": "execute_result" 341 | } 342 | ], 343 | "source": [ 344 | "dG.ndata['deg']" 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 83, 350 | "metadata": {}, 351 | "outputs": [], 352 | "source": [ 353 | "def message_func(edges):\n", 354 | " return {'x' : 2*edges.src['one'] }\n", 355 | "\n", 356 | "def reduce_func(nodes):\n", 357 | " tot = th.torch.sum(nodes.mailbox['x'], dim=1)\n", 358 | " tot = tot+nodes.data['deg']\n", 359 | " return {'tot' : tot}" 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": 84, 365 | "metadata": {}, 366 | "outputs": [], 367 | "source": [ 368 | "dG.register_message_func(message_func)\n", 369 | "dG.register_reduce_func(reduce_func)" 370 | ] 371 | }, 372 | { 373 | "cell_type": "code", 374 | "execution_count": 85, 375 | "metadata": {}, 376 | "outputs": [], 377 | "source": [ 378 | "dG.send(dG.edges())\n", 379 | "dG.recv(dG.nodes())" 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": 86, 385 | "metadata": {}, 386 | "outputs": [ 387 | { 388 | "data": { 389 | "text/plain": [ 390 | "tensor([6., 9., 6., 6., 9.])" 391 | ] 392 | }, 393 | "execution_count": 86, 394 | "metadata": {}, 395 | "output_type": "execute_result" 396 | } 397 | ], 398 | "source": [ 399 | "dG.ndata['tot']" 400 | ] 401 | }, 402 | { 403 | "cell_type": "code", 404 | "execution_count": 91, 405 | "metadata": {}, 406 | "outputs": [ 407 | { 408 | "name": "stdout", 409 | "output_type": "stream", 410 | "text": [ 411 | "tensor([0., 0., 0., 0., 0.])\n" 412 | ] 413 | } 414 | ], 415 | "source": [ 416 | "dG.ndata['tot'] = th.zeros(5)\n", 417 | "print(dG.ndata['tot'])\n", 418 | "dG.update_all()" 419 | ] 420 | }, 421 | { 422 | "cell_type": "code", 423 | "execution_count": 92, 424 | "metadata": {}, 425 | "outputs": [ 426 | { 427 | "data": { 428 | "text/plain": [ 429 | "tensor([6., 9., 6., 6., 9.])" 430 | ] 431 | }, 432 | "execution_count": 92, 433 | "metadata": {}, 434 | "output_type": "execute_result" 435 | } 436 | ], 437 | "source": [ 438 | "dG.ndata['tot']" 439 | ] 440 | } 441 | ], 442 | "metadata": { 443 | "kernelspec": { 444 | "display_name": "Python 3", 445 | "language": "python", 446 | "name": "python3" 447 | }, 448 | "language_info": { 449 | "codemirror_mode": { 450 | "name": "ipython", 451 | "version": 3 452 | }, 453 | "file_extension": ".py", 454 | "mimetype": "text/x-python", 455 | "name": "python", 456 | "nbconvert_exporter": "python", 457 | "pygments_lexer": "ipython3", 458 | "version": "3.7.4" 459 | } 460 | }, 461 | "nbformat": 4, 462 | "nbformat_minor": 2 463 | } 464 | -------------------------------------------------------------------------------- /GCN_hetero.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Graph Convolutional Netoworks and the ACM Heterogenous Graph\n", 8 | "this is a slightly modified version of the notebook from the DGL tutorial\n", 9 | "https://docs.dgl.ai/en/0.4.x/tutorials/hetero/1_basics.html\n" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "%matplotlib inline" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": {}, 24 | "source": [ 25 | "\n", 26 | ".. currentmodule:: dgl\n", 27 | "\n", 28 | "Working with Heterogeneous Graphs\n", 29 | "=================================\n", 30 | "\n", 31 | "**Author**: Quan Gan, `Minjie Wang `_, Mufei Li,\n", 32 | "George Karypis, Zheng Zhang\n", 33 | "\n", 34 | "In this tutorial, you learn about:\n", 35 | "\n", 36 | "* Examples of heterogenous graph data and typical applications.\n", 37 | "\n", 38 | "* Creating and manipulating a heterogenous graph in DGL.\n", 39 | "\n", 40 | "* Implementing `Relational-GCN `_, a popular GNN model,\n", 41 | " for heterogenous graph input.\n", 42 | "\n", 43 | "* Training a model to solve a node classification task.\n", 44 | "\n", 45 | "Heterogeneous graphs, or *heterographs* for short, are graphs that contain\n", 46 | "different types of nodes and edges. The different types of nodes and edges tend\n", 47 | "to have different types of attributes that are designed to capture the\n", 48 | "characteristics of each node and edge type. Within the context of\n", 49 | "graph neural networks, depending on their complexity, certain node and edge types\n", 50 | "might need to be modeled with representations that have a different number of dimensions.\n", 51 | "\n", 52 | "DGL supports graph neural network computations on such heterogeneous graphs, by\n", 53 | "using the heterograph class and its associated API.\n" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "Examples of heterographs\n", 61 | "-----------------------\n", 62 | "Many graph datasets represent relationships among various types of entities.\n", 63 | "This section provides an overview for several graph use-cases that show such relationships \n", 64 | "and can have their data represented as heterographs.\n", 65 | "\n", 66 | "Citation graph \n", 67 | "~~~~~~~~~~~~~~~\n", 68 | "The Association for Computing Machinery publishes an `ACM dataset `_ that contains two\n", 69 | "million papers, their authors, publication venues, and the other papers\n", 70 | "that were cited. This information can be represented as a heterogeneous graph.\n", 71 | "\n", 72 | "The following diagram shows several entities in the ACM dataset and the relationships among them \n", 73 | "(taken from `Shi et al., 2015 `_).\n", 74 | "\n", 75 | ".. figure:: https://data.dgl.ai/tutorial/hetero/acm-example.png# \n", 76 | "\n", 77 | "This graph has three types of entities that correspond to papers, authors, and publication venues.\n", 78 | "It also contains three types of edges that connect the following:\n", 79 | "\n", 80 | "* Authors with papers corresponding to *written-by* relationships\n", 81 | "\n", 82 | "* Papers with publication venues corresponding to *published-in* relationships\n", 83 | "\n", 84 | "* Papers with other papers corresponding to *cited-by* relationships\n", 85 | "\n", 86 | "\n", 87 | "Recommender systems \n", 88 | "~~~~~~~~~~~~~~~~~~~~ \n", 89 | "The datasets used in recommender systems often contain\n", 90 | "interactions between users and items. For example, the data could include the\n", 91 | "ratings that users have provided to movies. Such interactions can be modeled\n", 92 | "as heterographs.\n", 93 | "\n", 94 | "The nodes in these heterographs will have two types, *users* and *movies*. The edges\n", 95 | "will correspond to the user-movie interactions. Furthermore, if an interaction is\n", 96 | "marked with a rating, then each rating value could correspond to a different edge type.\n", 97 | "The following diagram shows an example of user-item interactions as a heterograph.\n", 98 | "\n", 99 | ".. figure:: https://data.dgl.ai/tutorial/hetero/recsys-example.png\n", 100 | "\n", 101 | "\n", 102 | "Knowledge graph \n", 103 | "~~~~~~~~~~~~~~~~\n", 104 | "Knowledge graphs are inherently heterogenous. For example, in\n", 105 | "Wikidata, Barack Obama (item Q76) is an instance of a human, which could be viewed as\n", 106 | "the entity class, whose spouse (item P26) is Michelle Obama (item Q13133) and\n", 107 | "occupation (item P106) is politician (item Q82955). The relationships are shown in the following.\n", 108 | "diagram.\n", 109 | "\n", 110 | ".. figure:: https://data.dgl.ai/tutorial/hetero/kg-example.png\n", 111 | "\n", 112 | "\n" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "Creating a heterograph in DGL\n", 120 | "-----------------------------\n", 121 | "You can create a heterograph in DGL using the :func:`dgl.heterograph` API.\n", 122 | "The argument to :func:`dgl.heterograph` is a dictionary. The keys are tuples\n", 123 | "in the form of ``(srctype, edgetype, dsttype)`` specifying the relation name\n", 124 | "and the two entity types it connects. Such tuples are called *canonical edge\n", 125 | "types*. The values are data to initialize the graph structures, that is, which\n", 126 | "nodes the edges actually connect.\n", 127 | "\n", 128 | "For instance, the following code creates the user-item interactions heterograph shown earlier.\n", 129 | "\n" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 2, 135 | "metadata": {}, 136 | "outputs": [ 137 | { 138 | "name": "stderr", 139 | "output_type": "stream", 140 | "text": [ 141 | "Using backend: pytorch\n" 142 | ] 143 | } 144 | ], 145 | "source": [ 146 | "# Each value of the dictionary is a pair of source and destination arrays.\n", 147 | "# Nodes are integer IDs starting from zero. Nodes IDs of different types have\n", 148 | "# separate countings.\n", 149 | "import dgl\n", 150 | "import numpy as np\n" 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "DGL supports creating a graph from a variety of data sources. The following\n", 158 | "code creates the same graph as the above.\n", 159 | "\n", 160 | "Creating from scipy matrix\n", 161 | "\n" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 3, 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "import scipy.sparse as sp\n", 171 | "# Creating from networkx graph\n", 172 | "import networkx as nx\n" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": {}, 178 | "source": [ 179 | "Manipulating heterograph\n", 180 | "------------------------\n", 181 | "You can create a more realistic heterograph using the ACM dataset. To do this, first \n", 182 | "download the dataset as follows:\n", 183 | "\n" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": 4, 189 | "metadata": {}, 190 | "outputs": [ 191 | { 192 | "name": "stdout", 193 | "output_type": "stream", 194 | "text": [ 195 | "['__header__', '__version__', '__globals__', 'TvsP', 'PvsA', 'PvsV', 'AvsF', 'VvsC', 'PvsL', 'PvsC', 'A', 'C', 'F', 'L', 'P', 'T', 'V', 'PvsT', 'CNormPvsA', 'RNormPvsA', 'CNormPvsC', 'RNormPvsC', 'CNormPvsT', 'RNormPvsT', 'CNormPvsV', 'RNormPvsV', 'CNormVvsC', 'RNormVvsC', 'CNormAvsF', 'RNormAvsF', 'CNormPvsL', 'RNormPvsL', 'stopwords', 'nPvsT', 'nT', 'CNormnPvsT', 'RNormnPvsT', 'nnPvsT', 'nnT', 'CNormnnPvsT', 'RNormnnPvsT', 'PvsP', 'CNormPvsP', 'RNormPvsP']\n" 196 | ] 197 | } 198 | ], 199 | "source": [ 200 | "import scipy.io\n", 201 | "import urllib.request\n", 202 | "\n", 203 | "data_url = 'https://data.dgl.ai/dataset/ACM.mat'\n", 204 | "data_file_path = 'ACM.mat'\n", 205 | "\n", 206 | "urllib.request.urlretrieve(data_url, data_file_path)\n", 207 | "data = scipy.io.loadmat(data_file_path)\n", 208 | "print(list(data.keys()))" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "The dataset stores node information by their types: ``P`` for paper, ``A``\n", 216 | "for author, ``C`` for conference, ``L`` for subject code, and so on. The relationships\n", 217 | "are stored as SciPy sparse matrix under key ``XvsY``, where ``X`` and ``Y``\n", 218 | "could be any of the node type code.\n", 219 | "\n", 220 | "The following code prints out some statistics about the paper-author relationships.\n", 221 | "\n" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": 5, 227 | "metadata": {}, 228 | "outputs": [ 229 | { 230 | "name": "stdout", 231 | "output_type": "stream", 232 | "text": [ 233 | "\n", 234 | "#Papers: 12499\n", 235 | "#Authors: 17431\n", 236 | "#Links: 37055\n" 237 | ] 238 | } 239 | ], 240 | "source": [ 241 | "print(type(data['PvsA']))\n", 242 | "print('#Papers:', data['PvsA'].shape[0])\n", 243 | "print('#Authors:', data['PvsA'].shape[1])\n", 244 | "print('#Links:', data['PvsA'].nnz)" 245 | ] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "metadata": {}, 250 | "source": [ 251 | "Converting this SciPy matrix to a heterograph in DGL is straightforward.\n", 252 | "\n" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": {}, 258 | "source": [ 259 | "Create a subset of the ACM graph using the paper-author, paper-paper, \n", 260 | "and paper-subject relationships. Meanwhile, also add the reverse\n", 261 | "relationship to prepare for the later sections.\n", 262 | "\n" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": 6, 268 | "metadata": {}, 269 | "outputs": [ 270 | { 271 | "data": { 272 | "text/plain": [ 273 | "Graph(num_nodes={'author': 17431, 'paper': 12499, 'subject': 73},\n", 274 | " num_edges={('paper', 'written-by', 'author'): 37055, ('author', 'writing', 'paper'): 37055, ('paper', 'citing', 'paper'): 30789, ('paper', 'cited', 'paper'): 30789, ('paper', 'is-about', 'subject'): 12499, ('subject', 'has', 'paper'): 12499},\n", 275 | " metagraph=[('author', 'paper'), ('paper', 'author'), ('paper', 'paper'), ('paper', 'paper'), ('paper', 'subject'), ('subject', 'paper')])" 276 | ] 277 | }, 278 | "execution_count": 6, 279 | "metadata": {}, 280 | "output_type": "execute_result" 281 | } 282 | ], 283 | "source": [ 284 | "G = dgl.heterograph({\n", 285 | " ('paper', 'written-by', 'author') : data['PvsA'],\n", 286 | " ('author', 'writing', 'paper') : data['PvsA'].transpose(),\n", 287 | " ('paper', 'citing', 'paper') : data['PvsP'],\n", 288 | " ('paper', 'cited', 'paper') : data['PvsP'].transpose(),\n", 289 | " ('paper', 'is-about', 'subject') : data['PvsL'],\n", 290 | " ('subject', 'has', 'paper') : data['PvsL'].transpose(),\n", 291 | " })\n", 292 | "G" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "**Metagraph** (or network schema) is a useful summary of a heterograph.\n", 300 | "Serving as a template for a heterograph, it tells how many types of objects\n", 301 | "exist in the network and where the possible links exist.\n", 302 | "\n", 303 | "DGL provides easy access to the metagraph, which could be visualized using\n", 304 | "external tools.\n", 305 | "\n" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 7, 311 | "metadata": {}, 312 | "outputs": [], 313 | "source": [ 314 | "import matplotlib.pyplot as plt\n" 315 | ] 316 | }, 317 | { 318 | "cell_type": "markdown", 319 | "metadata": {}, 320 | "source": [ 321 | "Learning tasks associated with heterographs\n", 322 | "-------------------------------------------\n", 323 | "Some of the typical learning tasks that involve heterographs include:\n", 324 | "\n", 325 | "* *Node classification and regression* to predict the class of each node or\n", 326 | " estimate a value associated with it.\n", 327 | "\n", 328 | "* *Link prediction* to predict if there is an edge of a certain\n", 329 | " type between a pair of nodes, or predict which other nodes a particular\n", 330 | " node is connected with (and optionally the edge types of such connections).\n", 331 | "\n", 332 | "* *Graph classification/regression* to assign an entire\n", 333 | " heterograph into one of the target classes or to estimate a numerical\n", 334 | " value associated with it.\n", 335 | "\n", 336 | "In this tutorial, we designed a simple example for the first task.\n", 337 | "\n", 338 | "A semi-supervised node classification example\n", 339 | "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n", 340 | "Our goal is to predict the publishing conference of a paper using the ACM\n", 341 | "academic graph we just created. To further simplify the task, we only focus\n", 342 | "on papers published in three conferences: *KDD*, *ICML*, and *VLDB*. All\n", 343 | "the other papers are not labeled, making it a semi-supervised setting.\n", 344 | "\n", 345 | "The following code extracts those papers from the raw dataset and prepares \n", 346 | "the training, validation, testing split.\n", 347 | "\n", 348 | "Note: in this version we look at 4 conferences: SOSP, SODA, Sigcom and VLDB." 349 | ] 350 | }, 351 | { 352 | "cell_type": "code", 353 | "execution_count": 83, 354 | "metadata": {}, 355 | "outputs": [], 356 | "source": [ 357 | "import numpy as np\n", 358 | "import torch\n", 359 | "import torch.nn as nn\n", 360 | "import torch.nn.functional as F\n", 361 | "\n", 362 | "pvc = data['PvsC'].tocsr()\n", 363 | "# sosp = 7\n", 364 | "# soda = 5\n", 365 | "# sigcom = 9\n", 366 | "# vldb = 13\n", 367 | "c_selected = [7, 5, 9, 13] \n", 368 | "\n", 369 | "p_selected = pvc[:, c_selected].tocoo()\n", 370 | "# remake 7,5,9,13 labels as 0,1,2,3\n", 371 | "labels = pvc.indices\n", 372 | "labels[labels==0] = 13\n", 373 | "labels[labels==7] = 0\n", 374 | "labels[labels==1] = 13\n", 375 | "labels[labels == 5] = 1\n", 376 | "labels[labels==2] = 13\n", 377 | "labels[labels == 9]= 2\n", 378 | "labels[labels == 3] = 13\n", 379 | "labels[labels == 13] = 3\n", 380 | "labels = torch.tensor(labels).long()\n", 381 | "\n", 382 | "# generate train/val/test split\n", 383 | "pid = p_selected.row\n", 384 | "shuffle = np.random.permutation(pid)\n", 385 | "\n", 386 | "train_idx = torch.tensor(shuffle[0:1400]).long()\n", 387 | "val_idx = torch.tensor(shuffle[1400:1500]).long()\n", 388 | "test_idx = torch.tensor(shuffle[1500:]).long()" 389 | ] 390 | }, 391 | { 392 | "cell_type": "code", 393 | "execution_count": null, 394 | "metadata": {}, 395 | "outputs": [], 396 | "source": [] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": 84, 401 | "metadata": {}, 402 | "outputs": [ 403 | { 404 | "name": "stdout", 405 | "output_type": "stream", 406 | "text": [ 407 | "332 662 648\n" 408 | ] 409 | } 410 | ], 411 | "source": [ 412 | "print( len(labels[labels==0]), len(labels[labels==1]), len(labels[labels==2]))" 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "execution_count": 85, 418 | "metadata": {}, 419 | "outputs": [ 420 | { 421 | "name": "stdout", 422 | "output_type": "stream", 423 | "text": [ 424 | "1400 719\n" 425 | ] 426 | } 427 | ], 428 | "source": [ 429 | "print(len(train_idx), len(test_idx))" 430 | ] 431 | }, 432 | { 433 | "cell_type": "markdown", 434 | "metadata": {}, 435 | "source": [ 436 | "Relational-GCN on heterograph\n", 437 | "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n", 438 | "We use `Relational-GCN `_ to learn the\n", 439 | "representation of nodes in the graph. Its message-passing equation is as\n", 440 | "follows:\n", 441 | "for each message type we will have a fully connected layer of trainable parameters W. Then for \n", 442 | "for each node and \n", 443 | "each edge type we compute\n", 444 | "\n", 445 | "\\begin{align} \\sum_{j\\in\\mathcal{N}_r(i)} W_r^{(l)}h_j^{(l)} \\end{align}\n", 446 | "\n", 447 | "following that sum over each edge type and apply sigma\n", 448 | "\n", 449 | "\\begin{align}h_i^{(l+1)} = \\sigma\\left(\\sum_{r\\in \\mathcal{R}}\n", 450 | " \\sum_{j\\in\\mathcal{N}_r(i)}W_r^{(l)}h_j^{(l)}\\right)\\end{align}\n", 451 | "\n", 452 | "Breaking down the equation, you see that there are two parts in the\n", 453 | "computation.\n", 454 | "\n", 455 | "(i) Message computation and aggregation within each relation $r$\n", 456 | "\n", 457 | "(ii) Reduction that merges the results from multiple relationships\n", 458 | "\n", 459 | "Following this intuition, perform message passing on a heterograph in\n", 460 | "two steps.\n", 461 | "\n", 462 | "(i) Per-edge-type message passing\n", 463 | "\n", 464 | "(ii) Type wise reduction\n", 465 | "\n" 466 | ] 467 | }, 468 | { 469 | "cell_type": "code", 470 | "execution_count": 86, 471 | "metadata": {}, 472 | "outputs": [], 473 | "source": [ 474 | "import dgl.function as fn\n", 475 | "\n", 476 | "class HeteroRGCNLayer(nn.Module):\n", 477 | " def __init__(self, in_size, out_size, etypes):\n", 478 | " super(HeteroRGCNLayer, self).__init__()\n", 479 | " # W_r for each relation\n", 480 | " self.weight = nn.ModuleDict({\n", 481 | " name : nn.Linear(in_size, out_size) for name in etypes\n", 482 | " })\n", 483 | "\n", 484 | " def forward(self, G, feat_dict):\n", 485 | " # The input is a dictionary of node features for each type\n", 486 | " funcs = {}\n", 487 | " for srctype, etype, dsttype in G.canonical_etypes:\n", 488 | " # Compute W_r * h\n", 489 | " Wh = self.weight[etype](feat_dict[srctype])\n", 490 | " # Save it in graph for message passing\n", 491 | " G.nodes[srctype].data['Wh_%s' % etype] = Wh\n", 492 | " # Specify per-relation message passing functions: (message_func, reduce_func).\n", 493 | " # Note that the results are saved to the same destination feature 'h', which\n", 494 | " # hints the type wise reducer for aggregation.\n", 495 | " funcs[etype] = (fn.copy_u('Wh_%s' % etype, 'm'), fn.mean('m', 'h'))\n", 496 | " # Trigger message passing of multiple types.\n", 497 | " # The first argument is the message passing functions for each relation.\n", 498 | " # The second one is the type wise reducer, could be \"sum\", \"max\",\n", 499 | " # \"min\", \"mean\", \"stack\"\n", 500 | " G.multi_update_all(funcs, 'sum')\n", 501 | " # return the updated node feature dictionary\n", 502 | " return {ntype : G.nodes[ntype].data['h'] for ntype in G.ntypes}" 503 | ] 504 | }, 505 | { 506 | "cell_type": "markdown", 507 | "metadata": {}, 508 | "source": [ 509 | "Create a simple GNN by stacking two ``HeteroRGCNLayer``. Since the\n", 510 | "nodes do not have input features, make their embeddings trainable.\n", 511 | "\n" 512 | ] 513 | }, 514 | { 515 | "cell_type": "code", 516 | "execution_count": 87, 517 | "metadata": {}, 518 | "outputs": [ 519 | { 520 | "data": { 521 | "text/plain": [ 522 | "[('paper', 'written-by', 'author'),\n", 523 | " ('author', 'writing', 'paper'),\n", 524 | " ('paper', 'citing', 'paper'),\n", 525 | " ('paper', 'cited', 'paper'),\n", 526 | " ('paper', 'is-about', 'subject'),\n", 527 | " ('subject', 'has', 'paper')]" 528 | ] 529 | }, 530 | "execution_count": 87, 531 | "metadata": {}, 532 | "output_type": "execute_result" 533 | } 534 | ], 535 | "source": [ 536 | "G.canonical_etypes" 537 | ] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": 88, 542 | "metadata": {}, 543 | "outputs": [ 544 | { 545 | "data": { 546 | "text/plain": [ 547 | "Graph(num_nodes={'author': 17431, 'paper': 12499, 'subject': 73},\n", 548 | " num_edges={('paper', 'written-by', 'author'): 37055, ('author', 'writing', 'paper'): 37055, ('paper', 'citing', 'paper'): 30789, ('paper', 'cited', 'paper'): 30789, ('paper', 'is-about', 'subject'): 12499, ('subject', 'has', 'paper'): 12499},\n", 549 | " metagraph=[('author', 'paper'), ('paper', 'author'), ('paper', 'paper'), ('paper', 'paper'), ('paper', 'subject'), ('subject', 'paper')])" 550 | ] 551 | }, 552 | "execution_count": 88, 553 | "metadata": {}, 554 | "output_type": "execute_result" 555 | } 556 | ], 557 | "source": [ 558 | "G" 559 | ] 560 | }, 561 | { 562 | "cell_type": "code", 563 | "execution_count": 89, 564 | "metadata": {}, 565 | "outputs": [ 566 | { 567 | "data": { 568 | "text/plain": [ 569 | "['author', 'paper', 'subject']" 570 | ] 571 | }, 572 | "execution_count": 89, 573 | "metadata": {}, 574 | "output_type": "execute_result" 575 | } 576 | ], 577 | "source": [ 578 | "G.ntypes" 579 | ] 580 | }, 581 | { 582 | "cell_type": "code", 583 | "execution_count": 90, 584 | "metadata": {}, 585 | "outputs": [ 586 | { 587 | "data": { 588 | "text/plain": [ 589 | "ParameterDict(\n", 590 | " (author): Parameter containing: [torch.FloatTensor of size 17431x10]\n", 591 | " (paper): Parameter containing: [torch.FloatTensor of size 12499x10]\n", 592 | " (subject): Parameter containing: [torch.FloatTensor of size 73x10]\n", 593 | ")" 594 | ] 595 | }, 596 | "execution_count": 90, 597 | "metadata": {}, 598 | "output_type": "execute_result" 599 | } 600 | ], 601 | "source": [ 602 | "embed_dict = {ntype : nn.Parameter(torch.Tensor(G.number_of_nodes(ntype), 10))\n", 603 | " for ntype in G.ntypes}\n", 604 | "for key, embed in embed_dict.items():\n", 605 | " nn.init.xavier_uniform_(embed)\n", 606 | "embed = nn.ParameterDict(embed_dict)\n", 607 | "embed" 608 | ] 609 | }, 610 | { 611 | "cell_type": "code", 612 | "execution_count": 91, 613 | "metadata": {}, 614 | "outputs": [], 615 | "source": [ 616 | "class HeteroRGCN(nn.Module):\n", 617 | " def __init__(self, G, in_size, hidden_size, out_size):\n", 618 | " super(HeteroRGCN, self).__init__()\n", 619 | " # Use trainable node embeddings as featureless inputs.\n", 620 | " embed_dict = {ntype : nn.Parameter(torch.Tensor(G.number_of_nodes(ntype), in_size))\n", 621 | " for ntype in G.ntypes}\n", 622 | " for key, embed in embed_dict.items():\n", 623 | " nn.init.xavier_uniform_(embed)\n", 624 | " self.embed = nn.ParameterDict(embed_dict)\n", 625 | " # create layers\n", 626 | " self.layer1 = HeteroRGCNLayer(in_size, hidden_size, G.etypes)\n", 627 | " self.layer2 = HeteroRGCNLayer(hidden_size, out_size, G.etypes)\n", 628 | "\n", 629 | " def forward(self, G):\n", 630 | " h_dict = self.layer1(G, self.embed)\n", 631 | " h_dict = {k : F.leaky_relu(h) for k, h in h_dict.items()}\n", 632 | " h_dict = self.layer2(G, h_dict)\n", 633 | " # get paper logits\n", 634 | " return h_dict['paper']" 635 | ] 636 | }, 637 | { 638 | "cell_type": "markdown", 639 | "metadata": {}, 640 | "source": [ 641 | "Train and evaluate\n", 642 | "~~~~~~~~~~~~~~~~~~\n", 643 | "Train and evaluate this network.\n", 644 | "\n" 645 | ] 646 | }, 647 | { 648 | "cell_type": "code", 649 | "execution_count": 92, 650 | "metadata": {}, 651 | "outputs": [ 652 | { 653 | "name": "stdout", 654 | "output_type": "stream", 655 | "text": [ 656 | "Loss 1.4772, Train Acc 0.2371, Val Acc 0.2300 (Best 0.2300), Test Acc 0.2281 (Best 0.2281)\n", 657 | "Loss 1.2481, Train Acc 0.6450, Val Acc 0.5300 (Best 0.5300), Test Acc 0.5410 (Best 0.5410)\n", 658 | "Loss 1.0722, Train Acc 0.7607, Val Acc 0.5800 (Best 0.5800), Test Acc 0.6203 (Best 0.6203)\n", 659 | "Loss 0.8221, Train Acc 0.8436, Val Acc 0.7400 (Best 0.7500), Test Acc 0.7955 (Best 0.7747)\n", 660 | "Loss 0.5720, Train Acc 0.8893, Val Acc 0.8100 (Best 0.8100), Test Acc 0.8150 (Best 0.8150)\n", 661 | "Loss 0.3809, Train Acc 0.9586, Val Acc 0.8600 (Best 0.8600), Test Acc 0.8428 (Best 0.8428)\n", 662 | "Loss 0.2482, Train Acc 0.9857, Val Acc 0.8700 (Best 0.8700), Test Acc 0.8707 (Best 0.8554)\n", 663 | "Loss 0.1638, Train Acc 0.9929, Val Acc 0.8500 (Best 0.8700), Test Acc 0.8790 (Best 0.8554)\n", 664 | "Loss 0.1139, Train Acc 0.9950, Val Acc 0.8800 (Best 0.8800), Test Acc 0.8971 (Best 0.8971)\n", 665 | "Loss 0.0848, Train Acc 0.9964, Val Acc 0.8900 (Best 0.8900), Test Acc 0.9068 (Best 0.8985)\n", 666 | "Loss 0.0661, Train Acc 0.9986, Val Acc 0.8900 (Best 0.8900), Test Acc 0.9068 (Best 0.8985)\n", 667 | "Loss 0.0535, Train Acc 0.9993, Val Acc 0.8900 (Best 0.8900), Test Acc 0.8985 (Best 0.8985)\n", 668 | "Loss 0.0450, Train Acc 0.9993, Val Acc 0.8900 (Best 0.8900), Test Acc 0.8985 (Best 0.8985)\n", 669 | "Loss 0.0385, Train Acc 1.0000, Val Acc 0.8800 (Best 0.8900), Test Acc 0.9026 (Best 0.8985)\n", 670 | "Loss 0.0335, Train Acc 1.0000, Val Acc 0.8800 (Best 0.8900), Test Acc 0.8999 (Best 0.8985)\n", 671 | "Loss 0.0298, Train Acc 1.0000, Val Acc 0.8800 (Best 0.8900), Test Acc 0.8887 (Best 0.8985)\n", 672 | "Loss 0.0270, Train Acc 1.0000, Val Acc 0.8600 (Best 0.8900), Test Acc 0.8790 (Best 0.8985)\n", 673 | "Loss 0.0247, Train Acc 1.0000, Val Acc 0.8600 (Best 0.8900), Test Acc 0.8790 (Best 0.8985)\n", 674 | "Loss 0.0230, Train Acc 1.0000, Val Acc 0.8600 (Best 0.8900), Test Acc 0.8748 (Best 0.8985)\n", 675 | "Loss 0.0216, Train Acc 1.0000, Val Acc 0.8600 (Best 0.8900), Test Acc 0.8734 (Best 0.8985)\n" 676 | ] 677 | } 678 | ], 679 | "source": [ 680 | "# Create the model. The output has four logits for four classes.\n", 681 | "model = HeteroRGCN(G, 10, 10, 4)\n", 682 | "\n", 683 | "opt = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)\n", 684 | "\n", 685 | "best_val_acc = 0\n", 686 | "best_test_acc = 0\n", 687 | "\n", 688 | "for epoch in range(100):\n", 689 | " logits = model(G)\n", 690 | " # The loss is computed only for labeled nodes.\n", 691 | " loss = F.cross_entropy(logits[train_idx], labels[train_idx])\n", 692 | "\n", 693 | " pred = logits.argmax(1)\n", 694 | " train_acc = (pred[train_idx] == labels[train_idx]).float().mean()\n", 695 | " val_acc = (pred[val_idx] == labels[val_idx]).float().mean()\n", 696 | " test_acc = (pred[test_idx] == labels[test_idx]).float().mean()\n", 697 | "\n", 698 | " if best_val_acc < val_acc:\n", 699 | " best_val_acc = val_acc\n", 700 | " best_test_acc = test_acc\n", 701 | "\n", 702 | " opt.zero_grad()\n", 703 | " loss.backward()\n", 704 | " opt.step()\n", 705 | "\n", 706 | " if epoch % 5 == 0:\n", 707 | " print('Loss %.4f, Train Acc %.4f, Val Acc %.4f (Best %.4f), Test Acc %.4f (Best %.4f)' % (\n", 708 | " loss.item(),\n", 709 | " train_acc.item(),\n", 710 | " val_acc.item(),\n", 711 | " best_val_acc.item(),\n", 712 | " test_acc.item(),\n", 713 | " best_test_acc.item(),\n", 714 | " ))" 715 | ] 716 | }, 717 | { 718 | "cell_type": "code", 719 | "execution_count": 93, 720 | "metadata": {}, 721 | "outputs": [ 722 | { 723 | "name": "stdout", 724 | "output_type": "stream", 725 | "text": [ 726 | "tensor(0.8707)\n" 727 | ] 728 | } 729 | ], 730 | "source": [ 731 | "logits = model(G)\n", 732 | "pred = logits.argmax(1)\n", 733 | "test_acc = (pred[test_idx] == labels[test_idx]).float().mean()\n", 734 | "print(test_acc)" 735 | ] 736 | }, 737 | { 738 | "cell_type": "code", 739 | "execution_count": 94, 740 | "metadata": {}, 741 | "outputs": [ 742 | { 743 | "name": "stdout", 744 | "output_type": "stream", 745 | "text": [ 746 | "[[array(['KDD'], dtype='\n", 915 | "\n", 928 | "\n", 929 | " \n", 930 | " \n", 931 | " \n", 932 | " \n", 933 | " \n", 934 | " \n", 935 | " \n", 936 | " \n", 937 | " \n", 938 | " \n", 939 | " \n", 940 | " \n", 941 | " \n", 942 | " \n", 943 | " \n", 944 | " \n", 945 | " \n", 946 | " \n", 947 | " \n", 948 | " \n", 949 | " \n", 950 | " \n", 951 | " \n", 952 | " \n", 953 | " \n", 954 | " \n", 955 | " \n", 956 | " \n", 957 | " \n", 958 | " \n", 959 | " \n", 960 | " \n", 961 | " \n", 962 | " \n", 963 | " \n", 964 | " \n", 965 | " \n", 966 | " \n", 967 | " \n", 968 | "
0123
sosp81.02.011.07.0
soda1.096.00.02.0
sigcom4.014.078.05.0
vldb1.09.00.091.0
\n", 969 | "" 970 | ], 971 | "text/plain": [ 972 | " 0 1 2 3\n", 973 | "sosp 81.0 2.0 11.0 7.0\n", 974 | "soda 1.0 96.0 0.0 2.0\n", 975 | "sigcom 4.0 14.0 78.0 5.0\n", 976 | "vldb 1.0 9.0 0.0 91.0" 977 | ] 978 | }, 979 | "execution_count": 101, 980 | "metadata": {}, 981 | "output_type": "execute_result" 982 | } 983 | ], 984 | "source": [ 985 | "import pandas as pd\n", 986 | "df = pd.DataFrame(mat, index =['sosp', 'soda', 'sigcom','vldb'])\n", 987 | "df" 988 | ] 989 | }, 990 | { 991 | "cell_type": "markdown", 992 | "metadata": {}, 993 | "source": [ 994 | "What's next?\n", 995 | "------------\n", 996 | "* Check out our full implementation in PyTorch\n", 997 | " `here `_.\n", 998 | "\n", 999 | "* We also provide the following model examples:\n", 1000 | "\n", 1001 | " * `Graph Convolutional Matrix Completion _`,\n", 1002 | " which we implement in MXNet\n", 1003 | " `here `_.\n", 1004 | "\n", 1005 | " * `Heterogeneous Graph Attention Network `_\n", 1006 | " requires transforming a heterograph into a homogeneous graph according to\n", 1007 | " a given metapath (i.e. a path template consisting of edge types). We\n", 1008 | " provide :func:`dgl.transform.metapath_reachable_graph` to do this. See full\n", 1009 | " implementation\n", 1010 | " `here `_.\n", 1011 | "\n", 1012 | " * `Metapath2vec `_ requires\n", 1013 | " generating random walk paths according to a given metapath. Please\n", 1014 | " refer to the full metapath2vec implementation\n", 1015 | " `here `_.\n", 1016 | "\n", 1017 | "* :doc:`Full heterograph API reference <../../api/python/heterograph>`.\n", 1018 | "\n" 1019 | ] 1020 | }, 1021 | { 1022 | "cell_type": "code", 1023 | "execution_count": null, 1024 | "metadata": {}, 1025 | "outputs": [], 1026 | "source": [] 1027 | } 1028 | ], 1029 | "metadata": { 1030 | "kernelspec": { 1031 | "display_name": "Python 3", 1032 | "language": "python", 1033 | "name": "python3" 1034 | }, 1035 | "language_info": { 1036 | "codemirror_mode": { 1037 | "name": "ipython", 1038 | "version": 3 1039 | }, 1040 | "file_extension": ".py", 1041 | "mimetype": "text/x-python", 1042 | "name": "python", 1043 | "nbconvert_exporter": "python", 1044 | "pygments_lexer": "ipython3", 1045 | "version": "3.7.4" 1046 | } 1047 | }, 1048 | "nbformat": 4, 1049 | "nbformat_minor": 1 1050 | } 1051 | --------------------------------------------------------------------------------