├── PCA ├── PCA.ipynb ├── PCA.py └── images │ ├── PCA.png │ ├── dataset.png │ └── eigen_vectors.png ├── README.md ├── decision_tree ├── decision_tree.ipynb ├── decision_tree.py └── images │ ├── data.png │ └── decision_boundary.png ├── k-means ├── images │ ├── dataset.png │ ├── k-means.png │ └── total_dist_vs_k.png ├── k-means.ipynb └── k-means.py ├── linear_regression ├── images │ ├── Error.png │ ├── cost.png │ └── data.png ├── linear_regression.ipynb ├── linear_regression.py ├── linear_regression_data.csv └── linear_regression_data_multi.csv ├── logistic_regression ├── images │ ├── cost.png │ ├── data.png │ ├── data_multi-class.png │ ├── decision_boundary.png │ ├── decision_boundary_multi-class.png │ ├── decision_boundary_overfitting.png │ ├── decision_boundary_regularization.png │ └── decision_boundary_underfitting.png ├── logistic_regression.ipynb ├── logistic_regression.py ├── logistic_regression_data.csv ├── logistic_regression_multi-class.ipynb ├── logistic_regression_multi-class.py └── logistic_regression_reg.py ├── neural_network ├── images │ └── decision_boundary_nnet.png ├── neural_network.ipynb ├── neural_network.py └── neural_network_mnist.py └── title.png /PCA/PCA.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "import seaborn as sns\n", 12 | "%matplotlib inline" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 3, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "x = np.array(range(25))\n", 22 | "y = x ** 1.3 + np.random.normal(10, 10, x.shape[0])\n", 23 | "X = np.stack((x, y), axis = 1)" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 4, 29 | "metadata": {}, 30 | "outputs": [ 31 | { 32 | "data": { 33 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhdZbn38e+duWnSIW2Slk6hpS0tWKYAFRCQAsoRbRWogGJVtOorKnouBYfjQT1K9Tgc31eOWoFjVRD6MhaVSQZBJpu2lA4pdKBD2jRN56ZD0iT3+WOv1rRJmqFZe2Xv9ftcV6+997PXyrpXd7t/Wc+z1rPM3RERkfjKiLoAERGJloJARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzCgKJBTNba2b7zWyPme00s5fN7LNm1uH/ATMrMzM3s6yQa0zKdkSOpiCQOHm/uxcCo4BZwC3AXdGWJBI9BYHEjrvvcvd5wIeBGWZ2qpm9z8wWmdluM9tgZre1WOWF4HGnmdWZ2TvNbIyZPWtm28xsq5ndY2YDDq1gZreY2cbgCORNM5sStGeY2a1mtjpYd66ZFbW3nZD/KkQABYHEmLv/A6gC3gXsBT4GDADeB3zOzKYFi14YPA5w9wJ3fwUw4HbgBGACMAK4DcDMxgM3AWcHRyDvAdYGP+OLwDTgomDdHcAdx9iOSOgUBBJ3m4Aid3/e3Ze4e7O7vwH8kcSXdZvcfZW7P+3u9e5eC/y0xfJNQC4w0cyy3X2tu68O3vsM8E13r3L3ehLhcbXGBSRKCgKJu2HAdjM718yeM7NaM9sFfBYY3N5KZlZiZvcF3T+7gT8cWt7dVwE3k/iS3xIsd0Kw6ijg4WDAeidQSSI4SsPaQZGOKAgktszsbBJB8HfgXmAeMMLd+wO/ItH9A9DWFL23B+2T3L0f8NEWy+Pu97r7BSS++B34YfDWBuAKdx/Q4k+eu29sZzsioVMQSOyYWT8zuxK4D/iDuy8BCoHt7n7AzM4Brm+xSi3QDIxu0VYI1JEY2B0GfLXFzx9vZpeYWS5wANhP4rd+SATM981sVLBssZlNPcZ2REKnIJA4eczM9pD4rfybJPr1PxG893+A7wbvfxuYe2gld98HfB94KejSmQx8BzgT2AX8GXioxXZySZyeuhXYDJQA3wje+zmJI4+ngm29Cpx7jO2IhM50YxoRkXjTEYGISMwpCEREYk5BICIScwoCEZGYS4mrGQcPHuxlZWVRlyEiklIWLFiw1d2LO1ouJYKgrKyMioqKqMsQEUkpZrauM8upa0hEJOYUBCIiMacgEBGJOQWBiEjMKQhERGIuJc4aEhGJkzW1dfz25bVs2nmAi8YN5tpzRpKdGd7v7QoCEZFeZHVtHdN+8RJ76hsB+GtlDa+9vZ1fXH9maNtU15CISC/y25fWHg6BQ/70RjVvb90b2jYVBCIivUj1rv1tt+9su70nKAhERHqRd41tPSNEYV4Wp48cENo2FQQiIr3I9eeO5H2Thh5+XZiXxU+nn05+TnhDuhosFhHpRbIzM7jj+jP5ymV1bN51gDNGDgg1BCDEI4LgBt6vt/iz28xuNrMiM3vazFYGjwPDqkFEJFWNKS7g/JMGhx4CEGIQuPub7n66u58OnAXsAx4GbgWecfexwDPBaxERiUiyxgimAKvdfR0wFZgTtM8BpiWpBhERaUOyguBa4I/B81J3rwYIHkvaWsHMZppZhZlV1NbWJqlMEZH4CT0IzCwH+ADw/7uynrvPdvdydy8vLu7wBjsiItJNyTgiuAJY6O41wesaMxsKEDxuSUINIiLSjmQEwXX8s1sIYB4wI3g+A3g0CTWIiEg7Qg0CM8sHLgMeatE8C7jMzFYG780KswYRETm2UE9Qdfd9wKCj2raROItIRER6AU0xISISsi27D/Dkss2srNkTdSlt0hQTIiIhuvvvb3P745UcbHIArjpzOP959SQyMiziyv5JRwQiIiHZsH0f//Hn5YdDAODBhVU8sWxzhFW1piAQEQnJa29vp9lbt7+8emvyizkGBYGISEhGFuW32T5iYNvtUVEQiIiE5JwTi3jX2MFHtA0b0IcPnz0ioorapsFiEem13ty8h9ysDMoG9426lG67c0Y5cyuqmP/2dkYX9+Wjk0cxID8n6rKOoCAQkV5n/bZ9zPx9BSs2J063PG/MIH75kbPon58dcWVdl5uVyQ2TR3HD5FFRl9IudQ2JSK/z1QcWHw4BgJdXb2PWEysirCi9KQhEpFfZW9/Ia29vb9X+7IqaNpaWnqAgEJFeJScrg8K81r3Wg/rmRlBNPCgIRKRXyc7M4MYLTmzV/pmLRkdQTTxosFhEep2bLx3HsAF9mLd4E7lZGVx/7kguObk06rLSloJARHqla8pHcE157zrfPl2pa0hEJOYUBCIiMacgEBGJOQWBiEjMhX3P4gFm9oCZrTCzSjN7p5kVmdnTZrYyeBwYZg0iInJsYR8R/Bx4wt1PBk4DKoFbgWfcfSzwTPBaREQiEloQmFk/4ELgLgB3b3D3ncBUYE6w2BxgWlg1iIhIx8I8IhgN1AL/Y2aLzOxOM+sLlLp7NUDwWNLWymY208wqzKyitrY2xDJFROItzCDIAs4EfunuZwB76UI3kLvPdvdydy8vLi4Oq0YRkdgLMwiqgCp3fy14/QCJYKgxs6EAweOWEGsQEZEOhBYE7r4Z2GBm44OmKcByYB4wI2ibATwaVg0iItKxsOca+gJwj5nlAGuAT5AIn7lmdiOwHrgm5BpEROQYQg0Cd38dKG/jrSlhbldERDpPVxaLiMScpqEWkdh6o2ond774Npt3HeCi8cXceMGJ5GVnRl1W0ikIRCSWlm3axTW/eoX6xmYA/rF2O4vW7+TOGW31Zqc3dQ2JSCzNeXnt4RA45K+VNayprYuoougoCEQklrbWNXSpPZ0pCEQkli45ufXsNoMLcjh9xIAIqomWgkBEYum6c0Zy7dkjyMwwAEr75fKL688kJyt+X4vm7lHX0KHy8nKvqKiIugwR6eV27T/I8k27OXFwX4b0z+vUOlt2H2DLnnpOHlJIVmZ6hYCZLXD3Dke/ddaQiKSF+/6xnu88tpz9B5vIzDA+cV4Z37pyYofrlfTLo6Rf50IjXaVX/IlILG3auZ9vPrKU/QebAGhqdu78+9s8t0JzWnaGgkBEUt7fV22lqbl1N/ff3tK9TDpDQSAiKW9oO+MBpTHv8uksBYGIpLzzxwxuddpnSWEu08uH9/i2Dhxs4q/La3jhrVoam5o7XiEFaLBYRFJeRobxh0+dy29fepuKdTs4qbiAT15wIoMKcnt0O0s37uLj//OPwxedjRqUzx9uPJcRRfk9up1kUxCISFooyM3ipkvGhrqNbz685Igrj9dt28esx1dwx0fODHW7YVPXkIhIJ+xvaGJx1a5W7a+s2RZBNT1LQSAi0gl52RmU9mvd1TQyxbuFQEEgItIpZsaXLx13RFtWhnHzpeF2RyVDqGMEZrYW2AM0AY3uXm5mRcD9QBmwFpju7jvCrENEpCdce85Iygb35dHXN5GblcHVZw3n1GH9oy7ruCVjsPjd7r61xetbgWfcfZaZ3Rq8viUJdYiIHLfJowcxefSgqMvoUVF0DU0F5gTP5wDTIqhBREQCYQeBA0+Z2QIzmxm0lbp7NUDw2HpScMDMZppZhZlV1NbqMnERkbCE3TV0vrtvMrMS4GkzW9HZFd19NjAbEtNQh1WgiITv/vnr+e/nV1Oz+wAXjyvhtg+c0ulpoiV8oR4RuPum4HEL8DBwDlBjZkMBgkdNDyiSxp6prOGWB5ewbts+Dhxs5ollm/nU7+ZHXZa0EFoQmFlfMys89By4HFgKzANmBIvNAB4NqwYRid7cig2t2pZu3M2yTa0vzpJohNk1VAo8bGaHtnOvuz9hZvOBuWZ2I7AeuCbEGkQkYu3dBDEFbo4YG6EFgbuvAU5ro30bMCWs7YpI73LVWcN5annNEW0nDylMi/Pv04WuLBaRUL3nlCF8b9qpnNA/j8wM49IJJdz18bOjLkta0OyjIhK6GyaP4obJo6IuQ9qhIBCJuZU1e/jzkmrysjOZdvowndYZQwoCkRh7bPEmbr7/9cP3+73j2VXc++nJvGO4+u/jRGMEIjHV3Ozc/pfKI276vqe+kZ88/WaEVUkUFAQiMbX7wEE27TrQqn1F9Z4IqpEoKQhEYqp/n2zKBrW+qcokdQvFjoJAJKbMjNs+cAq5Wf/8GhhckMPX3js+wqokChosFomxi8eX8OLX3s2Ty2vok53Je08dQkGuvhbiRp+4SMyV9MvTOf4xp64hEZGY0xGBSBLU1Tfy/T9X8qfFm+iTk8nH3jmKz7/7JIJJGVOKu/O3t2pZuH4nJ5UU8N5ThpCTpd8pU5mCQCQJbn3wDf70RjWQOFf/x0+9RV52Jp961+iIK+u6L9//Oo+8vunw67NGDeSeT51LXnZmhFXJ8VCMi4Ssrr6Rx5dubtXe1jz9vd3C9TuOCAGABet2MG/xpnbWkFSgIBAJmbcz8X4qzsdfWb27S+2SGhQEIiErzMvm8omlrdqvOmt4j29rx94G7nxxDbc/Xsmra7b1+M+fNGxA2+26CC2laYxAJAl+ePUk+uRk8qc3qsnPyeSGyaOY2cPjAxt37ueDd7zElj31APz6b2v48qXj+NKlY3tsG+8Y3p/rzx3Jva+tP9x2/kmDuHLSCT22DUk+a++wtTcpLy/3ioqKqMsQ6dX+/dGlzHll3RFtOVkZvPr1KRT1zenRbc1fu52F63YwtrSAi8eVkJGRemc/xYGZLXD38o6WC/2IwMwygQpgo7tfaWYnAvcBRcBC4AZ3bwi7DpF091ZNXau2hsZm1m7b2+NBcHZZEWeXFfXoz5ToJGOM4EtAZYvXPwR+5u5jgR3AjUmoQSTtnTGydf99QW4W40oLI6hGUkmoQWBmw4H3AXcGrw24BHggWGQOMC3MGkTi4tPvGs34Fl/6GQb/duUEzR0kHQr7X8h/AV8DDv3rHATsdPfG4HUVMKytFc1sJjATYOTIkSGXKZL6BvbN4U9fvIDnVmxhy556LhpXzIii1tNMixwttCAwsyuBLe6+wMwuPtTcxqJtjla7+2xgNiQGi0MpUiTNZGdmcPkpQ6IuQ1JMmEcE5wMfMLN/AfKAfiSOEAaYWVZwVDAc0CWJIiIRCm2MwN2/7u7D3b0MuBZ41t0/AjwHXB0sNgN4NKwaRESkY1FcWXwL8BUzW0VizOCuCGoQEZFAUk4ncPfngeeD52uAc5KxXZGwHGxq5pnKGmrrGrhYg7KS4nRemUgXbaur59rZr7JyS+ICrswM4/YPvYPp5SMirkykezTpnEgXzX5hzeEQAGhqdr732HL2NTQeYy2R3ktBINJFizbsbNW2p76RVVtaT/EgkgoUBCJdNLakoFVbblYGo4r6RlCNyPHrMAjM7CYzG5iMYkRSwWcvGkNxYe4RbV+45CT652dHVJHI8enMYPEQYL6ZLQTuBp70VJi7WiQkI4ryeermC3lwYRVb6xqYMqFEM3FKSuvU/QiCyeIuBz4BlANzgbvcfXW45SXofgQiIl3X2fsRdGqMIDgC2Bz8aQQGAg+Y2Y+Oq0oROaY9Bw6yYN0OttXVR12KpLEOu4bM7IskpoLYSmI66a+6+0EzywBWkphdVHqZ3QcOsrKmjjHFfRmQ37M3JZHkuH/+er772HL2NjSRk5nB5y4ew5cvGxd1WZKGOjNGMBj4kLsfcQ88d28OZhiVXmbOy2uZ9fgK9h9sIjcrgy9fNo7PXjQm6rKkCzZs38fXH1pCc9Bz29DUzM+fWcnk0YN455hB0RYnaafDriF3//bRIdDivcq22iU6q7bs4bbHlrH/YBMA9Y3NzHp8BYvbOPddeq+/vVV7OARaev7NLckvRtKeriNIM8+/WUtb4//P6QskpRx9empH7SLHQ0GQZtr7oigpzEtyJXI8ppxcwslDjrzXcGm/XK46c3hEFUk606RzaeY9pwxh9OC+rNm693DbsAF9eP9pQyOsSroqKzOD+2ZO5jcvrmHR+p2MKy3k0xeOZmBfDfxLz+vUdQRR03UEXbO1rp7fvLiGJVW7mDC0HzMvHE1pPx0RiMRNZ68j0BFBGhpckMvXr5gQdRkikiI0RiAiEnMKAhGRmAstCMwsz8z+YWaLzWyZmX0naD/RzF4zs5Vmdr+ZafRLelx9YxONTc1RlyGSEsI8IqgHLnH304DTgfea2WTgh8DP3H0ssAO4McQaJGa2723gs79fwMRvP8lp33mKH/ylkqa2rswSkcNCCwJPOHTLpuzgjwOXAA8E7XOAaWHVIPHzr3Nf54llm2lqdvY2NDH7hTX8+oWkTJIrkrJCHSMws0wzex3YAjwNrAZ2uvuhm7tWAcPaWXemmVWYWUVtbW2YZUqa2Lmvgeffav1v5ZFFGyOoRiR1hBoE7t7k7qcDw4FzgLbOaWzzuN3dZ7t7ubuXFxcXh1mmpImMDCPDrFV7VobOiRA5lqT8D3H3ncDzwGRggJkdun5hOLApGTVI+uuXl82Vk1pfQX3duSMjqEYkdYR51lCxmQ0InvcBLgUqgeeAq4PFZgCPhlWDxM+sD03i4+eVUVyYS9mgfL595URumDwq6rJEerXQppgws0kkBoMzSQTOXHf/rpmNBu4DioBFwEfd/Zi3X9IUEyIiXRf5FBPu/gZwRhvta0iMF8TWzn0N1NU3MnxgftSliIhorqFkamxq5luPLOWBBVU0NjsTh/bj/153BieVFERdWlJs3Lmf//fMShZX7eLkIYV84ZKTGF0cj30X6c10OkUS3fn3t7lv/gYagwucllfv5vP3LIy4quTY39DE9F+9wn3zN1BZvZuHF23kml+9wva9DVGXJhJ7CoIkemLp5lZtb9bs4e0W9w5oz9qte6ndc8yhlF7tiWXVbNy5/4i2bXsbdI6/SC+grqEk6tcnu1VbhkFBbvsfw5raOj5/7yIqq3eTYXDlpBP40dWTyMvODLPUHrdr38E223fub7tdRJJHRwRJ9Inzyzj6eqf3n3bCMe9D+6X7XqeyejcAzQ7zFm/iv59PvSkTpkwoJSvjyJ03g/ecUhpRRSJyiIIgid49voQ7P1bO+ScNYuLQftx86Vh+dPWkdpev3rWfJRt3tWp/alnrLqbebkRRPj+ZfhqDglst9svL4ntTT+WUE/pHXJmIqGsoyaZMKGXKhM79Fpyfk0V2pnGw6chrPQbkt+5iSgVTTx/GFacOpWrHPk4Y0CflurdE0pWOCHqx/n2yufqsEa3aP3n+iRFU0zNysjIYXVygEBDpRXRE0Mt9b+oplA3K5/GlmynMy+Lj55V1+ogiHTQ0NvPsihq27W3g4vElDBvQJ+qSRNKOguA4bNq5nzueW8XSjbuYeEJ/Pv/uMT1+tXBWZgafuWgMn7loTI/+3FSwta6eD//6FVbXJk6vzcww/vPqSXzozOERVyaSXhQE3bS3vpFrfvXK4XPjF1ft4rkVW3j6KxdSmJeaffi9za+eX304BACamp3b5i3jilOH0idHXUsiPUVjBN305yWtL5DavPsAjy2ujqii5KvetZ/fv7KWhxdVsa+hscPlu2rRhp2t2nYfaGR1bV0bS4tId+mIoJt2tDM1wva9qXv1b1c8uWwzX7h3EQ3BDeKH9HuTuZ95JyMH9VzX2LjSAhas23FEW152BiOKNFmfSE/SEUE3TZlQ2uriMDO4dGL6D+Q2NTv//uiywyEAiaOhn/31rR7dzucuOonBBTlHtH1xylj6t3GFtoh0n44IuumkkgJ+8MF38IO/VLLnQCOFuVl87YqTOXlIv6hLC13N7gNs3n2gVfviNrpyjsfIQfk8efOFPLRwI1v31nPphFLOLivq0W2IiILguFx3zkimnT6Mddv3MrIon/ycePx1Di7IZVDfHLYd1T02fkhhj29rUEEun75wdI//XBH5J3UNHac+OZmcPKRfbEIAEheF3XLFyUd0jfXvk83Nl46LrigR6bb4fHtJj5pePoJ3DOufuNAtN4upZ5xASWFe1GWJSDeEFgRmNgL4HTAEaAZmu/vPzawIuB8oA9YC0919R3s/R5KrudnJOGqW0PZMGNqPCUPTf0xEJN2F2TXUCPyru08AJgOfN7OJwK3AM+4+FngmeC0R++Xzqznre08z9luP8+nfVbCljcFgEUlPoQWBu1e7+8Lg+R6gEhgGTAXmBIvNAaaFVYN0ztyKDfzwiRVs29tAU7Pz9PIaPheTW2iKSJIGi82sDDgDeA0odfdqSIQFUNLOOjPNrMLMKmpra5NRZmw9tLCqVduCdTtYt63jW2iKSOoLPQjMrAB4ELjZ3Xd3dj13n+3u5e5eXlxcHF6BQmY7YwIZR18xJyJpKdQgMLNsEiFwj7s/FDTXmNnQ4P2hwJYwa5COTS9vfc+D88YM0lQOIjERWhCYmQF3AZXu/tMWb80DZgTPZwCPhlWDdM7U04fxvWmnMmpQPoW5WXzojGHccf2ZUZclIkli7t7xUt35wWYXAC8CS0icPgrwDRLjBHOBkcB64Bp3336sn1VeXu4VFRWh1Ckikq7MbIG7l3e0XGjXEbj734H2OpmnhLVdka56c/MeZj1eyeKqXYwrLeCr7xnPWaM0p5HEh6aYkFirq2/k+t+8ynNv1rJ9bwOvrtnOR+/8R6t7TYikMwWBxNqTSze3mjxv/8EmHlm0MaKKRJJPQSCx1vKeCi3VN7bdLpKOFAQSa5dPLCX/qPsfZ2YYHzhtaEQViSSfgkBibVBBLnfOKGdcaQEAI4vy+cV1Z3BSSc/fW0Gkt9I01BJ7540ZzFNfvoh9DY30yc7EdEW1xIyCQCQQp5sLibSkriERkZhTEIiIxJyCQEQk5hQEIiIxpyAQEYm5tA+CsGZXFRFJF2l7vtwvnl3J3S+tZff+g1x+SinfnXoqgwtyoy5LRKTXScsguOe1dfz4qbcOv/7Lks3s2n+Qez41OcKqRER6p7TsGnpwQeubsb+0ahvVuzS1sIjI0dIyCNq76bpuxi4i0lpaBsH0s1vfjP3i8cWU9suLoBoRkd4tzJvX321mW8xsaYu2IjN72sxWBo8Dw9j29PIR/NuVExk+sA+FuVlcdeZwfv7hM8LYlIhIygvz5vUXAnXA79z91KDtR8B2d59lZrcCA939lo5+lm5eLyLSdZ29eX1oRwTu/gKw/ajmqcCc4PkcYFpY2xcRkc5J9hhBqbtXAwSPJe0taGYzzazCzCpqa2uTVqCISNz02sFid5/t7uXuXl5cXBx1OSIiaSvZQVBjZkMBgsctSd6+iIgcJdlBMA+YETyfATya5O2LiMhRwjx99I/AK8B4M6sysxuBWcBlZrYSuCx4LSIiEQptriF3v66dt6aEtU0REem6XjtYLCIiyaEgEBGJOQWBiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzCgIRkZhTEIiIxJyCQEQk5hQEIiIxpyAQEYm5SILAzN5rZm+a2SozuzWKGkREJCHpQWBmmcAdwBXAROA6M5uY7DraU7unnrr6xqjLEBFJmtBuXn8M5wCr3H0NgJndB0wFlkdQy2Ebtu/j5vtfZ8G6HeRkZjD97OHc9v5TyMpU75mIpLcovuWGARtavK4K2o5gZjPNrMLMKmpra0Mv6qZ7F7Jg3Q4AGpqa+cOr6/nNi2+Hvl0RkahFEQTWRpu3anCf7e7l7l5eXFwcakFVO/axuGpXq/a/LKkOdbsiIr1BFEFQBYxo8Xo4sCmCOg7Lz8kio4146pubmfxiRESSLIogmA+MNbMTzSwHuBaYF0EdhxX1zeH9p53Qqn3GO8uSX4yISJIlfbDY3RvN7CbgSSATuNvdlyW7jqP98KpJjBiYz+NLq+nfJ5tPXnAiV7xjaNRliYiEztxbdc/3OuXl5V5RURF1GSIiKcXMFrh7eUfL6dxIEZGYUxCIiMScgkBEJOYUBCIiMacgEBGJOQWBiEjMpcTpo2ZWC6zr5uqDga09WE6qifP+a9/jK87733LfR7l7h3P0pEQQHA8zq+jMebTpKs77r32P575DvPe/O/uuriERkZhTEIiIxFwcgmB21AVELM77r32Przjvf5f3Pe3HCERE5NjicEQgIiLHoCAQEYm5tA4CM3uvmb1pZqvM7Nao60kmM1trZkvM7HUzS/s5vM3sbjPbYmZLW7QVmdnTZrYyeBwYZY1haWffbzOzjcHn/7qZ/UuUNYbFzEaY2XNmVmlmy8zsS0F72n/2x9j3Ln/2aTtGYGaZwFvAZSRujzkfuM7dl0daWJKY2Vqg3N1jcVGNmV0I1AG/c/dTg7YfAdvdfVbwi8BAd78lyjrD0M6+3wbUufuPo6wtbGY2FBjq7gvNrBBYAEwDPk6af/bH2PfpdPGzT+cjgnOAVe6+xt0bgPuAqRHXJCFx9xeA7Uc1TwXmBM/nkPhPknba2fdYcPdqd18YPN8DVALDiMFnf4x977J0DoJhwIYWr6vo5l9SinLgKTNbYGYzoy4mIqXuXg2J/zRAScT1JNtNZvZG0HWUdl0jRzOzMuAM4DVi9tkfte/Qxc8+nYPA2mhLz36wtp3v7mcCVwCfD7oPJD5+CYwBTgeqgZ9EW064zKwAeBC42d13R11PMrWx713+7NM5CKqAES1eDwc2RVRL0rn7puBxC/Awia6yuKkJ+lEP9aduibiepHH3Gndvcvdm4Dek8edvZtkkvgjvcfeHguZYfPZt7Xt3Pvt0DoL5wFgzO9HMcoBrgXkR15QUZtY3GDzCzPoClwNLj71WWpoHzAiezwAejbCWpDr0JRj4IGn6+ZuZAXcBle7+0xZvpf1n396+d+ezT9uzhgCC06b+C8gE7nb370dcUlKY2WgSRwEAWcC96b7vZvZH4GISU/DWAP8OPALMBUYC64Fr3D3tBlXb2feLSXQNOLAW+MyhPvN0YmYXAC8CS4DmoPkbJPrK0/qzP8a+X0cXP/u0DgIREelYOncNiYhIJygIRERiTkEgIhJzCgIRkZhTEIiIxJyCQEQk5hQEIiIxpyAQ6QYzOzuY1CsvuJJ7mZmdGui261EAAAChSURBVHVdIt2hC8pEusnM/gPIA/oAVe5+e8QliXSLgkCkm4I5rOYDB4Dz3L0p4pJEukVdQyLdVwQUAIUkjgxEUpKOCES6yczmkbjz3Ykkbhl4U8QliXRLVtQFiKQiM/sY0Oju9wb3x37ZzC5x92ejrk2kq3REICIScxojEBGJOQWBiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTm/hfQKPdnJZxhLAAAAABJRU5ErkJggg==\n", 34 | "text/plain": [ 35 | "
" 36 | ] 37 | }, 38 | "metadata": { 39 | "needs_background": "light" 40 | }, 41 | "output_type": "display_data" 42 | } 43 | ], 44 | "source": [ 45 | "sns.scatterplot(x = X[:, 0], y = X[:, 1], edgecolor = \"none\")\n", 46 | "plt.title('Dataset')\n", 47 | "plt.ylabel('y')\n", 48 | "plt.xlabel('x')\n", 49 | "plt.show()" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 14, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "def feature_normaliza(X):\n", 59 | " mu = np.mean(X, 0) \n", 60 | " sigma = np.std(X, 0) \n", 61 | " def get_norm(col):\n", 62 | " mu = np.mean(col) \n", 63 | " sigma = np.std(col)\n", 64 | " return (col - mu)/sigma\n", 65 | " return np.apply_along_axis(get_norm, 0, X), mu, sigma\n", 66 | "\n", 67 | "def drawline(p1, p2, color = 'r'):\n", 68 | " sns.lineplot([p1[0],p2[0]], [p1[1],p2[1]], color = color) \n", 69 | " \n", 70 | "def project_data(X_norm, U, K):\n", 71 | " Z = np.zeros((X_norm.shape[0], K))\n", 72 | " U_reduce = U[:, 0:K] \n", 73 | " Z = np.dot(X_norm, U_reduce) \n", 74 | " return Z\n", 75 | "\n", 76 | "def recover_data(Z, U, K):\n", 77 | " X_rec = np.zeros((Z.shape[0], U.shape[0]))\n", 78 | " U_recude = U[:, 0:K]\n", 79 | " X_rec = np.dot(Z, U_recude.T)\n", 80 | " return X_rec" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 6, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "X_norm, mu, sigma = feature_normaliza(X) " 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 16, 95 | "metadata": {}, 96 | "outputs": [ 97 | { 98 | "data": { 99 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAEGCAYAAAByy7CMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAbKklEQVR4nO3deXhV5b328e8v8wAhJIQ5EOahMggBBOeBOlRr1Worp1rU1jpd1Vo9p+d0fuvbqm31tdXWgvo6UaviXAfqjFJBAqLMgxEkjAlBCIEkJHnOHwkxyd7JXgl7Z+2d3J/rylX2k7X3+sWGm7We9QzmnENEJJQ4vwsQkdigsBARTxQWIuKJwkJEPFFYiIgnCX4X0Ba9evVyeXl5fpch0mktW7asxDmXE+x7MRUWeXl5FBQU+F2GSKdlZlta+p5uQ0TEE4WFiHiisBARTxQWIuKJwkJEPFFYiByF6ppauspkzJh6dCoSLbaWHuR/nlvJ+5tKyE5P4pqTh/G9E4f6XVZE+XZlYWa5Zva2ma01s9VmdqNftYi01fcfLeC9jSU4ByUHqrjt5bW8/MkOv8uKKD9vQ6qBHzvnxgDHAdeb2Vgf6xHxZNW2fazbWRbQ/txHRT5U03F8Cwvn3A7n3PL6P5cBa4EBftUj4lVCvAVtj48L3t5ZREUHp5nlAccCS/ytRCS00X0zmJibGdD+7SmDfKim4/geFmbWDXgGuMk5tz/I9682swIzKyguLu74AkWCmHt5PudP7E9GSgLDe3fjjxdP4NTRvf0uK6LMz8c+ZpYI/BNY4Jy7K9Tx+fn5ThPJRCLHzJY55/KDfc/PpyEGPAis9RIUIuIvP29DjgcuA04zsxX1X+f4WI+ItMK3QVnOufeBzt19LNKJ+N7BKSKxQWEhIp4oLETEE4WFiHiisBARTxQWIuKJ1rMQacWyLaX8/0Wb2Xuwiplj+vCd4waTEN81/41VWIi0YOnmUi6ds5jq2ropEYs27WHdzjJuv2i8z5X5o2tGpIgHcxcWNgTFEU8vK6LkQKVPFflLYSHSguIgoVBT6ygtr/KhGv8pLERacNqowCnng7LSGJ7TzYdq/KewEGnB908aytnH9G14PSAzlT9feixxnXxFrJaog1Ni0jvrd/PcR9uIjzO+lZ/LtKHZYT9HSmI8f/3OZLaWHuSLg4cZ2z+j0y+d1xqFhcScxxdv4WfPr2p4/dxH27hv1iTOGdcvIufLzUojNysiHx1TdBsiMcU5x5/e3NisDf781iafKuo6FBYSUw7XOHaXBT6l2Lb3oA/VdC0KC4kpSQlxTBsSeE9w4sgcH6rpWhQWEnP+7wXjyM1KbXg9sk83fva1MT5W1DWog1NizvDe3XjnllNZurmUxHhj0qCe1K3/LJGksJCYFB9nHBeBx6XSMt2GiIgnCgsR8URhISKeKCxExBOFhYh4orAQEU8UFiLiicJCRDxRWIiIJwoLEfFEYSEinigsRMQThYWIeKJZp9JlVByu4e11u6msruW0Mb3JSEn0u6SYorCQLuGzknJmzV3Mjn0VAHRPTuDB2VOYGmTVLQlOtyHSJdzx6rqGoAAoq6zm541WCJfQFBbSJRRs2RvQtn5XGWUVh32oJjYpLKRLGN47PaCtX48U0pN0J+6VwkK6hJtnjiI54ctfdzO49cxRXXYrwvbwNVbN7CHgXGC3c+4YP2uRzm3qkCxeu+kknl1eRGV1LeeO78f4gZl+lxVT/L4Gexi4F3jU5zqkk1i9fR93vLaelUVfMLZ/Bv955mgm5NaFwpBe6fz4q6N8rjB2+Xob4pxbCJT6WYN0HnvLq5g1dwkLNxSz9+BhFm3aw3ceWMLusorQb5aQor7PwsyuNrMCMysoLi72uxyJYi+v3MG+Q02fbpRVVvPSxzt8qqhzifqwcM7Ncc7lO+fyc3K0RZ20rKq6tk3t0jZRHxYiXp09rm+TJx4AifHG18b18/T+isM1OOciUVqnoLCQTqNfj1TmXJ7PsJy6MRVDeqXz1/+YzKDstFbft7JoH+ff+z6jf/4ax9/+FvOXFXVEuTHH/ExSM3sCOAXoBewCfumce7Cl4/Pz811BQUEHVSex7GBVNWkeBlwdqqrhhDveYk95VUObGcy/ZjqTB3e9eSNmtsw5lx/se74+OnXOXern+aXz8hIUAO9tLG4SFADOwfMfbe+SYdEa3YZIl5YYH/yvQEK8RnY2p7CQLu2EEb0Y2DO1SVtCnHHx5FyfKopeCgvp0hLj45j3vWmc+ZU+ZKYlMjE3kwe+m8/Y/hl+lxZ1/B7uLeK7wdnp/O2yoH160oiuLETEE4WFiHii2xCJWltLDzJnYSGFJQeYPDiL7504RIvs+khhIVFpd1kFF/xlESUH6sZALNq0h3fW7+b5647XgjU+0W2IRKWnC4oaguKIT4r28f6mEp8qEoWFRKXd+4OvQbG7rLKDK5EjFBYSlU4Z3TugLTHeOHFELx+qEVBYSJQ6dVRvrjl5GIn1w667Jydwx0Xj6ZOR4nNlXZc6OCVq/eTs0Vx1whA+Ly1ndN8M0pP16+on/deXDrNsSyk79lVw3NBsenVL9vSenO7J5HT3dqxElsJCIq7icA1XPbKURZv2AJAUH8ftF43jwkkDfa5M2kJ9FhJxj36wuSEoAKpqavnZ86vYr60DY4rCQiJuSWHgbg8Hq2pYWbTPh2qkvRQWEnGDswP3GTWDQVmtr40p0UVhIRF35Ql5ZKcnNWn79pRB5CosYoo6OCXiBvZM45UbT2Teks/Z8cUhTh6V43l5fokeCgvpEH0yUrh55ki/y5CjoNsQEfFEVxYS1M59Fdz9+gaWbillWE43fnjaCMYN7BGRc+05UMk764vpkZrIyaNyWlxxW/ylsJAAh2tquXTuYj4rKQegsLicf28q4bWbTgp7p+Qba3Zx/d+XU1m/H+nQnHT+8f3j6K05IFFHES4B3l1f3BAUR5RX1fB0mLf1q66p5afPr2wICqgLpnve3BjW80h4KCwkQFll8JGVZWEecVm09xC79geuT7Fsy96wnkfCQ2EhAU4Z2ZvUxPiA9nM8PO5cv7OMB94r5J+fbKeq0RVDMH0yUugeZCbpsN7dvBcrHSZkn4WZ9QRyGx/rnFseyaLEXz3Tk7j/ssn8z7Mr2fbFIXqkJvKjM0YwJa/1vT/nLPyU376yruH1qD7deeoH0+mRFnyR3dSkeG48YwS3vby2oa17cgI3nDo8PD+IhFWru6ib2W+A2cCnwJEDnXPutMiXFki7qHes2lrHjv0VZKcnkRLkSqOxPQcqmf67t6iqaXo1cePpI/hRiPEVH35WyqurdtAjNZGL83MZkJna6vESOUezi/olwDDnXFWI46QTioszz39x1+0sCwgKgJXbQk8Wmzoki6lDtGN5tAvVZ7EKyOyIQiS2jejdjYQgS/SP6dfdh2okEkKFxe+Aj8xsgZm9eOSrIwqT2NI7I4XrmvU1DM5O44rjh/hUkYRbqNuQR4A7gJVA613b0uXdPHMkJ4/sxbsbSujfI4XzJvTXupmdSKj/J0ucc3/qkEqkU5g8OIvJg9X/0BmFCotlZvY74EWgYfSMHp2KdD2hwuLY+v89rlGbA3x5dCoi/mkxLMwsDvirc+6pDqxHRKJUi09DnHO1wA0dWIt0pGeegfvv52BVNS+s2Mazy4vYd0irbUvLQt2GvG5mtwBPAg3TEJ1zgcs1t4OZnQXcA8QDDzjnbg/H50oIzsG8efDcc9zzyhr+NvZMALqnJPDwFVPUQSlBhRpncSVwPbAQWFb/FZbx1mYWD9wHnA2MBS41s7Hh+GwJwQyeeIKPJpzAf7/0Z77z0SsAlFVU86sX1/hcnESrVq8snHORHFEzFdjknCsEMLN/AOcD+m3tCMnJXHXuf3LngUpu+9dfqLDVzJ94Kyu37aOqupakBE1IlqZa/Y0ws0Qz+6GZza//usHMgk8hbLsBwNZGr4vq25rXcLWZFZhZQXFxcZhOLQB5/bP47oUn89JI+MbapVhtFXnZaQoKCSrUb8VfgcnAX+q/Jte3hUPgRIIvZ7Z+2eDcHOdcvnMuPycnJ0ynFoBJo9axPe0u/uOi0XzvovuJS0jmv84a7XdZEqVCdXBOcc5NaPT6LTP7OEznLqJunYwjBgLbw/TZEsLTq5/mV4t+QH7/aVww6F4SSOPrE/szum+G36VJlAoVFjVmNsw59ymAmQ0FasJ07qXACDMbAmwDvg3MCtNnSyueXv00lz5zKdNzp/PKrFfonqyZoRJaqLC4FXjbzAqpu20YDFwRjhM756rN7AZgAXWPTh9yzq0Ox2dLy8IVFGt37GffocNMGtRTfRxdRKinIW+a2QhgFHVhsc45F7jCajs5514BXgnX5/lt1bZ9HKisZvLgnlG590U4gqKs4jBXP7qMDwr3AJDTPZm/XTaZSYN6hrtciTJe5g9PBvLqj51gZjjnHo1oVTFm38HDXPXIUgrqV6Xum5HC3MvzI7YpT3uE64ri3rc3NQQFQHFZJTc/uYK3bzkFs2B91tJZhHp0+hjwB+AEYEr9V9D1+bqyu15f3xAUADv3V/Djp1f4WFFT4eyjWLihJKBt856DfF568GhKlBgQ6soiHxjrWlvVV3hvY+BfoA27DrBzXwV9e/i7s1a4OzP79Uhh7Y79TdqSE+LISk86qs+V6OdlDc6+HVFILOuXGRgI6Unx9EgN1/i19onEU49rTh5GYnzT243ZM/LonuLvzyqRF+rKohewxsw+pOniN1+PaFUx5rpThrOksJTq2i8vwK46cSipSa0vnx9JkXo8OnVIFs9eezyPfrCZfYcOc/a4vnxjYsDAW+mEQu0bcnKwdufcuxGrqBXRvG/Iiq1f8PjiLZRXVnPOuH6cN6G/b7VoHIW0V7v3DQkVCmb2gXNu+tEU11lMzM1kYq7/uyYoKCRSjnYwgL+9d9KEgkIi6WjDQk9JooSCQiIt+oYZSps1Doq558znUJWeTEj4tdpnUT93Y55zbm9Lh4S/JGmLI0Exud80Mst/wVl3F2AGZ4zpw93fmkg3bfIjYRLqyqIvsNTMnjKzsyxwPO9lEapLPGh8RTGE37CyqG7/aufg9TW7+MOC9T5XKJ1Jq2HhnPsZMAJ4EJgNbDSz35rZsPrvr4p4hRJU46CY/80XWVJ4KOCY11bt9KEy6axC9lnUD/XeWf9VDfQE5pvZnRGuTVrQvDMzK60HaYmBA8Ay09R3IeETaiLZD81sGXAnsAgY55y7lrqZqBd1QH3STLCnHonxcVw+Iy/g2CtP0A7mEj5ehntf6Jzb0rjROVdrZudGriwJprXHo7d+dRS9uyfz4sfbSU6IY9a0wXzdx1Gk0vm0Otw72kTzcO9I0zgK6QjtHu7dVVXX1LLks1Kcg+OGZpHg86pXCgqJBgqLZjaXlHP5Qx82LOYyIDOVR6+ayrCcbh1WQ8mBSn75wmpeX7MLl/pvCmt/xwwFhfhMIzib+fVLq5us+rTti0P88oWOXUf4unnLeXnlDva6hWyq/i2J1aP4wdi5CgrxlcKimUWf7glo+/enX66EtWVPOS+s2Maa7fsDjguHraUH+fCzUsrj3qck6U6Sa0fTu+pXvPzxFxE5n4hXug1pJrdnKp8Wlzdty0oD4O7XN/CntzZypE/4wkkD+OPFE8K+UG3zoIgjDa2FK37TlUUzP5o5sslfTDO46YwRbNhVxj1vfhkUAM8u38Yba3eH9fyLd7zMnuTfNwkKgG9OHhjW84i0la4smjl3fH/6ZqQwf1kRztVdPUwbms1ji7cEPX5J4R5mju0TlnMfeeoxbcA0xqfewTtry8hOT+KqE4dy4SSFhfhLYRFEfl4W+XlZTdqGZKcHPTavV/D2ttLjUYl2ug3x6Pjh2Rw/PLtJ24je3bjg2KNfrFZBIbGgy19ZVNfUehp0ZWY8NHsKzy7fxkef72Vkn+5cMiWX9KNcL6KtQVFcVsnji7eweU8504Zk883JA7XXqHSILjvc+401u/jtq2spLC5nbL8MfnneWKYNzQ79xjBqa1DsOVDJeX9+n+37KhraTh/dmwdnT4l0qdJFtDbcu0v+k1RYfIBr5y2jsP4R6Zod+7ny4aWUlld1WA3tufV44sPPmwQFwJvrdrNiq8ZgSOR1ybB46eMdHK5pekVVXlXDgtUds1jMkaA4ts9UZvT4A7c+tYEnl35OTW3rV3kt7Se6ZU950HaRcOqSfRYJ8cFHOMXHRX7k05GgmNhnKmXbbuHJz+pGh762eifLtuzlzm9OaPG904dl81RBUZO2hDhj2pCOvX2SrqlLXlmcP7E/KYlNf/TMtETOOiay27o2vvU4IfOPVFQlN/n+/GVFbP8icHm8I74+YUCTNSoS441fnDfW982XpWvoklcWA3um8cgVU7lzwXo27CxjQm4mPzl7NBkR3Ny3eR/FtY+tCTim1sGOfRX0z0wN+hnxccafLj2W604dxuaSg0we3JOc7slBjxUJty4ZFgDThmbzzLUzOuRcwTozTxzRi/c3lTQ5rmdaIscMyAj5eaP7ZjC6b+jjRMKpS96GdKSWnnp8d0Yep4/u3XBc95QE7rpkIskJ/u28LtKaLntl0RFaezyakhjPg7OnsH5nGbvLKsgfnEVqkoJCopfCIkIaB8W8bzzP4k8PMiDTMbZ/09uHUX27M6qvhndL9FNYREDjoLjmK3OZ+ccPqayuBeCrY/tw76xJGqItMceX31gzu9jMVptZrZkFHVoaqxoHxd8veJ7fvPRZQ1AA/GvNLp4q2OpjhSLt49c/b6uAC4GFPp0/Ipr3UWzcWU3F4dqA4z4IsnSfSLTz5TbEObcWCPtydH4K1pmZ2zP4sQOzgo+jEIlmunEOg5aeeozo051zx/drcmx2ehKXT8/zoUqRoxOxKwszewMINn76p865F9rwOVcDVwMMGjQoTNWFT6jZo//vWxM5aUQO728qYWDPVC6bPph+PXRlIbHH1/UszOwd4BbnnKdFKqJt+8JYWuFq9/4K1u8qY3TfDA0RlxZp+8IIiKWguPv1Ddz39iaqax2J8cZNZ4zk+lOH+12WxBi/Hp1eYGZFwHTgZTNb4Ecd7RVLQbH8873c8+ZGquvXyjhc4/j9gvWs3r7P58ok1vgSFs6555xzA51zyc65Ps65M/2ooz1iKSgA3ttQErx9Y/B2kZboaUgbxFpQAPTLDL7WRT+tgSFtpLDwKBaDAuC88f0Z2mxvk5F9ukV8oR/pfNTB6UGsBgVAalI886+dwcOLPmPNjv0cM6AHs2fkaSq8tJnCIoRYDoojstKTuPmro/wuQ2KcbkNa0RmCQiRcFBYtUFCINKWwCEJBIRJIYdGMgkIkOIVFIwoKkZYpLOopKERa1ykenVbX1PLmut1sLT3I9GHZfKV/jza9X0EhElrMh0V5ZTWzHljCx412Ev/h6SO4eeZIz5+xcvdKBYVICDF/G/LEh583CQqAe9/ayLZW9gxt7ten/Jo3LntDQSHSipgPi0+KAqda1zpYtc37FGwzIzlBC8KItCbmw2JMv8A9P81gtDbuEQmrmA+LWdMGMaJ3tyZts2fkMTg7vYV3iEh7xHwHZ4/URF684QRe+ng7W/fWPQ2ZMayX32WJdDoxHxZQNw37kim5fpch0qnF/G2IiHQMhYWIeKKwEBFPFBYi4onCQkQ8UViIiCcKCxHxRGEhIp4oLETEE4WFiHiisBARTxQWIuKJwkJEPFFYiIgnCgsR8URhISKeKCxExBOFhYh4orAQEU8UFiLiicJCRDzxJSzM7Pdmts7MPjGz58ws0486RMQ7v64sXgeOcc6NBzYA/+1THSLikS9h4Zz7l3Ouuv7lYmCgH3WIiHfR0GdxJfBqS980s6vNrMDMCoqLizuwLBFpLGI7kpnZG0DfIN/6qXPuhfpjfgpUA/Na+hzn3BxgDkB+fr6LQKki4kHEwsI5d0Zr3zez7wLnAqc75yISArW1jseXbOGfH+8gNSmey6cP5vQxfSJxKpFOz5e9Ts3sLOC/gJOdcwcjdZ7bX1vHnIWFDa/f3VDMfbMm8bXx/SJ1SpFOy68+i3uB7sDrZrbCzO4P9wkqDtfw2AdbAtrnvlcY5GgRCcWXKwvn3PBIn+NQVQ2HDtcEtO89WBXpU4t0StHwNCQieqYnMXlwz4D2M9RnIdIunTYsAO66ZAJj+mU0vD5jTB9unjnSx4pEYpcvtyEdZXB2Oq/eeCKbdh8gLSme/pmpfpckErM6dVgcMbx3N79LEIl5nfo2RETCR2EhIp4oLETEE4WFiHiisBARTxQWIuKJRWjCZ0SYWTEQOOEjevQCSvwuop1Uu3+iqf7BzrmcYN+IqbCIdmZW4JzL97uO9lDt/omV+nUbIiKeKCxExBOFRXjN8buAo6Da/RMT9avPQkQ80ZWFiHiisBARTxQWYWRmF5vZajOrNbOofxR2hJmdZWbrzWyTmf3E73q8MrOHzGy3ma3yu5a2MrNcM3vbzNbW/87c6HdNoSgswmsVcCGw0O9CvDKzeOA+4GxgLHCpmY31tyrPHgbO8ruIdqoGfuycGwMcB1wf7f/dFRZh5Jxb65xb73cdbTQV2OScK3TOVQH/AM73uSZPnHMLgVK/62gP59wO59zy+j+XAWuBAf5W1TqFhQwAtjZ6XUSU/9J2NmaWBxwLLPG3ktZ1iWX1wsnLtowxxoK06Xl6BzGzbsAzwE3Ouf1+19MahUUbhdqWMQYVAbmNXg8EtvtUS5diZonUBcU859yzftcTim5DZCkwwsyGmFkS8G3gRZ9r6vTMzIAHgbXOubv8rscLhUUYmdkFZlYETAdeNrMFftcUinOuGrgBWEBdJ9tTzrnV/lbljZk9AXwAjDKzIjO7yu+a2uB44DLgtPotPFeY2Tl+F9UaDfcWEU90ZSEinigsRMQThYWIeKKwEBFPFBYi4onCQnxTP4lNYoTCQgKY2RQz+8TMUswsvX4K9TFBjjvFzN4xs/lmts7M5tUPNsLMTjezj8xsZf1U8uT69s1m9gszex+4uP79d5vZwvrp2lPM7Fkz22hmt3Xwjy6tUFhIAOfcUupGcd4G3Ak87pxrac2IY4GbqJvePhQ43sxSqJs+/i3n3DjqphVc2+g9Fc65E5xz/6h/XeWcOwm4H3gBuB44BphtZtlh/eGk3RQW0pL/A8wE8qkLjJZ86Jwrcs7VAiuAPGAU8JlzbkP9MY8AJzV6z5PNPuPI8PKVwOr66duVQCFN562IjxQW0pIsoBvQHUhp5bjKRn+uoe4qIthM1sbKW/iM2mafV4smO0YNhYW0ZA7wc2AecEcb37sOyDOz4fWvLwPeDWNt4gOltgQws8uBaufc3+ufWPzbzE5zzr3l5f3OuQozuwJ42swSqJvZen8ES5YOoIlkIuKJbkNExBPdhkhIZjYOeKxZc6Vzbpof9Yg/dBsiIp7oNkREPFFYiIgnCgsR8URhISKeKCxExJP/BeJGs3i21aoDAAAAAElFTkSuQmCC\n", 100 | "text/plain": [ 101 | "
" 102 | ] 103 | }, 104 | "metadata": { 105 | "needs_background": "light" 106 | }, 107 | "output_type": "display_data" 108 | } 109 | ], 110 | "source": [ 111 | "m = X.shape[0]\n", 112 | "Sig = np.dot(X_norm.T,X_norm)/m \n", 113 | "U,S,V = np.linalg.svd(Sig) \n", 114 | "\n", 115 | "sns.scatterplot(x = X_norm[:, 0], y = X_norm[:, 1], edgecolor = \"none\")\n", 116 | "drawline((0, 0), S[0]*U[:,0], color = 'g')\n", 117 | "drawline((0, 0), S[1]*U[:,1])\n", 118 | "plt.ylabel('y_norm')\n", 119 | "plt.xlabel('x_norm')\n", 120 | "plt.axis('square')\n", 121 | "plt.show()" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 10, 127 | "metadata": {}, 128 | "outputs": [ 129 | { 130 | "data": { 131 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAEGCAYAAAByy7CMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUZdrH8e8zyaSHUAKhQwxVAgQISO8gLCiCgKAoFmR1xfauZXVFcdfV1WVdF1ERG4gIuArY6EpXxAQihN4xtIRACunJPO8fZwgtZZJMODOZ+3Ndc01m5sw598yV/HLKU5TWGiGEKI3F7AKEEO5BwkII4RAJCyGEQyQshBAOkbAQQjjE2+wCyiI0NFQ3bdrU7DKEqLJiY2PPaq1rF/WaW4VF06ZNiYmJMbsMIaospdSx4l6TwxAhhEMkLIQQDpGwEEI4xK3OWRQlLy+PhIQEsrOzzS7Frfn5+dGwYUOsVqvZpQgX5fZhkZCQQHBwME2bNkUpZXY5bklrTXJyMgkJCYSHh5tdjnBRbn8Ykp2dTa1atSQoKkApRa1atWTvrJJorckvsJldRoW5/Z4FIEHhBPIdOp/Nppm+ah/zfj5GVl4Bg9uE8cptbakZ6GN2aeVi2p6FUqqRUmqtUmqPUmqXUupxs2oRojJ8uOkw7647RHpOPrb8fJbtPM2fv4gzu6xyM/MwJB/4s9a6NdAVeEQpdaOJ9ZSbl5cXUVFRREZGcsstt5CSklJp21q3bh3Dhw8vcZm4uDiWLVtWaTUIxyzedgKAP/38BbMXv4JPfh7r9idxPiPX5MrKx7Sw0Fqf0lpvs/+cDuwBGlT2dm02TdzvKcSfSHXaOv39/YmLiyM+Pp6aNWvyzjvvOG3d5SFh4Rq8vYxDuxT/YAYe+pV3l76Kb34eFot7HvK5xAlOpVRToAPwS2Vu58jZDAb+Zz23vbOZ4W9v4taZm0hMd+5JvW7dunHihPEf5cKFCwwYMICOHTvStm1bvv76awDeeOMNZsyYAcCTTz5J//79Afjhhx+YMGHCNetcsWIFrVq1omfPnixevLjw+a1bt9K9e3c6dOhA9+7d2bdvH7m5ubz44ossWrSIqKgoFi1aVORyovLd0bkxAJ9HDeX5mx9h4KFf+d8PbxJicdOTnVprU29AEBALjCrm9clADBDTuHFjfbXdu3df81xxxr3/s27y7HdX3B5fsM3h9xcnMDBQa611fn6+Hj16tF6+fLnWWuu8vDydmpqqtdY6KSlJR0REaJvNpn/++Wc9evRorbXWPXv21J07d9a5ubl62rRpetasWVesOysrSzds2FDv379f22w2PWbMGD1s2DCttdapqak6Ly9Pa6316tWr9ahRo7TWWn/yySf6kUceKVxHcctdrSzfpXDMBxsO6Z6v/6A7/G2VXvrQVK1B6+HDtc7ONru0IgExupi/VVOvhiilrMBXwHyt9eKiltFazwZmA0RHR5d7wNCc/AJ+Ppx8zfPr9yeVd5WFsrKyiIqK4ujRo3Tq1IlBgwYBRhA///zzbNiwAYvFwokTJzhz5gydOnUiNjaW9PR0fH196dixIzExMWzcuLFwj+OivXv3Eh4eTvPmzQGYMGECs2fPBiA1NZWJEydy4MABlFLk5eUVWZ+jywnnm9TrBib1usH+aBBENYCHHoLRo+HLL8HX19T6ysLMqyEK+AjYo7V+s7K3Z7VYqFXEJauwan4VXvfFcxbHjh0jNze38JzF/PnzSUpKIjY2lri4OMLCwsjOzsZqtdK0aVM++eQTunfvTq9evVi7di2HDh2idevW16y/uMuaU6dOpV+/fsTHx/Ptt98W207C0eXEdfDHP8KsWfDdd0Zg5OSYXZHDzDxn0QO4G+ivlIqz3/5QWRuzWBQP94244jml4E/9mjltGyEhIcyYMYPp06eTl5dHamoqderUwWq1snbtWo4du9T7t3fv3kyfPp3evXvTq1cvZs2aRVRU1DXB0KpVK44cOcKhQ4cAWLBgQeFrqampNGhgnBOeM2dO4fPBwcGkp6eXupwwydWBkZUJx7fAqR1mV1YiM6+GbNJaK611O611lP1WqafwJ/W6gVkTOnJzmzCGtavHvPtv4tb29Z26jQ4dOtC+fXsWLlzIXXfdRUxMDNHR0cyfP59WrVoVLterVy9OnTpFt27dCAsLw8/Pj169el2zPj8/P2bPns2wYcPo2bMnTZo0KXztmWee4bnnnqNHjx4UFBQUPt+vXz92795deIKzuOWEiS4PjOi6MHswvN8LPhoMmefMrq5ISrvRvCHR0dH66sFv9uzZU+Suuyg7+S5NcFcz+PwQtPCGMf7grSD6ARhe6UfmRVJKxWqto4t6zSUunQrhkTLOQvMkGOYH+/Ph6yzj+YNrzK2rGFWib4gQbsk3GHyrQXSa8W871P6/u1qlt00sF9mzEMIs3r7QbYrxc0cfaOwNygI9nzS3rmLInoUQZur7LNSKgF1LwCcQOt0HTbqZXVWRJCyEMFvb0cbNxUlYCFFB5zNyeX/DYeJ+P0/zOsFM7n0DjWoGmF2W08k5Cye4vIv6mDFjyMzMLPM63nrrrWLft27dOkJCQujQoQOtW7fm5ZdfLnxt69at9O7dm5YtW9KqVSsmTZp0xXpGjBhBt26uuVtbFeQV2Lhj9s/MWn+ILYfPMW/LMUa995PbdkMvieeFhc0GCbFw6jenrfLyLuo+Pj7MmjWrzOsoKSzAaMS1fft2YmJi+Oyzz4iNjeXMmTOMGTOG119/nX379rFnzx6GDBlS2HozJSWFbdu2kZKSwpEjR8r9+UTxftybyP4zF/DNy2Ha6lnUyEwlKT2Hr7YlmF2a03lWWCQfgne6wIf94f3eMLsvpJ9x6iZ69erFwYMHAXjzzTeJjIwkMjKSt956C4CMjAyGDRtG+/btiYyMZNGiRcyYMYOTJ0/Sr18/+vXrV+L6AwMD6dSpE4cOHeKdd95h4sSJhXsOSilGjx5NWFgYAF999RW33HIL48aNY+HChU79nMKQlG707WiVdJRxO1bx+cK/FgZGVeNZYfHt45B84NLjk9th1QtOW31+fj7Lly+nbdu2xMbG8sknn/DLL7+wZcsWPvjgA7Zv386KFSuoX78+v/32G/Hx8QwZMoTHHnuM+vXrs3btWtauXVviNpKTk9myZQtt2rQhPj6eTp06FbvsggULGD9+POPHj7+iT4lwnj4tauNlUfxWvyUP3P4i4edP8tXCZ5h86AWY2RkWT4Zzh80u0yk8Jyzyc+Doxmufd0JruYtd1KOjo2ncuDEPPPAAmzZtYuTIkQQGBhIUFMSoUaPYuHEjbdu2Zc2aNTz77LNs3LiRkJAQh7axceNGOnTowODBg/nLX/5CmzZtSlz+zJkzHDx4kJ49e9KiRQu8vb2Jj4+v8GcVV2pUM4DXRrYlyNebzU2j+PPYpwk/f5JaryyF43thxyL4eCjkpJe+MhfnOVdDLFYICIXMs1c+X63iHckunrO4XHF9blq0aEFsbCzLli3jueeeY/Dgwbz44otXLLNkyZLCk5gffvghYBzefPfdd1cs16ZNG2JjYxkxYsQ121m0aBHnz58vnAckLS2NhQsX8sorr5TvQ7qhhPOZzP3pKMfPZdI9IpTxXRrj4+38/49jOzdiWLt6HEi8QIujR1Fe/rAgE+ZmwsQA4DTs/ho6XDsKmjvxnD0LiwV6/d9VT6pKay3Xu3dvli5dSmZmJhkZGSxZsoRevXpx8uRJAgICmDBhAk899RTbtm0DruxWPnLkSOLi4oiLiyM6usg+PQBMmTKFuXPn8ssvl0Yj/Oyzzzh9+jQLFixgxYoVHD16lKNHjxIbG+tR5y1OpmRx68zNfLDxCCt3neGlb3bxp/mxlba9QF9vohpVJ8CWCTd4w/gAuKDhnH0Ivey0Stv29eI5exYA3R6BkEbGrqGXFTpOhIiSTyiWV8eOHbn33nvp0qULAJMmTaJDhw6sXLmSp59+GovFgtVq5b333gNg8uTJDB06lHr16pV63uKisLAwFi5cyFNPPUViYiIWi4XevXvTsWNHjh8/TteuXQuXDQ8Pp1q1avzyyy/cdNNNzv/ALmbelmOcy8il9oXz3LFjJe90G8uaPYnsOplKm/qOHfqVS+vhsO5VIzAeDwIfBRZvaFVpQ7VcN9JFXRSqSt/l4wu383XcSR74dSlTf/yQeR3+wIuDHuL9ezozuE3dyt143OewaqpxyBsUBkNeg8jbK3ebTlJSF3XP2rMQHqNns1C+jjvJR9EjCM1I4eFfvsRLKbpMHVT5G4+6EyJHQ/op45yYV9WYbNpzzlkIjzKqY0Nui6oPSvF6n4l80H0Md277nurP/J/RMK+yeftAjSZVJiigiuxZaK1lrs4KcqfDUUd4WRRvjevAowOak3A+iw7TBsPLN8DrrxsLzJxpnPQWDnP7sPDz8yM5OVlmUq8ArTXJycn4+VV8pHNXE1E7iIjaQcaD114z7iUwysXtw6Jhw4YkJCSQlFTx+T88mZ+fHw0bNjS7jMqllARGBbh9WFit1sKGR0KUSgKj3Nw+LIQoMwmMcpGwEJ5JAqPMJCyE55LAKBMJC+HZJDAcJmEhxNWBkZMOtzeEtOMQ3hs63WsM2+/hJCyEgEuBkZ0G/30PdljhD36w9zs4tBbu9Jweu8WRfS0hLlIK/hAKPXwgJg+WZYPWsH85nN5pdnWmk7AQ4nJpCTDA1wiMHXmXxqNIrXoD8JaVHIYIkXYStn0KF85AYG1jD2OAL3TygRoW8PaHxl1LX08VJ2EhPNu5w/DBAMg6d+m50JZwdh/UUGANhFtngH8N82p0ERIWwnNpDaunGUGRboPD+dDexwiQB1YbV0Uadga/amZX6hIkLIRnOh0PX9x9aZj+zbnwSy7kAF1gzZ4zNGrXl5Z+waaW6UokLIRn+uoBIygybeCvYJAvpNhgeTZZWJmSnUf2jxu4v0c4L95yo9nVugS5GiI8z7kjkLQXcjR8nAnfZBt/CWP8KWhpxX95OmO2rQbg481H2JmQam69LkLCQniWjGSwBhjzyPgqiPSGuDz4JpvEgAi63/Ihq5p35e+rZ9HtmDEfbsyxc6Ws1DPIYYjwDKfj4es/GRNi+1aDeu3gRCz0tY8Otj4XW+0QEjvU4JERzzJ2x2q2NG4LQLM6QSYW7jokLETVZyuABeMh9TjkayDNCIqouyDlODwUAu19qDtjDrPTApnc5yHmdzDm+ejVPJSezULNrd9FmBoWSqmPgeFAotY60sxaRBWW8KsRFBdsxpSCXX2MBlfZqXCvfUrIcUCNJgx6+WXW1A5k3v0v0K5JTYa3qy9ju9qZvWcxB5gJfGpyHaIqswYY937KaJH5XTYAyy1pvDF9Hfd0a8J9PcJh2jQAIl5+mWm1g+DDD6Wr+mVMDQut9QalVFMzaxAeoF47aNgFErbCWH/4Igu+y+ZYvh9H2mfw8re78bYo7u7WtDAwsE9MLYFxict/C0qpyUqpGKVUjIzgLcrtzkXQ6T5stZrw6+gofotowUMrvmRc3AoAPt/6+6Vlp02Dl16CTz6BSZOuz6REbsDsw5BSaa1nA7PBmOvU5HKEuwqoCbe8RXpWHmP/tgrrbXnMWvIP6qWfBSA3v+DK5WUP4xouHxZCOFOIv5V+Levw495EHrx9KgXKCICRHRpcu3ARgZFn/3dl9fK84JCwEB5n+pj2PLd4B2v2JBLgbWF8l8Y81Cei6IUvC4xfj55jQrcH0crCiPb1eXlEGwJ8POdPyOxLpwuAvkCoUioBeElr/ZGZNYmqr2agD+/fHU12XgFeFlX6XsK0afy4N5H+i97j72czeHboY/wvNgGrt4VXR7a9PkW7AFP3pbTW47XW9bTWVq11QwkKcT35Wb0cPpx4vNUI3uoxnrE71/D68hkobWPp9hOVXKFr8Zx9KCEqwOpl4a2edwHgZbOhUXhbPKuxloSFEA4Y36UR76w9ZASG1qAU429qbHZZ15WEhXA/2Wmw51soyIFWwyGoTqVv8smBLfC2WPgyNgGtNbd3asjjA5pX+nZdidLafZouREdH65iYGLPLEGZK3AtzhkGm0T4Cb38YvwAi+plbVxWhlIrVWkcX9ZrnXSwW7m3NNCMo0m2QWAD5WbD8GbOr8ggSFsK9nIg17pdkGT1IEwvg7H7j0ERUKgkL4V7qtDbuh/mBFzA3k4yUWtz92W7+b1GcDIFXiSQshHsZ8CL4BEEtL5gYgPYC2+wznPk5lsXbT3D7rJ/47fcUs6uskiQshHtpGA2PbIWB00gf9iz33vkyGV5+LFjwPC2SjpKbb+PjzUfMrrJKkrAQ7iekAfR8kqORj7G+ekfGjX+NPC9vbt2zAYDEtByTC6yapJ2FcFut6wUTVs2XozTglon/JSmwOgB9W9Y2ubKqSfYshNvy9rLw9viO1An2JSmoBsqiGNaunjFEnnA62bMQrkNr4zKoXwgE13XoLV3Ca7L5L/2JP5FKaJAvjWoGVHKRnkvCQriG0/Hwv3sh+QAaCykRt1Jj/Afg7VPqW61eFjo0llnOK5schgjznY6HjwZC8gHI0ihdQI1DS1nyzrPkFcj4l65CwkKYKy8LPr3VuM+ywUcZsDwbtCY8eT1LtnnWmBGuTMJCmOvAashMNvp6fJYJdS3wax5sySVZV2PL4WSzKxR2EhbCdeQAB/LhJiu2KB8+KhhK09BAs6sSdhIWwlzNB0FgHQi2wMQACLagt+cx+9RQjgZHc6eHDTDjyiQshLms/nDPUmjaC13NSvrDrUmrUYf7v/ye5dEWQoN8za5Q2MngN8L1nDwJ/foZ9ytWQI8eZlfkMWTwG+Fe6teHtWuN+yFDYPNmsysSSKMsUQFaaz7ZfJTF2xOwKMXY6EZM6Nrk2gWzU2HXUsjPNsbMDCli9q+rXQyMfv1gyBAKli1nQ+0WJGfk0rtFKHWC/Zz/gUSJJCxEub215gD//eFA4eMdCalk5RbwYO8bLi109ZiZq6bCHfOgxc2lb8AeGAV9+5IzaDAzR79MbMMb8fGy8O+x7bmlfX0nfyJREjkMEeU29+ejAEzaupi7ti8DYM5PR69c6Me/Xxoz83SBMSL38meMfiCOqF+fd6d+wKnAmsz930t0SthNboGNqV/Hk51XUPr7hdNIWIhy0VqTkZOP0ja6Ht/JP1a9y13bl5GWnXflgie2GfdfZ8HcDDhVAOePQtZ5h7e17oKV8eNe5UxQTT753zRqZKaSkpnHwcQLzvtAolQSFqJclFLc3KYuWln4023PsyaiM/9Y9S5/O7nRWOBCEuRlQ91I4/Fwf/BV8GkGtrSaLIpPY87mI5xKzSp1W81qB5EYXIvx417lL0Mf43xACL7eFhrVkB6m11Opl06VUjWARlx2fkNrva2S6yqSXDp1LeczcnnyizjW70/CNz+PL9b8m3Zxm2B8BLRIAp9gaDsa4hdDTiqk2NBzM8nK8WHM2H+yq24zfLwszLq7I/1bhRW7nSNnMxj17mbOZ17aa3liYHOeGNjienxMj1LSpdMSw0Ip9XfgXuAQcHFBrbXu7+wiHSFh4ZqSL+RgUYoalnyIrgu7043Rt6Pt3cuHvWlcCcnLZPpP1Rj32osE5WZy1x2vsKtuMxrXDGD9031Rqvi5QxPTsvki5nfOXshlYOswejYPvU6fzrOUFBalXQ0ZC0RorXOdX5aoKmpdbGV5fDuMBPK94ftsUEAnHzj4A4z/HIAvN/3A0jtfY+HnzzF78T/oN3k2x89lkpKZR43A4seuqFPNjyn9PWu6QFdTWljEA9WBxOtQi3B3PgHgrWCMv3FCs7b9lJjPpc5gresFszYtjHF3vkad9HPkelupH+JHiL/VpKKFo0oLi9eA7UqpeIw+gQBorW+t1KqEe6rbFhp1hd+3wO32k4/KAtH3Fy7y58EtiTl2ngTCSAgJw8ui+OuwG7FYij8EEa6htLCYC7wO7ARkyCJRuvELYO0/4OAaqGYM2U+TboUvRzYI4Yc/9+GbuJNk5xUwtG09ImoHmViwcFRpJzjXa637XMd6SiQnOIWoXBU5wRmrlHoN+IYrD0NMuXQqhDBPaWHRwX7f9bLnNGDKpVPhAtJPw/Z5RqOrlkMgQn4VPEWxYaGUsgDvaa2/uI71CFeWfAg+HAhZ54zHW9+HPn+Bfs+ZW5e4Lopt7q21tgFTrmMtwoVtPXKOI1++YARFug122VtTbvoPZJ4ztzhxXZTWN2S1UuoppVQjpVTNizdnbVwpNUQptU8pdVAp9RdnrVc4T36BjWkfLCLgk36EnzJ6lrIhB77Mgu25Ri/SlGPmFimui9LOWVy8QP7IZc9p4IYili0TpZQX8A4wCEgAflVKfaO13l3RdQvnWb7zBJMSnqehSoLjBdDYG272g/M2+CYbrAFQu5XZZYrroMQ9C611eBG3CgeFXRfgoNb6sL05+UJghJPWLZwkcc9mGqqzsD0PPsmE2Fyjlea4AGhmhcXnYd4Cs8sU10GJYaGUsiqlHlNKfWm/TVFKOatdbgPg98seJ9ifu7qGyUqpGKVUTFJSkpM2LRxVL7SW8UM7K0R6FzbhvlCvI/y0HwYPhkmT4OOPTaxSXA+lnbN4D+gEvGu/dbI/5wxFte+9poWY1nq21jpaax1du3ZtJ21aOKpfn/7s8rrR2Ju4PQAae2PDQtDw16B2U1i6VALDQ5R2zqKz1rr9ZY9/VEr95qRtJ2CMk3FRQ+Ckk9YtnMTfx4sbHvuW/V89T+jpTejgelQf/MylJtx+fkZg3HabERgA999f/AqF2yotLAqUUhFa60MASqkbAGcNfPgr0FwpFQ6cAMYBdzpp3cKJ/ENCaXH/7OIXkMDwCKWFxdPAWqXUYYzDhibAfc7YsNY6Xyk1BVgJeAEfa613OWPdwgRFBEby2LvYezqdZnWCCKsmQ/e7uxLDQmv9g1KqOdASIyz2aq1zSnpPWWitlwHLnLU+d5CYns2+0+m0rBtc9ea+uCww9KRJTF+8kwWRA/G2KB7uG8GfB7c0u0JRAY7MG9IJaGpftr1SCq31p5VaVRU180djno28Ao3VS/Fo/+Y8NqCKjf7k58eB9+dxst8Q/vH9f8m32fhfu8G8/eNB+rSoTXRTp7XpE9dZaZdO5wHTgZ5AZ/utyO6romQ7E1KZvmo/eQWaBqmJ5BVo3ly9nx0JKWaX5nTrj6czedQLbAzvwL+Wz2DQgS3G8/vl0rc7K23PIhq4UbvT7MkuasMB4w+l9+FYPvrqbzwz9HGWRPZnw/4k2jWsbnJ1zlW/uj853j5MHvUCT2z6nJ8atwOgXoi/yZWJinBkDM66wKnrUEuVVr+6cX5ia6M2bG3Uhn9//x8A6o1pX9Lb3NLA1mG0qhvM3tPpvN73XgCa1Arg1iiZbtCdlTZS1logCtiKC4zB6c4jZWXnFTD87U0cTLyAX142H331N7od30n+R5/gc99Es8tzutTMPOb8dJQdCSm0qhfMfT3CCb04CrhwWRWZN6TIIfW01uudVFuZuHNYgDEpz5yfjrLrZCrtavrw8L+fwLphPcydC3ffbXZ5QpR/WL3SQkEp9bPWultJy4hLagT68OSgy2bRGvA93HILTLTvWUhgCBdW0blOq1hDgessIAC+/Rb69TMCY968ytlO4h74fBxMbwGfjoAE9907E+ZxpJ1FSeQqSUVdDIzK2sPIToM5wyHzrPH4whkjLB7ZCiHXdPIVolgyi7orqMw9jD3fGkGRboNPM+CcDXIvkLxlPjabZL1wXGmNsqbYZ1EvdhEn1+O5Kisw8rON+ywNZ2wwxwiMuRv20Wf6Wn49KuNnCseUtmdRF2O4uy/s42VeHQ5yRs6ZKiMwWg0Hb3+o4wX3BEAB6DkZbE8O5/dzWTw0L5acfGd1JBZVWWnD6r0ANAc+Au4FDiilXlVKRdhfj6/0Cj2NswMjOAzGzYdazSHMi9MTGnAh35835r9FvbQkkjNyiTl63jm1iyqt1HMW9qbep+23fKAG8KVS6o1Krs1zOTswmg2AR2OY1mYlXWu8x5hxr7M2IpqkQOMIU2YwF44o7ZzFY0qpWOANYDPQVmv9MEZP1NuvQ32eqxIOScb2uBEfby/21gnn+SGPku/lTZfwmkQ2CHFCwaKqK+3SaSgwSmt9xcQQWmubUmp45ZUlAKdfVr2xfjUWPNiVWesPcTIli57NQ5nSr5mTihVVXYnNvV2Nuzf3LrfMTCMw1q6VpuGiUpXU3FvaWVTAwcR0Vu06TWJ6duVuqLhDkoJ8OPYznIyr3O0LQcVbcHokm03z9Jc7+GpbAgBWL8ULw25kYvemlbfRqw9JTu0kL3suVpsRVEf8WhNy/2Jq1pFu4KJyyJ5FOSyLP2UEhdbckJxAXoHmb9/t5mRKVuVu+GJg3NQO/ey/sG5Pgws2yNeEZ+8hbs7/Ve72hUeTsCiHnw4lAzD+t5Ws+HgK/Q9upcCm+eWI8XxWbgEr4k/z494z5BXYnLtxP18Yeg4V7gVLs+G9C7AwE/I1rTJ+4URlB5bwWBIW5dCoRgAA37fqye6wcGYteZX+B7fSqEYAOxJS6PH6jzz0WSz3z4mh3/R1HEvOcM6GT8fDjCiwpcI4f/ABMoF84yT1eR0s7e9FpZGwKIc7OjeiQXV/0vyCuGfs39kdFs7spa8SHf8TLyyN51xGLvXTEgnJSifhfBavLdvrnA0vfQhSjkOGDbwUjA8AK5Cu0RmalbXvpX51GedSVA4Ji3KoGejD11N68PiA5vS5qQV753yJpUMUetQoQtetxstWwNwvXmL+ohcIyUovPDypkAuJcHon5GljNvMvs6CR0d9DX9DkfKa57+bbKr4dIYohYVFOoUG+PDmoBW+P78C4we2wrF4NUVG8v/RV+hyO5ZX+k2h+9jjzF71ApL8TOmr5BoNPEFgVdPGBvflGYDSthfrsbfxyLNQYdjMkJFR8W0IUQcLCWapXR61aRXrLNsxa8ipetgImj3qB5meP8+6cZ+FcBbuCW/2h68PGz118YKifERjrG8GIybBqFZw5Y7TFkMAQlUBacDqgwKaxKLi2h34RUlLI6Nsf3107+fK5/9KjVV0a3X8n3HgjrFkDNR2ckSs/Bw6vA+UFN/QBL3tnr98Wwa4l4BMIu0Pgpa4Si2AAAA6jSURBVP8Y84suWgTbtsHgweiwMFbN/JwVKVbCqvlx102NaVQzoNyfX3iOco/u7Wqud1gkpmXz16Xx/Lg3kWA/b+7rHs5jA5qVHhopKTB4MMTFweLF4O1t/EE7GhhndsFnt0O6fbqWkMZwz1KoFXHtsjNnwqOPXhEY2f0HcsqvGuPHvcbpaqGE+Fv5+pEeNA0NLN8XITyGNPcupykLtrN69xl0fj4pmXn8Z81+Pt96vPQ3Vq9uHBZERcGoUZCfb0wYvHs3DBxY+iHJ908ZQZFug0wbpB6Hlc8XU+QUePttY/133MGJG1pz5+iXCc1IYcHC56ibdpbUrDw+2nSk7F+AEJeRsChGwvlMth45h19eNp9+8SITY78FYMm2E46toLyBYSuA4z+B1jA/Ez7NhEwb2QfW8dryPZxOLaIfymWB4T/hTnaGRXDP2L8TmpHC3du/B+DYuczyfA1CFJKwKIa3xfhqCixeXPAN4OU17zMx9lu8LGVo9lSewLB4QfXGoBQM8oNkG3yaybH0Wry//jC3ztxEUnrOte+zB0bN1cuY9e2/iK8bwYh7/sO/e00AoHtErfJ8DUIUkrAoRt0QP/q0qE2el5VHb32GFS268fKa93n+8Jqyrag8gdF/KqAgwhvGBaCTbQTMO0eNzFQS03NY9Gsxh0L2wBiw7yfe/fYNfq8ehs3iRc9modxbmZ3chEeQsCjBjHEdGBvdkMDgAKbfO43jfW6m/RsvGrv8ZVFSYPTtBXPuhne7wdI/wflj0G4s3L+SI03vYG6Tobx0+8PUPneem/f/DMCJlBK6xNsDY9C+n9m07X2+ebAzn026CT+rVwW+CSHkakjZ5ObCuHGwZAnMmGFchSiLq6+SZJ+H8fdAqAXuCQR/BdUawJRfwSeQM2nZ9Pjnj+TbNA1TTpNQvS4A793VkaFt65W8rauvkvj4lPNDC08iV0OcxccHFi6EkSPhsccqtocxciQsfhjuCIAk+wRAWRrSThgTAwFh1fz4+22R+HpbSKheF6VgfJfGDImsW/q2rrpKQm5uOT6wEJfI4DdldTEwxo0zAgPKtodRvTos+w6iGsCiDLjVz7itzDFmC2vgBdmphYuP79KYoZF1+S0hlfBagTSuVYbGVVOmXKrvjjtkD0NUiIRFeVQ0MFJ2wAR/+PSCMSZFsIL7A6CWFzblTX7EzVz+J109wIc+LWqXr1YJDOEkchhSXhU5JLF4g5+Ce4KgnsU4/Pg8C1uajcdyHqbX7IPsPpnmvFrlkEQ4gSlhoZQao5TapZSyKaWKPJniFsobGE17QVBdIzAeDLR3M7eRMtfC1tQWnEnL4fklO51bqwSGqCCz9izigVHABpO27zzlCQyLFzywCsJ7g7cfyY1q8fG4UfhcyGPeF1PxshUQ93sKWblOnoNUAkNUgCnnLLTWe8DBXpzuoDznMGo0gYnGVY/7Z27it+xUvh/TjdDMFAosXtQO9sXXuxKyXM5hiHKScxbOUtweRvoZyC25X8bjA5vjZVFsa9iaVS26Gc8NaI6lLE3Ly0L2MEQ5VFqjLKXUGqCoBgF/1Vp/bV9mHfCU1rrYllZKqcnAZIDGjRt3OnbsWHGLuobLG26NDYfWyWANNAauGTC12Lf99nsKi2J+Jzffxm1RDejZPLTya5WGW+IqLjuehSNhcTnTW3A6KjsLutSFnWkwxA9usv8Rjnwf2o8zt7arXRYYeZ8vIPZUBsF+3rSpL5Mle6KSwkLaWVSGs7tghIY8b1iRbRzsdfYxRrhytbC47BzG5k4DeXDYU+R5WenStCYfTIwmxN9qbn3CZZh16XSkUioB6AZ8r5RaaUYdlcYn0Biqf7Q/tLNCmP1r9gkyt67iTJnCjNseo++ezcz8+nWsBXlsPXqO/645YHZlwoWYEhZa6yVa64Zaa1+tdZjW+mYz6qg0dVobbSm8FIz0h8beoCzQ+QGzKyvSiZQs3mw5mBcH/pGbD2zhtRUzAdh4IMnkyoQrkcOQynLHZ7DuNTiwGoLrQc8noEl3s6sqUnV/K/5WLz7tdAv5Xt7sDGsGQD2ZsEhcRrqoCwCmr9zHzLUHCx97WxTzHriJbjLClkeRE5yiVE/d3JLmYUEs33maYD9vJnRtQvtG1c0uS7gQCQtRaERUA0ZENTC7DOGipAWnEMIhEhZCCIdIWAghHCJhIYRwiISFEMIhEhZCCId41KXTfafT2XggifrV/Rl0YxhWL8lKIRzlMWExa/0h/rl8b+Hj1vWqseiPXanmJ70qhXCER/xrTUrP4d+r9gHQ7dgOfPNz2XMqjU9/OmpuYUK4EY8Iiz2n0sgrMKYA/PSLqby35FV883P5LSG19DcLIQAPCYvmYUF4WRQJ1evy4qCH6X84hveWvEpkTRlGTghHeURY1Avx54+9bwBgQdQQnrt5Cv0Px/DwzGchu4QZyYUQhTzmBOczQ1rRt2Ud42rIqKfJueVGfKf8CW6/Hf73BZzdCdYAqNfO7FKFcEkeExYAXcJr0iW8pv3Bw2D1gj/+EaLrwigF3goadoE7F0FATXOLFcLFeMRhSLEmT4a7msGeC/BFFuRrSNgKP/zN7MqEcDmeHRYZydAsEYb7wYF8WJplPH/oB3PrEsIFedRhyDV8g8C3GnRKAwWE2rOzWkNTyxLCFXn2noW3L3S3z03a0efSKNw9nzS3LiFckGfvWQD0eRpqRcCuxcbVkE73QZNuZlclhMuRsACIHGXchBDF8uzDECGEwyQshBAOkbAQQjhEwkII4RAJCyGEQyQshBAOqVqXTs8fhS3vwbkj0LQndHkQrDITuBDOUHXCIjUBZveDrHPG4wMr4fBauHuJuXUJUUVUncOQXz8ygiLdBj9mg03DoR/hRKzZlQlRJVSdsEg7adzvyYeNufCtPTBST5hblxBVRNUJi2YDjPsuPtDHF+Ly4Ls8aCT9PIRwhqoTFpGjoeM9gIK+vtA/GLZnw2PPQEGB2dUJ4faqTlhYLHDr2/B4HNzzDSw7AdOmwZw5MGmSBIYQFVR1roZcVKOpcQN46SXjfto04/7DD8HLy4SihHB/VS8sriaBIYRTmBIWSql/AbcAucAh4D6tdUqlbVACQ4gKM+ucxWogUmvdDtgPPFfpW3zpJTmHIUQFmLJnobVeddnDLcDo67Jh2cMQotxc4ZzF/cCi4l5USk0GJgM0bty44luTwBCiXCotLJRSa4C6Rbz0V6311/Zl/grkA/OLW4/WejYwGyA6Olo7pTgJDCHKrNLCQms9sKTXlVITgeHAAK21c0KgLC4LjOSMHKYOf4JTF/Lo06I2D/WJwM8q4SHE5cy6GjIEeBboo7XONKMGAF56iTNp2YS9+U/67Uni2aGPsv14CjsTUvno3s6mlSWEKzLrnMVMwBdYrZQC2KK1fsiMQt64aRwNexznyc2fA/Ds0Ef5YW8ih5IuEFE7yIyShHBJZl0NaWbGdotyPjOXr3reCcAdO1YRmpFCYnAtzmXkElHb5OKEcCFVp29IOQ26MQyA//a8k6H3vU1icC1qB/sS1ai6yZUJ4Vo8PizuiG7EhK6N8bYoUv2DaVDdn/fu6ojVy+O/GiGuoMy4EFFe0dHROiYmplLWffZCDskXcmlWJwgvi6qUbQjh6pRSsVrr6KJec4VGWS4hNMiX0CBfs8sQwmXJvrYQwiESFkIIh0hYCCEcImEhhHCIhIUQwiESFkIIh7hVOwulVBJwDAgFzppcTnlJ7eZx5/qvV+1NtNZFdnRwq7C4SCkVU1zDEVcntZvHnet3hdrlMEQI4RAJCyGEQ9w1LGabXUAFSO3mcef6Ta/dLc9ZCCGuP3fdsxBCXGcSFkIIh7htWCil/qWU2quU2qGUWqKUcpuhrZRSY5RSu5RSNqWUW1zKU0oNUUrtU0odVEr9xex6ykIp9bFSKlEpFW92LWWllGqklFqrlNpj/5153Kxa3DYsMGMKROeJB0YBG8wuxBFKKS/gHWAocCMwXil1o7lVlckcYIjZRZRTPvBnrXVroCvwiFnfvduGhdZ6ldY63/5wC9DQzHrKQmu9R2u9z+w6yqALcFBrfVhrnQssBEaYXJPDtNYbgHNm11EeWutTWutt9p/TgT1AAzNqcduwuMr9wHKzi6jCGgC/X/Y4AZN+YT2ZUqop0AH4xYztu/Swes6aAtEMjtTuRooalFSuuV9HSqkg4CvgCa11mhk1uHRYuPwUiCUorXY3kwA0uuxxQ+CkSbV4HKWUFSMo5mutF5tVh9sehlw2BeKtpk6B6Bl+BZorpcKVUj7AOOAbk2vyCMqYsu8jYI/W+k0za3HbsMCYAjEYYwrEOKXULLMLcpRSaqRSKgHoBnyvlFppdk0lsZ9IngKsxDjB9oXWepe5VTlOKbUA+BloqZRKUEo9YHZNZdADuBvob/89j1NK/cGMQqS5txDCIe68ZyGEuI4kLIQQDpGwEEI4RMJCCOEQCQshhEMkLIRp7B3UhJuQsBDXUEp1tnf991NKBdq7RkcWsVxfpdQ6pdSX9uEC5tsbEaGUGqCU2q6U2mnvIu5rf/6oUupFpdQmYIz9/f9RSm2wd8PurJRarJQ6oJR65Tp/dFECCQtxDa31rxgtNF8B3gA+01oXNxZEB+AJjK7rNwA9lFJ+GN3C79Bat8XoVvDwZe/J1lr31FovtD/O1Vr3BmYBXwOPAJHAvUqpWk79cKLcJCxEcf4GDAKiMQKjOFu11glaaxsQBzQFWgJHtNb77cvMBXpf9p5FV63jYtPxncAue7fsHOAwV/ZJESaSsBDFqQkEYTSp9ythuZzLfi7A2Isoqpfq5TKKWYftqvXZcPHOjp5EwkIUZzYwFaPr/+tlfO9eoKlSqpn98d3AeifWJkwgqS2uoZS6B8jXWn9uv2Lxk1Kqv9b6R0fer7XOVkrdB/xPKeWN0WvVbTr6iaJJRzIhhEPkMEQI4RA5DBGlUkq1BeZd9XSO1vomM+oR5pDDECGEQ+QwRAjhEAkLIYRDJCyEEA6RsBBCOETCQgjhkP8H40r2aScjd2MAAAAASUVORK5CYII=\n", 132 | "text/plain": [ 133 | "
" 134 | ] 135 | }, 136 | "metadata": { 137 | "needs_background": "light" 138 | }, 139 | "output_type": "display_data" 140 | } 141 | ], 142 | "source": [ 143 | "K = 1 \n", 144 | "Z = project_data(X_norm, U, K) \n", 145 | "X_rec = recover_data(Z, U, K) \n", 146 | "\n", 147 | "sns.scatterplot(x = X_norm[:, 0], y = X_norm[:, 1], edgecolor = \"none\")\n", 148 | "sns.scatterplot(x = X_rec[:, 0], y = X_rec[:, 1], edgecolor = \"none\")\n", 149 | "plt.legend(['Raw data', 'Post-PCA'])\n", 150 | "\n", 151 | "for i in range(X_norm.shape[0]):\n", 152 | " drawline(X_norm[i,:], X_rec[i,:])\n", 153 | "\n", 154 | "plt.ylabel('y_norm')\n", 155 | "plt.xlabel('x_norm')\n", 156 | "plt.axis('square')\n", 157 | "plt.show()" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": null, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [] 166 | } 167 | ], 168 | "metadata": { 169 | "kernelspec": { 170 | "display_name": "Python 3", 171 | "language": "python", 172 | "name": "python3" 173 | }, 174 | "language_info": { 175 | "codemirror_mode": { 176 | "name": "ipython", 177 | "version": 3 178 | }, 179 | "file_extension": ".py", 180 | "mimetype": "text/x-python", 181 | "name": "python", 182 | "nbconvert_exporter": "python", 183 | "pygments_lexer": "ipython3", 184 | "version": "3.7.4" 185 | } 186 | }, 187 | "nbformat": 4, 188 | "nbformat_minor": 2 189 | } 190 | -------------------------------------------------------------------------------- /PCA/PCA.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import seaborn as sns 4 | import os, sys 5 | 6 | def feature_normaliza(X): 7 | mu = np.mean(X, 0) 8 | sigma = np.std(X, 0) 9 | def get_norm(col): 10 | mu = np.mean(col) 11 | sigma = np.std(col) 12 | return (col - mu)/sigma 13 | return np.apply_along_axis(get_norm, 0, X), mu, sigma 14 | 15 | def drawline(p1, p2, color = 'r'): 16 | sns.lineplot([p1[0],p2[0]], [p1[1],p2[1]], color = color) 17 | 18 | def project_data(X_norm, U, K): 19 | Z = np.zeros((X_norm.shape[0], K)) 20 | U_reduce = U[:, 0:K] 21 | Z = np.dot(X_norm, U_reduce) 22 | return Z 23 | 24 | def recover_data(Z, U, K): 25 | X_rec = np.zeros((Z.shape[0], U.shape[0])) 26 | U_recude = U[:, 0:K] 27 | X_rec = np.dot(Z, U_recude.T) 28 | return X_rec 29 | 30 | def PCA(X, K): 31 | X_norm, mu, sigma = feature_normaliza(X) 32 | 33 | Sig = np.dot(X_norm.T,X_norm)/X_norm.shape[0] 34 | U,S,V = np.linalg.svd(Sig) 35 | 36 | Z = project_data(X_norm, U, K) 37 | X_rec = recover_data(Z, U, K) 38 | return X_rec 39 | 40 | if __name__ == '__main__': 41 | images_dir = os.path.join(sys.path[0], 'images') 42 | if not os.path.exists(images_dir): 43 | os.makedirs(images_dir) 44 | 45 | x = np.array(range(25)) 46 | y = x ** 1.3 + np.random.normal(10, 10, x.shape[0]) 47 | X = np.stack((x, y), axis = 1) 48 | 49 | sns.scatterplot(x = X[:, 0], y = X[:, 1], edgecolor = "none") 50 | plt.title('Dataset') 51 | plt.ylabel('y') 52 | plt.xlabel('x') 53 | plt.savefig(os.path.join(images_dir, 'dataset.png')) 54 | plt.clf() 55 | 56 | X_norm, mu, sigma = feature_normaliza(X) 57 | Sig = np.dot(X_norm.T,X_norm)/X_norm.shape[0] 58 | U,S,V = np.linalg.svd(Sig) 59 | 60 | sns.scatterplot(x = X_norm[:, 0], y = X_norm[:, 1], edgecolor = "none") 61 | drawline((0, 0), S[0]*U[:,0], color = 'g') 62 | drawline((0, 0), S[1]*U[:,1]) 63 | plt.ylabel('y_norm') 64 | plt.xlabel('x_norm') 65 | plt.axis('square') 66 | plt.savefig(os.path.join(images_dir, 'eigen_vectors.png')) 67 | plt.clf() 68 | 69 | K = 1 70 | X_rec = PCA(X, K) 71 | sns.scatterplot(x = X_norm[:, 0], y = X_norm[:, 1], edgecolor = "none") 72 | sns.scatterplot(x = X_rec[:, 0], y = X_rec[:, 1], edgecolor = "none") 73 | plt.legend(['Raw data', 'Post-PCA']) 74 | 75 | for i in range(X_norm.shape[0]): 76 | drawline(X_norm[i,:], X_rec[i,:]) 77 | 78 | plt.ylabel('y_norm') 79 | plt.xlabel('x_norm') 80 | plt.axis('square') 81 | plt.savefig(os.path.join(images_dir, 'PCA.png')) 82 | plt.clf() 83 | -------------------------------------------------------------------------------- /PCA/images/PCA.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/PCA/images/PCA.png -------------------------------------------------------------------------------- /PCA/images/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/PCA/images/dataset.png -------------------------------------------------------------------------------- /PCA/images/eigen_vectors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/PCA/images/eigen_vectors.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ML Algorithm with Python 2 | 3 |

4 | 5 |

6 | 7 | ## Table of Contents 8 | 1. [Linear Regression](#linear_regression) 9 | 2. [Logistic Regression](#logistic_regression) 10 | 3. [Neural Network](#neural_network) 11 | 4. [Decision Tree](#decision_tree) 12 | 5. [K-Means](#k-means) 13 | 6. [Principal Component Analysis](#principal_component_analysis) 14 | 15 | ## Being constantly updated .. 16 | 17 | Machine learning algorithm written in Python 18 | - Requires only basic linear algebra 19 | - Uses Numpy for matrix operation to avoid costly looping in Python 20 | - Usually includes notebook for easy visual 21 | - Simple examples provided 22 | 23 | to run each algorithm, `cd` to corresponding directory and run the following (replace `***` with the corresponding algorithm) in terminal: 24 | 25 | ``` 26 | python ***.py 27 | ``` 28 | 29 | (I also have plan to package each algorithm into a class) 30 | 31 | ## credit 32 | Most equations are from [CS 229](http://cs229.stanford.edu/syllabus-autumn2018.html) class of Stanford. 33 | 34 | 35 | ## 1. [Linear Regression](/linear_regression/linear_regression.ipynb) 36 | 37 | ### Cost function 38 | 39 | ``` 40 | def cost(X, y, theta): 41 | h = np.dot(X, theta) 42 | cos = np.sum((h - y) * ( h - y))/(2 * len(y)) 43 | return cos 44 | ``` 45 | 46 | ### Gradient descent 47 | ``` 48 | def gradient_descent(X, y, theta, alpha, num_iters): 49 | m = len(y) 50 | costs = [] 51 | for _ in range(num_iters): 52 | h = np.dot(X,theta) 53 | theta -= alpha * np.dot(X.T, (h - y))/m 54 | costs.append(cost(X, y, theta)) 55 | return theta, costs 56 | ``` 57 | ### Feature normalization 58 | ``` 59 | def feature_normaliza(X): 60 | mu = np.mean(X, 0) 61 | sigma = np.std(X, 0) 62 | def get_norm(col): 63 | mu = np.mean(col) 64 | sigma = np.std(col) 65 | return (col - mu)/sigma 66 | return np.apply_along_axis(get_norm, 0, X), mu, sigma 67 | ``` 68 | 69 | ### Main function 70 | 71 | ``` 72 | def linear_regression(X, y, alpha = 0.01,num_iters = 100): 73 | X = np.append(np.ones((X.shape[0], 1)), X, axis = 1) 74 | theta = np.zeros((X.shape[1], 1), dtype = np.float64) 75 | theta, costs = gradient_descent(X, y, theta, alpha, num_iters) 76 | predicted = np.dot(X, theta) 77 | return predicted, theta, costs 78 | ``` 79 | ### Plot an example 80 | ``` 81 | predicted, theta, costs = linear_regression(X, y) 82 | 83 | plt.plot(X, predicted, 'b') 84 | plt.plot(X, y, 'rx', 10) 85 | for i, x in enumerate(X): 86 | plt.vlines(x, min(predicted[i], y[i]), max(predicted[i], y[i])) 87 | plt.ylabel('Y') 88 | plt.xlabel('X') 89 | plt.legend(('linear fit', 'data')) 90 | plt.show() 91 | ``` 92 |

93 | 94 |

95 | 96 | 97 | ## 2. [Logistic Regression (with Regularization)](/logistic_regression/logistic_regression.ipynb) 98 | ### Sigmoid function 99 | ``` 100 | def sigmoid(z): 101 | return 1/(1 + np.exp(-z)) 102 | ``` 103 | 104 | ### Cost function 105 | 106 | ``` 107 | def cost_reg(theta, X, y, lam = 0): 108 | h = sigmoid(np.dot(X, theta)) 109 | theta1 = theta.copy() 110 | theta1[0] = 0 111 | cos = -(np.sum(y * np.log(h)) + np.sum((1 - y) * np.log(1 - h)))/len(y) + lam * np.sum(theta1 * theta1)/len(y) 112 | return cos 113 | ``` 114 | 115 | ### Expand features 116 | ``` 117 | def expand_feature(x1, x2, power = 2): 118 | #expand a 2D feature matrix to polynimial features up to the power 119 | new_x = np.ones((x1.shape[0], 1)) 120 | for i in range(1, power + 1): 121 | for j in range(i + 1): 122 | new_x = np.append(new_x, (x1**(i-j)*(x2**j)).reshape(-1, 1), axis = 1) 123 | return new_x 124 | ``` 125 | 126 | ### Gradient descent 127 | ``` 128 | def gradient_descent_reg(X, y, theta, alpha, lam = 0, num_iters = 100): 129 | m = len(y) 130 | costs = [] 131 | 132 | for _ in range(num_iters): 133 | h = sigmoid(np.dot(X, theta)) 134 | theta1 = theta.copy() 135 | theta1[0] = 0 136 | theta -= alpha * (np.dot(X.T, (h - y)) + 2 * lam * theta1)/m 137 | costs.append(cost_reg(theta, X, y)) 138 | return theta, costs 139 | ``` 140 | 141 | ### Prediction 142 | ``` 143 | def predict(theta, X): 144 | return (sigmoid(np.dot(X, theta)) > 0.5).flatten() 145 | ``` 146 | ### Main function 147 | ``` 148 | def logistic_regression_reg(X, y, power = 2, alpha = 0.01, lam = 0, num_iters = 100): 149 | X = expand_feature(X[:, 0], X[:, 1], power = power) 150 | theta = np.zeros((X.shape[1], 1), dtype = np.float64) 151 | theta, costs = gradient_descent_reg(X, y, theta, alpha, lam, num_iters) 152 | predicted = predict(theta, X) 153 | return predicted, theta, costs 154 | ``` 155 | ### Examples 156 |

157 | 158 | 159 | 160 |

161 | 162 | 163 | ## 3. [Neural Network](/neural_network/neural_network.ipynb) 164 | ### Initialize parameters 165 | ``` 166 | def init_para(D, K, h): 167 | W = np.random.normal(0, 0.01, (D, h)) 168 | b = np.zeros((1, h), dtype = float) 169 | W2 = np.random.normal(0, 0.01, (h, K)) 170 | b2 = np.zeros((1, K), dtype = float) 171 | return W, b, W2, b2 172 | ``` 173 | ### Softmax 174 | ``` 175 | def softmax(scores): 176 | exp_scores = np.exp(scores) 177 | return exp_scores / np.sum(exp_scores, axis = 1).reshape(-1, 1) 178 | ``` 179 | 180 | ### Main function 181 | ``` 182 | def nnet(X, y, step_size = 0.4, lam = 0.0001, h = 10, num_iters = 1000): 183 | # get dim of input 184 | N, D = X.shape 185 | K = y.shape[1] 186 | 187 | W, b, W2, b2 = init_para(D, K, h) 188 | 189 | # gradient descent loop to update weight and bias 190 | for i in range(num_iters): 191 | # hidden layer, ReLU activation 192 | hidden_layer = np.maximum(0, np.dot(X, W) + np.repeat(b, N, axis = 0)) 193 | 194 | # class score 195 | scores = np.dot(hidden_layer, W2) + np.repeat(b2, N, axis = 0) 196 | 197 | # compute and normalize class probabilities 198 | probs = softmax(scores) 199 | 200 | # compute the loss with regularization 201 | data_loss = np.sum(-np.log(probs) * y) / N 202 | reg_loss = 0.5 * lam * np.sum(W * W) + 0.5 * lam * np.sum(W2 * W2) 203 | loss = data_loss + reg_loss 204 | 205 | # check progress 206 | if i%1000 == 0 or i == num_iters: 207 | print("iteration {}: loss {}".format(i, loss)) 208 | 209 | # compute the gradient on scores 210 | dscores = (probs - y) / N 211 | 212 | # backpropate the gradient to the parameters 213 | dW2 = np.dot(hidden_layer.T, dscores) 214 | db2 = np.sum(dscores, axis = 0) 215 | # next backprop into hidden layer 216 | dhidden = np.dot(dscores, W2.T) 217 | # backprop the ReLU non-linearity 218 | dhidden[hidden_layer <= 0] = 0 219 | # finally into W,b 220 | dW = np.dot(X.T, dhidden) 221 | db = np.sum(dhidden, axis = 0) 222 | 223 | # add regularization gradient contribution 224 | dW2 = dW2 + lam * W2 225 | dW = dW + lam * W 226 | 227 | # update parameter 228 | W = W - step_size * dW 229 | b = b - step_size * db 230 | W2 = W2 - step_size * dW2 231 | b2 = b2 - step_size * db2 232 | return W, b, W2, b2 233 | ``` 234 | 235 | ### Example 236 |

237 | 238 |

239 | 240 | 241 | ## 4. [Decision Tree](/decision_tree/decision_tree.ipynb) 242 | ### Gini impurity/Entropy 243 | ``` 244 | def gini_impurity(y): 245 | # calculate gini_impurity given labels/classes of each example 246 | m = y.shape[0] 247 | cnts = dict(zip(*np.unique(y, return_counts = True))) 248 | impurity = 1 - sum((cnt/m)**2 for cnt in cnts.values()) 249 | return impurity 250 | 251 | def entropy(y): 252 | # calculate entropy given labels/classes of each example 253 | m = y.shape[0] 254 | cnts = dict(zip(*np.unique(y, return_counts = True))) 255 | disorder = - sum((cnt/m)*log(cnt/m) for cnt in cnts.values()) 256 | return disorder 257 | ``` 258 | 259 | ### Information gain 260 | ``` 261 | def info_gain(l_y, r_y, cur_gini): 262 | # calculate the information gain for a certain split 263 | m, n = l_y.shape[0], r_y.shape[0] 264 | p = m / (m + n) 265 | return cur_gini - p * gini_impurity(l_y) - (1 - p) * gini_impurity(r_y) 266 | ``` 267 | 268 | ### Find best split 269 | ``` 270 | def get_split(X, y): 271 | # loop through features and values to find best combination with the most information gain 272 | best_gain, best_index, best_value = 0, None, None 273 | 274 | cur_gini = gini_impurity(y) 275 | n_features = X.shape[1] 276 | 277 | for index in range(n_features): 278 | 279 | values = np.unique(X[:, index], return_counts = False) 280 | 281 | for value in values: 282 | 283 | left, right = test_split(index, value, X, y) 284 | 285 | if left['y'].shape[0] == 0 or right['y'].shape[0] == 0: 286 | continue 287 | 288 | gain = info_gain(left['y'], right['y'], cur_gini) 289 | 290 | if gain > best_gain: 291 | best_gain, best_index, best_value = gain, index, value 292 | best_split = {'gain': best_gain, 'index': best_index, 'value': best_value} 293 | return best_split 294 | ``` 295 | ### Create leaf and decision node 296 | ``` 297 | class Leaf: 298 | # define a leaf node 299 | def __init__(self, y): 300 | self.counts = dict(zip(*np.unique(y, return_counts = True))) 301 | self.prediction = max(self.counts.keys(), key = lambda x: self.counts[x]) 302 | 303 | class Decision_Node: 304 | # define a decision node 305 | def __init__(self, index, value, left, right): 306 | self.index, self.value = index, value 307 | self.left, self.right = left, right 308 | ``` 309 | ### Training (build decision tree) 310 | ``` 311 | def decision_tree(X, y, max_dep = 5, min_size = 10): 312 | # train the decision tree model with a dataset 313 | correct_prediction = 0 314 | 315 | def build_tree(X, y, dep, max_dep = max_dep, min_size = min_size): 316 | # recursively build the tree 317 | split = get_split(X, y) 318 | 319 | if split['gain'] == 0 or dep >= max_dep or y.shape[0] <= min_size: 320 | nonlocal correct_prediction 321 | leaf = Leaf(y) 322 | correct_prediction += leaf.counts[leaf.prediction] 323 | return leaf 324 | 325 | left, right = test_split(split['index'], split['value'], X, y) 326 | 327 | left_node = build_tree(left['X'], left['y'], dep + 1) 328 | right_node = build_tree(right['X'], right['y'], dep + 1) 329 | 330 | return Decision_Node(split['index'], split['value'], left_node, right_node) 331 | 332 | root = build_tree(X, y, 0) 333 | 334 | return correct_prediction/y.shape[0], root 335 | ``` 336 | ### Prediction 337 | ``` 338 | def predict(x, node): 339 | if isinstance(node, Leaf): 340 | return node.prediction 341 | 342 | if x[node.index] < node.value: 343 | return predict(x, node.left) 344 | else: 345 | return predict(x, node.right) 346 | ``` 347 | 348 | 349 | ### Example 350 |

351 | 352 |

353 | 354 | 355 | ## 5. [K-Means](/k-means/k-means.ipynb) 356 | ### Initialize centroids 357 | ``` 358 | def init_centroid(X, K): 359 | m = X.shape[0] 360 | idx = np.random.choice(m, K, replace = False) 361 | return X[idx, :] 362 | ``` 363 | ### Update labels 364 | ``` 365 | def update_label(X, centroid): 366 | m, K = X.shape[0], centroid.shape[0] 367 | dist = np.zeros((m, K)) 368 | label = np.zeros((m, 1)) 369 | 370 | for i in range(m): 371 | for j in range(K): 372 | dist[i,j] = np.dot((X[i, :] - centroid[j, :]).T, (X[i, :] - centroid[j, :])) 373 | 374 | label = np.argmin(dist, axis = 1) 375 | total_dist = np.sum(np.choose(label, dist.T)) 376 | return label, total_dist 377 | ``` 378 | ### Update centroids 379 | ``` 380 | def update_centroid(X, label, K): 381 | D = X.shape[1] 382 | centroid = np.zeros((K, D)) 383 | for i in range(K): 384 | centroid[i, :] = np.mean(X[label.flatten() == i, :], axis=0).reshape(1,-1) 385 | return centroid 386 | ``` 387 | ### K-Means function 388 | ``` 389 | def k_means(X, K, num_iters = 100): 390 | m = X.shape[0] 391 | centroid = init_centroid(X, K) 392 | 393 | for _ in range(num_iters): 394 | label, total_dist = update_label(X, centroid) 395 | centroid = update_centroid(X, label, K) 396 | 397 | return centroid, label, total_dist 398 | ``` 399 | ### Example 400 |

401 | 402 |

403 | 404 | ### Determine K 405 |

406 | 407 |

408 | 409 | 410 | ## 6. [Principal Component Analysis](/PCA/PCA.ipynb) 411 | 412 | ### SVD (Singular Value Decomposition) 413 | ``` 414 | Sig = np.dot(X_norm.T,X_norm)/X_norm.shape[0] 415 | U,S,V = np.linalg.svd(Sig) 416 | ``` 417 | 418 | ### Data projection 419 | ``` 420 | def project_data(X_norm, U, K): 421 | Z = np.zeros((X_norm.shape[0], K)) 422 | U_reduce = U[:, 0:K] 423 | Z = np.dot(X_norm, U_reduce) 424 | return Z 425 | ``` 426 | 427 | ### Data recovery 428 | ``` 429 | def recover_data(Z, U, K): 430 | X_rec = np.zeros((Z.shape[0], U.shape[0])) 431 | U_recude = U[:, 0:K] 432 | X_rec = np.dot(Z, U_recude.T) 433 | return X_rec 434 | ``` 435 | 436 | ### PCA function 437 | ``` 438 | def PCA(X, K): 439 | X_norm, mu, sigma = feature_normaliza(X) 440 | 441 | Sig = np.dot(X_norm.T,X_norm)/X_norm.shape[0] 442 | U,S,V = np.linalg.svd(Sig) 443 | 444 | Z = project_data(X_norm, U, K) 445 | X_rec = recover_data(Z, U, K) 446 | return X_rec 447 | ``` 448 | ### Example (2D -> 1D) 449 |

450 | 451 |

452 | 453 | ... 454 | -------------------------------------------------------------------------------- /decision_tree/decision_tree.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from math import log 3 | import matplotlib.pyplot as plt 4 | import seaborn as sns 5 | import os, sys 6 | 7 | def gini_impurity(y): 8 | # calculate gini_impurity given labels/classes of each example 9 | m = y.shape[0] 10 | cnts = dict(zip(*np.unique(y, return_counts = True))) 11 | impurity = 1 - sum((cnt/m)**2 for cnt in cnts.values()) 12 | return impurity 13 | 14 | def entropy(y): 15 | # calculate entropy given labels/classes of each example 16 | m = y.shape[0] 17 | cnts = dict(zip(*np.unique(y, return_counts = True))) 18 | disorder = - sum((cnt/m)*log(cnt/m) for cnt in cnts.values()) 19 | return disorder 20 | 21 | def test_split(index, value, X, y): 22 | # split a group of examples based on given index (feature) and value 23 | mask = X[:, index] < value 24 | left = {'X': X[mask, :], 'y': y[mask]} 25 | right = {'X': X[~mask, :], 'y': y[~mask]} 26 | return left, right 27 | 28 | def info_gain(l_y, r_y, cur_gini): 29 | # calculate the information gain for a certain split 30 | m, n = l_y.shape[0], r_y.shape[0] 31 | p = m / (m + n) 32 | #return cur_gini - p * entropy(l_y) - (1 - p) * entropy(r_y) 33 | return cur_gini - p * gini_impurity(l_y) - (1 - p) * gini_impurity(r_y) 34 | 35 | def get_split(X, y): 36 | # loop through features and values to find best combination with the most information gain 37 | best_gain, best_index, best_value = 0, None, None 38 | 39 | #cur_gini = entropy(y) 40 | cur_gini = gini_impurity(y) 41 | n_features = X.shape[1] 42 | 43 | for index in range(n_features): 44 | values = np.unique(X[:, index], return_counts = False) 45 | for value in values: 46 | left, right = test_split(index, value, X, y) 47 | if left['y'].shape[0] == 0 or right['y'].shape[0] == 0: 48 | continue 49 | gain = info_gain(left['y'], right['y'], cur_gini) 50 | if gain > best_gain: 51 | best_gain, best_index, best_value = gain, index, value 52 | 53 | best_split = {'gain': best_gain, 'index': best_index, 'value': best_value} 54 | return best_split 55 | 56 | class Leaf: 57 | # define a leaf node 58 | def __init__(self, y): 59 | self.counts = dict(zip(*np.unique(y, return_counts = True))) 60 | self.prediction = max(self.counts.keys(), key = lambda x: self.counts[x]) 61 | 62 | class Decision_Node: 63 | # define a decision node 64 | def __init__(self, index, value, left, right): 65 | self.index, self.value = index, value 66 | self.left, self.right = left, right 67 | 68 | def decision_tree(X, y, max_dep = 5, min_size = 10): 69 | # train the decision tree model with a dataset 70 | correct_prediction = 0 71 | 72 | def build_tree(X, y, dep, max_dep = max_dep, min_size = min_size): 73 | # recursively build the tree 74 | split = get_split(X, y) 75 | 76 | if split['gain'] == 0 or dep >= max_dep or y.shape[0] <= min_size: 77 | nonlocal correct_prediction 78 | leaf = Leaf(y) 79 | correct_prediction += leaf.counts[leaf.prediction] 80 | return leaf 81 | 82 | left, right = test_split(split['index'], split['value'], X, y) 83 | 84 | left_node = build_tree(left['X'], left['y'], dep + 1) 85 | right_node = build_tree(right['X'], right['y'], dep + 1) 86 | 87 | return Decision_Node(split['index'], split['value'], left_node, right_node) 88 | 89 | root = build_tree(X, y, 0) 90 | 91 | return correct_prediction/y.shape[0], root 92 | 93 | def predict(x, node): 94 | if isinstance(node, Leaf): 95 | return node.prediction 96 | 97 | if x[node.index] < node.value: 98 | return predict(x, node.left) 99 | else: 100 | return predict(x, node.right) 101 | 102 | if __name__ == '__main__': 103 | images_dir = os.path.join(sys.path[0], 'images') 104 | if not os.path.exists(images_dir): 105 | os.makedirs(images_dir) 106 | 107 | N = 100 # number of points per class 108 | D = 2 # dimensionality, we use 2D data for easy visulization 109 | K = 3 # number of classes 110 | X = np.zeros((N * K, D), dtype = float) # data matrix (each row = single example, can view as xy coordinates) 111 | y = np.zeros(N * K, dtype = int) # class labels 112 | 113 | for i in range(K): 114 | r = np.random.normal(i + 0.5, 0.3, (N, 1)) # radius 115 | t = np.linspace(0, np.pi * 2, N).reshape(N, 1) # theta 116 | X[i * N:(i + 1) * N] = np.append(r * np.sin(t), r * np.cos(t), axis = 1) 117 | y[i * N:(i + 1) * N] = i 118 | 119 | sns.scatterplot(x = X[:, 0], y = X[:, 1], 120 | hue = y, palette = sns.color_palette('deep', K), edgecolor = "none") 121 | plt.title('Dataset') 122 | plt.xlabel('X') 123 | plt.ylabel('Y') 124 | plt.savefig(os.path.join(images_dir, 'data.png')) 125 | plt.clf() 126 | 127 | overfit_accuracy, overfit_model = decision_tree(X, y, float('inf'), 1) 128 | accuracy, model = decision_tree(X, y, 6) 129 | 130 | print('The accuracy of a model with unlimited split is {:.2f} %'.format(overfit_accuracy*100)) 131 | print('With limited split, the accuracy becomes {:.2f} %'.format(accuracy*100)) 132 | 133 | u = np.linspace(min(X[:, 0]),max(X[:, 0]), 400) 134 | v = np.linspace(min(X[:, 1]),max(X[:, 1]), 400) 135 | 136 | models = [overfit_model, model] 137 | titles = ['Overfit DB', 'DB with limited split'] 138 | 139 | fig, axs = plt.subplots(ncols = 2, figsize = (12, 5)) 140 | for k, ax in enumerate(axs): 141 | z = np.zeros((len(u),len(v))) 142 | for i in range(len(u)): 143 | for j in range(len(v)): 144 | z[i,j] = predict([u[i], v[j]], models[k]) 145 | z = np.transpose(z) 146 | 147 | ax.contourf(u,v,z, alpha = 0.2, levels = K - 1, antialiased = True) 148 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = y, 149 | palette = sns.color_palette('deep', K), edgecolor = "none", ax = ax) 150 | ax.set_title(titles[k]) 151 | ax.set_xlabel('X') 152 | ax.set_ylabel('Y') 153 | plt.savefig(os.path.join(images_dir, 'decision_boundary.png')) 154 | plt.clf() 155 | -------------------------------------------------------------------------------- /decision_tree/images/data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/decision_tree/images/data.png -------------------------------------------------------------------------------- /decision_tree/images/decision_boundary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/decision_tree/images/decision_boundary.png -------------------------------------------------------------------------------- /k-means/images/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/k-means/images/dataset.png -------------------------------------------------------------------------------- /k-means/images/k-means.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/k-means/images/k-means.png -------------------------------------------------------------------------------- /k-means/images/total_dist_vs_k.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/k-means/images/total_dist_vs_k.png -------------------------------------------------------------------------------- /k-means/k-means.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "import seaborn as sns\n", 12 | "%matplotlib inline" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "metadata": {}, 18 | "source": [ 19 | "K-means is a type of unsupervised learning method, therefore there are no classes per say. However I generate the data with distinctive classes for easy visulization. Later we will find that the no. of centroids we choose will be similar to the no. of classes for generated data." 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 13, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "N = 80 # number of points per class\n", 29 | "D = 2 # dimensionality, we use 2D data for easy visulization\n", 30 | "K = 3 # number of classes \n", 31 | "X = np.zeros((N * K, D), dtype = float) # data matrix (each row = single example, can view as xy coordinates)" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 22, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "for i in range(K):\n", 41 | " r = np.linspace(0.05, 1, N).reshape(-1, 1) # radius\n", 42 | " t = np.random.normal(0, 0.4, (N, 1)) # theta\n", 43 | " X[i * N:(i + 1) * N] = np.append(r * np.sin(t) + i/2, r * np.cos(t) - 2*i/2, axis = 1)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 23, 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "data": { 53 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3hUVfrA8e+ZSe+9B1IJoZfQO6iA2BV7L1jXturq6rquu65li7o/XVcsaxcLKnVBUHoPEHpNCKSQ3vuU8/tjkiGTuZMCCQlwPs/jg3PvnTsnlHnvae8rpJQoiqIoiiO67m6AoiiK0rOpQKEoiqK0SgUKRVEUpVUqUCiKoiitUoFCURRFaZUKFIqiKEqrVKBQFEVRWqUChaK0IITIFELUCiEqhRBlQoiNQogHhBBt/nsRQsQIIaQQwqmL23hWPkdRQAUKRXHkcimlN9AbeA34HfBR9zZJUbqHChSK0gopZbmUciFwA3CHEGKAEGKWEGKnEKJCCJElhHip2VvWNv5aJoSoEkKMEULECyF+FUIUCyGKhBBfCiH8mt4ghPidECKnsQdzSAgxrfG4TgjxrBAivfG93wohAhx9Thf/VigXMBUoFKUdpJRbgWxgAlAN3A74AbOAB4UQVzVeOrHxVz8ppZeUchMggFeBCCAZiAZeAhBCJAGPACMaezDTgczGezwKXAVManxvKfBuK5+jKF1CBQpFab9cIEBKuVpKuUdKaZZS7ga+xvJlrklKeVRKuUJKWS+lLAT+2ex6E+AK9BNCOEspM6WU6Y3n7geel1JmSynrsQSX69S8hHK2qUChKO0XCZQIIUYJIVYJIQqFEOXAA0CQozcJIUKEEPMah5cqgC+arpdSHgUexxIEChqvi2h8a2/gx8YJ9TLgAJbAEtpVP6CiaFGBQlHaQQgxAkugWA98BSwEoqWUvsB/sAwvAWilY3618fggKaUPcGuz65FSfiWlHI8lMEjg9cZTWcBMKaVfs//cpJQ5Dj5HUbqEChSK0gohhI8Q4jJgHvCFlHIP4A2USCnrhBAjgZubvaUQMANxzY55A1VYJp4jgaeb3T9JCDFVCOEK1AG1WHoNYAlArwghejdeGyyEuLKVz1GULqEChaJoWySEqMTyVP88lnmFuxrPPQS83Hj+ReDbpjdJKWuAV4ANjUNGo4E/AcOAcmAJ8EOzz3HFsvy2CMgDQoDfN557G0vP5efGz9oMjGrlcxSlSwhVuEhRFEVpjepRKIqiKK1SgUJRFEVplQoUiqIoSqtUoFAURVFadV7u8AwKCpIxMTHd3QxFUZRzxvbt24uklMFa587LQBETE0Nqamp3N0NRFOWcIYQ47uicGnpSFEVRWqUChaIoitIqFSgURVGUVqlAoSiKorRKBQpFURSlVSpQKIqiKK3q1kAhhPhYCFEghNjr4LwQQvxLCHFUCLFbCDHsbLdROTMms+SzTZnc/vFWnvgmjV1ZZd3dJEVROqi791F8ArwDfObg/EwgsfG/UcB7jb8qjaSU1BvNuDnru7spmp7/cQ/ztmVZXy/ZfZKv54xmeG//bmyVoigd0a09CinlWqCklUuuBD6TFpsBPyFE+NlpXc+3IC2H8a+vou8flnHZ/61jx4nS7m6SjcLKer7bnm1zrMFk5s+L93dTixRFOR09fY4iEkvhmCbZjcfsCCHmCCFShRCphYWFZ6Vx3Wl3dhmPf5NGTlktAHtzKrjz461U1Ru7uWWnFFfXYzLb1ztJyyojvbCqG1qkKMrp6OmBQmgc06y0JKWcK6VMkVKmBAdrpis5ryxMy6VlzamKOiOrDhZ0T4M0JIZ44+vurHlu2d48h+/bklHMG8sO8vmmTCrqDF3UOkVR2qu75yjakg1EN3sdBeR2U1t6FGcn7RjvrNeKrd1DrxPcPDKa99Zk2J1zdzCn8o+fD/F/vx61vn5/bQY/PDSWEG+3Lmunoiit6+k9ioXA7Y2rn0YD5VLKk93dqJ7g2mGRuOht//jCfNyYnBTSTS3S9uCUBAI8bHsVvu7OXDEkwu7agoo63ludbnMsu7SWj9dndugzCyrrePLbNEa8spIr31nPiv35HW63oiindPfy2K+BTUCSECJbCHGPEOIBIcQDjZcsBTKAo8AHWIraK0BCiDdv3zSEpFDL8M5FyaF8ed+oHrf6ycfNmW8fGMOM/mFE+rlzUXIo8+aMJsjL1e7ao4VVGDXmNA7lVXToM+/67zZ+2JFDYWU9u7LLuf/zVLZltrZmQlGU1nTr0JOU8qY2zkvg4bPUnC5XbzTx71XpLNubh6+7M3eNi2HmwNNbxPXuqqO8/csRGoxm3Jx1jEsIJD7Yq5Nb3DkSQrz5z23D27wuOcwHVycd9UazzfHB0X7t/qy0rDL25doGFrOEr7eeYERMQLvvoyjKKT196Om88tz8Pbz9yxEO5VeyNbOEB7/cwbK9HR9JS8sq42/LD9HQ+IVaZzDzp0X7OZJf2dlNPqv8PV343Yy+iGbTLH3DvLlrbGy779HQIsg0aRl8FEVpv54+mX3eKK1uYMEu+3n4TzceZ8aAjvUqVh/SXtm0+lAhiaHep9W+nuLu8bFMSgpm3eFCwnzdmJYcirO+/c8zw3v7E+nnbl023OTKwfZzIoqitI8KFGdJjcGkuaegsr7jyz8drQAK8bEf9z8XxQd7nfYwml4n+O9dI/jd/N3sPFFGgKcLD02O55L+YZ3cSkW5cKhAcZZE+rkzOMqXXdnlNsdnavQmNhwtYmN6EdH+HlwxJAIPF9s/piuGRPDv1UfJLj311BwX7Mn0Dn4Z1hlMlNUYCPPt+qWn24+XsCAtFxe9jtkp0SSFdV3Pp0+oNz8+NI6qeiPuznr0up6zZFhRzkVCtty1dR5ISUmRPbFm9vHiah6bl0ZaVhnOesF1w6N4+coBNkMrf1q0j/9uyLS+TgjxYv4DY/FtscS0oKKOD9ZlcOBkJQOjfLl3fCyBGiuJHHl75RHmrk2nusFEYogXf589uM1J480ZxSzenYu7s57rU6LbPcz1XWoWz8zfbd0g6KwXfHznCCYknv8bIxXlXCGE2C6lTNE8pwLFmTlaUMnSPXm4Oeu4akgkIT5tP53nldfh4arHx832y/94cTWT/77absf109OTeHhKQqe1eemekzz05Q6bY27OOq5PieaWUb01n/Y/35TJHxbss752cdLx+d0jGRUX2OpnSSkZ8+qv5FXU2Rwf1suPHx4ad/o/hKIonaq1QKFWPTVjNJlZdbCAJbtPtitn0qJduUx/ax3/XHGYvy49yLR/rGFvTnmb7wvzdbMLEgCH8irtggTAwbzOXc20ZI/9Sqs6g5nPNh3n8nfWsyWj2OacwWTmzZVHbI41GM3861fbY1pqDSa7IAGQWVzTwVYritJdVKBolFtWy8VvruWuT7bx8Fc7GPPqL2w95niTltks+evSAzYT1JX1Rv7x86HTbsOASF/N8fTBUb6nfU8t3q6Op6YajGbeWXXU5lhFrYGS6ga7azOL2v6y93BxYkCkj93xUbHnxp6GI/mVfLH5OBuOFuGo9320oJIFaTlkqESHynlKTWY3+vvyQxwrqra+rqwz8twPu/nlt5M1ry+vNXCy3P5Juenpv7iqnndXpbP9RCnxwZ48NDmBhJDWV/JE+Lnz6NRE3lx52HpscLQfN43sdRo/kWO3jOrN/B3ZGEzaX3yZxdU2rwO9XEkM8eJIge0XYXu/7F+5aiB3/ncrpTWWFV5R/u48NzP5NFre+RbtyuXzTcepbjAya1A4cybE4dQ4Z/TmisO8/cupXtP4hCA+vnMELs3ybL24YC+fbToOgBBw7/hYnp/V7+z+EIrSxVSgaLRFo/eQXlhNUVW9ZroJPw9negd6cLzFEMqgKF+MJjM3fbCZw/mWL9ZdWWX8cqCA5Y9PbHOF0UX9QnB30VHbYCI53IepfUOsX1ydZWCUL5/ePZL3VqezOaPYLmCM0Zh3ePWagdzzaSrltZYv+7ggT56antSuzxsc7cfGZ6ex5nABLk46JiQGd2hvRFdZkJbDY/PSrK/35VZwsqyOP181gOPF1XZDa+uPFnHtexv59y3DiA7wYGN6kTVIAEgJH6w7xqUDwxnaSxVmUs4fKlA0ig3ytNukFeDp4jBNthCCly7vz/1fbLfuBg7ycuHp6UmsPVJoDRJNymsNfJeaxW+mJWrez2gy8+i8nSzdY0m/7aQTvHh5v3YFCYPJzOH8SsJ93QnwdGnzeoCx8UGMjQ9i+/ES7vk0lbLGp/0+oV48dcmpAJBZVE1pTQNDe/mz8dmprDtSiJuznvEJQR0KYO4u+g5vLOxqHzdbXdbkm9Qsnp3Zl7SsMs35oj055dz60RZ+eXKSw6HJzRklKlAo5xUVKBo9dlEi2zJLbFI9PHFxn1affKf0DWHdM1NYvi8PN2c9MweE4e3mTFqW9oR2cbNxfqPJjBDCOifx484ca5AAMJolLy/az4z+Ya2upFq0K5cXF+yltMaAs15wx5gYXris/UMfw3sHsOnZaaw/WoSni57RcYHodILaBhOPfLWDXxrrW0T4uvHercN73Jf9maistd/s2GA0U2cwtTpMeLy4hg3pxcQEemqejw3y6LQ2KkpP0P39/x5iREwASx+bwP2T4rhtdG++mTOa20b3bvN9oT5u3D4mhutTovFuXMk0qU+wXQpwgEv6h1JZZ+DxeTtJfnEZA/64nJcW7qPBaGZzhv3TqdEs2eog66nRZOap79L4zdc7rWP/BpPkw/XHWLy7YyU73F30XNwvlLEJQegaA9d7q49agwRAbnkdj83b6XBC91w0Y4D9BsWRMQEEernSP8KXq4dqFlMEoN5gYubAMLuJ+iHRflyUHNrpbVWU7qR6FM3EB3t1yiRrsLcrb904hBd+2ktJdQPuznoemZrA2PggHv16Jwsbcz4ZTCY+2ZiJh4ue3oHaT6GOnlo/23Sc77fnaJ5buT+fywadWW6j1Yfty8lmFtcw+/1NlFQ1MCY+kCcv7tOhTX49zaPTEsktq2XR7pOYzJJhvfz45w2Dref/MXswCSFe/G257Uq2AE8XJiQG4+qk55s5Y/g2NYv9uRUMjPJl9vDoTp9TUpTupgJFF7l0YDjTkkM4VlRNhJ87Pm7O1BtNLNXYw/DTzhwW/mY887aeILfZSqpL+oUyIFJ7aezP+x2XEu2ML+8Qb+17pGaWApBRVE1aVhmLfzMeIVpPkbEpvZj9JysYFOXb4VTfJdUNFFXVEx/s1empONyc9bx141D+eHl/6owmwn3dbc7rdIKHpyQQ4OnCP1ccprCynv4RPrxy9UDcXSx1PzxdnbhrXPuz2yrKuUgFii7k6qSnb9ipoQmBZU6iZXEeJ72OIC9XFv5mPF9uPsHx4mpGxwVy9TDHQx9+7tqT1u7Oem4ZdebLae+bEMfqQ4WahYSa7MutIPV4qfXLv95o4o1lh/hxZw46Ibg+JZL0wmqW7ztVYe6aoZH884YhbX6+lJI/LdrPl1uOYzBJIv3cefOGIYzsgv0X/m0sALhpZC9uSImmusFoHV5UlAuJ6iOfRS5OOq4dHmV3/MaRlrLgQV6uPHZRIv+8YQjXj4hudSL9rnExdk/YEX5u/PDQWOI6oYDRqLhAvn1gDFcOiWBSn2CGOMgDVVV3agf7K0sO8NH6Y9ZewL9XZ9gECYAfduawucXOby3fbc/mk42Z1qW7OWW1PPTlduqNpjbfW1LdwK6sMmob2r62pePF1aw6WEBxVb3NcZ1OqCChXLBUj+Is++Pl/fBydeLHnTm46HXcPKoXD0yM7/B9RsUF8vk9I/lw3TGKq+qZ2jeU+yfFOSyFKqXEaJYd2r8wrJc/wxqXef5yIJ97PrXNn+Xv4cyY+EDr/b9LzW7XfXdllTG6jRxRWnWui6oa2HG8zPqZWv6+/BBz12bQYDLj4+bEn68awJVDHPfMmkgpef6nvXy99QRSgotexwuXJXP7mJg236so5zsVKM4yVyc9v780md9feuaT5k17IdrynzXpzF2bQWlNAxMSg/nr1QOI8j81eW4yS/6zJp35O7IRwOyUaOZMiLOugAKYlhzKMzOSeG9VOpX1RuKCPXnj2kHWwFRa04C5nSui+rQjxXiAh/ZwUKCX42GiNYcLbdKPVNQZeeq7XYyJD3RYw6PJz/vz+WrLCevrBpOZlxbuY2rfEJvfK0W5EKlAcZ77aWcOr/3voPX12sOFzPlsO0sfm2A99uaKwzZfsK/97yB1BhOPX9SHspoGFu8+SU2DkcsGRnD3uFibGhYVdQae/m4XK/bnozWdEeHrZjNBPzkpmEntSC9++9je/JiWY1PadEJiEH1aSW3+6wH7XojBJFl3uEhzyK+5DUeL7I6ZpWUifnaKChTKhU0FinbIK6/jl4P5eLs5c0m/UIfDO53paEEl87ZmUd1g4rJB4YxLaLvnoOXHnfZLaPefrOBQXqU1nfgXW47bXfPllhPMGhjODXM3WxMCvrHsEP+6aSiXDjy16e6VxQfs5iH0QljSlo+I5qlLkvj1YAH7T1YwMNKXS/qF2vRUHOkf4cs3c0bzwboMcsvqmNgnmAcmxbX6Hq1UKwBBDlZwNRfp56593F/7uKJcSFSgaMOyvXk8+vVOGkyWJ9tIP3e+uX90lw5HbMss4ZYPt1ifpr/eeoIXL+vH3eM7vgzTycGXspP+1PE6g/2kb0l1A898v9sma6zRLPnz4v1M7x9mnUjXWu6rE7D3T9Oty2YvHxzB5adRs3poL3/+fcvwdl9/w4ho/rsx06bNfUK9mdCOIHt9SjSfbsy06f2MjA3QzHulKBcateqpFUaTmT8u3GsNEmBZffP2yrbrMJyJf/1yxGbIBeBtjWPtccOIaLtjI2L8bWpSX66xOc9kluzMKrM7frK8juLqUyuCvNzsnzW83Jza3FvRFUJ83PjgtuF4N2tTRmElS/faB7OW/D1d+OnhcTw8JZ6ZA8J4YVYyn941slt+DkXpaVSgaMXJ8jryK+rtjqdpfIF2ppYZacGSVLCsxr4mRFsu6R/G364bREKIF34ezlwzLJL/3Gr7lP7i5f2YNbB9OZz83J3xdj21TPSOsTF213TnSqGf0nKpbLZk12iGPy7Y165ltSE+bjw9vS/v3TqceyfEWTfVKcqFrluHnoQQM4C3AT3woZTytRbn7wT+BjQNtL8jpfzwbLUvxMcVPw9na2bVJq1NqHaGkbEBnCixDRZxwZ4EtzHWnl5YRXFVA0Oi/WxqJsxOiWZ2in3Poom3mzN/nz1Ys/KdENhkUS2rNXDzh5v5+r7RuDnreWBSPJ4uer5NzcZkllw7PIq7x8W07wftAjuzSu2OFVc3kFVSQ0JI1/65Kcr5qtsChRBCD7wLXAxkA9uEEAullPtbXPqNlPKRs95ALEtZn56exAs/7bV+Wfq6O/Oog1ThneXp6UmkZZVxtLFQkK+7M69dM8jhMEidwZLpdeUBSxK/IC8X3r15GKPiAjGZJUv2nGTrMUu209kp0Tap07NLa/hxRw61BhNDov3sekuT+wSz6pBt3qedJ8pYkJbDDSMsO8BvGxPDbT1kv0GfEG/25lTYHPNydSLCwWS1oiht684exUjgqJQyA0AIMQ+4EmgZKLrVLaN6MyjSj2X7TuLt5sw1QyNbTfvdUWazZMeJUswShvf2R68ThPq4sfzxiWxML6K63sTEPkF4uDj+o/pgbYY1SIBlY9rj36Sx/ndTeWzeThbvPtVT+HLLCX56aBy+Hs6kZZVx8webqWncwSywVJ/LLrXU5ZiQGMTERPtAAbA/t8LuWFerqjfy/pp01h8tItLPnQcmxdvlwnp4agKrDhVYM+oCPH5RYqu/f4qitK47//VEAlnNXmcDozSuu1YIMRE4DDwhpczSuAYhxBxgDkCvXp1bOnRglC8DO7luNVie5u/4eCvphZbSozGBHnxy10higjzR6wQT2rHfACwbzVo6WV7Hkt25NkEC4FhRNfO2neD+SfG8ueKwNUgASCxFkFY8MRE3Zz3RAR7sy9WurTEwSjulhyPHiqp5edE+NmeU0CvAgycuTuxwbYt7PtlmrUS480QZKw/ks+iR8SQ2GwqMD/bi5ycm8cOObEpqGrikXxjDe6siQopyJrpzMltrHKXllq1FQIyUchCwEvjU0c2klHOllClSypTg4PZ9wXa3Py3abw0SYEnj/YcFezt8H63yqnqdsJYtbSm90DKkdSS/0u5cfkU9gV6uRAdYlv/2j/Dl1tG2gXdMXCBXdGC5q8Fk5raPtrDqUCG1BhOH8it56MsdHVoUsDu7zK5cbZ3BzBeb7feABHu7cv+keJ6bmayChKJ0gu7sUWQDzWdYowCbijtSyubZ4z4AXj8L7Tpr1h+x3w28/mgRUsoOLcu8b0IcP+/Pt1k+e31KNJOTQtCJfXY7plMas70O7eVPbosJ7NggT/w9bJPf/eWqgVw9NJLUzFISQryYkhTSrk1zTTamF1uHs5qYJXy/PcthssGWmu+NaK7YwXFFUTpPdwaKbUCiECIWy6qmG4Gbm18ghAiXUjZ9k10BHDi7TexaEX5uNj0KgAhf9w6v3R8c7ccPD47ls02ZHMqrxNvNiaQwL/w9XXhqehJ/W37IOhk/tW+ItXLb09OT2H68lLwKyyYzd2c9L1/ZX/Pzh/cOYHjv9qf4ziyqZuWBfHzcnW32NTRn6sC2kFGxgfi4OVHRbOkrwMX9VDU5Relq3RYopJRGIcQjwHIsy2M/llLuE0K8DKRKKRcCjwohrgCMQAlwZ3e1tys8Oi2Rx79Js1l++ui0BLvrth4rYXd2Gf3CfRjrYJfxgEhfKuuM7Mq2zCmsP1rMZxuP8+ND45g1MJytx0qIDfIkJSaA4qp65m3LIqukhqemJ+GsE9QbzUxLDumUokc/7szmqe92Y2rsyoR6uxLi7UpB5ak9KULAtQ7qbdQbTby69CDzd2SjE4LZw6P43UzL/oYnv00jv6IeF72O28b07tAQmKIop0ecTzWQm6SkpMjU1NS2L+wBNqYX8X1qNiYpuXZYFBP72M6vPPFNmk2+phn9w/j3LcPshn4+25jJiwv32d3/mRlJPDT5VPApqKxj1tvrKKw6NWQza2AY73YgVUZr6o0mxrz6q91Q0SX9QqmoM7DlWAnR/h48eXEfrnJQk/qPC/by6SbbuYf7J8bx3KXJGE1mjhZWEert1mbBIUVR2k8IsV1KmaJ1Tq0Z7GatpQrflF5sl9Rv2b48Vh0qYFqy7ZBL8+yvzR3Jr7J5/d7qdJsgAbBkTx6/OVlB33Af2lJcVU+d0ewwiV5Oaa3mfMKh/EpuGdWLu8fFMi05tNWypt9vt69r8d32bJ67NBknvc6maqCiKF1PBYoebHe29qqgXdnlNoHCYDLbDOs0N7SX7WTxxqPa1eWW7DnZaqCoM5h4+vvdLNmdi1nCkGg/3rl5KN5uzuSW1RIf7IWLk44IP3fN3ezHi2v461JLuvMRMf58fs+oDmXhPR97vopyrlCBogdLclDgJ7nFcWe9jv4RPuxrsQnOx92J61uk7mi5oqlJUGNBoILKOjZnlBDp52Yzef3WyiMs2nVqUVpaVhk3vL+Zoqp66o1mAj1deOXqAcwYEM5zM/vy7A97cPTdvi2zlB935pAQ4sXKA/kEerpw7bAo6/zIdcOj7IaeWktBoihK11KBogeb1CeYqX1D+PXgqV3XY+MDNVf6vHRFf+78eCvVjRvovFz1fH3vaLun9gcnx7O5xX4EvU5w2aAIvt2WxfM/7bHWqR6fEMSHd6Tg5qznfxoZWHPKTi15La5u4LF5aWx6LpAbRvRiSLQ/P+/L41hRNT9o1MSYt/WEdeIdYO7aDOY/OJbegZ48d2kyQgjmb89GCMtS36cuSWrPb5miKF1ATWb3cCazZMX+PPbklJMc7sP0/mEO616XVjewfF8eTnod0/uH4u2m3Xt4d9VR3l55hAaTGX8PZ169ZhCj4wIY/eov1Bls16z+4bJ+3DM+lv4vLrMGoda8dcMQm0nqAycrmPn2OrvrnHQCY4sNHjeOiOa1awe1+RmKonQ+NZl9DtPrBDMGhLcr3YW/pws3jmw7fcnDUxK4fUxvTpbXERPoiYuTjjWHC+2CBMCWjGIGR/m2K0g0tffTjZkYzZJLB4aRHO7DjSOimbftVOaVhBAva8LD5g5r7BRXFKX7qUBxgfJ2c7bpccQEetilFAfLTu2iKu2J8pZiAj14bv5uqhqDyhvLDvLB7Sm8du0gLhsUwdZjxcQGezKtbygT3lhll2JkaC+VbkNReiJVuEgBoHegJ9cPt50wDvR0oaS6gY/WHcPRatYBET4MifZjzsQ4grxcrUECoN5o5s+LLcmAxycG8eQlSVw9NAofd2deuqKfTZnW+GBPHpwc3/k/mKIoZ0z1KBSrV68ZyIQ+Qaw7XESApzPfb8/hO409DU1c9Dreu3W4NYHgwJeW211zpKCKeqMJVyfbSfWrh0YxKjaQVYcKCPR0ZVpyiMO5F0VRupcKFOeZzzcf5+P1xyivNTC9fxjPXdoXHweT2i3pGlc/TewTzHfbsijUGHJqSsWRGOLF87OSrUECoH+ED5szbFdUJYR42QWJJhF+7tw0ohdmKXFSQUJReiwVKM4j87dn84efTqUp/3rrCQor6/jwjhHtev/6I0W8uHAvGYXVeLlq/9V4ZGoCN4/spfnF/rsZfbnto61U1VsS97k46XhhVrLmfUxmyevLDvLl5uPUG83MGBDGK1cNxNfBPg9FUbqPChTnkXnbTtgd++VgAQUVdZpV+XLLalmQlovJbGZcfBD3fZZKrcEyx9D0Zd+cTsDkPiEOn/6H9vJn1VOTWbQrF6PZzKxBEQ5Tfby3+ihz12ZYXy/efZIGo5m5t2uuzlMUpRupQHEeabkvASyrmLSOb8ss4faPtloDw5viCCaNPTV6ITBJiaeLnhcu60evQA+7a5oL9nbl7vGxbbZVaxPeygP5VNYZHO7/UBSle6hAcR65akgkO0/Y5ocaFRtARONTvdks+WRjJgt35ZJeUGUNEoBmkGg6Hubjyl+uHsBFyWGd1lYnjWVUOiHQdbAWh6IoXU/NIJ5HKmpts7bqBTwy5VSK8deXH+TlxftJyyqjUiavMkUAACAASURBVGNoyZG8inoe+mKnw9Kqp+PGEfYbA68YHIGng7kRRVG6jwoU54mqeiPvrcmwOWaS8HXjvEWD0cwXm+zrS7dXg8nM11tt50BSM0t4d9VRFu/OxdCRcnXAXeNieG5mX6L83QnycuHOsTG8cvXA026foihdRz2+nScKKuqo0UizcayoBrCkIq8xOE7DoZV7qaX0wioO51diNJl57oc9Nkn9Bkf58vWc0Xi4tO+vlBCC+yfFc/8ktclOUXo6FSjOE70DPQnzcbPWv24yKtaSKtzT1YnxCUGsO1Kk+f64IA/MCM0cTE02HCnku1TtDXi7ssv5dlsWd45reyJbUZRzixp6OsekF1ax/kgR1S3mGPQ6wevXDcLT5dTmtuRwHx6dlmh9/cZ1gxjWopBRk8MF1bx0RT/m3jacly7vR4i3be1sN2cdueWt53za26IehqIo5wfVozhHGExmHp+XxpI9lroQXq5O/H32YGYMOLUSaVKfYDb9fhrrDhfh5+HMmLhAm9ra4b7u/PDQOB75cgeL99jXl6ioNXLpQEuW2iuGRPL+mnRWHsjHaJYcL65ps439I1SJUkU5H6kexTli3tYT1iABlsnrp7/bZdez8HFzZtagcMYlBNkEieaa14to4uGiZ1zCqdrdnq56fj1YQHphdbuCRK8AD7tqeoqinB9UoDhHrNWYW6isN9rtm2iPi/qF8pupCbg6Wf74g71deefmofi6n9ro9vO+fI60Ml/RXLS/OyufnKSWtirKeUr9yz5HRPjap+AACPfTPt6W316SxL3j4zhZUUt8sJdd5ta88joH77QXHeDB8L+swM1Zzy2jevHo1ESHvRlFUc49qkdxjrhrXCzebrZxfdagcOKDvU77nr4ezvQN89FM7z0pKbhd9wj0dGFjejGVdUYKK+t5a+UR3l+b0fYbFUU5Z3RroBBCzBBCHBJCHBVCPKtx3lUI8U3j+S1CiJiz38qeISbIk4WPjOeOMb25uF8of76yP2/dMKTLPq9PqDfPX5qMi5PjvyLR/u6Ut9gNDvBdapbG1YqinKu6behJCKEH3gUuBrKBbUKIhVLK/c0uuwcolVImCCFuBF4Hbjj7re0ZYoM8+dOVA87a5903MY5rh0dxMK+C5fvy+HSj7c7uJy9O4unvdwG2G/XMDvJGKYpyburOOYqRwFEpZQaAEGIecCXQPFBcCbzU+P/fA+8IIYSU6pvobAnwdGFsfBBj4gJJCvVh0a5c3F303Da6N1P6hrDqUAELd+XavOfaYVHd1Nqe58DJCrJLa0np7Y+/p0t3N0dRTkt3BopIoPkYRTYwytE1UkqjEKIcCATslgAJIeYAcwB69bJPOKecGSEEN4/qxc2jbH9v/3rNQFyddCzclYurk45bRvfmoWaJCE9XeY2Bo4VVJIR42azGOlcYTGZ+89VOlu3LAywpUu4aF8PvL01GtCNDbnZpDS8u2Meaw4WEervy8NQEbhnVu6ubrSiaujNQaP1radlTaM81loNSzgXmAqSkpKgex1ni5erE32YP5m+zB3faPT9cl8Hffz5EncGMu7Oep6cntavGRU/yXWq2NUiApSbIB+uOcTCvkk/uGom+lVVhUkru+SSVQ/mVAOSW1/H8j3sJ9Xbjon6hXd52RWmpOyezs4HmO7SigFxH1wghnABfoATlvLUvt5y/LDlAncGSjbbWYOLlxfs5mHdupQfZkK6dU2vdkSJW7M/TPNdkT065NUg0N3+Hdp4tRelq3RkotgGJQohYIYQLcCOwsMU1C4E7Gv//OuBXNT9xflt9qFDz+KqD2sd7qmh/x5UAdzfLutukoKKOeVtP8L89Jx2mbNcasao3mli+L4+le05S09D+GiOK0hHdNvTUOOfwCLAc0AMfSyn3CSFeBlKllAuBj4DPhRBHsfQkbuyu9ipnR7CXq/Zxb+3jPdUdY3vz1ZbjVNTZf3knh9vmxFq2N49Hv95JQ2OA6B3oQWKIl93O+NnDbVOkpBdWceuHWzjZuDnSy9WJz+8eydDe/p35oyhK9+6jkFIulVL2kVLGSylfaTz2YmOQQEpZJ6WcLaVMkFKObFohpZy/Zg0KJzrA3eZYrwAPZjUmKzxXhPu6s/yJicQFedocHx0XYJPI0WAy8+KCvdYgAXC8uIZ+ET7M6B+Gm7OOmEAP3rhuEFP6htjc65UlB6xBAiz5v275cItd/i9FOVMqhYfSo3i6OjH/gbHMXZvBruwyzBI8XHT8e/VR7hoXS0AnLTEtrzHwn7XppGaWEBvkyQOT4ok7g13uWsJ9LTmwfj1YwO7sMpLCfLikf6jNTvic0loKKu3Ttx/Kq2TZ4xNbvf/WY/bTdTUGE59tyuTByWe+8kxRmqhAofQ4IT5uPDOjL1e8s56DeZZJ3XVHilm8+yRLHh3f7ip6jpjNkls+2szeHMsE+bbMUpbvy2fZ4xMI93Vv490do9MJLuoX6nC1UqiPG95uTlS2GKJKCGk7aAV7uVCl0XvYeqyEByefVnMVRZPK9aT0SMv35VmDRJNjRdUs2tVyYVzHbUgvsgaJJuW1Bh6fl8ZGB6uVuoq7i57fXtzH5piPm5NNwSlHHp6q3WvYeqyE8lpDp7RPUUD1KJRulltWy4frjpFeWMXw3v7cNS4GbzdnTpbXOri+/VltHSmuss9PBbDlWAk3f7CFO8fG8NIV/c/4c9rrznGxDIr2Y/nePHzcnblueBShPm1nBb5ueDRfbTnBjhap5qsbTCzenas26CmdRgUKpdsUVdVz5bsbKGwco19zuJBfDhbw44NjmZAYDBy0e8/EPkF2xzpqfGIQLk46Gozay1A/3ZTJHWNjiG0xEQ2WYau1RwopqKhnfGIQEX6dM1Q1rJc/w3p1fLXSzAHhdoECoLRaOxh2twMnK/h0YyZFVQ1clBzC9SnRKiX9OUANPSnd5rvUbGuQaLIrq4z1R4tIDvfhmRlJOOstXyJOOsGjUxMY3jvgjD83yMuVN68fgp+HdmoQKeHgSfsNfhV1Bq58dwN3/ncbz8zfzYQ3VvHttu7NlDs1OYSW37NCwMX9wrTf0I32ZJdz1bsbmLcti5UH8nn2hz08/9Pe7m6W0g6qR6F0m/wK7WGkvMbjD01OYPbwaA7mVdAn1LtdwzHtNWtQONOSQ/j78kN8uP6YzTmdgAGRvnbv+WjdMfbknNosZzJL/rRoHzMHhuHt5syCtBz+tycPbzcnbh3dm8HRfp3WXkfig724bng083dkYzJL3Jx1/GFWP5LCvNv1/nqjiR3Hy/DzcLbb39HZPliXQX2LXty3qVk8cVEiIZ34Z6t0PhUolG4zqU8wn2zMtDmm1wkmJJ4aXgr2diXYu31FlDrKzVnPExf3YdvxUnZlnRq+eWhyAtEB9jurtx8vtTtW3WDiYF4law4V8s6qo9bjP+7M4bN7RjI2/syHylrz+aZMvm1W/6POYLbZW9GabZklPPjFdooa52zGJQQy97aULitpq1U10WSWFFTWq0DRw6mhJ6XbTOkbwn0TYq0J8nQCkPDIVzs1v5S7gmXfxhj+c+twfn9pXxb/ZjxPTU/SvFZryaqTThDs5cpHLXolRrPkwS920P/FZVz33kZSM7smRdnHGzLtjn26KROzufVMN2az5Ilv0qxBAmDD0WL+vfpoK+86M1rzS6E+rvRtZ+9H6T4qUCjd6vlZ/fjhwbG46AVmCSYp2X68lDs+3kpxlf1GtM5iMkv++fMhRryykpRXVrI5o5jbx8RoDjk1uXdCrF0qkXsnxOHspKPWYLK7vrzWQHWDidTjpdz+8VaHK7nORIXGMtjqeiPGNgJFRlE12aX27Vl72LI8uKreyOebMvnz4v38vC+Pzkixdu+EOKY1210e6OnCWzcMxUmjFK/Ss6ihJ6Xbbc4opsFk+0VUVW9k6d48bhvdNUs83111lH/9eurp+ZONmdQbTbx6zSCH74ny92DZYxP4bns2+RV1TEkKYWKfYGobTLg66ezG35uraTCxMC2X+yfFd+rPMWNAGF9uOWFzbGrf0FZL2AIEebngotfZpA4BCPd1o7zWwDX/3kB6YTUAH60/xrXDovjH9WeWSt7NWc9Hd47gaEElRVUNDO3lh6uT/ozuqZwdKlAo3c7hs2oXJgr+RmO10vwdOdw6qjfvrDrKgZMVDIry48mL+xDTbJlsoJcrD7T4sv9+R3arQaKJqQt+nmdn9qWwsp4VB/KREsbEBfLXa9oul+vn4cJtY3rbDJm5OOm4f1I832w7YQ0STebvyObeCbGdMuGdEOJNQkjb1yk9hwoUSre7bFA4b644bPNl6+miZ8aArksEqDWUIqXkpg82WzO+ZhbXsDmjmF+fmoxXKxO8e7Lt9zG05KLXcfmgiNNvsAPebs7MvT2Foqp6jCZJmG/7J4VfmJVMv3Afft6fh39j4Ogf4cu8rSc0rz9SUNXlK6OUnkkNDirdLsrfg4/vHEH/CB90AgZH+fLJ3SO7NLX4tcPt63r3DfOxSwteUFnPsr2tFxrqHWi/Ma85N2cd798+nOgADyrrDG1ONJ+OIC/XDgUJsJS3vXZ4FO/flsJr1w6if4RlfmaoxsY/nYChZ2G5r9IzqR6F0iOMSwhiyaMTztrnPTotkTqDiXlbszCYzVw5OJIgL1ebfRJN2krbffe4GP71yxGHw091BjO1DUZmvLWWg3mVhPq48ti0RGanRNtkkq1tMPHWysMs25eHr7szd4+L5aqhkWf2g7ZBSsn8HTksb/zMO8bEcO3wSJbsyWXD0WLrdY9N66O5ZFi5MIjzsWBcSkqKTE1N7e5mKOcIKSVCCA7nVzLjrbU0f+B31gvWPD2l1VQdh/IqOZRfwQs/7aWiVjuoeLnqqaq3XRnlrBfcOqo3z89Kxkmv4+GvdrBk90mba969eRizBnXdENxfFu+32XDorBd8dd9oUnr7szG9mGNF1YyOCyAhRC1hPd8JIbZLKVO0zqmhJ+WCJxprjPYJ9eZv1w0mqLHKXrivG/930zCHQeJEcQ0z3lrL9LfW8ujXaSSH+XDfhFi76/qGedsFCQCDSfLfjZm8vHg/RVX1/G/PSbtrPt+ceQY/WevKaw18tvm4XZveX5OBEIJxCUHcOrq3ChKKGnpSlOauHR7FFUMiKK5qINjb1boZUMsz83fZpELfcqyEuGBPHp4Sz1dbTlBvNDNzQBjjE4J44ttdDu8zb2sWcybGoTV1UdNgH2A6S0l1g2ZixK7Y76Gc21SPQrkgldca7BISNnHW6wjzdWs1SNQ0GNmcYb/bemFaLu+uSqe0xkBNg4nM4hqm9g0lspWhqwaTGWe9jiEak8VdsVKqSUygB70D7ecdJvbpmpQpyrlLBQrlglJnMPHkt2kM//MKRryykhvnbtLMQdQWF70OHzf7Dnl1ix7A9uOl/JSWw1f3jeLSgWEIjdjjpBN4uTrxzs1DGRVryY7r5qzjrnEx3D3efiirswghePOGITary8YnBPHwFFVGVbGlJrOVC8rryw7y3up0m2MTEoP4/J5RHb7X2yuP8ObKw21ed83QSP55wxAAlu09yQNf7LA5f+/4WF64rJ/1dVlNA65Oetxdzs6uZYPJzI7jpfh5uLQ762xbsktrWLrnJHqdjssHhaukf+eA1iaz1RyFckFZqjFhvO5IEeW1BnzdtetTOPLYRYlE+LmxcFcurk46JieF8IJGfYXmX74zBoTz5b2j+HzTcarqjVw6MJybRkYDsDennKV7TuLhoueaYVG4u3Ru/W5HnPU6RsUFdtr9Vh8qYM7n263zH2+uOMxn94w8rcJMSs+gAoVyQdHaYe3qpMPlNBPTzU6JZnZKtPX1wbwKvth8amdzYogXN43qZfOecQlBjEuwzaT6/fZsnv5+lzVryX/WZPDVfaMYFHXubXJ7ZckBm0nyqnojr//vIN/cP6YbW6WcCRUolB4tv6KOeVuzKKyqY1rfUKb07ViSoHqjiV8OFFBea2Bq3xDuGBvDM9/vtrnm+pToDg3z1BlMbD9eiq+7s1222b9cNZBL+oWxKaOYmEAPrhgc2ea9jSYzr/3voE1qq6p6I2+uOMx/7xrZ7nb1BA1GM0cKquyO79eoGKicOxwGCiHEUuAhKWVmZ3+oECIA+AaIATKB66WUdgUIhBAmYE/jyxNSyis6uy1Kz5VZVM3V/95AaY0llfYXm0/w4OR4fjejb7ven1dexw1zN3G8uAawTEC/feMQ/jF7MJ9vPk6dwcRlg8Ltkvy1ZusxS7Gf4saa1KPjAvjwjhE2PZWJfYI7tHKorNZAkUZK9cP59l+4PZ2Lk46kUG8O5VfaHB8Q4Th9u9Lztdbf/gT4WQjxvBCiY4O3bXsW+EVKmQj80vhaS62UckjjfypInMO+2Hyc6W+uZfzrv/Lq/w5Qp1G/oaW3Vh62BokmH67LaHedin/9esQaJMCyDPXFhfu4YkgEPz08jmWPT+SRqYntrofQVOynKUgAbM4o4f9+OdKu9zsS6OlCL430GNEB7ry98ggL0nKoN9r+fhlNZpbtzeODtRnsy7VPO9KdXrgsGddmac69XZ14dmb7grvSMznsUUgpvxVCLAFeBFKFEJ8D5mbn/3kGn3slMLnx/z8FVgO/O4P7KT3Y11tP2Ezyvr8mg+KqBv4+23F9g3+uOMxPabl2xw0mSVZpLYFebScM3HnCPqtrYWU9WSU1xAXbV6trS0ZRNTll9pvRvtxygucuTe7w/ZoIIXj5yv488MV26gyWf2IeLno2Z5RY92oMivJl3pzReLg4UV1v5KYPNrM7+1SAeHRaIk9e3Oe029CZJiQGs+bpKSzdcxJnveDSgeHt+vNSeq62HqUMQDXgCni3+O9MhEopTwI0/upo4NlNCJEqhNgshLiqtRsKIeY0XptaWFh4hs1TOtMXLdJEACxIy6HKQbK9tKwy/uXgKd3HzYmk0Pb99UvUKF3q7epEuO/prSYK9nJFaw9eVb2R3e1INd6ayUkhrP/dVF67ZiDPzexrtyN7d3Y532/PBuCrLSdsggTAO78e0Qxi3SXM1427x8dy25gYFSTOA63NUcwA/gksBIZJKWscXevg/SuBMI1Tz3fgNr2klLlCiDjgVyHEHillutaFUsq5wFyw7KPoSFuVrqU1zGQ0S4wm7Wyr649oB3q9EPzl6oHtnnj+zdQE1hwupLxZudAnL+lz2vsTfD2ciQ/20pyszSmtPeMVSkFertw4shc/7szWPP/Bugw+Xn9MM+2GWVqW17a2A1xRTldrq56eB2ZLKfedzo2llBc5OieEyBdChEspTwohwoECB/fIbfw1QwixGhgKaAYKpee6YnCk3ca0CYnB+Hm4aF7vKAnfS1f244rB7U9pkRjqzYonJjJ/Rw5ltQ3M6B+mWWuhIx6cHMeT39qumnJx0jGycUf1mSqpbiBNY8gMIKvEcY9BCEgOU0WFlK7R2hxFVxYHWAjcAbzW+OuClhcIIfyBGillvRAiCBgHvNGFbVK6yENT4imprueb1CzqjWamJIXw+rWOa1NfOjCc91an2zy59wn14vpm+xXaK8THjQcnx2M2y8YqcOZ2T14fK6rmnV+Pcji/ksHRvvxmaiJXDYliw9ES5u+wPPW7Oet45aqBnTK8UlbTwJXvrm81IDhy19hYemnkbVKUztAtKTyEEIHAt0Av4ASWnkuJECIFeEBKea8QYizwPpYJdB3wlpTyo/bcX6Xw6JnqDCaMZtlqWdEm5TUGPt2Uyb7ccgZE+HL7mBh8PU5v8d3K/fn8ceE+cspqCfJy5bmZfW0q3NUZTCzclcuR/EqGRPszY0AYZTUNXPLmWpsVTr0CPPj5iYm4Oes5VlTN8eJqhkb7n3a7WvpgbQavLD1gd3xCQiDrmhURas7VScfj0xJ5UOVnUs5Qj0vhIaUsBqZpHE8F7m38/43AwLPcNKULuTm3f27A18OZR6clnvFn5pXX8dCXO2honA8pqqrn6e930T/Sh75hPtQZTNwwdzO7spqGe45xUXIoo+MCbIIEwImSGlbsz+fywRHEBnlSWFnPp5sy6R3owYwBYbg6nVluphMl2tOAY+KD2JhRgkkjD3m90cx7a9K5c1zsWcsNpVx41M5s5ZzwyYZjfLb5OLUNlk1yv70kqV2BZ8WBfGuQaGKWsHRPHn3DfFi0K7dZkLBYeSDfYa+nrMYSPP68eD8fNasM1y/ch28fGNOu3pKW7NIaEkPtV2npBFw+OIKoAA/+sng/BRqp0SvqjOzMKmVsfJDdOUXpDCpQKD3e55syeWnRfuvrD9Ydo7TG0Oo+jCaeDp6ym45rrWAC8HS1f58QMDU5lGNF1Xy84ZjNuf0nK5i39QT3Tohrs03NFVXV8/CXO9hyzLJfItjLlcLGDYVOOsGzM/sSHeBBdIAHlw4I4+nvd/HjTvv9JaEqO6vShVSgUHqsVQcLmLftBBs0xucXpOXwpyv649nGE/z0/mGE+hwkv+LUk7iPmxNXD40EYKhGsSCwDOm0JKVlR/T+3Aq0pvb252rnMyqqqufPi/fz68ECgr1deWBSvHVi/o8L9lmDBEBhVT1TkoK5YUQ0Q3v52wQAJ72Oh6ck8PO+fJu6FzP6hxF/GhsIFaW9VKBQeqQfdmTzZCvlQ41miVGrdmgLnq5OfDNnDP9YcZhdWWUkhXnzxEV9rPURLukfxiX9Qvl5f771PX7uzuw8YZd6DLDszu4f4YMQ2AWL/pHa+YzmfJbKjsYlr5V1Rp75fjf+Hi5c3C+UFc0+t8nmjBKHyQATQrz56eFxfLzhGLlldUzsE8xto3u3+fugKGdCBQqlR/r36ta3y0xJCmmzfoSUkoW7clmxP58ATxc+vCOFPi12det1grm3p/CvXw7zzxWW3eBltQbKag1293Nx0jE4yo8ATxfmTIjj/bUZ1nODony5cYT98t0j+ZXWINHcN9tOcHG/UPw8nO3mHfzbWEWVGOrNq9c4Xl6sKJ1NBQqlRyqo0C5PqhNwUXIor7WyD6PJX5ce4IN1p+YSvk3N4tv7x9jsoJZS8t6adN5Z1Xpg0usEL17WjwBPyybB5y5NZsYASzrx3gGeXNI/FGeN/RmOej1NK5junRDLX5cetDl3TwfnORSlq6lAofRIU/qGsKBFUsBxCYF8dMeIdq12Kqtp4NONtjmm6gxm3l+Twbu3DLMe+2BdBm8sO+TwPu/cPJR6g5mxCYF2OaKG9vJvc6d3crgPyeE+HGhRj+HqYZZ9HHMmxuPv4cL327OpqDOQHObDmE6sNqconUEFCqVHemFWPzKLa6xLV5NCvXntmkHt3otRUFlvtywWILusljqDie+3Z7Mrq4xfDmpmjwFgcJQvlw1ynDLkYF4Fy/fm4+vuxJVDIvH31E5J8sHtw3nhp72sOVxIoKcLD0yKt0lFcuWQSBak5XLgZCUHTlbyw84cHpgUr1JzKz1Gt+zM7mpqZ/b542BeBUaTtKsk15bqeiNjX/vVJiEgwEOT40k9XsrWZiuNtAyO8uX/bhrmMC3G11tP8Psf91gntF2ddNwxNoaHpyQ4nDtxlD7k29Qsu6p7ACufnESCRgZcRekKre3MPr1CwYpylvQN8+lwkKgzmLj5g812QWJkbAADonzbDBLTkkNY8Mh4h0GizmCyK11abzQzd20GV/97A5V1But1OWW1mBvnIxzlmErL0k4C2HIjoKJ0FzX0pJx3luw+ya5s+6pvD09JIKNQe4Ods95SaGLWwHBevmpAq/fPLau1C0JNMgqr+WFHDiaz5K2Vh6moMxLp585frxnIJAflUfs46DUkhZ1p2RdF6RyqR6GcVzIKq/hq6wnNc0cLqhgVqz1R7O3qxBWDI/j9pcn4uLW+PDXS351AB/MRAB+tz+DlxfupqLMUZsopq+XBL7Zb03+0NDslmr4tgsLVQyM73JNSlK6iehTKeWPN4ULu+zRVcxIbYHhvf/pF+PDIlATeXX3UZuiopMbA/B057MkpZ9ljE9FplbLDssv6vdXpeLs5UVLdgNYM3wmNNOE1DSZWHyrkqsYd4c15ujrx40PjWJCWw7GiakbEBDC1r6Oij4py9qlAoZw3Xv/fQYdB4s6xMQxpTNfx1PQkZqdE8Yef9rL2SJHNdYfzq9hyrIQx8fY9j2V7T/Lo12k2n+GkE+3aIQ7g7eb4n5u7i54bR/Zq130U5WxTgUI5bxwpqLQ7JgQs+c0E+kXYVn/rHejpMJFercG+lndqZgkPfbGDlmGovUEiNsjT4RyFovR0ao5COW8Mjbbf/Da8l79dkGgya1C43bEATxe7dN0/78vjxrmb7YJEk2AH1e3cnHTEBXly44hovr5vdLsr6ylKT6P+5irnjRcuS7bZw+Dj5sQfLuvn8PrJSSG8MCsZv8bcSkmh3nx4R4rNpr46g4mnv9/das/hjesGaqYlrzOaeeLiPrx27SDCfFUacOXcpYaelPPGoCg/1v1uCiv25SOBS/qHtrmC6d4Jcdw+JoaKOgNBXq4cyqvkme93cbK8jomJwQyI8nG4FBYgxNuFvyw5wIAIX5t04U0c1btQlHOJChTKecXHzdmmHnZr0gur+Mvi/aRmlhIb7MkNKdG8svQANY21HtYdKWJinyB0wlIVrzkPZz01BhMFlQ0UVDaQXlit+RlDe2nXu1CUc4kKFMoFqd5o4pYPtpDXmKV2d3Y5e3PK7QLC2sNFDO/lx/ZmqcKddAI/T2dqykw217YMKFcMjmCymsBWzgMqUCgXpDWHCq1BoomjaYjdLXZ5G82S3DL7NOgS+Pq+UaQXVpMc7sPw3q1nllWUc4UKFMoFydTOZa0AhnZeOzExmDHxQYxpsWpKUc51atWTckGanBRiLULUGTxc9Pxttqo6p5yfVKBQLkjuLno+vWukdbI5OsCd6f1C7a7TTuRh77JB4YR4qyWwyvmpW4aehBCzgZeAZGCklFKzeIQQYgbwNqAHPpRSvnbWGqmc9wZG+fLjQ+OsvIK5rgAAIABJREFUdSLqDCYen5fG8v15SAlxQZ64OOk4mGe/47u5AE8XHr+oz1lqtaKcfd01R7EXuAZ439EFQgg98C5wMZANbBNCLJRS7j87TVQuFE07pt2c9fzntuHklNVSWWcgKdSbbZml3PnfrdYlsy15uOj5+YmJBDnYnX02VdcbyS6tpXegR7srASpKe3RLoJBSHgAQotWO/UjgqJQyo/HaecCVgAoUSpeK9HMHLPWxR8YG8OtvJ7N4dy4msyTS352P1x/jcH4VQ6L9eO7Svj0iSHyy4Rh///kwVfVGfN2d+cNl/biunftJFKUtPXnVUySQ1ex1NjDK0cVCiDnAHIBevVQWTqXzhPm6ce+EOOvr1upod4c92eW8tOjU81N5rYFnvt/FyJgAh1X6FKUjuixQCCFWAmEap56XUi5ozy00jjlcpyilnAvMBUvN7HY1UlHOAysO5NsdM0tYeSCfu8fHdkOLLGoajMzfkcOR/EqGRPtx+eAInJslRtxxopRXlhxgd3YZfcN8eG5mX8YmqKXFPVGXBQop5UVneItsILrZ6ygg9wzvqSjnnQAP7XxWnbn8t6NqG0xc994m9p+sAOCzTcdZkJbLJ3eNQAhBcVU9t3+0lap6S0r3PTnl3PXJNn757SSi/FUvqKfpyctjtwGJQohYIYQLcCOwsJvbpCg9zlVDIwn2tp0niQ5wZ8YArQ792bEgLccaJJqsOVzIxvRiAP63N88aJJrUG80s2nXyrLVRab9uCRRCiKuFENnAGGCJEGJ54/EIIcRSACmlEXgEWA4cAL6VUu7rjvYqSk/m5+HC/AfGckNKNIOjfLltdG++u39st658yijSTpKYXth6Nl3ZOLqcWVTNqkMFlFRr1xnvbLuzy9hxohRzB3bsX0i6a9XTj8CPGsdzgUubvV4KLD2LTVOUHsVslrz9yxG+3HKCeoOJywZH8MKsZDxdbf/p9gr04PXres7O8GG9tPNcpfQOAGDmgDBeXXqA6mbLjp10gssHhfPs/N18k5qFlODqpOOPl/fn5lFds0Alv6KOu/67zdr7iQvy5OM7RxAT5Nkln3eu6slDT4pywXt/bQZv/3KEoqp6KuuNfL31BM//uKe7m9WmS/qFclmLCoJzJsZZqw0GermSEOptc95olny8IZN52yxBAizDUS8u2Et+hX0Sxs7w58X7bYbIMoqq+f058Pt7tvXk5bGK0qMYTGY+23Sc1YcKCPZ25e5xsQyI9O3Sz/wuNcvu2JI9J3n1mkG4u5ze0FK90cSyvXnkldcxsU8wyeHapWLPhE4neOfmYdwzvpQj+VUM6eVHn2aBIbesll1ZZXbvW7Lbfo7CaJZsSi/mqqGRgGWivMFktqlmeLrWHC60O7YxvRiDyWyzQutCpwKForTTk9/uYtGuUwvvluw+yfwHx3ZpsNAaMZdnMIxeWt3A9e9vslbee/V/B3luZl/unxR/+jdtxdBe/gzVGIZylL3XWa+9CTfS350Go5mXFu3j++3ZGExmxicE8Y/ZgwnxOf0cW+G+blTW2c6bBHu7qiDRgvrdUJR2yCqpYfFu29XZ9UYz/92Q2aWfq7W7eubA8NPuTfx3wzG78qz/WHGY4qr607rf6YoO8CBFo17HHWNjCfWxXcE1Nj6QETEBvLvqKF9tOUGD0YyUlgqEv/1u1xm14+EpCXbHHtE4dqFTPQpFaYeCynrNJ/mCyq4ZO2/ywKR4ahtMfLHlOHUGE5cPiuCPV/Q/7fvtzim3O9ZgNHMov5KxZzkVybu3DOP5H/fw68ECfNyduXtcLAGeLsQEeeLhrCfAy5VZA8OtE9mLdttvo1p3pIiymgb8PE5vz8iVQyIJ8nLlm21ZGM1mrh4axcUaWYQvdCpQKEo7DIz0JcjLlaIWT95TkkK69HP1OsFT05N4anrSGd+rsLKeuCBPVh+yHZd31v9/e3ceF3Wd/wH89Z6B4RJEDgER5BYFUhHxQM3UTG07tHK71Q43Szu37dxfu+2vtv2123ZZ6ZZlaYe1qR1aHpVXXmCoKAqoCAhyyC3XHJ/fHzOMM3y/MwwyB+D7+Xj0kPnOd77z4fugec/ner8J8QN9Lbyqe7Q6gfyKBgT6eEj2eoT4eeL9+WOg1urgpiC8ua0AfzTpIZw+34R7MqKNy3y9ZXpR7krq9jBRRlwQMnhHuFUcKBizgcpNgTdvG4mHP8tGVWMriIDrrhiEu8YPcXXTOlXZ0IrHvsjGroIqEPQfuKbZcJdOjZd8iNtDZmE1Hvk8G2drm6FUEOalReB/b0yGUmE+D+GuVECt1eH9Xack11i+4ySuNayeSgrzQ85Z8018c0aFS5YKM/vjO8yYjSbEBuHXp6cip7QOwf08EBFwaakmGlrU+HRfEQ6frcPwMD/cOW6IXVbwWPLM10ewq6AKgH5yvKlNiwmxgUiPDsCVCcGyk83dpdbq8OCag6ho0PfAtDqBz/YXIWmQ/vftqEWtRUOLRnK8ol7/+sMltVibVWL2nEpJ+NPMRLu3nUlxoGCsC1RuCoubyWzRptHh1hV7cbRU/834+8Nl2JB9Ft8smeiQndStGi1+Oi5NGphf0YhP7x9n1/cqrm7CyxtzsffUeQT28zAGCVNbc8tlA4WvpzvGRA3AgcIas+NTh+mH9r49VCqZI2rTCuzKrzIum2WOw6ueGHOiLcfKjUGiXV55IzYecUyOIzeFAt4q6fdBX0/7fkfUaHW484N92JRzDjVNahRUyKfqsJao8P9uHoH4gf2Mj8fHBOKpa/Q9BktBdOXu09Bodd1oObMF9ygYs0FJTRP8vVXo183x8OKaJtnjRdXyx7tLqSDcMS4Sy7ebj/8vmBBl1/f59eR5nDlv/XdQKRVYOMFy2vPoIB9sfmwyjpXVo+aCGinh/dHfkBl3bupgvLf9JNRa827F4ZI6rN57BnkVjcZ05g9cGYvAHlBMqi/hQMGYFUdK6vDY2mwUVDTC012BBROi8dTMoZ1VZ7QoI1Z+dY0jV908dU0ignw8sD77LNyUCtyeHoHfj7Fv7iS1hW/1KeH9oXJTIMTPA/dNikHKYOubE3PLGvDo59nIr2iESqnArekReGZWIl7ZlCsJEu3+8cMJNKv1k/MHCmvw0/EKbHpkMlRuPGBiLxwoWJ9QWHUBTW1aDAvzveQP8Y40Wh3+8EkmSuv0eyVa1Dq8t/0k4gf2w02XWGY0ZXB/PDw1Dst+OQmtTkBBwP2TYjAmKsAubZajUBDunxyD+yfHdH7yJcqIC5JdPvz8tcMwNibQpmvodAKL12QZeyZthpQp5fUt+PGodJ6lXXuQaHey8gK25ZZjVkqYhVewruJAwXq1+hY1HlpzEDvz9at64gb2w3t3jkacyVj3pTpUUmsMEqY25ZRdcqAAgMdnDMXv0yNxrLQeiaG+l7x6qifxdFfio4Vj8Pz6HGQX1yLUzxOPXR1vc5AAgGNl9bLDVx33fZgK8FHJpiKvclJ68ssFBwrWq722Oc8YJACgoKIRf/zyENY/lNHta/t6yi9ZtXS8K8L9vRDu79Xt6/QkyeH9sf6hDLSotfBwU5j17BpbNXjx26PYkF0KlZsCt6dH4slrhsLNZLOcpSXCrRr5YS0vN4VskFAqCFMSgrv52zBTPIjHerWfjldIjmUX19ql4E1CiC8mxJp/I3ZTkOzyTnaRp7tSMvz3/LojWJtZglaNDg0tGizfcQpv/VRgdk5EgLckfYbCyihis0wA8fV0wytzU/pEL60n4UDBerWgftLllt4qpWy6h0ux/K7RuG9iNBJC+mFSfBBW3ZOO0TLJ7C4HtU1tOF11AaKL6Wtb1Fp8L7P89+vfSiTH3rx1FJZOjcPwMD9MTgjGRwvTEWpjdthnZydi37PTcEtaRJfaxzrHQ0+sV1s0ORaL12SZbcZaMCHKbpvXfD3d8fzvhtvlWr2VTifwwjdH8fmBIqi1AjFBPnjj1lGdrmAC9BXkSiwsCSZIuwteKiWemDEUT8y4mNvqX/NGYPHqLNQbdm6H+HmgvF66mS8x1E92zwjrPr6rrFebmRyKVQvTsXrvGTS1aXHdiDDM64PfKOtb1Fh7oBiF5y8gPToQ16aESXImOcqa/UX4ZO8Z4+NTVReweE0Wdjx5FRQW2qDVCTy37gjWZhZDJwBPNwXUHaprzEuzbUFARlwQVtydhte35qG5TYsJcYH4YFch2kyGnmKCfTixnwNxoGC93uSEYEzuw5OXdU1qzH5zJ87WNgMAVu8twg85ZXjnjtFOef/NR89JjpXUNCOntA5XDPaXfc3azGJ8fuBidb4WjQ4qNwW83JXwcFPgtvRIPDjFtroPB4tqcPcH+9Fm2KtxqKQOVyYEQ6PTobCqCWNjAvDkNUOdFjgvRxwoGOvh3tiWbwwS7TYeOYcjJXU2Df90l5+F1UjWEhnKLTJo0+jw0cIxmGBh06ElH+w8bQwS7XbmV2LPM9MQ0o3qdsx2PJnNWA+3TSapH6Bf3eUMCydESb6tTx8WgiGBPhZfE2QhhcbAS0hnXimTXFAnINncxxyHAwVjPZxCZtIXAAb5O+fbdFpUAFbfOxZTEwciOdwPD0+Nw1u3jbL6mvkThsCzQwoNfy939PfqeiW6KYnSYcVB/T2RGOrX5WuxS8OBgrEe7roRgyTHCMADq7Pwh08ycU5m97i9jY8NxMoFY/Dd0kl4fMZQY83uwqoLePTz3zD1X79gyacHjVljE0P9JJPLtc1qPL/+SJff+96J0bg2JQztWzMGeLtjydQ4npNwIpcECiK6hYiOEpGOiNKsnFdIREeIKJuIMp3ZRsZ6igemxGBihw9dAUCtFfjxaDnu/1j6v4Zaq8MPOWX4YNdpnDjX0OX3rG9R4++bcnHdW7vw4JosHCmR1tpuaFFj3vI9WJ9dilOVF/Dd4TLMW74HNYbNjvsLqyWv2ZpbAZ2ua/swPNyUWHZHKv52fRJUSkJNkxrPrsvBvPf24EKrtNiRJRqtDufqWqDt4vsz101m5wCYC2C5DedeJYSo6vw0xpyv5kIbvsoqQVldCyYnBGGKA2poe6vcsPq+scg5W4c/r8/Bbx3mJo6crcPxc/XGoZiGFrVZcSQAeGpmIhZPibX5PRes3I+DRbXG6/98vBLfLp1olkNr05FzkuJE1Rfa8M2hUsyfEAV/b3dJ1br+Xu4Wl9Ra06LW4tXNeWgzySC7v7AaK3edxtJp8Wbn/nqyCu/+chLn6lowKT4Yj0yPxy8nKvDS97moaGhFWH9PvHDdcMxM5qSBtnJJj0IIkSuEOOGK92bMXsrrWzD7zZ14aWMuVu4+jQUfHsBL3x9z2Pslh/e3WGfBdMPhx3vOSIojvbblBCoabBuiyjpTYwwS7ZrVWqzZd8bsWH2LWvb17T2KezOktSfuyYiyqQ0dHSurR12z9P32nDpv9vhQcS3u/mA/duZXIb+iESt3n8ad/9mHx9ceMga1sroWLP3sN4sbAZlUT5+jEAA2E1EWES2ydiIRLSKiTCLKrKy0nG2SMXtZues0yjrMD6zcXYiyumYLr+i+m0dLy34mDfLDsLCLE7uHS6SrodRagWMdgocldc3yebJqm8w/qK9JCoWbTO9ga245dDqBBRnReG3eCKRHByBtyAC8PCcFS6bGS863xWB/L9k5iSGB5jmdVu89A02HoaUjpXWS4Sa1VmCzldTll+pkZSOWbz+JtQeK0diFYbGezmFDT0S0FUCozFPPCSE22HiZDCFEKRENBLCFiI4LIXbInSiEWAFgBQCkpaXxIGQvVdnQisMltYgO8kFMcPdThTtSvky5T61O4HTlBYT1d0xm2JnJYfjbjcl475eTqGhowZShA/HiDUlm5wwN9ZPUb1AqCAkhvja9x7iYQPh5uhlTZrSb0SFhX0SAN25KHYwvMovNjueU1mN7fiWuGjoQc1MHY26q+Q7svPIG/PXbo9h/uhpRgT54YsZQzEyW+6i4aKCfJ+4aNwQf/VpoPObn6Yb7JpnX2Og41GVNd6sVdrQ2sxhP//cw2mPSv7fm4csHxmPwgN6foNBhgUIIMd0O1yg1/FtBROsApAOQDRSs91v1ayFe+j7XuLnq1jER+PvcFLsVIrK31Eh/ycYyT3cFkgY5dhPcXeOG4C4rGWwXTIjCt4dKcbrqgvHYfZOiMciQ1vzEuQb8Z+cplNU1Y1J8sCQ3lrfKDe/eORp//PIQyupa4OGmwL0To2ULAfl7y2+6K7JQFrVFrcWd7+8zDgPlVzTioU8PYsNDGUgOt37fXrhuONKjA/DT8QoM9PXA7WMjJR/Cs1JC8UOHneTB/VRQawVqTYaugvp5YFaK9eDUFS1qLV76PhemHZeyuha8/VMBXrnpCru9j6v02J3ZROQDQCGEaDD8PAPAiy5uFnOQkpom/PXbo2b/o31+oBhThgb32EnH+ROisCW3AocMk8sKAp6/drixzrOrBPio8N3SidiQXYrS2mZMjA/COEMBoYKKBsx5Zzea2vRV4XYXnEdmYTXenz/G7BoZcUHY9dRUnK5qRLCvp8Vd2ONiA7F8xynJcdP07EdK6lDd1Ib0qADszK+UTIBrdQJfHzzbaaAgIsxOCcNsK5XrbhgZjrzyBqzcVYhmtRaxwT54bd5IeKmU+PeWPBwrq0dyeH88cXWCXeqKtCuubpKdQ8kpla4W641cEiiIaA6AtwAEA/ieiLKFENcQ0SAA7wshZgMIAbDO8G3SDcCnQogfXNFe5nh7T1VDbtXi7oLzPTZQ+Hq6Y93iCdiRX4myuhZMjAtySB0EjVaHzDM1cFcqkBrpb1MPy8fDDbePldbFXvXrGWOQaLc1twL55Q2I7zA0pVQQ4gZaH66akhCM28dG4tN9RQD0wfLR6QmID/FFY6sG9350APtO65fJ+nu7Y8H4KNnrCNhvtPjJaxLx4JQ41DarzYpDvXun43JjRQR4yw7XJYU5PsWKWqvDwTM16O/t7rBNiC4JFEKIdQDWyRwvBTDb8PMpACOc3DTmIpaqvQ0e0LOrwCkU5JAlse3yyxuw4MMDxlxPiaG+WHVP+iXnOLK08qmioVUSKGxBRHh5TgruyYhGfnkDUgb3Nw4HvftLgTFIAPrJ8De25UuuoSBg7qhLLy0rx8fDDT52noOwxtNdiWdmD8Oz644YV6CF+HlgyVTbEh9equziWiz6ONPYSxsfE4gVd4+2a28J6PmrnthlYlxMAMZ3qK8c7u/VJ1OGy6ltasP+09WS/EVPf33ELCHg8XMNeHlj7iW/z1UyQc3P0w2pkd0rxhQ3sB9mpYSZzRnsKjgvOa9jvyFigBfeuSPVKckNHe229Ej8+OhkPHnNULw0JxlbHr/SoZX2hBB47Itss6G8PafO4+0OlQPtocfOUbDLCxHhw4Vj8Nn+ImQW1iA22Ad3jY/CAJ+u5wbqbT7cfRqvbDqOVo0O7krCkqvi8cj0eLSotcg6UyM537RGeFfdkhaBg0U1+DKzBAKAu5KwMCPamJLDnlTKzofIrhsxqMcOLV6KhBBfm1eXdVdxdbPZgoV22/Mq8czsYXZ9Lw4UrMfwdFdiYUY0Fsps1OqrCioa8eJ3x4zDFWqtwL+35mFifCBGRQxAUD8VqhrN9zV4qZTYkH0WVw8P6XJFN6WC4OPhZvxmr9YKvLEtH0NDfa1OEtuiqU2D97afwq78SgT185DsIJdTWtsMnU5Y3K296UgZVu0pRGOrBrNTwrBoUgzclDwQAgADfNzh4aZAa4fa4WH97Z8skgMFYy60I68SciWofzlRidFDArDkqjj85Vvz3d5na5rxyOfZCOrngc/uH9uluYWmNo1x4tnU+ztPmQWKVo0Wq/cWYWd+JcL9vbAwI9osfYecRR9nYVdB13o767NLodYJLLs9VfLcd4dLseTT34yPc87W42xNM16ak9Kl9+irfD3dsSAjCsu3X1x15q4kPHCl7alabMWhmTEXsjQpPdBwfEFGNFYuSMN1IwbBz9P8e11VYyte2XS8S+/X3KaVfAMFpLuul376G/723TH8cqISa/YVYc6y3bLDHO2OldbbFCRCZX7f7w+XGZcYm/pwd6Hk2JeZJX1qx3N3PT0zEa/NG4EZw0Nw8+jB+OqBCRjbYa7PHjhQMOZCVw8PQUKI+Tf1cH8v3DjyYmrxqYkh+MdNKZKllwBsGt4xFdjPA6mR0vKlVydd3HWdV96AzcfMd3Y3tGqwymRX9Pa8Sjy05iAWr87ClmPlqL4gn/bDlI9KibExAbLPye1yl9uX0KbVoamNA0U7IsLc1MFYcXca/nnLCIyIkC9N21089MSYC6ncFPhi0Xis3H0a2cW1GBbmh/smRkuWN3q5KzF4gBdKaszzSMV3MhzUUc2FNjR0+EY+esgAPDotwfjYUn2Lc3UtEELgy6wS/Omrw8bjm3LO4X+uGw5/b3dJz6RdgI8KqxamI7+iARuyS82eIwJGyQSva5JCjPUtTNs60JfLnzob9ygYc6ELrRq8sS0fXx88i/ONbRga4mscdjJFRHh6VqJZYjxPdwWemDG0S++37OcC5Jebf/gWVzfB3WSF0ughA+ArswdBJ3QY9bctZkGi3Yrtp/DOHamyQ0sAUNPUhq+yivHS97mSyncPTolFrExer6VT43HjyEHG33lEhD9e//3Izn9JZnck5GbSerm0tDSRmcl1jljPt+jjTMkwz7LbU3HtFfIrkI6fq8e3h0qhUioxNzW8y+v0r397Fw7LFCHa8thks0nxrcfK8fjabNS3aPTf+CP8JanHTSkIKHhpNnRCYPpr21HYIdeTgiDZeT8rORSPXZ3Q6XLS6gttaFZrLW7KZPZBRFlCCNlCcjz0xJiLVNS3YEuuNNX1mn1nLAaKxFC/bqVpGBLoIwkUnu4KhHRYUjl9eAj2PTsdh0tqMcjfC3/88pDV606KD4ZCQVCA8PC0eDy+1vx8ufQsJ8obbNpzEHAZ7KXp6XjoiTEXadXoZJfGNqu10oN28uCUWEl67T9MjoWfTMoHL5USY2MCUVLTjDMWssEC+l3Z/3tjsvHx3NTBWHHXaExNHGg1lbdG2/dGM/oq7lEw5iIRAd4YGeGP7A4rl64fMcjCK7pvWJgfNj48CZ8dKEJtUxtmDA/FVYmWc1V9sqcQf95wVPa5kYP98ZcbkuCjUiLnbB2IYEzhMSMpFDOSQpH8wo8Wr33jKGkRJtYzcaBgzIWW3ZGKP311CLsLzsNHpcSd44dgvoUMq/YSGeiNp2YmdnqeRqvD61ulSfwAIKifCh8uTMOrm/MkmWMfNqlhPSrSX5JyREHAwoxoLO2QMK+pTYOCikZEBnjD35uHm3oSDhSMuVC4vxfW3DcODS1qqNwU8HCzf86lS1HZ0IpXfzyO8xb2R1Q1tmHT0XNmu7x1AnhtSx5mJoca5x6ev3Y47nh/rzENia+HEisXpmNMlPl+iv9mleAv3xxFQ6sGKjcFHpoSh5vTBmPfqfOICPCWnM+ci1c9McbMaHUCM1/fIbsJztQV4X44fFZah/uv1ydh/oQo4+OmNg225lZAq9Nh2rAQyXxIaW0zJv/fz5Ja16YrpSbGBeH9+WlmlfiYffGqJ8aYzXbkV3YaJJIG+aGgUj6lR2Sg+ZJdb5Wb1XmX7XmVkiABmK+U2lVQhbWZxbjbwcNyTB6vemKMmamol9+Z3Z609cqEYLx9W6qkUh6g7wVcGR9sduxkZSP+vD4Hiz7OxJp9Z6DRmueaCurnYVO79psUQWLOxT0KxpgZhYVSqyoFYddz0xDoo/9gnxQfJJmo1gng3lUHcM/EaGTEBuFUVSNuXParMZHf5mPlyCqswWsmO6yvGhqMoSG+OFHeYLVdMTK7t5lzcI+CMWYm1EI9g2aNwPrfLuZpenlOCkL8pL2Bn09U4q4P9mP2mzux7OeTkmyv67LPorj64r4MN6UCny0ahz9cGYP0qADcPjYSs5JDzV4zqL8n7ho3pDu/FusG7lEwxsxMiA1CiJ8HyutbJc+drLw4dxER4I2JccH478ES2escP9eABpmMt0IAZXUtZulHAnxUeGbWxapsOp3Appxz2H2yChEDvPH7MRG8Q9uFuEfBGDOjVBDekSkkBACpkf7YmV+JPSfPQ6cTSA63nk5ELhOtv7c7ruikRrZCQbj2ijC8PCcFi6fEcpBwMe5RMMYkRkcF4LHpCXh9W54xzcjYmAD888cTOGfoacQE+2D5naPhriSoLaTj0HZYfu/r6YbX5o3gZa69DO+jYIxZdLrqAg4UViMmyAd/33QcWWdqzJ6fnRKKsroW/GYls2y7B6fEYsnUuC7X+WbOwfsoGGOXJDrIB9FBPmjVaCVBAgB2F5xHk42lSdfsK8Kj0xM6P5H1OC6ZoyCiV4noOBEdJqJ1RCRbv4+IZhLRCSIqIKKnnd1OxpieSqmQXeEU4KOCWi6HuIy6ZjX2njpv76YxJ3DVZPYWAMlCiCsA5AF4puMJRKQEsAzALADDAdxGRMOd2krGGAB9hb1Hppn3BpQKwpKrYqFSSj9GgvrJTz57qXhuojdyydCTEGKzycO9AG6WOS0dQIEQ4hQAENHnAG4AcMzxLWSMdXT72EgMCfTGut/Owl1JmJcWgVGRA1DR0IZ//HDceF5MkA/+en0SFny4H6Zz3ImhvkgbMsAFLWfd1RPmKO4B8IXM8XAAxSaPSwCMdUqLGGOyMuKCkBEXZHZs8ZRYTE4Iwva8SoT6eWJ2Shg83ZX4cGE63vopHyU1zZgQG4SnZg4FWdj1zXo2hwUKItoKIFTmqeeEEBsM5zwHQANgjdwlZI5ZHAwlokUAFgFAZGRkl9vLGLt0SYP6I2mQ+d6IyQnBmJwQbOEVrDdxWKAQQky39jwRzQfwOwDThPwa3RIAESaPBwMolTmv/f1WAFgB6JfHdrnBjDHGZLlq1dNMAE8BuF4IYakY7wEA8UQUTUQqALcC+MZZbWSMMabnqlVPbwPwBbCFiLKJ6D0AIKJBRLQRAIQQGgBLAPxfEywWAAAErUlEQVQIIBfAWiGEfPFexhhjDuOqVU9xFo6XApht8ngjgI3OahdjjDEpTgrIGGPMKg4UjDHGrOJAwRhjzKo+mT2WiCoBnLHjJYMAVHV61uWB74Ue34eL+F5c1JvvxRAhhOzGlz4ZKOyNiDItpd+93PC90OP7cBHfi4v66r3goSfGGGNWcaBgjDFmFQcK26xwdQN6EL4XenwfLuJ7cVGfvBc8R8EYY8wq7lEwxhizigMFY4wxqzhQyCCiACLaQkT5hn9ly3IRkdaQ1DCbiPpMZtvOapUTkQcRfWF4fh8RRTm/lc5hw71YQESVJn8H97minY5GRCuJqIKIciw8T0T0puE+HSaiVGe30VlsuBdTiKjO5G/if5zdRnvjQCHvaQDbhBDxALYZHstpFkKMNPx3vfOa5zg21iq/F0CNIbnjvwH8w7mtdI4u1G3/wuTv4H2nNtJ5PgIw08rzswDEG/5bBOBdJ7TJVT6C9XsBADtN/iZedEKbHIoDhbwbAKwy/LwKwI0ubIuzGWuVCyHaALTXKjdlen++AjCN+maNS1vuxWVBCLEDQLWVU24A8LHQ2wvAn4jCnNM657LhXvQ5HCjkhQghygDA8O9AC+d5ElEmEe0lor4STORqlYdbOsdQN6QOQKBTWudcttwLALjJMNzyFRFFyDx/ObD1Xl0uxhPRISLaRERJrm5Md7mkHkVPYK2mdxcuEymEKCWiGAA/EdERIcRJ+7TQZWypVd6leua9mC2/57cAPhNCtBLRA9D3tKY6vGU9z+XyN2GLg9DnTWokotkA1kM/JNdrXbaBwlpNbyIqJ6IwIUSZoftcYeEapYZ/TxHRLwBGAejtgcKWWuXt55QQkRuA/uibXfFO74UQ4rzJw/+gj87X2KBLNe77MiFEvcnPG4noHSIKEkL01mSBPPRkwTcA5ht+ng9gQ8cTiGgAEXkYfg4CkAHgmNNa6Di21Co3vT83A/hJ9M2dm53eiw7j8NdDX7b3cvQNgLsNq5/GAahrH7693BBRaPucHRGlQ/85e976q3q2y7ZH0YlXAKwlonsBFAG4BQCIKA3AA0KI+wAMA7CciHTQ/yG8IoTo9YFCCKEhovZa5UoAK4UQR4noRQCZQohvAHwA4BMiKoC+J3Gr61rsODbei4eJ6HoAGujvxQKXNdiBiOgzAFMABBFRCYAXALgDgBDiPehLFs8GUACgCcBC17TU8Wy4FzcDWExEGgDNAG7t7V+kOIUHY4wxq3joiTHGmFUcKBhjjFnFgYIxxphVHCgYY4xZxYGCMcaYVRwoGHMwIoogotNEFGB4PMDweIir28aYLThQMOZgQohi6LOpvmI49AqAFUKIM65rFWO2430UjDkBEbkDyAKwEsD9AEYZMtIy1uPxzmzGnEAIoSaiJwH8AGAGBwnWm/DQE2POMwtAGYBkVzeEsa7gQMGYExDRSABXAxgH4LG+WtSH9U0cKBhzMEMm0XcBPCqEKALwKoB/urZVjNmOAwVjjnc/gCIhxBbD43cAJBLRlS5sE2M241VPjDHGrOIeBWOMMas4UDDGGLOKAwVjjDGrOFAwxhizigMFY4wxqzhQMMYYs4oDBWOMMav+H9pOe+RIsn1qAAAAAElFTkSuQmCC\n", 54 | "text/plain": [ 55 | "
" 56 | ] 57 | }, 58 | "metadata": { 59 | "needs_background": "light" 60 | }, 61 | "output_type": "display_data" 62 | } 63 | ], 64 | "source": [ 65 | "sns.scatterplot(x = X[:, 0], y = X[:, 1], palette = sns.color_palette('deep', K), edgecolor = \"none\")\n", 66 | "plt.title('Dataset')\n", 67 | "plt.ylabel('Y')\n", 68 | "plt.xlabel('X')\n", 69 | "plt.show()" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 68, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "def update_label(X, centroid):\n", 79 | " m, K = X.shape[0], centroid.shape[0] \n", 80 | " dist = np.zeros((m, K)) \n", 81 | " label = np.zeros((m, 1)) \n", 82 | " \n", 83 | " for i in range(m):\n", 84 | " for j in range(K):\n", 85 | " dist[i,j] = np.dot((X[i, :] - centroid[j, :]).T, (X[i, :] - centroid[j, :]))\n", 86 | " \n", 87 | " label = np.argmin(dist, axis = 1) \n", 88 | " total_dist = np.sum(np.choose(label, dist.T))\n", 89 | " return label, total_dist\n", 90 | " \n", 91 | "\n", 92 | "def update_centroid(X, label, K):\n", 93 | " D = X.shape[1]\n", 94 | " centroid = np.zeros((K, D))\n", 95 | " for i in range(K):\n", 96 | " centroid[i, :] = np.mean(X[label.flatten() == i, :], axis=0).reshape(1,-1)\n", 97 | " return centroid\n", 98 | "\n", 99 | "\n", 100 | "def init_centroid(X, K):\n", 101 | " m = X.shape[0]\n", 102 | " idx = np.random.choice(m, K, replace = False)\n", 103 | " return X[idx, :]" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 69, 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "def k_means(X, K, num_iters = 100):\n", 113 | " m = X.shape[0]\n", 114 | " centroid = init_centroid(X, K)\n", 115 | " label = np.zeros((m, 1))\n", 116 | " \n", 117 | " for _ in range(num_iters):\n", 118 | " label, total_dist = update_label(X, centroid)\n", 119 | " centroid = update_centroid(X, label, K)\n", 120 | "\n", 121 | " return centroid, label, total_dist" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 51, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [ 130 | "centroid, label, _ = k_means(X, K, num_iters = 100)" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 48, 136 | "metadata": {}, 137 | "outputs": [ 138 | { 139 | "data": { 140 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3hUVd7A8e+ZnjLpCamQ0HvvTZqKKCqiLjbUdWV11S3urq7vvq/i7uq6xbaru6vs2tvaQRQQpPcivQcIpJLeM5Mp5/0jyZDJ3EkhCQlwPs/DE+bce889E2V+c0/5HSGlRFEURVH80XV0AxRFUZTOTQUKRVEUpVEqUCiKoiiNUoFCURRFaZQKFIqiKEqjVKBQFEVRGqUChaIoitIoFSgUpQEhRJoQokoIUSaEKBZCbBZCPCCEaPLfixAiWQghhRCGdm7jBbmPooAKFIriz2wppRXoBjwHPA78p2ObpCgdQwUKRWmElLJESrkE+AFwtxBioBDiWiHEbiFEqRAiXQixsN4l62t/FgshyoUQ44QQPYQQq4UQBUKIfCHE+0KIsLoLhBCPCyEya59gjgohpteW64QQvxFCnKi99mMhRIS/+7Tzr0K5jKlAoSjNIKXcDmQAk4AKYD4QBlwLPCiEuLH21Mm1P8OklMFSyi2AAP4IxAP9gCRgIYAQog/wMDCq9gnmaiCtto6fAjcCV9ReWwS82sh9FKVdqEChKM2XBURIKddKKfdLKd1Syn3Ah9R8mGuSUqZKKVdKKe1SyjzghXrnuwAz0F8IYZRSpkkpT9Qe+zHwWyllhpTSTk1wuVmNSygXmgoUitJ8CUChEGKMEGKNECJPCFECPABE+btICBEjhPiotnupFHiv7nwpZSrwc2qCQG7tefG1l3YDvqgdUC8GDlMTWLq01xtUFC0qUChKMwghRlETKDYCHwBLgCQpZSjwL2q6lwC00jH/sbZ8sJQyBLiz3vlIKT+QUk6kJjBI4E+1h9KBa6SUYfX+WKSUmX7uoyjtQgUKRWmEECJECHEd8BHwnpRyP2AFCqWUNiHEaOD2epfkAW6ge70yK1BOzcBzAvDrevX3EUJME0KYARtQRc1TA9QEoGeEEN1qz40WQtzQyH0UpV2oQKEo2r4SQpRR863+t9SMK9xbe+wnwO9qjz8JfFx3kZSyEngG2FTbZTQWeBoYDpQAXwOf17uPmZrpt/lADhAD/E/tsZepeXL5tvZeW4ExjdxHUdqFUBsXKYqiKI1RTxSKoihKo1SgUBRFURqlAoWiKIrSKBUoFEVRlEZdkis8o6KiZHJyckc3Q1EU5aKxa9eufClltNaxSzJQJCcns3Pnzo5uhqIoykVDCHHa3zHV9aQoiqI0SgUKRVEUpVEqUCiKoiiNuiTHKBRFUTqCw+EgIyMDm83W0U3xy2KxkJiYiNFobPY1KlAoiqK0kYyMDKxWK8nJyQghmr7gApNSUlBQQEZGBikpKc2+TnU9KYqitBGbzUZkZGSnDBIAQggiIyNb/MTToU8UQog3gOuAXCnlQI3jgpoMmrOASuAeKeX3F7aVSmu43W6+PbGeXVn7CTEHc02vqfSMTO7oZilKu+msQaLO+bSvo7ue3gJeAd7xc/waoFftnzHAP2t/KrWklDhcDkwGU0c3RdPruz5g9clNntdb0r9n4dRf0DtKbaOgKBeLDu16klKuBwobOeUG4B1ZYysQJoSIuzCt6/w2nt7Ow0v/lzs/+xmPf/ssx/JPdnSTvBTbSll7aotXmdPt5O3dn3RQixTl8rB8+XL69OlDz549ee6551pdX2cfo0igZuOYOhm1ZT6EEAuEEDuFEDvz8vIuSOM60onC0/x961vkVdbE2VNF6fxx/StUOTrPbItSWxlu6fYpP16YRlZpTge0SFEufS6Xi4ceeohly5Zx6NAhPvzwQw4dOtSqOjt7oNDqTNPcaUlK+bqUcqSUcmR0tGa6kkvKptM7kA1+FRWOKnZnH+igFvlKDIkjyBioeWxrxm6/1x3KPc4H+75kxfF1VFZXtVfzFKXDrd2Vzg//8C3X/3IxP/zDt6zdld70RU3Yvn07PXv2pHv37phMJubNm8fixYtbVWdHj1E0JQNIqvc6EcjqoLZ0Kga99n86vU5/gVvin06nY0aPiSw+8q3PMYvBrHnNR/uX8PmhZZ7XS46u5JnpvyYsILTd2qkoHWHtrnRe+WQvdkfNFul5RVW88sleAKaMSGrs0kZlZmaSlHTu+sTERLZt29aqtnb2J4olwHxRYyxQIqXM7uhGdQaTk8dg0HkHi4iAMIbF+Uwe61Bz+s3EagryKgsyBTKh60ifc4uqSlh8eIVXWV5FAV8fW92iexZXlfDKtrdYsPhxnlj5HDsz97a84YrSzt5ZdtgTJOrYHS7eWXa4VfVqbW/d2plYHRoohBAfAluAPkKIDCHEfUKIB4QQD9Se8g1wEkgFFlGzqb1CTbfOz8b9kKTQeIKMgYyMH8yTU36GSd/81ZYXQqApgKen/5LRiUOJDoxgZPxgFk79BaGWEJ9zM0tzcGmMaZwpadlD5B/Xv8r6tG0U20o5UXiav2x6jSN5qef9HhSlPeQXaXer+itvrsTERNLTz3VhZWRkEB8f36o6O7TrSUp5WxPHJfDQBWpOu3O4HHxxeAXbMnYTbArkml5TGZs0/Lzq+uLQcj49+DUOtxOT3sjALn2ID4lt4xa3jcSQOH414cdNntctLAGj3ojD5fAq7xnRrdn3Si1I41Sxdz+vlJJVJzfSN7pns+tRlPYWFR5AnkZQiAoPaFW9o0aN4vjx45w6dYqEhAQ++ugjPvjgg1bV2dm7ni4pr+14n08Pfk16SRaH81J5YfMitjUyqOtPakEaH+5fjMPtBKDa5eCt3Z+QUXJx98pZzcHcMfhGRL05DN1CE7im99Rm1+FwO7TLXc5Wt09R2tL8a/phNnqPKZqNeuZf069V9RoMBl555RWuvvpq+vXrx6233sqAAQNaV2errlaarcxezsYzO3zKlx9fy5jEYS2qy9/Mpt3ZB0kMvbiXmczqPY2hsf3Zm3OYiMAwRsQPxtCCAfo+kT2IDozwTBuuM7HbqLZuqqK0St2A9TvLDpNfVEVUeADzr+nXqoHsOrNmzWLWrFmtrqeOChQXiN1ZrbmmoNLR8v7IcD8zgPyVX2ziQ2LPuxtNp9Pxm8kP8a8d73G84BRWczBz+l3NqIQhbdxKRWm9KSOS2iQwtDcVKC6QqKAIekR040Sh926DYxN9xyj2nz3CgbNHiQmKZEK3UT5TSSd0HcUXh1eQV1HgKYu3dmF04tAWtanaWU15dSURgWEtuu58HM0/wcbTOzDqDExJGUfXMM11k20iKTSeZ2Y8RpXDhllvQqdTPayK0hoqUFxAPxt3H3/f8gbHC9PQ6/RMSR7H7L5Xep3z1vcf883xNZ7XS499x++n/4rgelNMA4wW/jD913x1dBWnizPoEdGN63pPb9GMp08Pfs2SIyuxOe0khsTxk9Hzm0zWdyj3GJvP7MJkMDEtZXyzu7nWntrCP7e/61kguCx1LU9MeojBsa3ri21KgNHSrvUryuVCBYpWyijNZmv6bkx6I5O6jW60+yc2OJpnrnycwspiLAYzgSbv2Q055XksO77WqyyzNIeVqRuY03+mV3l4QCjzh849rzZvTf+ejw8s9XoPC9e8wLSU8czoMUnz2/6K4+v4z/cf1Xu9lt9e8VP6x/Rq9F5SSj7av8RrFbnL7eLjA0vbPVAoitI21DN5PS63i++zDrAlfVezciZtPrOTXy3/Ax8f+Ir39n7Oz5ct5GThmSaviwgM8wkSAOklWT5pOQBOl2Q27w0005Z030zt1S4Hy1PX8cTK5ziUe9zrmNPt4uODS73KHG4nnx36usl72V3VFFYV+5Rnl+e2sNWKonQU9URRK7+ykN+vednzARZoDODxSQ/SL1r7G7Nbunl3z+deA9RVDhv/PfAVT0w+v6UfKeFJ6ITOZ9C7JesImqOxLhmH28nnh5Z5PSlUVldSZi/3OTe7rOnkixaDmZTwJE4Vea9t6O/n99rZZJRkcyjvOHHWGAbG9NFc4ZpRmk1aUQbdI7oSb+3SAa1UlPalAkWtj/Yt8fqWW+mo4vUdH/DirKc0z6+orqSgqsin/Exxzbf/UlsZnx9ezrH8k8SHdGFOv5kkNDGTJyowgpsHzPLqFuoZkcyM7hPP5y35dVWPSaxL24rL7dI8ntPg236IxUpiSBwZpd7rNJr7YX//iNv54/pXKKuuACA6KJI7h8w5j5a3vc1ndrIidR02h51xXUcwu88MT76sjw8s5dOD556aBnXpyxOTHvLKs/WfXR+xInUdAALBtX2mn3eXoKK0hR/+8IcsXbqUmJgYDhxomyShKlDUOpx33KcssyyHElupZrqJYFMQXYKjOVvu/a26R0Q3XG4XT695kfTaD9bUwjR2Ze3n+av/r8kZRiPjB2PWm7A5q0kOT2R43MA2T/TXPaIbv538CF8eXsGh3GM4pXfAGBDTx+eaBSPv4E8b/0FFdSUAcdYY5g2+vln36xmZzD9mP8uenIMYdQYGx/Zv0dqI9rLx9A7+tvUNz+tTxenkVxbyoxG3kVOex2cHv/E6f//ZI/zvd3/h0QkLiAmK5MDZo54gASCRLD26inFJw+kV2fz9iBWlLd1zzz08/PDDzJ8/v83qVIGiVqw1xmeRltUcTFCDhHZ1hBDcO+xWnt/0mmeFdKjZyrzB17M355AnSNSpqK5kzanNzB2gvQjG5Xbx8pY32JpRM36gFzruHnZLs4KE0+0ioySLiMBwQszBTZ4PMLBLHwZ26cPR/BP8acM/Ka/9tp8UEse8QecCQE5ZLmXVFfSOTOGf1z3D3rOHMetNDOrSt0UBzGwwtXhhYXv7RiPZ4JqTm7lz8BxSC05pjhedLDrD79e+zEvXPKX55QLgYO4xFSiUZik7sJ6iNe/jLC3AEBJJ+NQ7sA6c3Ko6J0+eTFpaWts0sJYKFLVuGXAtR/JPeOUZ+sHA6xr95js8fiB/v+73bM/Yg1lvYkzSMAKNAaQWpGmeX1qvn9/ldiEQnjn+G05v9wQJAJd08/buTxiTOKzRmVSbzuzkjV0fUVZdgV6n55qeU5g/7Obmvm36RPXgn7OfZf/Zw1gMZvrH9EYndNid1by45d98n7UfgMjAcH45fkGn+7BvjQpHpU+Zw+2k2uUgMcT/1N+z5XnsP3uU2OAYzeNxVu1yRamv7MB68r/+F9JpB8BZmk/+1/8CaHWwaGsqUNTqG92Tv1z1P6w+tQWb08b4pJFNTv2EmtTeM3tN8SobGjcAg86A0+2dX2hUwhAqHVX8e9dHbEnfhUHomdp9PPOHzOVg7jGful3SzeG8VMZ3HeF7zO3iXzveY13aVq+ypce+o2dkiuY1/pgNJkY2WLn85eEVniABUFBZxN+2vsHLs57u9JvHN9eYxGF82SCteb/onoRYrIRYrEzqNpoNp7drXutwOxibNIylx1Z5DdT3ikhmZPzgdm23cmkoWvO+J0jUkU47RWveV4GiM4sPiW2TQdYwSwg/HXsvi3Z9SJm9HLPexJz+MxnYpQ8vb/kPm87sBMCFi+XH12IxmOkSrL0rX6yf8hWp67yCRH07s/a1KFBo2ZN90KcspzyPJ1c/T6m9jIExffjBwNmEWKytuk9HunnAteRXFrH5zE7c0k3vyO48POYez/GHxtxNYkgsH+5f4nWd1RzM4C79MOqNPD31UVaf2kxacQbdw7syNWV8p9o8Sum8nKUFLSrvSCpQtJOxScMZET+I7LJcogIjCDQF4HA52KqxhmHD6e08d+Vv+O7kRgoqz82kGpUwhO4RXTXr39HIZjyh5tZ/eIcFhIDvpC6O5p8AILssl9SCNJ676okmnzAO5h4jrSidHhHdWpzqu9ReTomtlARrbJun4jDpjfx07L3cO+wWql0OIgPDvY7rhI45/a/Barby8YGvKLaVkhKWxP0jb8dsMAFgMVqY1Xtam7ZLuTwYQiJxluZrlnc2KlC0I6Pe6LXKuWZMQo/L5b1OwiD0hFpCeO7K37DyxAZyyvLoH9Obyclj/NYdZNLei9qsN3Flz0mtbvvsPleyJ/ug5kZCdU4Vp3M0/4Tnw9/hcvDBvsWsP70NndAxNXkcWWVn2Z65x3PN5G5jeHjsPU3eX0rJm7s/ZuWJDbjcLqICI3hk7D1+17W0hrWJCQAzekxkWvfx2Jx2Ao2t2ytAUeqET73Da4wCQBjMhE+9o1X13nbbbaxdu5b8/HwSExN5+umnue+++1pVpwoUF5BBb+CK5LGsOrHBq3x6j5p1EqGWEG4ecG2z6prVaxo7M/d5Lc6LCozg8UkPtsmir/4xvXh62i9Zfnwt5dUVlDsqNQfpK+utYH9nz2de00W/PLLC5/z1p7cxrft4+sf0bvT+a09tYXm9dCb5lYW8sGkR/5j9DMYmclqV2svJLc8nKTTe882/uXLK88gqPUvPiG5e3Wo6oVNBQmlTdeMQbT3r6cMPP2yL5nlRgeICu3fYLQQYzGw4vR2jzsD0HhO5vkFiwOboH9OL/73ipyw9uooSexkj4gdxfd+r/CYGlFLiku4WrV/oHdWd3lHdAdiVtZ8/bfiH13GrKYiBtR/4UkrWntrSrHpTC9OaDBQ7svb5lJXYyzhWcIoBjVz70f7FLDmyCqfbSZAxgPtG3NasvSiklCza9SHfndiIRGLQGZg/dK7PRAVFaUvWgZM73cC1FhUoLjCj3shdQ+dyVxus3q1bC9GUxYe/ZcnRlZTbKxgc248FI28nOuhcP6jb7WbxkW9Zl7YVgWBKyjhm952BTpwbExgRP4jbB9/IF4eXU+WwEW/twgOj7sJU+429zF6uud+GlqTQpvfvDfGzfqWxdSJ7sg/x+aHlntcVjipe3f42A2N6E9bEXh07Mvd6Pek53U7e3P0xI+IHef2uFOVypALFJW5D2nbe3/eF5/XenEP8ZeO/+PPVv/WUfXzwK68P2Pf3fUG1q5pbBl5Hub2Czem7sDntjEsazqxeU732sKisruIf299hR9ZearY49xYZGO41QD8sbgBDYvs32e6Zvaaw4fR2z2JGgCGx/RoNMvWn89ZxuV3szTnMFSljG73f/rNHfMqklBzMPcaUlHFNtldRLmUqUDRDYWUxu7L2E2iyMCp+iOdbdHvKKM1m9YlNNR/QXUcwqEvf86pnw+ltPmVpxRmcKc70DLR/m7rB55yVJzYwLmkET615wZMQ8MN9X/KzcfcxNuncZkvv7PnUa7AaavrzTXoj01LGM2/QDXyfvd8zfXRUwhCvJxV/ksOTWDjtUb46soqCykKGxPXn+r5XNXpNqJ+pulopWBqKDorQLI8K1C5XlMuJChRN2J6xh5e2/MezeC46MIKF0x5t1+6II3mp/H7ty55v06tObuSeYbec1zRMnZ8xifpjFdWuap/jpfZy3vruX5Q56q0ml27e3v0poy1J6KJr1ndsyfCd7isQvH3Ti55ps+O7jmR815EtbnuvyBQenXB/s8+f1n0C3xxf45XpNikknsGxTQfZqSnjWXZ8rdfTT7/oXo2OhyjK5ULtR9EIl9vFf77/yGuFdV5lIZ8cbHofhtb47NA3Xl0uAJ8c/Bqny+nnCv+md5/gU9Y3qofXntRaH+JzP97GT370V+KyvPeSMJ04BUOGwMKFAAQafGcCBRotHbJ6OzwglMcmPECg4Vwa9cyyHLam727yWqs5mGdnPM6cfjMZkziM+UNv5n8mP3zJrEJXlNZQgaIRBZVFFFWV+JT7y+XUVnI09nmoqK70pOluiVEJQ/jJ6PkkhMQSbApicvIYfjXhx17n3DPsFsbV60665ZMd3PLZLiKKKnnq90s8wSIuq5iFf1iKLjsbnn4aFi7UnBV0dQfOFNpwZjuVznNTdt3SzRvff+SVw8uf8IBQbht8A7+csIDr+kxv8dRaRekM0tPTmTp1Kv369WPAgAG8/PLLra6zQ7uehBAzgZcBPfBvKeVzDY7fA/wFqNvi7RUp5b8vVPvCA0IJNgV5MqvWae5e0eerX3QvzlZ4r9iMt3YhrIm+9qzSHErsZfSKSPHaM2FKyrhGB2QDjQH8ZPTdbEn/3hMk6tQFi9fuv4IfL1pHeFG9RHpPP80NPIXljnmsObUZt3RzRfLYDl2pfLzglE9Zqb2csxX5jSb6U5RLhcFg4Pnnn2f48OGUlZUxYsQIrrzySvr3b3oSid8627B9LSKE0AOvAlcCGcAOIcQSKeWhBqf+V0r58AVvIDVTWW8bdAP/3vWhJ+V0kCmQm/trpwpvK/MGX8/xwlNkluZ47vnjUXf47Qaprs30uqt21k+o2covxt9P/5heuN1utmTs4nBuKrHWGKamjPNa1Z1XUcD6tG3YXdUMMkQzffVhn/ojiip54s/LtBu7aBFXP/IIV/e6opXvum0khcT77KYXYLCoQWmlU9pwejsf7ltMQWUhkYER3Db4BiZ1G92qOuPi4oiLq/lSZLVa6devH5mZmRdnoABGA6lSypMAQoiPgBuAhoGiQ13ZcxI9IrqyLWMPgcYAJiePaTTtd0u5pZtj+aeQuOkT2QOdTkdEQBjPX/1/HMg9is1pZ3BsPywGs986vjq6yhMkoGZh2t+3vsmr1/2Bv219g83p554QVqau55krHyPYFERqQRpPr30Je70UAn//43weeeIdIop8U3D7iI+HNWsg8sKsM6hy2Fh85Fv2nz1CdGAE1/e9yicX1k39Z7I7+4BXN90tA69r9PenKB1hw+ntvLbjfc9kkvzKQl7b8T5Aq4NFnbS0NHbv3s2YMf7TATVHRwaKBKD+V78MQOvdzBVCTAaOAb+QUqZrnIMQYgGwAKBrV+1Eeuere0Q3urfxvtVQ823+2XWvkFlW8+QQGxzN/0x+mFhrDDqdjsGx/ZpVz54c39haUFXE5vRdXkECILs8l9UnN3F936v4+MBXXkECICsulOqVK3DNvgV9do7fe9q7RGFeswZ6N29WUHZZLm/t/oRDuceICY7i1oHXtXhviz9t+AeHajcLOl5wip1Z+3juyie8ugLjQ2J5/ponWZ+2lVJ7BaMThnhWlytKZ/LhvsU+Mw6rXdV8uG9xmwSK8vJy5s6dy0svvURISNNTxBvTkYPZWv0oDVdsfQUkSykHA6uAt/1VJqV8XUo5Uko5MjpaOzV3Z/Pm7k88QQJq8gz95/uPWlxPZIDv9qo6ofNsW9pQZulZAJ9d+ACKqkoIHDgU/aLGh4IMr/+n2UHC6Xbxh3V/Y3f2AeyuatJLsnhh86IWTQo4UXjaEyTqVLscfJu63ufcMEsI1/e9ijuHzFFBQum0ChrsqNlUeUs4HA7mzp3LHXfcwU033dTq+joyUGQASfVeJwJZ9U+QUhZIKeu+8i4CWrfJQiezP8d3PGBfzhHNFc6Nua7PDIw674fDaSnjGRY/UHNco29UDwDN7TrjgmOwpmXCggWN3lP/4INwzHezJS0Hzh4lr8I7x35LckMBXmsj6iu1lzW7DkXpTCL9jJv5K28uKSX33Xcf/fr149FHH21VXXU6MlDsAHoJIVKEECZgHuC1Q4wQov40lesB30/Wi5jWAGtUYHiL5+73jEzmDzMeY2rKOHpEdGNQTF+SQuOxmoK4bdANiHoPb8PjBjKpNn357YNuIKLe04hZb+LBsLGIadMgK8vnPl6ysmDqVL/BIqcsl6VHV7Hm5GZsDbq36jQ3NxTUzAQL0sjeOjJB7SanXJxuG3wDJr33FGyT3sRtg29oVb2bNm3i3XffZfXq1QwdOpShQ4fyzTfftKrODhujkFI6hRAPAyuomR77hpTyoBDid8BOKeUS4KdCiOsBJ1AI3NNR7W0Pcwdcw9+3vuWZUVVT5juj6nDecU4UniE5LIGBflJ5pIQnUeWwcaLwNAD7c4+wPHUtz8x4jLFJwzmSl0pscAx9o3tQaitj1cmN5FYUMG/Q9Rh0eqpdTkaZ47GOntB0kKhTFyz27fMa0F6fto1/bH/HEwjCLaGEWUIotpV6zhEIJidr519yuBy8t/eLmiSFQjA1eRy3D5nDoxMW8Mq2tyiqKsGgM3B1zyuY0LXpzLCK0hnVjUO09ayniRMntrhXoikduo5CSvkN8E2Dsifr/f0J4IkL3a4LZWK30YRZQll7akvNGoSUsT4J8/6+9U2vfZtHJw7l0fH3++RLWnF8LVszvFcgZ5flsurERm7sd7VnS9XiqhIe+/ZZz4f26pObGJc4nF/Upcq4//6axXT1xcfD66/XdEc1DCL33+8VJBwuB2/v+dTraaHIVuLZL/xQ7nFigiK5deBs+kb30Py9vLvnc5anrvW8XnrsO3Q6HXcOuYl/XPcMmaU5hAeENrnhkKJ0dpO6jW6zGU7tSeV66mCNpQo/mHvMK0hATe6p3dkHGRE/yKv8s0Pa6xzSS7w/2L88vMLrmz3U5GuaW5cksDY1hydY1E2B7d0b1qzBPWVKzcpsgKeeOnd+rbzKQs3xhPSSLGb0mMSsXlMZET+40W1N16b5jl2sObWFO4fchF6n99o1UFGU9qdSeHRiJwrTmlXudLt8PvzrNByw3p97VPM8r2m0CxfWBIF6QaLaWc1LBRt49LEpFIYH8t2d08n79SNUVFdyujjDk4cqKjCCYI29JHLK83hv7+f8ZdNrLFzzAtXNSKnhpY0fpRWlvbR1t09bO5/2qSeKTqxrqPY354blBp2elLAkThV7LzEJMgYyLWW8V5nVpN1dU5cepLiqhIN5x4h6+C76PPKIp1vpk4Nfs/nMTogL5Vd/vpVyq4Wo1S9QYi/D4XIQYg7m/pG3MyZxGHcOmcNrO973Gnup70j+CdanbSMxJJadWfsJMQczJXmsZ+vRKcnjvLqeALUnhHJRsFgsFBQUEBkZ2SkTSkopKSgowGKxNH1yPSpQdGJDYvszPG4g32cf8JQNjOnDqIQhPufeO/xWnl3/imeGUYDBzJNTfu6zd8aN/a7kUJ73TCWd0DE+aQSrT25m0a4PcLldAAzq0pfHJz6IyWDyGv8ot9b8T5Zfb753qb2cv219k3/O7sW07hPoGZHMjsy9ZJflsl5jT4zvTm70DLwDfHVkJb+f8Wtigx0IJzwAACAASURBVKO5c+hNIGBd2lZ0CKamjOe2Qa2bCaIoF0JiYiIZGRnk5fkm9uwsLBYLiYmJLbpGdPbHpPMxcuRIuXPnzo5uRptwu93syNrLycIzdAtLZHTiUL/7XpfZy9mesQe9Ts/oxKEEakwnBfji0HI+Pfg1DrcTqymIH4+6k/7RvXjgqyd8uoTuHnoz1/aZzvzPfu53mmt9j4y5l0nJ5wbnThdn8OsVz/icpxc6XA2mx07rPoEHRt3Z5D0URWl7QohdUkrNjWPUE0Unp9PpGJM4rFnpLqzmYKb3mNjkeXP6z+TqXldQUFlEXHAMBr2BPdmHNMcNDuUdp0dEcrOCRE17BcuPr8XldjE2aTjdwhKZ1n0Cq09u8pyTEBLrSXhYX0aJ70pxRVE6ngoUl6lAYwCBoeeeOGKt0QiEz7hCnDWGErv2QHlDscHRvLbjfU9Q+WD/Yh6b+AAPjLqT8UkjOJyXSpw1hhFxg3j4m//zSTGitVJcUZSOp2Y9KUDNh/zUBgPGIeZgSm1lfH30O6/V3fWlhCfRKyKZ2X1mEGoJ8XrycLgcvL37UwAGx/bjB4NmMzl5DEHmQO4ddiv6emtBEqyx3Niv8T2xFUXpGOqJQvFYMOoOBsf2Z1/OIULMwaxN28ratK1+zzfoDPxy/AJigqMAuOdz37wyGaXZOFwOjHqjV/nk5DH0j+nF7qyDhFqsDI8f5HfsRVGUjqUCxSXm29R1fH10NeWOSsYkDOXOITcRaNIe1G5IJ3SM7zqCobH9WX1qs+bajDBLKMW2EhJD4pg/dK4nSEDN08XBXO8ZVQkhsT5Bok5UYATTe0xASoleBQlF6bRUoLiErDu1lX/vOpemfNXJjRTZSnh80k+adf2+nMO88f1/ySo7S4BBe571Tf1ncmWPSZof7LcPvpE/rP0bVbV7Vht1Bu4eerNmPW63m/f3fcHKExtwuByMSRzGj0beprlYT1GUjqUCxSXku5Mbfcq+zzpAUVWJ5q58+ZWFbDy9A7d0MyimD3/Z+C/stRup1H3Y1yeEYFjcAL/f/ntFpvDytU+z+cxOnG4X45NGEBWknTL5yyMr+OroKs/rzem7cLid/HriA816r4qiXDgqUFxCGq5LAJBIXNLlU34kL5Vn1v3dExg+FjrNtN+62nKLwcz8oTfTJbjxTaHCLCHM6j2tybauT/NdhLczax+Vjiq/6z8URekYKlBcQiZ1G83xglNeZf2je3n2vXBLN8uPr2XT6R1kluV4gkTdMS1u6SY8IIwFI25jRBvu/aAXvhPudAh0fmZXKYrScdT02EtIw3UJOqHjpv7XeF5/sG8xb+3+hOOFaVQ6fLuW/CmqKub5zYv8bq16PrQWBk7oOgqLsWU5aBRFaX8qUFwiqhw2vjzyrVeZW7pZdaJm3MLpcvJt6rrzrt/pdrLqxAavsiN5J/j80DI2n9mF0+3bvdWYa3pN5c4hc4gOiiTUbGVmryncP/L2826foijtR3U9XSKKbCXYNdJsZJfnAjUf9HZntc/xOlq5lxrKKj1LekkWTreL13e+75XUr0dEN56a+gssBnOz2iuE4Pq+V3F9X7XITlE6OxUoLhGxQdFEBIRRWFXsVd4/uhcAFqOFQV36su+s9rbj8cFdcAupmYOpzv6zR1ijsakQwInC06w5uZlrek89z3egKEpnpbqeLjJZpTnsyzmMrcEYg06n44FRd3l9o+8Wlui1B/eDo++id2R3zXrTy7L54bBb+fXEB7h32K2EWbyn05r0JvKrihpt26mi9EaPK4pycVJPFBcJp9vF37a+wdb07wEIMFh4aMzdjE4c6jlnaFx//jX7j+w9e4hgUxADYnp77a0dGRjOH2b8mhc3LWJLxvc+96hwVDE2aTgAE7qNYsnhb9mZtQ+3dJNT3nR+/eTwluW4VxTl4qCeKC4S353Y6AkSULMg7tXtb/s8WQSaAhiXNIJBXfp6BYn6JiWP8SkzG8wM6tLX8zrAYOb77ANklZ1tVpDoEhTls5ueoiiXBhUoLhJ7NcYWqhw2jjVYN9EcIxMGc1P/azw5mMIsIfxi3H0EmQI95+zI3EtGafP2h4gJiuTFa55SU1sV5RKlup4uElEB4drlgdrlTZk36Hqu6zOdwspi4kNifTK3FlQW+7nSV0xQFD9a/BgmvZEre0xi7oBZfp9mFEW5+Kh/zReJWb2n+qS2GJc0gviQ2POuM9gURNewBM303kPj+jerjhCzlQO5R6l0VFFsK+WTg1+z5MjK826ToiidT4cGCiHETCHEUSFEqhDiNxrHzUKI/9Ye3yaESL7wrewcYq0x/PHK3zCz5xRGJgzhvuHzeGTsve12v6TQeO4aMhejzv9DZ0xQJOXVFT7la05tbrd2KYpy4XVY15MQQg+8ClwJZAA7hBBLpJSH6p12H1AkpewphJgH/An4wYVvbecQZ43hhyMu3Nuf3XcGV6SM5UxxJtsz97D8+Fqv47cOnM0/t7/jc52U0qdMUZSLV0eOUYwGUqWUJwGEEB8BNwD1A8UNwMLav38KvCKEEFJ9El0wIeZgBnbpw4CY3nQNjWfTmZ2Y9Sau6nkFw+MHsjv7AJvO7PS65orksR3U2s7HfjYNZ0kelsS+6AOtHd0cRTkvHRkoEoD6K7QygIbzNj3nSCmdQogSIBLIb1iZEGIBsACga9eu7dHey5oQghk9JjGjxySv8gUj78CoN7LpzE5MOgNX9pzMnH4zW30/V1U5joJMjFGJ6C0X32ZG0uXk7BcvUHm0Np26Tk/oqFlETL8bIZrOkOsoyaVg+b+pPLEbvTWC8AlzCRmu0p0oHaMjA4XWv5aGTwrNOaemUMrXgdcBRo4cqZ44LpAAo4WfjJ7PT0bPb7M6i7ctoWjth0hnNcJoJmLK7YSOvq7N6r8QyvauPhckANwuSrZ9RXXuGWLn/RbRyNavUkpy/vtHHHlnAHCV5pO/7DX0weEE9R7V3k1XFB8dOZidASTVe50IZPk7RwhhAEKBwgvSOqVD2HNOUbjqbWRtAkPpsFOw8k2qc083cWXnUpW2X7v81F4qj+1o9Nrq7BOeIFFf+f61bdE0RWmxjgwUO4BeQogUIYQJmAcsaXDOEuDu2r/fDKxW4xOXtsoTvqlFACpTtcs7K0NYjN9j9uwTPmXOsiJKd6+i/MgWpN+U7b4P2NLpoOLoNsoPb8Fd3fw9RhSlJTqs66l2zOFhYAWgB96QUh4UQvwO2CmlXAL8B3hXCJFKzZPEvI5qr3Jh6IPCtMuDtcs7q9CRsyjbvRK3zXf6sKlLstfriiPbOPvlC+ByAmAIj8UYlYgjP8PrPOsQ78y81QWZZL//NK6yAgCEKYC425/EktC7Dd+JonTwOgop5TdSyt5Syh5Symdqy56sDRJIKW1SyluklD2llKPrZkgpl67g/uN9vo0bwroQ1O/iyiNlCIkk8f4XMUTEe5Vbug0gqM+5ORvS5SR/xSJPkABwFuVg6pJCYJ8xCIMJQ3gs0dc9RGDPEV51Fa562xMkAGR1FdnvL8RdXdVO70q5XKkUHkqnojMFED//WUq2LcaWlQrSjc5ooXjT54SOvhZ9YEib3MdVVU7J1i+xpR/BGBFH6Lg5mCLjm76wBQwhkSQ98DKVx3dhz07FFNONoN6jEfpz/+ycJXm4yn3TtzvyTpN4/4uN1l915pBPmXTYKdm5jPDxN7X+DShKLRUolE7HYA0nYuodZL7xGNW5NYO6Vaf2UnF4Ewn3/RWdqXXJB6V0k/3B01Tn1Dyg2tIPU3F0O4n3v4AhJLLV7a9PCB1BvUf5na2kt0agMwfitnvvR26MStI83+va4DCchb5PD7Yzh+HiegBTOjmV60nplCqObvcEiTqOwmzKD21sdd1Vp/Z7gkQdt62c3MUv+Z2t1F50RjPhV9zmXWYJInzizU1eGzZhrma5Lf0QLo2xEUU5X+qJQulQztJ8ird9hSM/E0tiH0JHX4vOHIiz1GdNZe35BZrlLeGqLNEst505RPb7CwkZNYuoq+5r9X2aK3TULMzxPak4shWdJRjr4CkYrBFNXhcyeCpl33+LPfOYV7mstlFxaJNaoKe0GRUolA7jqigh843HcVXUpDSvOrmbytRdxN/zLAEpQzSvCeyuXd4SgSlDEHoj0uXQPF66YxmhI6/BGOE7ZiGlm6qTe3GWFRLYfQiGkKhWtwfAktD7vGYrBfUd5xMoAFyVpW3RrDZnP5tG6c5luCpKCOw1EuvQaQiVkr7TU4FC6TBle1d7gkQde9Zxqk7tI7D7UCKm3kHhuv+C2wk6PWHj52BJ7OuntubTB4USfcNPyV/2Gu6qco0zJPbc0z6Bwm2rIOv9p6nOqV0HIXREzXqAkKHTW92m8xXYawSFq98F6a5XKgjqPbrD2uSPPfsEWe/8r2cxZeXxHdizU4me9UAHt0xpigoUSodxlmkvsnfVloeNv4ngwdOozj2NKTqpWd0xzRXcbzyBvUZStPYDSrZ95X1Q6DDH9vC5pnj70nNBAkC6KVj5BsH9xqEzB1J+cAPlh7egMwcRMuJqLPE926y9/pgiE7AOnkLZvrUg3QiDiYgZ92CKaV6+M+l0YMs8is4SjLnB+o62VrxtiSdI1Cnb8x3hk36AwXp+G3ApF4YKFEqHCewxlNKd33gXCp1Xt5MhOAxDOy220xlMhE/+Abb0I9izjnvKw8bPwaixstqeccSnTFbbqM49TeWJ3RRv+sxTXn5gHXG3/R8ByYPape11SnYup2zv6nPtcVZ7ra1ojC39MGc/+wuuipoxm4DkQXS55XF0poAmrjw/Lq3xJenGVV6kAkUnpzoHlQ4T2HMEoWOuh7o+6tqsqme/eB5bxtEL0gadKYD4u5+hy9zHiJg+n4Qf/oWIKbdrnmuMTNSoQI8uKIyS7Uu9y90uzn72V0795Q4y3/4ttnTfINMWSncs9S3buQzp1RXlS0o3uYv/5gkSUJOfqnjT523exjoB3Yf6lOmDIzB16dZu91TahgoUSoeKnHE38fc8C3oDSAnSjT3jKNkf/t7rQ6ytSbeLwnUfcvql+zj90g+pOn2AkBEzMcd193tN6NjZPilGQsfMRqc3IB12n/PdtnJktQ17xhGyP/x9m8zYakhrGqy72gZ+80XVcBRk4SzJ9SmvPLm3pg57FSU7l1Ow8k0qjm5vk82oQsfM9lpdrgsMIebGnzWaSVfpHFTXk9LhbKcPeqWwgJp0FBVHthAyovV7W2gp3vQ5xRs/9bwu3fkN0llN9LUP+r3GGBpD4v0vULZvTc2sp57DCew+FLfDDnoj+JlFBSAdNsoPbiBs3I1t+j6C+o6l7PtvvcoCe45A6I2NXqcPCq0Jzg1+74aQSFy2CrLeegJHQSYAJduXEjx4CjGzH2lVW3VGM7E/+B+q8zNwVZRgSeiNMDTeTqVzUIFC6bTaM09w2d7vfMrK968jZMRMijd9iv1sGub4nkRMnocxIs5zjj4o1OfDvnzfmkaDhEcT3UHnI3LaXbjKi2tTl0ss3QYSNevHTV6nD7ASOmKmV5eZ0BsJG3cjZXtWeYJEnfJ9awkdPbtNBrxNUYkQpdGNp3RaKlAoHS6o/3iK1v/Xa0aMMFkI6tt+W6pqdaVIJFnvPYW013TnOItysJ0+SNIDf0dn9j/Aa9NIG+5DbyCo/4Tzbq8/OnMgsbc8jquiBOlytigFScSMezB1SaHi2Hb0AdaarrfYFEp3r9I835Gf0e4zo5TOSY1RKB3OGBpD7K1PYOqSUjM1Na4ncfP+t91mOwFYB03xKTPFdPMEiTqu8iIqjm5ttC5jeGyjx4XBROzNj2MM64LbXtnkQPP50AeFtjhPlRAC6+ApxN78GNHXPog5NgUAS0IvjZN1mLXKlcuCeqJQOoWAlMEk/uivF+x+4ZNuQTqrKd2zClxOggdMQh8cSrXG04Hb3nja7tDR11G88ROkU7v7STqrcTtsZCx6lOrc0+iDIwibfCshg6d6ZZJ1O+wUrf8vFUe3obcEETL6OqwDJ7fujTZBSkn5/rVUHN2GzhJM6MhrsA6eSsXhzV55r8In3YIxrEu7tkXpvMSluGHcyJEj5c6dOzu6GcpFQkqJEILqvDNkLPql91iCzkDXh15tNFVHde4Z7HlnKFj2Om67djI+YQ5ANgw4ej0hw2cSOeNuhE7P2c+fp+LwZq9TYm76JcHtuBdHwaq3vBcc6gzE37kQc2JfbGn7cRRmY+k2oGZcQbmkCSF2SSlHah1TTxTKZU/Urt8wRXcl+rqHKFz9Lq6KYvTWSKKuus9vkHAU5XD20z979vM2dx2AOa47pQ1WehtjuuHQ2vPb5aJ0x9cg3YRPvIWKI75dXKW7lrdboHDZKijdudy70O2keMtiYm/tR0DKYAJSBrfLvZWLiwoUilKPdfAUggdMxFVRgj44rNE5/nlf/8MTJADsZw5iiogjdPxcynZ/i3Q6COo7loCUweQt+Zvfekp3ryJ07PWas6JkO+6D7a4s0UyM6C9zr3L5UoPZymXJZavAWV6seUzoDRhCIhsNEu5qW836jwbKD22kZPNnuKvKkA4bjqJsAnuOaDzLrMuB0Bkwx/sOFgf1n9j0mzlPhvA4DBoD8YE9fFdQK5c3FSiUy4rbWU3ukr9z+sV7OfPyfWS99+R5rZgWBiM6S5BPecMnAHvGUcoPrCfuzqcJ6jvOk6bEi06PzhxAzE2PYuk6oLZ+EyGjriV09LUtbltzCSGIueFnXqvNA1IGEzZee0Mk5fKlBrOVy0rhmvco3vyFV1lAyhDibn+yxXUVbfiYovX/bfK84EFXEHP9TwGoOLyVs5//xet46JjZRM64x/PaVVWGMJjQGc0tbtP5kC4ntsyj6C3WZmedbYqjJJeKw1sQOj1B/SaopH8XATWYrSi1yg9v8SmrOrUXl60CvcYTQmPC+0/HEBJF+cGNCIORgB7DKFi+CF2lHXfguQ95U/S5D9+gfmOJu/0pSnYtR1ZXEdR3HNZhVwJgzzlZ8+FqNGMdPOWCBQqhNxBQ+yTTFipP7ObsJ3/yjH8UrvuIuNufPK+NmZTOQQUK5bKilUJbGExe6xmaZeFCWLQI65o1WOs9jbi+30HIL5+lbGQyRdP7Y4xKJKQ2ENTRmk1Utm8NeV+9CtQ84Rdv+ZL4OxZivgB7WrS1glVveQ2Sy+oqCle/R/xdv+vAVimtocYolE7NWVZI0YaPyV/2OpWpu1p8vXQ6KD+8hdLdK3GWFRI6apbPOdYh01r07V0++X/w9NOQlQVTp8Kx2q1Ijx0jYuG/MJTZCF9zhIT0IBJ++GfNsQyv+twuCle/R12QgNoP12Z0a3U20uXAkZ/hU1599lQHtEZpK36/RgkhvgF+IqVMa+ubCiEigP8CyUAacKuUskjjPBdQtzz0jJTy+rZui9J5OQqzyXzrCdxVZQCUfr+CsPFziJh6Z7Oud5YW1AxWF+XUFOgNxNz4c6JnP0LpruVIp52gfhMIGz+n+W36xUMYX/rHuYKsLOTUKYjXF8GCBTXBo5b5tXchtnvN00cj3FXlPlvCAjjy05vdrs5C6I0Yo7viyDvjVW6K9Z++Xen8GnuieAv4VgjxWyFEW+cC/g3wnZSyF/Bd7WstVVLKobV/VJC4iJXuWkH667/gzCsPUrD6XdwNtsTUUrT+v54gUad461fN3qeiaOOn54IEgMtJwfJ/EzxgIgn3Pkfi/S8SPvHmZu+HIPPzEG++5VMusrLhuuu8goTHokVQ0PisKl1gCAaN9BiGsBiKNnxM+cENPulBpNtFxZFtFG9dgj2nc31bj5xxD8Jg8rwW5kAipt3VgS1SWsvvE4WU8mMhxNfAk8BOIcS7gLve8Rdacd8bgCm1f38bWAs83or6lE6sdPdK8pe/7nldsuVLXBUlxMx+2O81hes+ovzgBt8DbieO4tya/RSaYM885lPmqijGUZyLKTK+eY2vx4Gds/dOIO4/GzCUNWMhXHw8rFkDkY0n6xNCEDXzfs5++mdPBl1htGA7fdCzVsMc14O4O3+HzmTBXV1F9ntPYa+Xlyps4i1EXDGvxe+pPQR2H0LSg69QcWQL6AwE9xvXrP9eSufV1BiFA6gAzIC1wZ/W6CKlzAao/em7QXENixBipxBiqxCi0R1fhBALas/dmZeX18rmKW2pdNcKn7LyAxv8JtuzZR6neOMnmsd0lqBmT+E0RvvmJxLmwBZnWa2jDwrDER1K9n2TcFotjZ9cFyR6N2+mT2CPYXR9+F9EzXqQiGl3IR0N1mNkn6Bs3xoASr9f6RUkAIo3fYazpPP8f28IiSR09HWEjpypgsQloLExipnAC8ASYLiUsrIlFQshVgFa+Zd/24Jqukops4QQ3YHVQoj9UkrN5P9SyteB16FmHUVL2qq0L6nVzeR2Id1O33JqpqtqEjqiZi5o9sBz+ISbqTqxB7et3FMWccW88552qg8IxhgZj0O6ybtxGHHv+k619Xj99WYHCU/9QaGEDJtB2f51msdLti2hZPtSpEvj9ybd2HNOYgiNbtE9FaU5GpsT+FvgFimlb56CZpBSzvB3TAhxVggRJ6XMFkLEAb6b99bUkVX786QQYi0wDGjGLjFKZxI8YKLPwrSA7kPQB2g/mBpCtdNdRF11H8EDmp/SwhSdROKClyjfvxaXrZygPmNaPZc/dNwcSt58lugvdzd+4oIFLXqiqOOqLMWedVzzmLNY859JLYEppluL7qUozdXYGMWkdrzvEuBu4Lnan4sbniCECAcqpZR2IUQUMAH4czu2SWknYeNvwlVZStme75BOB4E9hxN17U/8nh/UbzzFm7/wmmZpjE7COnR6i+9tsIYTNn4OUrpxlZcg3a5mD147CrMo2vQZ1bnpWOJ7EjbxZqzmWILe2Y6+qTGKuqmzLQgWrqoyMt98vImAoC1k9LVNbqCkKOerQ1J4CCEigY+BrsAZap5cCoUQI4EHpJQ/EkKMB16jZgBdB7wkpfxPc+pXKTw6J7ezGlyuRrcVreOqKqd05zLsZ09hju1OyIiZ6AOCz+u+Fcd2UPDtf3CW5KEPCiNi2l1YB0/xalfFwY1U56VjTuhFUJ8xuKvKSX/957grSz3nmQyhJPxtBUJrdpM/8fGwb1+TA9oAxVuXUPjd2z7llpTB2E7t077IYCR80q2Ej7+p+W1SFA2dLoWHlLIA8Pl6KKXcCfyo9u+bgUEXuGlKO9IZTM3+P04fEEz4pFtafU9naQFnP/8r1PbruyqKyVv6KubYFEwx3XA7q8l+90mv7p7AXqOwdOvvFSQAqp0lOG68GtM/3vQqd8dEIRb9G/HgT3ynyN5/f7OCBICz+KxmeUC3QdjSDmimIcfpoGTLl4SOuvaCpfxQLj9qZbZyUSjZ8Q3p/3qE039bQMGqt3E77M26rvL4Dk+Q8JBuT86nioMbfcYEKo/voNrP2oSqe2+Fp57yvHZaLWTePozM3A24VyyreYKo89RTTS62q+MoydWcpYXQETxgQk2W12DtxHpuW4XmVGBFaSsq15PS6ZXsXE7Bt+d6HUu2LcFVVdboOow6wqQ9jVVXW17tZ/WzMGp1jwmCeo3A8dPBlK//GOvOU2TfNwlHlBXOnqK0Io2wNWtqxibuv79ZQcJVUcLZz5/HdqZmzoguKAx33SptnZ6IaXdiDOuCMawLQf3GkffVq5Qf8J0VpbdGNHkvRTlfKlAonVZl6i5K93xHlUb/fPmBDURdfZ9mkr/6gvqMoTD4fVzlhZ4ynSWI4IFXAGD2MwtKurSeWCTS7cJ+No2i6f0oGdfdK0ts9dk0GHO9z5iEq6KEgpVvUpm6C31wGGHj5mAdMg2A/BWLPEECwF1RTECP4YQMnY45oTeGegFA6PSETbiJimPbvPa9COwzBlNkQqO/B0VpDRUolE6pbP9a8pb83f8JbhfSrdFn34DOFED8Xb+jcN2H2LNSMcV0JXzyPM/+CEG9RxPYezSVx7afu8YSjD1Te4qqoyALc5cUQHgFCQBzXT6jBmMSOZ/8CXvm0Zpm2yvJW/oqugArQb1HUXFsh889bGcOEjdPe7mRKSqRhHueo2T7UpylBQT2GErIiKub/D0oSmuoQKF0Sg03F2oosOfwJvePkFJScWgjFcd2oA+wEnvrb7z2hoCab+mxtzxO0YZPKFr/EQBuW7nXIj3PuXoj5vie6ANDCB17PSVbz83qNsf10Jy+W52X7gkS9ZXtWUVQ71HoA6y4yr3zYer8rC+pY4pOIvraBxs9R1HakgoUSqfU8MPTQwgCe41q1gdl4XfvULJtied12d7VxN/1e689HqSUlGz5guJNnzVemdAReeW96ANDAIicPp+gvmOxnT6AITyWoN6jNfe0kG6XZnV1T0OhY2ZT+N07XsfCxsxu8r0pyoWkAoXSKQX2GO6TFNCSPIjYW59o1jRQV1UZJTu/8SqTzmqKt35Jl5t+5Skr2baEwjXv+60nZs6jSKeDgORBPjmiLAm9m1zpbe6SjCkmmercNK9y66CaMZKwsTegD7BStm8NblsFpphkLN0GNvn+FOVCUoFC6ZQiZtyDoyjHM3XVGN2V6GsfbPZaAVd5ke+0WMBZko/bWU353jXYslKpTPW/MNMc15Pg/hP8Hq/OPU3F0W3oLMEED5iEPlC7y6jLrY+Tv+x1qk7sQR8UQui4G71SkQQPnET5wQ1U556mOvc05QfWETruRiJVam6lk+iQldntTa3MvnRU555Gul3nBoqbyWWvIv2VH+O2VXiVh46fgz3jKLYzhxq93hzfk5gbf+E3LUbp7pXkf/MadbvSCYORkJGzCJsw1+/Yib/0IWV7V5O39FWf8sQfv4wpSmNthaK0g8ZWZqsFd0qnZorp1uIg4XZWk/P+Qp8gYUnqjzm2R5NBIqDnSBLu/ZPfIOF2VvtuXep0ULJ1MVlv/Qa3vdJznrMkD1m7otpfjimbnxlW9qzURtupKBeK6npSLjkVhzZjz/b9kA2bZBCASwAAIABJREFUcBOOQj95mvQGkBDcfzxRV/+o0fqdJfmas6KgZvps2b61IN0UbfgYt60CQ2g0Udf8mMAewzSvMUUn+Slv3r4bitLe1BOFckmpLsiidPdKzWOOgkwsXQdoHhOmAIIGTCRi2nx0TUy7NYZGo6ud/aSlePtSCla+6XmicZbkcfazv+JqsK1rHeuQqT4pwoMHTsYcp/aZVjoH9UShXDIqT+wm55PnNAexAcwJfTB3SSZswlyKN32OV9dRVRkV+9dSnXOCxPtfQAjt71CuihKKN3+OzhzkkzTQc45Gcj/psFF5YjfWgZN9julMAcTf80fKD2zAUZiFJakfgb1GNOMdK8qFoQKFcskoXPO+3yARMnIWloReAERMuR3rkGnkL3+dqpPeu+k58tKxnTlEgMYU1fIjW8n98kXve+h00IwV4gA6c6D/Y0YzIcP87vWlKB1KBQrlklGdp5XgT5Dwo79i7pLsVWoMj0UfrJ3+W1b75nmypR8m97O/Uv8pBGh2kDBGxPkdo1CUzk6NUSiXjLonhvrMiX18gkSd4P7jfcp0gSFYkr2fJiqObifrvafwCRK19EFh2g0ymDBGxGMdOoO4O55u9s56itLZqEChXDIiZ9yDznJuFzydJYjIK+/1e35gj2FEzLgHXe3Oecborj4rv90Oe80aBz+pOACirvsJQiuLrbOa8CvmEX3tgz6ruhXlYqK6npRLhjm+J10f+kdtRlZJUO/RTc5gChszm9CRM3HbKtEHhVKde4a8pa/iLC0goPtQzLHd/U6FBdAFh1G46m1MsSnYNdZnaHeHKcrFRQUK5ZKiswR57YfdmOqCTApWvoUt4wimiDiCh0yn8Lt3kI6avR6qTu3F0n0oCJ3vNqRGCzhsuMuLcZcXQ0Gm5j2aygWlKBcDFSiUy5J0Osh+fyGuspoNjezZJ7DnnIQGKW1sJ/dgTuyDPaNeqnCdviY9uMPmdS5CeF0fNGAiAWoAW7kEqEChXJYqT+z2BAkPP3nP7FknvAvcLlyleb4nSoi7YyGOgixMXZKxJPZpo9YqSsdSgUK5LEnpf3Dah1t7bUZDAd2HEpA8iIDkQefZKkXpnNSsJ+WyFNhjeKNpOFpKGM1Ez36ozepTlM5EBQrlsqQzmomb97+YawebDWExBPYerXGmaFZ9wf0nYAgOb8MWKkrn0SFdT0KIW4CFQD9gtJRSc/MIIcRM4GVAD/xbSvncBWukcskzx/Ug4Z4/evaJcDvs5C5+mcqj2wGJMSIeDEYcuacbrUcXGEL45B9cmEYrSgfoqDGKA8BNwGv+ThBC6IFXgSuBDGCHEGKJlLLxzQQUpYXqVkzrjGZib34MZ0kebnslxuiu2NIPk/PRM54psz7Xmiwk3f8i+mA/q7MvoCq7k9zCSmKjgjAb1Spwpe10SKCQUh4GEKLRx/rRQKqU8mTtuR/B/7d35+FRlWfjx7/PTPaErCSEhCUkIWETEJBF9lVFxA23vir6U2xt7Wttbau1RezyaldbW7WodatWrTuKGy6Iyr4jEEJYshCy72Sdmef3xySTDGdmMglJJoH7c11cyZw5c86TQzL3nGe5by4HJFCIbuUXEev4PnjIKAZ/7zFqDn4DNht+EbFUbl1LY3EOQYnDiZ53c68IEu99dZR/f3iQugYLYcH+3H75GOZfIPUsRNfozbOeEoG2y1rzgCnudlZK3QHcATBkiPyBiK7jFx5D5JSljsee6mj7QlZuBU+9s8/xuKauicde28Xo5BjiYzyvTBfCG90WKJRSnwKuakk+oLV+15tDuNjmtsC31vop4Cmw18z2qpFCnAW27C8wbLNp2Lq/gKWzUnzQIrv6Bguf78glp6CatCFRzDo/ET9z6/yZjOwynl2zn8O55SQNDOeWJaMZNzzWwxGFr3RboNBan2ly/TygbY3IQYCbOpZCnLvCQwM6tL0n1Dda+Pk/vuZofiUAa785xpe78lh1+1SUUlTWNLBy9SbqGuxrVLLyKvn1M5t58ufziYt2X7dD+EZvnh67DRiulBqmlAoArgfW+LhNQvQ6cyYOIqpfoNO2AdEhTBub4KMWwZc7TziCRIudGUXsPVwCwMa9+Y4g0aLRYmPDbtc5s4Rv+SRQKKWuVErlAdOAtUqpj5u3JyilPgDQWluAu4CPgYPAf7XW+33RXiF6s34hAfzhhzNZOHkIwwdHsvjCJH5/1wyfznw6Uew6425ekeu64S10cxqV/JIath8spLLGWESqOxzOLScjuwybTXqtXfHVrKe3gbddbM8HFrd5/AHwQQ82TYhexWbTvLruEB9uOk5jk5WZ4xO5bekYggOd/3TjY0L53+t6TwLCkUlRxj9wYFSyvS7HhWMTeO79/dQ1tKZSMZsUM8cn8vf/7mbd1my0hgA/EyuuOI+LpyV1SztLK+v49TNbHHc/ibGhrLx9Kgn9w9p55bmlN3c9CXHOe2t9Fq98coiK6gZq6y18vDmbJ97Y0/4LfWzK6IHMHJ/otO3KOakMS4gAICIskMEDnFOoWG2aNV8d5ZMt2Y78jI0WG/98ay+llXXd0s5/rdnv1EV2ovgUj7/e+69vT+vN02OF6FUsVhtrvznG9oOFRIcHsXRmMimDuncNxadbjavCv95zgh9cM46ggM79+TZZrGzce5LSyjrOT49zvHl3JZNJ8bObJnH5rGT7rKehUQyNbw0MxeV1ZOaUG173zR7jfBWrTbMvq4Q5E+1zW+obLVismrBg/zNu586MQsO2vVklWKw2pxla5zoJFEJ46dH/7HQabP169wn+8MOZ3RosXGU+d5MN3StVpxq57/GvyS20jxU89/4Bbl0yiqvmGuuNd4X0odGkD402bLfabC72BrPZ9SLc2KgQmiw2nnpnH59ty8FitTF+eCw/umEC0eFBnW5fTGQwpwqcx02i+gVKkDiNXA0hvFBQeoqv9jjPyGm02Fjz1dFuPa+r1dXTxyZ0+m5izVdHHEGixUsfZfTYoHGL+JhQRiYZA8iS6cMMb/xjU/szOjmG1z/L5KNNx2my2NAadmUW8+grO8+oHdfON1YgvHaBVCU8ndxRCOGFiuoGl5/ky6tc54DqKlfPG05Dk5UPNx6jocnGrPGJrLhiTKePl5VbYdjWZLGRXVDF2NSeXez285sn8cQbe9l+sIDQYH+WzkohIiyQhNhQggLMRIQFMmNcAhc1D2Rv2GWcOrs7s5jq2kb6hXRuzcjsCYOIDAvkk63ZWK2auRMHMWXMwDP5sc5KEiiE8ELKoEgi+wVSUe38yXvSyAHdel6zSXHTJSO56ZKRZ3ys8up6EuPC2JFR5LTdz6wYPKDfGR/fFatNk1tYTURYAFH9nO8UYiKC+dVtU7BYbZhNilfXZfLXV3c5ns8vOcXSWcmOab5Bgcbpvn5mdcbdROPSYhmXJivCPZFAIYQX/P1M/PTGifzxpR1UVDegFMwcn8ji6cN83bR2lVfX85eXd7L7cDEKCAowU9/YOi31uoXphjfxrnDgWCl/enkHxeV1mEyKhZOHcOfV4zCbnMch/MwmLFYb736ZZTjGm19kMWOcffZUckIER/KcF/HNnTjYMFVYdD25wkJ4aWxqLM/+chFHTlQQ1S+IAZ1MNVFb38RHm45zOLeC5MQILrlwWJfM4HHn8df3sPuwvca3BuobrYxN7c+Y5BgmjIhzOdh8pixWG79/cRtlVfY7MJtN8/HmbFKaf97TNTZZOVVvLDnb0rV3OLecT7flOD3nZ1bcvPjM77RE+yRQCNEB/n4mRpzBG2uTxcb9T3zD0RP2T8Zf78ln/c48/vKj2d2ykrrJYmXbAWPSwNzCan53Z9dmwS0oPcVz7+9nX1Ypkf0CHEGirS37C1wGipAgf0YNi+bAsTKn7ReMsucV3bDrhGGMyGLV7M4sdkybFd1HZj0J0YO27D/pCBItcgqqXa4f6Aomk4lAFzOkQoK69g7GarWxcvUmNu49SXVtI7mFrlN4RIQFutwOcPd15zuNlYxN7c/y5jsGd0F0zVdHsVpdT7UVXUfuKITwQlFZLWEh/mf8BltYWutm+6kzOq47ZpNi8YVJvPmFc///ZTO6dmxlT1YJJ9v5GfzMJi6bkez2+YTYMB7/6VyO5VdRdaqB1EGRhDXPZpo3aTBvfnEYi9X5tuJwbgUfbDxGTmENOQVVpA2JYtm84R4Dkug4CRRCeJCVW8FfXtlBbmENAf5mLpsxjOWXjmqvOqNb7uotjO3GOgw3Lx5FRFgg63fm4WdWXDQ1iUVThnbpOSxuPtWnDorA389MdHgQV8xJIXWw58WJx09W8aeXd5BbWI2f2cRFU4dy65JRPL/2gCFItHhh7UEamuyD8weOlbH9YCGP/WQu/n7SYdJVJFCIs0J+SQ0NjVaSBoZ3+k38dFarjd89v5WSCnueocYmK29+kcWQ+H7Mm9S5KoqpgyO5bmEar392GJtNY1JwxexURjcny+sOJpPiyjmpXDkntdvOMX54rMvpw7ctHcOYlP5eHcNm0zz8/DbHnUlLypTSyno2f3vS7etagkSLvKIath4oYLoP06yfbSRQiD7tVF0Tv39xG7sy7bN6Bg8I4/7lk7tkXUBmToUjSLT1zZ6TnQ4UADdePJJFU4Zy7EQlSQkRnZ491ZsE+JtZdftUnnxzL4dyyomJCOI7F43wOkgAHMuvdNl9tcNFPqYW4aEBVJ1qNGzv6ZXmZzsJFKJPe+mjg44gAZBbWMPfXt3Fn+6edcbHDgl2/efhbntHxEWFEBfV9wNEWymDIvnT3bNoaLIS4GdyurOrrW/imXe/5cudefj7mbhoahI3Lx6Juc1iuVA3U4SbLK67tQL8zC6DhMmkmDiiexdCnmukE0/0adsPGj9tHsop75JPlEPjwxmb6vyJ2GxSXOpieqdoFehvNnT/PfnmXtZtzaHRYuNUvYW31mfx2qeZTvvEx4QyZXS80zZPvYiNFqthW2iQHz+8ZvxZcZfWm0igEH1apIvZLUEB5i5brfvArZO5YnYKQ+L7cX5aLA+tmMYIF8nszgXVtY3kF9c4qtB5q7HJytd7jHmaPt+ea9h2740TuW5BGskJEUxIj2PV7dOIifBu1fitS0bz/MqLWDC5892CwjXpehJ92lVzU3n4hW1Oi7Eum5lMQBctXgsJ8ue2pWO4rUuO1jfZbJrVb+/lky3ZWKyaxNhQ7v2fSe3OYAJ7BbmislpAYV8X3srV3UJQgB83XjKSG9vktrrn+gk8/OI2TtU1ARAdHkSZi2SMSQnhBEk6j24hV1X0adPOS2DViml8uPEY9Q1WZp6fyMKz8BPlqbom1m3NIb+khjHJMUwfl2jImdRdPtp8nA82Hnc8PlF8iodf3MbT9y/A5KYNVpvmiTf28OnWbGzNJU1P5+0n/3FpsTxw62T+81EGDU1Wxg7vz5oNR53GLhJjw9xOPRZnTgKF6PMmpMcxIT3O183oNjW1jdz9l/UUldtnYH248Tgb957kvuUX9Mj5N+0zTk0tKqvlyIkKhg+OcvmaT7dm88mW1up8jRYb/n4mAv3NBPjbB7Ovmedd3YeM7DJWrt7kWKtxOLeCCSNisVo1+SWnOC+lPzddMrLHAue5SAKFEL3cq+sOOYJEi2/25pOVW+FV98+ZcpewMCzYfQ2IbQeMkwyaLDZWrZja4boX73x5xLCgb/ehYp791SJiIoI7dCzROTKYLUQvt2W/MakfwKGcMpfbu9plM5MNXUxTRsczsH+o29dE9nOdQqMz6cxPX8QHYNNQWWOcGiu6hwQKIXo5d+MAsZE9MwV01LAYfvvdC5k0cgApgyK4aUoc99440bhjaanj2yUzkgnwd3576RfiT1hIx3NlTRxh7FbsHxnM0IHhHT6W6BwJFEL0crPGDzJsU8DDL2zl/57fSmmlcfV4VzsvtT8P3j6Vv1au59p7lhF03F4rPL+khj+/vIOVP3mB6uEjqbz3fgCSBoYbBpera5t48s29HT73FbNTmD4uwTFLql+IP9fNT5MxiR7kk0ChlLpGKbVfKWVTSk3ysN9xpdQ+pdRupdT2nmyjEL3F1XNTGX/am67GXo9h076T/PbZLYbXWKw2Nu7N590NR8g+WdXhc56qa+L59/dzz6PreeSFbfZa26tWwUMPQX4+zJ1L3b793P/412R+toW7V/+YfuXFRPz5ERp+8SsADhwtNRx3y/4CbLaOrcPw9zNz380X8N2rxuJnNlFd28Tjb+7hvse/pq7BWOzIHavVRmllHdYOnl/4bjD7W+AqYLUX+87VWpd0c3uE6JSqU418ti2Hkso6JqTHdUvqiKBAP37zvQs5klfBk2/t5VB2udPzWXmVHD9ZRVJzV0xtfZNTcSSA5ZeOYtm84V6fc9XTm8hoPk9WXiXJTz9K6jevtO6Qn4+aP5/k6Su4a90TxJxqHS8JfPi3EGAmLGS6oWpdWLC/2640TxqarLz0wUGnQe39R0tZs+EI1y1Md9p3b1Yxb3x2mJLKes5Pj+WGRSPYfrCQ5977lrKqBvpHBLHiivO4UJIGes0ngUJrfRDosiyfQvhCaWUd9/5tAyWV9sVfazYc5YrZKdy2dEy3nC9lUKTLleiA02rptd8cMxRHevmjg8yfNJio8PYHkzOOlzmCBEC/uirm7/7YsF9QcSEPvvNb1wd5+mmWrb6Mx79wrr+xdJb7ehSeHMuvpKZ5wV1be7NKnAJFZk45K1dvctw15BZWc+BoGUfzKx13MiWV9fzxpe2svm8BcZLqwyu9fYxCA58opXYope7wtKNS6g6l1Hal1Pbi4mJPuwrRJdZsOOoIEo5tXx11mXG2q8ybZCz7mZwYwbCECMfjw7kVhn0sVs3R/ErDdleqa51nE1UHh/OLa39DdZR301orwvtj++xzLl4ykXtumMDo5BhGJkXzg2XjuG5BevsHcCEuKsTlncjpM68+3Hjc0LWUlVdh6O6yWLXH1OWdlVdUzVtfHGbdlmxq642Bra/qtjsKpdSnQLyLpx7QWr/r5WGma63zlVJxwDqlVIbWeoOrHbXWTwFPAUyaNEk6Ifuo8up6DudWkBgbRmJsmK+b41FOYbVhm82mOVFcQ//I7pnff+HYBO68eixvfH6Y8qp6Jo4YwPeuGuu0z9D4cMMiOZNJMWSAd7OEzkvpT2iwvyNlBkB+VCKZz7/JxDuvt49RuFEaGs0vrn6IFTqSSdgD2+nBLbugiqff2cf+o6UM7B/GTZeMYNp5nruBosODuHT6MN776qhjW2iwP1fMTnHa71QH3py7Kh9Yi0+3ZvP3/+6mJSb95+MMfn/XzLPirqXbAoXWekEXHCO/+WuRUuptYDLgMlCIvu/9r4/yrzX7Hf3Qi6YM5a5rxvXaLsoRSVGG7LUB/mZSEiPcvKJrLL5wGIs9ZLBdMmMYG3blkV/SWtvhytkpxEbZg1f2ySre/jKLkoo6zk+LM+TGCgr04/6bL+Cvr+6kpLKeAD8Tl89OYeLiUWB+CpYscXvufyz8PvlRiRS4KYva2GRl5eqNlFXZ10bkFlbzyIvb+fPds0gd5Hnx4IrLxzB6WAzbDhYQHR7ExVOTDG/C08cmGIJkZFggFpuNmtrWIBLZL5Dp47pujKKhycq/1uyn7Y1LSWU9r32ayQ+vHd9l5/GVXrsyWykVCpi01tXN3y8Cfu3jZoluUlRWy9Pv7HP6Q/tkSzYTR8T12kHHy2Yks3V/AZk59q4ek4Lbl4521Hn2lYiwQP764zl8uTOP4oo6xqfFcl5zAaHcwmrufWwD9Y32FN17Dpdw4FgZv7ptitMxxqXF8swvF5FfXENUeJB9dXZmJtzhsQeYu9Y9wS+u/Q1jU+c6tmXlVlB1qpFRydHszix2BIkWNpvmix257QYKpRTTxyV4fIOfPWEQ2QVVrPnqKA2NVgbFhXHPDRMIDDDzn48zOHaiipRBEdx4ycgzrn/eVmHpKZdjKEdOGLsB+yKfBAql1JXA34FYYK1SarfW+iKlVALwjNZ6MTAAeLv506Qf8B+t9Ue+aK/ofvuOlOBq1uKew8W9NlCEBPnzxx/OYldmESUVdYxPi+uWOghWq40Dx8vwN5tIHxrl1R1WcKAfF09LMmx//+ujjiDRYuuBAnIKqhgS79w1ZTap1kqBmZkwd67HbieAmFNlPLrmIULuW0BtZDC/eXYL3x6xT5PtF+LPkhluBrO7sLP45sWjuGZ+GtW1jU7Foe5fPrnrTnKaATGhhu46gOSE7r27BPtU6IzjZYSFBDhmvnU1X816eht428X2fGBx8/dHgXE93DThI+6qvfX2AjTdXU0tp6CKh57Z7Mj1lDQwnFUrpnY6x1G5i3QYAOVVDQxxNaII9hXXXgSJFiGlRTB3Lu/9+XVHkAD7grtXPzlk2N+kYO5E4yD9mQgO9OvyMQhPAv3N3LpkFI+/sceR8j46PIhrF3iX+LCzMnPK+d1zWxx3aWNT+/PArZO79G4Jev+sJ3GOGJMSY6gmFxsVzILJQ33Uop5VXdvI/qOlhrxG/3h9j1NCwOMnq3juvQOdPo+roBYa7E96kusssADExMCKFcbtCQnw/vv2r6dbsYKthcaumNNvHAZEh/Dzmy/okeSG3e2iqUn8/d653Lx4JN9fNo4nfjaP+Bj3+bDOlNaaP7+8w6krb29WCa+ty/Twqs7ptWMU4tyilGLl7VP5ePNxDhwrY1BcGJdOH0Z4qG/7+3vCmq+O8ML7B2i02PAzK65dkM4Ni9JpaLJy8Lgx8d+uzKJOn2vB5CFkZJfx2dYcNOBnViydMYyggHbeClatsn996CH714QE+OILSEuzf217x/Hgg7BqFX7/+Krd9sw6P7HXdi12xtD4cIbG90wOqsKyWqcJCy12Hiri1stGd+m5JFCIXiPQ38zSmSksnZnS/s5nidzCap5591tHd4XFqvnPxxmMHx5L+tAoIsMCqTit/ndQgJn1O/OYOjq+wxXdzCZFSKCf45O9xap5ZV0mQwdGtD8LqCVYPP10a5AAR7DQc+eyb+6V/Dt6HpHPbeFQTrnbQ7UoLq/DZtNuV2t/szeftV8fo7ahieljE7hqTipms3SEAISHBhDgZ6LR4pyCvTumZssVF8KHdh0qwlUJ6h0ZhZhMymUfd1F5HX9+eQcrHv6UnIKO5XGqb7Dw0abjhu3vfJnl9LjJYuXdDUdY9fQmnnhjD7kta0ZWrYK9e1uDRIu0NB6+5xkeGLCQjOxyNn9bgNXa/gj1+p15/PEl12ncvtp9gkde2Ma+IyUcyavkxQ8OsvrtfV78lOeGkCB/LpvpPDnAz6y4em5ql59L7iiE8KHoCNcpNVq2XzYzmfiYENbvyGPHoSKnWTUV1Q08v/YAK2+b6vX5Gpqshk+gYFyN/Yd/b2fzt611ML7clcejP5pNQmyYfcziNMfyK9l0sv36EDERQZSetpr96z35XJlTTtoQ53GStovrWqzbmsMtS0Z1+WBtX7X80lEMiQ9n0758QoP9WXzhMMN17AoSKITwoSmjBzIkvh85Ba2rvOOigpkzoTW1+AWj4jkvpT/X/GKt4fWnJwhsT0RYICOGRjnlcgKYOmag4/vsgiqnIAFQW2/hva+P8t0r7avAd2YU8cnWbLTWzJs4mMAAM+0JDjRzXkp/1u/MMzyXW1hteIOrqTMGHovVRkOjVQJFM6WUy9XvXU0ChRA+5O9n4uHvz2DNhiNk5pSTlBDBFbNTDG+EgQFm4qJDKCpzTrI3JL5fh85XdaqR2gbn2Ugjk6K4flFrDqbTP/G33a615tNtOTz22m7H9o17T7LiijH0C/GnutZ1Co3w0AAeWjGNnMJqQ6BQCtKHGj8FTx0zkNzCw6e1NdqrxIaia8kYhRA+VNdg4dV1h/h8Ry6VNY0kDexHtIs3QqUUt1w6ymnQN8DfzI0Xj+zQ+V7/LJOcghqnbYVltfi3GSAemRRNSJDxM6RNa/5n5YdOQaLFW19kcd/yC4hx05VWXWtPx/7se98aKt8tmzecQXHGgHf9wnTmTBjk+JnThkTy4+9MaP+HFF1OaVcjaX3cpEmT9PbtUudI9H6/e26LoZvn5zdPYsa4RJf7Hz9ZxYZdefj7mZk7cVCH5+nf89cv7UWITvP4T+c6rczeur+Av7yyk1N1TSgFaUOiPHZzmRS8/YelaK35/h8+N0zbNCkMK+8vPG8g37l4RLvTSStrGmhosrpdlCm6hlJqh9baZSE56XoSwkfKqurZsr/AsP3DjcfdBoqkgeEkDRzV6XMmxIQaAkWAv9mw0nvy6HieX7mIrNwK+kcG87fXdnk87vj0uOZP/orrFqbz6Cs7nZ53lZ4lu6DaqzUHEW5qcIieI11PQvhIY5PV5dTYhiarcWMXWTZ/uCG1xdVzUwkNNg4OBwX4MSalP8XldZx0sbCrxeABYdzZJtX5vEmDeeDWyUwaOcBlF1YLq804+0r0TnJHIYSPxMeEkj4kyrAwbdb5ru8musKwhAge+8kcPt6cTXVtI1PHDGTSSPe5qtZ+c4x/vrXX5XNpgyP57lVjCQowc+REJSalHGm/p44ZyNQxA7nWxUytFnMmdO9MHdF1JFAI4UM/u3kSj722iz2HSwgONLP4wmEsmd65cqHeio8JZfml7XdfWa02Xvkkw+VzkWGBPLhiKv/+MMOxgM+k4IaLRnB9m9KkI4ZGsSvTueKkUrB0ZgrXLXRetFffYCG3qJr4mFD6+ThVu3AmgUIIH4qLCuG335tObX0T/n4m/P3aX4/QE8qr6/n3BweprHG9iK6ipoGNe086rfK2aXj5owymnTfQMfZw29Ix/PKfGx1pSEKC/Hjw9qmMGua8aO/z7TmsfnsftfUW/P1MXDM/jfkXDObbI6UMiA5hdLJxkZ/oOTLrSQjhxGrT/PBPX7Sm7XAjdVAEWXnGOtzfvfI8p7oT9Q0Wth4owGrTTB4VbxgPKS6vY8X/rTPUum47U2r88Fh+ddsUp0p8omvJrCchhNd2HSrvGRiXAAAIxElEQVRqN0gkJ0aQV1Tj8rnTp+wGBfox6/xBLvcF2Hmo0BAkwHmm1O7Dxazbks2l7gofiW4ls56EEE7Kqupcbjc1v1tMGBHHz26aZKiUB/a7gAnpcU7b8oqqefLNPfzuuS18uOk4VqvzbKdIL6e/7j9mTLkueobcUQghnJiU68+P/ibFvx68yLGuYXxaLLtPG6i2afjNv7awdFYy44bHcqK4hp/8bQN1DRYANn9bwMFjpfz4OxMdr5k0cgBD4/uRXeD5LiYxNuxMfixxBuSOQgjhxF0ajgaLdsrT9INl41ymG9meUcjKpzZx91/W8/pnmY4g0WL9zjwKSlvXZZjNJn5353SunpvK6OQYLp6WxIVjBzq9pn9kMIunJ53BTyXOhNxRCCGcjB0eS3R4EGVVxuSAbccl4mNCGZ8Wy+fbc10e5/jJKmrrXZRD1fYEg23HMiLCArllSWtVNptNs3FfPnsOlzAgOoSFk4fICm0fkjsKIYQTs0lx33KXk19IHxrFrkNF7MsqwWbTpAyK8HiskgpjsOkX4t9ujWyTSTFjXCI/WDaOZfOGS5DwMbmjEEIYjEyK4TsXjeCVTzIcaUbGpMTw0gcHKK2yr4lIjA3jgVsm42c2YbG6TsdhO236fWiQH/fcMIFAmebap0igEEK4dMOidGafn8iBY6UkxIbx/PutQQLgRHENL3+cQeqgCEMhJFeWzU/luvnpHa7zLXxP/seEEG4lxIaREBtGk8XKwePG6al7DhcbBqvd+WhjNt9Z1LH6GaJ38MkYhVLqj0qpDKXUXqXU20oplx2WSqmLlVKHlFJZSqn7erqdQgg7P7PJ5Qyn8NAAl4vlXKmpa2LfkZKubproAb4azF4HjNFajwUygftP30EpZQYeBy4BRgE3KKU6n4hfCNFpSiluaFMuFewDztctTMPPbHwbiQxzndRPxib6Jp90PWmtP2nzcDOwzMVuk4EsrfVRAKXUq8DlwIHub6EQ4nQXT0tiYEwoX+zMxc9sYuHkIaQPjaasqoEX1rb+WSbGhvLdK8ey6ulNTmk4kgaGM2pYtA9aLs5Ubxij+H/Aay62JwJtJ2jnAVN6pEVCCJfGpcUyLi3WaduyecOZkB7HjoxCYiKCmT4ugUB/Mw+umMZr6w5RVFbL2OGxLL90FEopN0cWvVm3BQql1KdAvIunHtBav9u8zwOABXjZ1SFcbHPbGaqUugO4A2DIkCEdbq8QovOSEyNITnReUzEhPc6Q90n0Td0WKLTWCzw9r5RaDiwB5mvXuc7zgLYlsAYB+R7O9xTwFNjTjHe4wUIIIVzy1ayni4GfA0u11rVudtsGDFdKDVNKBQDXA2t6qo1CCCHsfDXr6R9AP2CdUmq3UuqfAEqpBKXUBwBaawtwF/AxcBD4r9Z6v4/aK4QQ5yxfzXpKdbM9H1jc5vEHwAc91S4hhBBGkhRQCCGERxIohBBCeCSBQgghhEfK9czUvk0pVQxkd+Eh+wOSpMZOroWdXIdWci1a9eVrMVRrHevqibMyUHQ1pdR2rbXrSi7nGLkWdnIdWsm1aHW2XgvpehJCCOGRBAohhBAeSaDwzlO+bkAvItfCTq5DK7kWrc7KayFjFEIIITySOwohhBAeSaAQQgjhkQQKF5RS0UqpdUqpw81fo9zsZ21OarhbKXXWZLZtr1a5UipQKfVa8/NblFJJPd/KnuHFtbhFKVXc5vfgdl+0s7sppZ5VShUppb5187xSSj3WfJ32KqUm9HQbe4oX12KOUqqyze/Eyp5uY1eTQOHafcBnWuvhwGfNj12p01qPb/63tOea1328rFV+G1DenNzxUeD3PdvKntGBuu2vtfk9eKZHG9lzngcu9vD8JcDw5n93AE/2QJt85Xk8XwuAr9r8Tvy6B9rUrSRQuHY58ELz9y8AV/iwLT3NUatca90ItNQqb6vt9XkDmK/OzhqX3lyLc4LWegNQ5mGXy4EXtd1mIFIpNbBnWtezvLgWZx0JFK4N0FqfBGj+6q6eY5BSartSarNS6mwJJq5qlSe626e5bkglENMjretZ3lwLgKubu1veUEoNdvH8ucDba3WumKaU2qOU+lApNdrXjTlTPqlH0Rt4qundgcMM0VrnK6WSgc+VUvu01ke6poU+402t8g7VM+/DvPk53wNe0Vo3KKW+h/1Oa163t6z3OVd+J7yxE3vepBql1GLgHexdcn3WORsoPNX0VkoVKqUGaq1PNt8+F7k5Rn7z16NKqfXA+UBfDxTe1Cpv2SdPKeUHRHB23oq3ey201qVtHj7NWTpe44UO1bg/m2mtq9p8/4FS6gmlVH+tdV9NFihdT26sAZY3f78cePf0HZRSUUqpwObv+wPTgQM91sLu402t8rbXZxnwuT47V262ey1O64dfir1s77loDXBz8+ynqUBlS/ftuUYpFd8yZqeUmoz9fbbU86t6t3P2jqIdjwD/VUrdBuQA1wAopSYB39Na3w6MBFYrpWzYfxEe0Vr3+UChtbYopVpqlZuBZ7XW+5VSvwa2a63XAP8C/q2UysJ+J3G971rcfby8Fv+rlFoKWLBfi1t81uBupJR6BZgD9FdK5QEPAv4AWut/Yi9ZvBjIAmqBW33T0u7nxbVYBtyplLIAdcD1ff2DlKTwEEII4ZF0PQkhhPBIAoUQQgiPJFAIIYTwSAKFEEIIjyRQCCGE8EgChRDdTCk1WCl1TCkV3fw4qvnxUF+3TQhvSKAQoptprXOxZ1N9pHnTI8BTWuts37VKCO/JOgoheoBSyh/YATwLrADOb85IK0SvJyuzhegBWusmpdRPgY+ARRIkRF8iXU9C9JxLgJPAGF83RIiOkEAhRA9QSo0HFgJTgXvO1qI+4uwkgUKIbtacSfRJ4Eda6xzgj8CffNsqIbwngUKI7rcCyNFar2t+/AQwQik124dtEsJrMutJCCGER3JHIYQQwiMJFEIIITySQCGEEMIjCRRCCCE8kkAhhBDCIwkUQgghPJJAIYQQwqP/DwiS9TVmqrnpAAAAAElFTkSuQmCC\n", 141 | "text/plain": [ 142 | "
" 143 | ] 144 | }, 145 | "metadata": { 146 | "needs_background": "light" 147 | }, 148 | "output_type": "display_data" 149 | } 150 | ], 151 | "source": [ 152 | "sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = label, palette = sns.color_palette('deep', K), edgecolor = \"none\")\n", 153 | "sns.scatterplot(x = centroid[:, 0], y = centroid[:, 1], marker = \"x\", facecolor='red', s = 10**2, linewidth = 5)\n", 154 | "plt.title('Dataset')\n", 155 | "plt.ylabel('Y')\n", 156 | "plt.xlabel('X')\n", 157 | "plt.show()" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 70, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "dist = [0]*10\n", 167 | "for i in range(10):\n", 168 | " _, _, dist[i] = k_means(X, i + 1, num_iters = 100)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": 71, 174 | "metadata": {}, 175 | "outputs": [ 176 | { 177 | "data": { 178 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAbL0lEQVR4nO3de5RcZZ3u8e/TaRpCCBKSJgcTMMJAFC8EaVkcAw4KMogwgIogKOBwc8ARHGc5gB4QR4XRg85hEJTbkChGEMjACHpgAgrC4dLcQQgicmmISQghhCTQJP07f+y3dipNdXWl6apd3fV81upVVe++1K82pJ7a77svigjMzMwA2oouwMzMmodDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FG9UkbSQpJE19C+s4U9J5w1mXWbNyKFjDSXq17K9P0qqy14cPsuw+kp4cxlrulPSapOWSXpF0j6R/krRBaZ6IOCMivlTjuj43XLU1k/6fTdLekl6WdFCRddnwcyhYw0XEJqU/4Flg/7K2ywso6ZiIGA+8HTgVOAq4toA6RgRJ+wFXAodHxNyi67Hh5VCwpiNprKQfSVogqUfS9yVtIGkiMBfYpmzPYqKkmZLukrRM0guSfiipfX3fNyJejYj/Bg4A9pS0V6rnbEkXp+fjJP1C0kvpl/JdkiZIOgf4IHBxquucNP8F6TO8IuluSbuWfc6zJV0uaU7aU3lI0oyy6dMkXSvpxfR3Ttm04yXNT3VcL2nKANvyt5KO6dc2X9K+ksZIOk/S4rTtHpQ0vdo2kvRJ4KfApyPi+vXbwjYSOBSsGZ0JvB94H7AzsAfwtYhYAhwEPFW2Z7EEeAP4ErA5sDuwP3BMpRXXIiL+BDyY1tXfMUA7MAWYlN63NyK+CtxDttexSXoN8P/S55hItvfxy/KuqfR5LgU2A+YB/waQ5vk18BiwNbAVcHWadihwcvqck4H7gZ8N8HF+Dny29ELSzmTb6UZgP7Ltuy0wATgMWFpl03wKuAQ4IIWnjUIOBWtGhwNnRMSLEbEQ+Dbw+YFmjoi7I+KeiFiTvtAvBv76LdbwAtmXZ39vAJ3AthGxOr3viiq1zY6IpRHxBvBdsnDYpmyWmyPipohYQ/YLvLSnsBuwKXBaRKyMiFURcUeadjzw7Yh4Iq33TGA3SZMrlHAV8CFJW6bXhwG/jIjV6bNsCrwrKzUejYhFVbbJnsCjwF1V5rERzqFgTUWSgP8BPFPW/AzZL/OBltlB0q8lLZT0CnA62a/4t2IK8FKF9kuA3wFXpW6h70oaU6W2U1N3zTKyX+Eb9avtL2XPVwKbpOdbAX+OiL4Kq30H8OPUffUysBhYDbzpCKuIeAm4CfiMpDbgEKA0bvPr9Hl+AiyUdL6kTfqvo8w/k+0lXdVvb8dGEYeCNZXILtv7F7IvvpKtgedLs1RY7CLgPrJf75sC3wI01BokbUPWfXVbhfpej4jTI+JdwIeBg4FDK9Um6WPAP5B1EW1GtuexqsbangOmpS/yStOOiojNyv7GRsS9A6xrDlkX0l+Thccd6bNERPwgInZKn3dH4KQqNS0H9iEbkJ9TLQxt5HIoWDOaA5yRBpG3AL7O2j7zhcAW/X7RjgeWRcSrkt4DHDuUN02DyB8F/hP4baV+c0l7pT2TNuAVsi/ZNWW1lXcNjSfrolkMdJCF1UY1lvN7si/hf5G0cRp8/1Ca9mPgG6VB4TTQ/akq67oWeA/ZdpyTghdJu0rqSoPyK4Dess9SUUS8DOwNbA/MHiC0bATzf1BrRqcDfyDrv34AuB34Xpr2IHAd8EzqPtkc+ApwjKRXgR8BV6zn+10saTnZHsr3ybpX9h9g3ilkX7LLgUeAG8gOzwT4IXCEpKWSvgf8F3Ar8CfgKeBFsoAYVBor2Jfs13sP2aG7n0zT5gDnAdek7rIHgI9VWddKsm22J9nAc8lmwGXAy6m+Z4Bza6htCbAXsBPZthvyXpk1H/kmO2ZmVuI9BTMzyzkUzMws51AwM7OcQ8HMzHLrfX2YZjJp0qSYNm1a0WWYmY0o995774sR0Vlp2ogOhWnTptHd3V10GWZmI4qkZwaa5u4jMzPLORTMzCznUDAzs5xDwczMcg4FMzPLjeijj4aqry9YsqKX3tVr6Ggfw8RxHbS1+ZpeZmYtFwp9fcH8hcs5dnY3PUtXMXXCWC46oovpk8c7GMys5bVc99GSFb15IAD0LF3FsbO7WbKit+DKzMyK13Kh0Lt6TR4IJT1LV9G7uuq9RczMWkLLhUJH+ximThi7TtvUCWPpaPedBc3MWi4UJo7r4KIjuvJgKI0pTBzXUXBlZmbFa7mB5rY2MX3yeOaeMNNHH5mZ9dNyoQBZMHSO37DoMszMmk7LdR+ZmdnAHApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmubqFgqStJN0i6TFJj0o6KbVvLukmSX9MjxNSuySdK+lJSQ9J+kC9ajMzs8rquaewGvhqRLwb2BU4UdIOwCnAvIjYDpiXXgN8HNgu/R0HXFDH2szMrIK6hUJELIiI+9Lz5cBjwBTgAGBWmm0WcGB6fgAwOzJ3AptJ2rJe9ZmZ2Zs1ZExB0jRgJ+AuYHJELIAsOIAt0mxTgOfKFutJbf3XdZykbkndixcvrmfZZmYtp+6hIGkT4Grg5Ih4pdqsFdriTQ0RF0ZEV0R0dXZ2DleZZmZGnUNB0gZkgXB5RFyTmheWuoXS46LU3gNsVbb4VOCFetZnZmbrqufRRwIuAR6LiB+UTboOODI9PxK4tqz9iHQU0q7AslI3k5mZNUZ7Hdc9E/g88LCkB1LbacDZwJWSjgaeBQ5O024A9gWeBFYCX6hjbWZmVkHdQiEifk/lcQKAPSvMH8CJ9arHzMwG5zOazcws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHI1hYKkqZI+kp5vKGlcfcsyM7MiDBoKkv4OuA64ODW9A7i2nkWZmVkxatlT+DKwK/AKQEQ8AWxRz6LMzKwYtYTCaxHRW3ohaQyg+pVkZmZFqSUUbpf0NWCjNK5wBfCr+pZlZmZFqCUUvgYsBx4HTgLmAV+vZ1FmZlaMWkJhA+AnEXFQRBwI/AToGGwhSZdKWiTpkbK2b0p6XtID6W/fsmmnSnpS0nxJfzOUD2NmZm9NLaFwC1B+COo44OYalrsM2KdC+w8jYkb6uwFA0g7AocB70jLnp7ELMzNroFpCYWxELC+9SM83HmyhiLgVeKnGOg4AfhERr0fEn4EngV1qXNbMzIZJLaGwUtKOpReSZgCvvYX3/JKkh1L30oTUNgV4rmyentT2JpKOk9QtqXvx4sVvoQwzM+uvllD4CjBX0i2SbgGuJjt3YSguALYFZgALgHNSe6VDXKPSCiLiwojoioiuzs7OIZZhZmaVtA82Q0TcJendwLvJvrwfLT9vYX1ExMLSc0kXsfbQ1h5gq7JZpwIvDOU9zMxs6Gq9IN6OwPZkwfBpSYcN5c0kbVn28iCgdGTSdcCh6bpK7wS2A+4eynuYmdnQDbqnIOkyYAfgAWBNag7g54MsNwfYA5gkqQc4A9gjjUkE8DRwPEBEPCrpSuAPwGrgxIhYU2m9ZmZWP4qo2HW/dgbpcWCHiOhrTEm16+rqiu7u7qLLMDMbUSTdGxFdlabV0n30KDBpeEsyM7NmNGj3EfA24DFJdwKvlxoj4pN1q8rMzApRSyicVfcqzMysKdRySOq8RhRiZmbFq+XOax+UdKekZZJek/S6pFcaUZyZmTVWLd1H5wOfA35Bdj2io1j3RDMzMxslajn6qC0i5gPtEfFGRFwE7FXnuszMrAC17CmskNQBPCjpu2TXLNqkvmWZmVkRatlTOCrN9yWyM5q3Az5Vx5rMzKwgtYTCvhHxWkS8HBH/KyK+DPjOaGZmo1AtofB3FdqOHu5CzMyseAOOKUg6hOwWme+UdE3ZpE2Bl+tdmJmZNV61gea7gSVk9zb4UVn7cuD+ehZlZmbFGDAU0r2S/yzpDmBVRISkbYHpDHBXNDMzG9lqGVO4FRibbpDzO+DvgUvrWpWZmRWi1pPXVpIdhnpeROwPvL++ZZmZWRFqCgVJHwQOY+09lcfUryQzMytKLaHwj8CZwPUR8YikbYDb6luWmZkVoZZLZ98M3Fz2+inghHoWZWZmxah2nsI5EfFVSXOpcLSR77xmZjb6VNtTuCI9nteIQszMrHjVzlO4Oz3OkzQhPV/aqMLMzKzxqg40S/qGpIXA08Czkv4i6bSGVGZmZg03YChIOgn4KLBbRLwtIsYDHwY+KunLjSrQzMwap9qewpHAIRHxx1JDRDxBdr7CUXWuy8zMClAtFDoiYnH/xohYBGxYv5LMzKwo1UKht8q014e7EDMzK161Q1J3lPRShXbhezSbmY1K1UKho2FVmJlZU6h2nsKaRhZiZmbFq+WCeGZm1iIcCmZmlnMomJlZrtpVUpdS+V7MAiIiNq9bVWZmVohqRx9NalgVZmbWFGo++kjS5sBGZU0v1KsoMzMrxqBjCpI+IekJoAe4Kz3eXH0pkHSppEWSHilr21zSTZL+mB4npHZJOlfSk5IekvSBoX8kMzMbqloGmr8DzATmR8RWwN8Av61hucuAffq1nQLMi4jtgHnpNcDHge3S33HABTWs38zMhlktobA6XRivTZIi4iZg0F/yEXEr0P8yGQcAs9LzWcCBZe2zI3MnsJmkLWv6BGZmNmyqDTSXLJM0Dvg9MFvSIqBviO83OSIWAETEAklbpPYpwHNl8/WktgX9VyDpOLK9CbbeeushlmFmZpXUsqdwIPAacDJZt9HzwH7DXIcqtFU6HJaIuDAiuiKiq7Ozc5jLMDNrbbWEwqkRsSYi3oiISyLiB8A/DvH9Fpa6hdLjotTeA2xVNt9UfHSTmVnD1RIK/QeLAT4xxPe7juyObqTHa8vaj0hHIe0KLCt1M5mZWeNUO6P5eOCLwPaS7iubNB7oHmzFkuYAewCTJPUAZwBnA1dKOhp4Fjg4zX4DsC/wJLAS+MJ6fxIzM3vLqg00X0l22OhZrD10FGB5uiVnVRHx2QEm7Vlh3gBOHGydZmZWX9XOaF4KLAUOlvReYLc06TbWjgWYmdkoUssZzSeS7TVsnf6ulHRCvQszM7PGq+U8heOBXSLiVQBJ3wXuAM6vZ2FmZtZ4tRx9JOCNstdvUPm8AjMzG+GqHX3UHhGrgZ8Cd0q6Ok06iLWXqjAzs1GkWvfR3cAHIuJ7km4BdifbQ/hiRNzTkOrMzKyhqoVC3kWUQsBBYGY2ylULhU5JA17OIl3uwszMRpFqoTAG2AQPKpuZtYxqobAgIr7VsErMzKxw1Q5J9R6CmVmLqRYKb7pGkZmZjW4DhkJE9L+VppmZjXK1nNFsZmYtwqFgZmY5h4KZmeUcCmZmlnMomJlZrpb7KVgd9PUFS1b00rt6DR3tY5g4roO2Np8aYmbFcigUoK8vmL9wOcfO7qZn6SqmThjLRUd0MX3yeAeDmRXK3UcFWLKiNw8EgJ6lqzh2djdLVvQWXJmZtTqHQgF6V6/JA6GkZ+kqelevKagiM7OMQ6EAHe1jmDph7DptUyeMpaN9TEEVmZllHAoFmDiug4uO6MqDoTSmMHFcR8GVmVmr80BzAdraxPTJ45l7wkwffWRmTcWhUJC2NtE5fsOiyzAzW4e7j8zMLOdQMDOznEPBzMxyDgUzM8s5FMzMLOdQMDOznEPBzMxyDgUzM8s5FMzMLOdQMDOzXCGXuZD0NLAcWAOsjoguSZsDVwDTgKeBz0TE0iLqMzNrVUXuKXwkImZERFd6fQowLyK2A+al12Zm1kDN1H10ADArPZ8FHFhgLWZmLamoUAjgRkn3SjoutU2OiAUA6XGLgmozM2tZRV06e2ZEvCBpC+AmSY/XumAKkeMAtt5663rVZ2bWkgrZU4iIF9LjImAusAuwUNKWAOlx0QDLXhgRXRHR1dnZ2aiSzcxaQsNDQdI4SeNLz4G9gUeA64Aj02xHAtc2ujYzs1ZXRPfRZGCupNL7/zwifiPpHuBKSUcDzwIHF1CbmVlLa3goRMRTwI4V2pcAeza6HjMzW6uZDkk1M7OCORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs1xR91OwJtHXFyxZ0Uvv6jV0tI9h4rgO2tpUdFlmVhCHQgvr6wvmL1zOsbO76Vm6iqkTxnLREV1MnzzewWDWotx91MKWrOjNAwGgZ+kqjp3dzZIVvQVXZmZFcSi0sN7Va/JAKOlZuore1WsKqsjMiuZQaGEd7WOYOmHsOm1TJ4ylo31Mw2vp6wsWL3+d55euZPHy1+nri4bXYGYOhZY2cVwHFx3RlQdDaUxh4riOhtZRGts46Pzbmfmvt3DQ+bczf+FyB4NZARQxcv/hdXV1RXd3d9FljGjNcPTR4uWvc9D5t6/TlTV1wljmnjCTzvEbNrQWs1Yg6d6I6Ko0zUcftbi2NhX+xeuxDbPm4e4jK1wzjW2YtTqHghWuWcY2wAPeZu4+ssK1tYnpk8cz94SZhY5tNMvJfM0wzmOty3sK1hRKYxtTJmxM5/gNC/kSbIaT+XwklhXNoWCWNMOAdzMEU7Nxl15jufvILCkNePc/NLaRA97NEEzNpFm69Eq1FN2t14gavKdgljTDgHczHYnVDL/Qm2XPqRm69RpVg0PBLCkf8L79nz/C3BNmNvwXaTMEEzTHlyA0z55TM4RTo2pw95FZmaJP5muWI7EG+gJq9FnmzdClB80RTo2qwXsKZk2mGY7EaoYvQWiePadm6NZrVA0OBTN7k2b4EoTm6NKD5ginRtXgC+KZ2Zs001E/zWI0HX3kC+KZ2XpplrGNZlL0eFOjanAomFlFzfAlaI3nMQUzM8s5FMzMLOdQMDOznEPBzMxyDgUzM8uN6PMUJC0Gnim6jrdoEvBi0UU0EW+Ptbwt1uXtsdZb3RbviIjOShNGdCiMBpK6BzqJpBV5e6zlbbEub4+16rkt3H1kZmY5h4KZmeUcCsW7sOgCmoy3x1reFuvy9lirbtvCYwpmZpbznoKZmeUcCmZmlnMoFETSVpJukfSYpEclnVR0TUWTNEbS/ZJ+VXQtRZO0maSrJD2e/h/5n0XXVBRJX0n/Rh6RNEfSRkXX1EiSLpW0SNIjZW2bS7pJ0h/T44Thej+HQnFWA1+NiHcDuwInStqh4JqKdhLwWNFFNIn/A/wmIt4F7EiLbhdJU4AvA10R8V5gDHBosVU13GXAPv3aTgHmRcR2wLz0elg4FAoSEQsi4r70fDnZP/opxVZVHElTgU8AFxddS9EkbQp8GLgEICJ6I+LlYqsqVDswVlI7sDHwQsH1NFRE3Aq81K/5AGBWej4LOHC43s+h0AQkTQN2Au4qtpJC/RvwNaCv6EKawDbAYuA/UnfaxZLGFV1UESLieeB/A88CC4BlEXFjsVU1hckRsQCyH5jAFsO1YodCwSRtAlwNnBwRrxRdTxEk7Qcsioh7i66lSbQDHwAuiIidgBUMY/fASJL6yg8A3gm8HRgn6XPFVjW6ORQKJGkDskC4PCKuKbqeAs0E/lbS08AvgI9K+lmxJRWqB+iJiNKe41VkIdGK9gL+HBGLI+IN4BrgQwXX1AwWStoSID0uGq4VOxQKIklkfcaPRcQPiq6nSBFxakRMjYhpZIOIN0dEy/4ajIi/AM9Jmp6a9gT+UGBJRXoW2FXSxunfzJ606KB7P9cBR6bnRwLXDteK24drRbbeZgKfBx6W9EBqOy0ibiiwJmse/wBcLqkDeAr4QsH1FCIi7pJ0FXAf2RF799Nil7uQNAfYA5gkqQc4AzgbuFLS0WTBefCwvZ8vc2FmZiXuPjIzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwZqepFfLnu+brgy5dZX595AUkvYva/uVpD2Gua4NJf23pAckHVJh+j+lq5w+IulBSUcM8X1mSNp3CMt1STp3gGlPS5o0lHpsdPN5CjZiSNoT+Hdg74h4dpDZe4CvA/9Vx5J2AjaIiBn9J0j6IvAxYJeIeEXS2xj6RctmAF3Am85hkdQeEasrLRQR3UD3EN/TWpT3FGxEkLQ7cBHwiYj4Uw2LPAgsk/SxCuvaM11o7uF0rfoNB3nvzSX9p6SHJN0p6f2StgB+BsxIewrb9lvsNOCE0vWsImJZRMxK69tZ0u8k3Svp/5ZdruC3kv5V0t2SnpC0ezp57VvAIaU9EknflHShpBuB2ZI2kvQf6fPcL+kjaX17lO5NIWmipBvT9J8ASu3jJF2f9mQeqbTHY63FoWAjwYZkp/EfGBGPr8dy3wa+Ud6QbtByGXBIRLyPbG/57wdZz5nA/RHxfrIv+9kRsQg4BrgtImaUB5Wk8cD4SuGVrnf178CnI2Jn4FLgO2WztEfELsDJwBkR0QucDlyR3ueKNN/OwAERcRhwIkD6PJ8FZlW4Ec0ZwO/TBfauA0rdb/sAL0TEjul+Bb8ZZFvYKOdQsJHgDeAO4Oj1WSgiboN8L6NkOtkF1p5Ir2eR3bugmt2An6Z13gxMTN1BAxEw0KUCpgPvBW5Klzf5BjC1bHrpwoj3AtOqvMd1EbGqQn2PA88A2/eb/8NkezZExPXA0tT+MLBX2kPZPSKWVXlPawEOBRsJ+oDPAB+UdNp6LvsdsrGFEg3h/SstM+D1YVKX0QpJ2wywrkfTr/4ZEfG+iNi7bPrr6XEN1cf8VgxSX8XSKtT6BNlex8PAWZJOr3FdNko5FGxEiIiVwH7A4ekiYEg6SNJZgyx3IzCB7JaWAI8D0yT9VXr9eeB3g7z9rcDh6T33AF6s4d4XZwE/SndRQ9Kmko4D5gOdSvdclrSBpPcMsq7lwPga69uerGtofpV5Pk62TZD0dmBlRPyM7GY2rXqJbkt89JGNGBHxkqR9gFslvQhsC9RyY6LvkC4tHBGvSfoC8Etlt3e8B/gxgKRvAd0RcV2/5b9Jdhe0h4CVrL1kcTUXAJsA90h6g6wL7JyI6JX0aeDc1AXVTnbXuUerrOsW4JTU3VQpBM8HfizpYbIriR4VEa9nV5rOnQnMkXQfWQiWjt56H/B9SX2pxsHGV2yU81VSbcRSdiOer0TE4qJrMRstHApmZpbzmIKZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlvv/WANbot2LtLEAAAAASUVORK5CYII=\n", 179 | "text/plain": [ 180 | "
" 181 | ] 182 | }, 183 | "metadata": { 184 | "needs_background": "light" 185 | }, 186 | "output_type": "display_data" 187 | } 188 | ], 189 | "source": [ 190 | "sns.scatterplot(range(1, 11), dist)\n", 191 | "plt.title('Total Distance vs K')\n", 192 | "plt.ylabel('Total Distance')\n", 193 | "plt.xlabel('K, No. of Centroids')\n", 194 | "plt.show()" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": null, 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [] 203 | } 204 | ], 205 | "metadata": { 206 | "kernelspec": { 207 | "display_name": "Python 3", 208 | "language": "python", 209 | "name": "python3" 210 | }, 211 | "language_info": { 212 | "codemirror_mode": { 213 | "name": "ipython", 214 | "version": 3 215 | }, 216 | "file_extension": ".py", 217 | "mimetype": "text/x-python", 218 | "name": "python", 219 | "nbconvert_exporter": "python", 220 | "pygments_lexer": "ipython3", 221 | "version": "3.7.4" 222 | } 223 | }, 224 | "nbformat": 4, 225 | "nbformat_minor": 2 226 | } 227 | -------------------------------------------------------------------------------- /k-means/k-means.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import seaborn as sns 4 | import os, sys 5 | 6 | def init_centroid(X, K): 7 | m = X.shape[0] 8 | idx = np.random.choice(m, K, replace = False) 9 | return X[idx, :] 10 | 11 | def update_label(X, centroid): 12 | m, K = X.shape[0], centroid.shape[0] 13 | dist = np.zeros((m, K)) 14 | label = np.zeros((m, 1)) 15 | 16 | for i in range(m): 17 | for j in range(K): 18 | dist[i,j] = np.dot((X[i, :] - centroid[j, :]).T, (X[i, :] - centroid[j, :])) 19 | 20 | label = np.argmin(dist, axis = 1) 21 | total_dist = np.sum(np.choose(label, dist.T)) 22 | return label, total_dist 23 | 24 | def update_centroid(X, label, K): 25 | D = X.shape[1] 26 | centroid = np.zeros((K, D)) 27 | for i in range(K): 28 | centroid[i, :] = np.mean(X[label.flatten() == i, :], axis=0).reshape(1,-1) 29 | return centroid 30 | 31 | def k_means(X, K, num_iters = 100): 32 | m = X.shape[0] 33 | centroid = init_centroid(X, K) 34 | 35 | for _ in range(num_iters): 36 | label, total_dist = update_label(X, centroid) 37 | centroid = update_centroid(X, label, K) 38 | 39 | return centroid, label, total_dist 40 | 41 | if __name__ == '__main__': 42 | images_dir = os.path.join(sys.path[0], 'images') 43 | if not os.path.exists(images_dir): 44 | os.makedirs(images_dir) 45 | 46 | N = 80 # number of points per class 47 | D = 2 # dimensionality, we use 2D data for easy visulization 48 | K = 3 # number of classes 49 | X = np.zeros((N * K, D), dtype = float) # data matrix (each row = single example, can view as xy coordinates) 50 | 51 | for i in range(K): 52 | r = np.linspace(0.05, 1, N).reshape(-1, 1) # radius 53 | t = np.random.normal(0, 0.4, (N, 1)) # theta 54 | X[i * N:(i + 1) * N] = np.append(r * np.sin(t) + i/2, r * np.cos(t) - 2*i/2, axis = 1) 55 | 56 | # sns.scatterplot(x = X[:, 0], y = X[:, 1], edgecolor = "none") 57 | # plt.title('Dataset') 58 | # plt.ylabel('Y') 59 | # plt.xlabel('X') 60 | # plt.savefig(os.path.join(images_dir, 'dataset.png')) 61 | # plt.clf() 62 | 63 | centroid, label, _ = k_means(X, K, num_iters = 50) 64 | 65 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = label, palette = sns.color_palette('deep', K), edgecolor = "none") 66 | sns.scatterplot(x = centroid[:, 0], y = centroid[:, 1], marker = "x", facecolor='red', s = 10**2, linewidth = 5) 67 | plt.title('Dataset') 68 | plt.ylabel('Y') 69 | plt.xlabel('X') 70 | plt.savefig(os.path.join(images_dir, 'k-means.png')) 71 | plt.clf() 72 | 73 | dist = [0]*10 74 | for i in range(10): 75 | _, _, dist[i] = k_means(X, i + 1, num_iters = 100) 76 | 77 | sns.scatterplot(range(1, 11), dist) 78 | plt.title('Total Distance vs K') 79 | plt.ylabel('Total Distance') 80 | plt.xlabel('K, No. of Centroids') 81 | plt.savefig(os.path.join(images_dir, 'total_dist_vs_k.png')) 82 | plt.clf() 83 | -------------------------------------------------------------------------------- /linear_regression/images/Error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/linear_regression/images/Error.png -------------------------------------------------------------------------------- /linear_regression/images/cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/linear_regression/images/cost.png -------------------------------------------------------------------------------- /linear_regression/images/data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/linear_regression/images/data.png -------------------------------------------------------------------------------- /linear_regression/linear_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import seaborn as sns 4 | import os, sys 5 | 6 | def cost(X, y, theta): 7 | h = np.dot(X, theta) 8 | cos = np.sum((h - y) * ( h - y))/(2 * len(y)) 9 | return cos 10 | 11 | def gradient_descent(X, y, theta, alpha, num_iters): 12 | m = len(y) 13 | costs = [] 14 | for _ in range(num_iters): 15 | h = np.dot(X, theta) 16 | theta -= alpha * np.dot(X.T, (h - y))/m 17 | costs.append(cost(X, y, theta)) 18 | return theta, costs 19 | 20 | def linear_regression(X, y, alpha = 0.01,num_iters = 100): 21 | X = np.append(np.ones((X.shape[0], 1)), X, axis = 1) 22 | theta = np.zeros((X.shape[1], 1), dtype = np.float64) 23 | theta, costs = gradient_descent(X, y, theta, alpha, num_iters) 24 | predicted = np.dot(X, theta) 25 | return predicted, theta, costs 26 | 27 | if __name__ == '__main__': 28 | images_dir = os.path.join(sys.path[0], 'images') 29 | if not os.path.exists(images_dir): 30 | os.makedirs(images_dir) 31 | 32 | X = np.array(range(25)) 33 | y = (X ** 1.3 + np.random.normal(10, 10, X.shape[0])) 34 | X, y = X.reshape((-1, 1)), y.reshape((-1, 1)) 35 | # data = np.loadtxt(os.path.join(sys.path[0], 'linear_regression_data.csv'), delimiter = ',', dtype = np.float64) 36 | # X, y = data[:, :-1], data[:, -1].reshape((-1, 1)) 37 | 38 | predicted, theta, costs = linear_regression(X, y) 39 | 40 | sns.scatterplot(X[:, 0], y.flatten()) 41 | plt.title('Dataset') 42 | plt.ylabel('Y') 43 | plt.xlabel('X') 44 | plt.savefig(os.path.join(images_dir, 'data.png')) 45 | plt.clf() 46 | 47 | sns.lineplot(range(100), costs) 48 | plt.title('Cost vs Number of Interations') 49 | plt.ylabel('Cost') 50 | plt.xlabel('No. of Interations') 51 | plt.savefig(os.path.join(images_dir, 'cost.png')) 52 | plt.clf() 53 | 54 | sns.lineplot(X[:, 0], predicted.flatten()) 55 | sns.scatterplot(X[:, 0], y.flatten()) 56 | for i, x in enumerate(X): 57 | plt.vlines(x, min(predicted[i], y[i]), max(predicted[i], y[i])) 58 | plt.ylabel('Y') 59 | plt.xlabel('X') 60 | plt.title('Error') 61 | plt.legend(('linear fit', 'data')) 62 | plt.savefig(os.path.join(images_dir, 'Error.png')) 63 | plt.clf() 64 | -------------------------------------------------------------------------------- /linear_regression/linear_regression_data.csv: -------------------------------------------------------------------------------- 1 | 6.1101,17.592 2 | 5.5277,9.1302 3 | 8.5186,13.662 4 | 7.0032,11.854 5 | 5.8598,6.8233 6 | 8.3829,11.886 7 | 7.4764,4.3483 8 | 8.5781,12 9 | 6.4862,6.5987 10 | 5.0546,3.8166 11 | 5.7107,3.2522 12 | 14.164,15.505 13 | 5.734,3.1551 14 | 8.4084,7.2258 15 | 5.6407,0.71618 16 | 5.3794,3.5129 17 | 6.3654,5.3048 18 | 5.1301,0.56077 19 | 6.4296,3.6518 20 | 7.0708,5.3893 21 | 6.1891,3.1386 22 | 20.27,21.767 23 | 5.4901,4.263 24 | 6.3261,5.1875 25 | 5.5649,3.0825 26 | 18.945,22.638 27 | 12.828,13.501 28 | 10.957,7.0467 29 | 13.176,14.692 30 | 22.203,24.147 31 | 5.2524,-1.22 32 | 6.5894,5.9966 33 | 9.2482,12.134 34 | 5.8918,1.8495 35 | 8.2111,6.5426 36 | 7.9334,4.5623 37 | 8.0959,4.1164 38 | 5.6063,3.3928 39 | 12.836,10.117 40 | 6.3534,5.4974 41 | 5.4069,0.55657 42 | 6.8825,3.9115 43 | 11.708,5.3854 44 | 5.7737,2.4406 45 | 7.8247,6.7318 46 | 7.0931,1.0463 47 | 5.0702,5.1337 48 | 5.8014,1.844 49 | 11.7,8.0043 50 | 5.5416,1.0179 51 | 7.5402,6.7504 52 | 5.3077,1.8396 53 | 7.4239,4.2885 54 | 7.6031,4.9981 55 | 6.3328,1.4233 56 | 6.3589,-1.4211 57 | 6.2742,2.4756 58 | 5.6397,4.6042 59 | 9.3102,3.9624 60 | 9.4536,5.4141 61 | 8.8254,5.1694 62 | 5.1793,-0.74279 63 | 21.279,17.929 64 | 14.908,12.054 65 | 18.959,17.054 66 | 7.2182,4.8852 67 | 8.2951,5.7442 68 | 10.236,7.7754 69 | 5.4994,1.0173 70 | 20.341,20.992 71 | 10.136,6.6799 72 | 7.3345,4.0259 73 | 6.0062,1.2784 74 | 7.2259,3.3411 75 | 5.0269,-2.6807 76 | 6.5479,0.29678 77 | 7.5386,3.8845 78 | 5.0365,5.7014 79 | 10.274,6.7526 80 | 5.1077,2.0576 81 | 5.7292,0.47953 82 | 5.1884,0.20421 83 | 6.3557,0.67861 84 | 9.7687,7.5435 85 | 6.5159,5.3436 86 | 8.5172,4.2415 87 | 9.1802,6.7981 88 | 6.002,0.92695 89 | 5.5204,0.152 90 | 5.0594,2.8214 91 | 5.7077,1.8451 92 | 7.6366,4.2959 93 | 5.8707,7.2029 94 | 5.3054,1.9869 95 | 8.2934,0.14454 96 | 13.394,9.0551 97 | 5.4369,0.61705 98 | -------------------------------------------------------------------------------- /linear_regression/linear_regression_data_multi.csv: -------------------------------------------------------------------------------- 1 | 2104,3,399900 2 | 1600,3,329900 3 | 2400,3,369000 4 | 1416,2,232000 5 | 3000,4,539900 6 | 1985,4,299900 7 | 1534,3,314900 8 | 1427,3,198999 9 | 1380,3,212000 10 | 1494,3,242500 11 | 1940,4,239999 12 | 2000,3,347000 13 | 1890,3,329999 14 | 4478,5,699900 15 | 1268,3,259900 16 | 2300,4,449900 17 | 1320,2,299900 18 | 1236,3,199900 19 | 2609,4,499998 20 | 3031,4,599000 21 | 1767,3,252900 22 | 1888,2,255000 23 | 1604,3,242900 24 | 1962,4,259900 25 | 3890,3,573900 26 | 1100,3,249900 27 | 1458,3,464500 28 | 2526,3,469000 29 | 2200,3,475000 30 | 2637,3,299900 31 | 1839,2,349900 32 | 1000,1,169900 33 | 2040,4,314900 34 | 3137,3,579900 35 | 1811,4,285900 36 | 1437,3,249900 37 | 1239,3,229900 38 | 2132,4,345000 39 | 4215,4,549000 40 | 2162,4,287000 41 | 1664,2,368500 42 | 2238,3,329900 43 | 2567,4,314000 44 | 1200,3,299000 45 | 852,2,179900 46 | 1852,4,299900 47 | 1203,3,239500 48 | -------------------------------------------------------------------------------- /logistic_regression/images/cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/logistic_regression/images/cost.png -------------------------------------------------------------------------------- /logistic_regression/images/data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/logistic_regression/images/data.png -------------------------------------------------------------------------------- /logistic_regression/images/data_multi-class.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/logistic_regression/images/data_multi-class.png -------------------------------------------------------------------------------- /logistic_regression/images/decision_boundary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/logistic_regression/images/decision_boundary.png -------------------------------------------------------------------------------- /logistic_regression/images/decision_boundary_multi-class.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/logistic_regression/images/decision_boundary_multi-class.png -------------------------------------------------------------------------------- /logistic_regression/images/decision_boundary_overfitting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/logistic_regression/images/decision_boundary_overfitting.png -------------------------------------------------------------------------------- /logistic_regression/images/decision_boundary_regularization.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/logistic_regression/images/decision_boundary_regularization.png -------------------------------------------------------------------------------- /logistic_regression/images/decision_boundary_underfitting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/logistic_regression/images/decision_boundary_underfitting.png -------------------------------------------------------------------------------- /logistic_regression/logistic_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import seaborn as sns 4 | import os, sys 5 | 6 | def sigmoid(z): 7 | return 1/(1 + np.exp(-z)) 8 | 9 | def cost(theta, X, y): 10 | h = sigmoid(np.dot(X, theta)) 11 | cos = -(np.sum(y * np.log(h)) + np.sum((1 - y) * np.log(1 - h)))/len(y) 12 | return cos 13 | 14 | def expand_feature(x1, x2, power = 2): 15 | #expand a 2D feature matrix to polynimial features up to the power 16 | new_x = np.ones((x1.shape[0], 1)) 17 | for i in range(1, power + 1): 18 | for j in range(i + 1): 19 | new_x = np.append(new_x, (x1**(i-j)*(x2**j)).reshape(-1, 1), axis = 1) 20 | return new_x 21 | 22 | def gradient_descent(X, y, theta, alpha, num_iters): 23 | m = len(y) 24 | costs = [] 25 | for _ in range(num_iters): 26 | h = sigmoid(np.dot(X, theta)) 27 | theta -= alpha * np.dot(X.T, (h - y))/m 28 | costs.append(cost(theta, X, y)) 29 | return theta, costs 30 | 31 | def predict(theta, X): 32 | return (sigmoid(np.dot(X, theta)) > 0.5).flatten() 33 | 34 | def logistic_regression(X, y, power = 2, alpha = 0.01, num_iters = 100): 35 | X = expand_feature(X[:, 0], X[:, 1], power = power) 36 | theta = np.zeros((X.shape[1], 1), dtype = np.float64) 37 | theta, costs = gradient_descent(X, y, theta, alpha, num_iters) 38 | predicted = predict(theta, X) 39 | return predicted, theta, costs 40 | 41 | if __name__ == '__main__': 42 | images_dir = os.path.join(sys.path[0], 'images') 43 | if not os.path.exists(images_dir): 44 | os.makedirs(images_dir) 45 | 46 | data = np.loadtxt(os.path.join(sys.path[0], 'logistic_regression_data.csv'), delimiter = ',', dtype = np.float64) 47 | X, y = data[:, :-1], data[:, -1].reshape((-1, 1)) 48 | 49 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = y.flatten()) 50 | plt.title('Dataset') 51 | plt.xlabel('X') 52 | plt.ylabel('Y') 53 | plt.savefig(os.path.join(images_dir, 'data.png')) 54 | plt.clf() 55 | 56 | predicted, theta, costs = logistic_regression(X, y, alpha = 0.15, num_iters = 4000) 57 | print('The accuracy is {:.2f} %'.format(sum(predicted == y.flatten())/len(y)*100)) 58 | 59 | sns.lineplot(range(4000), costs) 60 | plt.title('Cost vs Number of Interations') 61 | plt.ylabel('Cost') 62 | plt.xlabel('No. of Interations') 63 | plt.savefig(os.path.join(images_dir, 'cost.png')) 64 | plt.clf() 65 | 66 | u = np.linspace(min(X[:, 0]),max(X[:, 0]), 50) 67 | v = np.linspace(min(X[:, 1]),max(X[:, 1]), 50) 68 | 69 | z = np.zeros((len(u),len(v))) 70 | 71 | for i in range(len(u)): 72 | for j in range(len(v)): 73 | z[i,j] = np.dot(expand_feature(u[i].reshape(1,-1),v[j].reshape(1,-1)),theta) 74 | z = np.transpose(z) 75 | 76 | plt.contour(u,v,z,[0,0.01], cmap = "Reds") 77 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = y.flatten()) 78 | plt.title('Decision Boundary') 79 | plt.xlabel('X') 80 | plt.ylabel('Y') 81 | plt.savefig(os.path.join(images_dir, 'decision_boundary.png')) 82 | plt.clf() 83 | -------------------------------------------------------------------------------- /logistic_regression/logistic_regression_data.csv: -------------------------------------------------------------------------------- 1 | 0.051267,0.69956,1 2 | -0.092742,0.68494,1 3 | -0.21371,0.69225,1 4 | -0.375,0.50219,1 5 | -0.51325,0.46564,1 6 | -0.52477,0.2098,1 7 | -0.39804,0.034357,1 8 | -0.30588,-0.19225,1 9 | 0.016705,-0.40424,1 10 | 0.13191,-0.51389,1 11 | 0.38537,-0.56506,1 12 | 0.52938,-0.5212,1 13 | 0.63882,-0.24342,1 14 | 0.73675,-0.18494,1 15 | 0.54666,0.48757,1 16 | 0.322,0.5826,1 17 | 0.16647,0.53874,1 18 | -0.046659,0.81652,1 19 | -0.17339,0.69956,1 20 | -0.47869,0.63377,1 21 | -0.60541,0.59722,1 22 | -0.62846,0.33406,1 23 | -0.59389,0.005117,1 24 | -0.42108,-0.27266,1 25 | -0.11578,-0.39693,1 26 | 0.20104,-0.60161,1 27 | 0.46601,-0.53582,1 28 | 0.67339,-0.53582,1 29 | -0.13882,0.54605,1 30 | -0.29435,0.77997,1 31 | -0.26555,0.96272,1 32 | -0.16187,0.8019,1 33 | -0.17339,0.64839,1 34 | -0.28283,0.47295,1 35 | -0.36348,0.31213,1 36 | -0.30012,0.027047,1 37 | -0.23675,-0.21418,1 38 | -0.06394,-0.18494,1 39 | 0.062788,-0.16301,1 40 | 0.22984,-0.41155,1 41 | 0.2932,-0.2288,1 42 | 0.48329,-0.18494,1 43 | 0.64459,-0.14108,1 44 | 0.46025,0.012427,1 45 | 0.6273,0.15863,1 46 | 0.57546,0.26827,1 47 | 0.72523,0.44371,1 48 | 0.22408,0.52412,1 49 | 0.44297,0.67032,1 50 | 0.322,0.69225,1 51 | 0.13767,0.57529,1 52 | -0.0063364,0.39985,1 53 | -0.092742,0.55336,1 54 | -0.20795,0.35599,1 55 | -0.20795,0.17325,1 56 | -0.43836,0.21711,1 57 | -0.21947,-0.016813,1 58 | -0.13882,-0.27266,1 59 | 0.18376,0.93348,0 60 | 0.22408,0.77997,0 61 | 0.29896,0.61915,0 62 | 0.50634,0.75804,0 63 | 0.61578,0.7288,0 64 | 0.60426,0.59722,0 65 | 0.76555,0.50219,0 66 | 0.92684,0.3633,0 67 | 0.82316,0.27558,0 68 | 0.96141,0.085526,0 69 | 0.93836,0.012427,0 70 | 0.86348,-0.082602,0 71 | 0.89804,-0.20687,0 72 | 0.85196,-0.36769,0 73 | 0.82892,-0.5212,0 74 | 0.79435,-0.55775,0 75 | 0.59274,-0.7405,0 76 | 0.51786,-0.5943,0 77 | 0.46601,-0.41886,0 78 | 0.35081,-0.57968,0 79 | 0.28744,-0.76974,0 80 | 0.085829,-0.75512,0 81 | 0.14919,-0.57968,0 82 | -0.13306,-0.4481,0 83 | -0.40956,-0.41155,0 84 | -0.39228,-0.25804,0 85 | -0.74366,-0.25804,0 86 | -0.69758,0.041667,0 87 | -0.75518,0.2902,0 88 | -0.69758,0.68494,0 89 | -0.4038,0.70687,0 90 | -0.38076,0.91886,0 91 | -0.50749,0.90424,0 92 | -0.54781,0.70687,0 93 | 0.10311,0.77997,0 94 | 0.057028,0.91886,0 95 | -0.10426,0.99196,0 96 | -0.081221,1.1089,0 97 | 0.28744,1.087,0 98 | 0.39689,0.82383,0 99 | 0.63882,0.88962,0 100 | 0.82316,0.66301,0 101 | 0.67339,0.64108,0 102 | 1.0709,0.10015,0 103 | -0.046659,-0.57968,0 104 | -0.23675,-0.63816,0 105 | -0.15035,-0.36769,0 106 | -0.49021,-0.3019,0 107 | -0.46717,-0.13377,0 108 | -0.28859,-0.060673,0 109 | -0.61118,-0.067982,0 110 | -0.66302,-0.21418,0 111 | -0.59965,-0.41886,0 112 | -0.72638,-0.082602,0 113 | -0.83007,0.31213,0 114 | -0.72062,0.53874,0 115 | -0.59389,0.49488,0 116 | -0.48445,0.99927,0 117 | -0.0063364,0.99927,0 118 | 0.63265,-0.030612,0 119 | -------------------------------------------------------------------------------- /logistic_regression/logistic_regression_multi-class.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import seaborn as sns 4 | import os, sys 5 | from logistic_regression_reg import * 6 | 7 | # prediction function is different from binary classification 8 | def predict_multi_class(theta, X): 9 | return np.argmax(sigmoid(np.dot(X, theta)), axis = 1) 10 | 11 | def logistic_regression_reg_multi_class(X, y, power = 2, alpha = 0.01, lam = 0, num_iters = 100): 12 | X = expand_feature(X[:, 0], X[:, 1], power = power) 13 | theta = np.zeros((X.shape[1], y.shape[1]), dtype = np.float64) 14 | theta, costs = gradient_descent_reg(X, y, theta, alpha, lam, num_iters) 15 | predicted = predict_multi_class(theta, X) 16 | return predicted, theta, costs 17 | 18 | if __name__ == '__main__': 19 | images_dir = os.path.join(sys.path[0], 'images') 20 | if not os.path.exists(images_dir): 21 | os.makedirs(images_dir) 22 | 23 | N = 80 # number of points per class 24 | D = 2 # dimensionality, we use 2D data for easy visulization 25 | K = 3 # number of classes, binary for logistic regression 26 | X = np.zeros((N * K, D), dtype = float) # data matrix (each row = single example, can view as xy coordinates) 27 | y_ = np.zeros(N * K, dtype = int) # class labels for plotting 28 | y = np.zeros((N * K, K), dtype = int) # class labels for training 29 | 30 | 31 | for i in range(K): 32 | r = np.random.normal(i + 0.5, 0.3, (N, 1)) # radius 33 | t = np.linspace(0, np.pi * 2, N).reshape(N, 1) # theta 34 | 35 | X[i * N:(i + 1) * N] = np.append(r * np.sin(t), r * np.cos(t), axis = 1) 36 | y_[i * N:(i + 1) * N] = i 37 | y[i * N:(i + 1) * N, i] = 1 38 | 39 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = y_, palette = sns.color_palette('deep', K), edgecolor = "none") 40 | plt.title('Dataset') 41 | plt.xlabel('X') 42 | plt.ylabel('Y') 43 | plt.savefig(os.path.join(images_dir, 'data_multi-class.png')) 44 | plt.clf() 45 | 46 | num_iters = 2000 47 | predicted, theta, costs = logistic_regression_reg_multi_class(X, y, alpha = 0.3, lam = 0, num_iters = num_iters) 48 | print('The accuracy is {:.2f} %'.format(sum(predicted == y_)/len(y_)*100)) 49 | 50 | gridsize = 200 51 | u = np.linspace(min(X[:, 0]),max(X[:, 0]), gridsize) 52 | v = np.linspace(min(X[:, 1]),max(X[:, 1]), gridsize) 53 | 54 | gridx, gridy = np.meshgrid(u, v) 55 | grid = np.array([gridx.reshape(-1, ), gridy.reshape(-1, )]).T 56 | 57 | z = predict_multi_class(theta, expand_feature(gridx.reshape(-1, 1), gridy.reshape(-1, 1))).reshape(gridsize, gridsize) 58 | plt.contourf(u, v, z, alpha = 0.2, levels = K - 1, antialiased = True) 59 | 60 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = y_, palette = sns.color_palette('deep', K), edgecolor = "none") 61 | plt.title('Decision Boundary') 62 | plt.xlabel('X') 63 | plt.ylabel('Y') 64 | plt.savefig(os.path.join(images_dir, 'decision_boundary_multi-class.png')) 65 | plt.clf() 66 | -------------------------------------------------------------------------------- /logistic_regression/logistic_regression_reg.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import seaborn as sns 4 | import os, sys 5 | from logistic_regression import * 6 | 7 | def cost_reg(theta, X, y, lam = 0): 8 | h = sigmoid(np.dot(X, theta)) 9 | theta1 = theta.copy() 10 | theta1[0] = 0 11 | cos = -(np.sum(y * np.log(h)) + np.sum((1 - y) * np.log(1 - h)))/len(y) + lam * np.sum(theta1 * theta1)/len(y) 12 | return cos 13 | 14 | def gradient_descent_reg(X, y, theta, alpha, lam = 0, num_iters = 100): 15 | costs = [] 16 | 17 | for _ in range(num_iters): 18 | h = sigmoid(np.dot(X, theta)) 19 | theta1 = theta.copy() 20 | theta1[0] = 0 21 | theta -= alpha * (np.dot(X.T, (h - y)) + 2 * lam * theta1)/len(y) 22 | costs.append(cost_reg(theta, X, y)) 23 | return theta, costs 24 | 25 | def logistic_regression_reg(X, y, power = 2, alpha = 0.01, lam = 0, num_iters = 100): 26 | X = expand_feature(X[:, 0], X[:, 1], power = power) 27 | theta = np.zeros((X.shape[1], y.shape[1]), dtype = np.float64) 28 | theta, costs = gradient_descent_reg(X, y, theta, alpha, lam, num_iters) 29 | predicted = predict(theta, X) 30 | return predicted, theta, costs 31 | 32 | if __name__ == '__main__': 33 | images_dir = os.path.join(sys.path[0], 'images') 34 | if not os.path.exists(images_dir): 35 | os.makedirs(images_dir) 36 | 37 | data = np.loadtxt(os.path.join(sys.path[0], 'logistic_regression_data.csv'), delimiter = ',', dtype = np.float64) 38 | X, y = data[:, :-1], data[:, -1].reshape((-1, 1)) 39 | 40 | # overfitting without regularization 41 | power, num_iters = 10, 100000 42 | predicted, theta, costs = logistic_regression_reg(X, y, power = power, alpha = 0.6, lam = 0, num_iters = num_iters) 43 | print('The accuracy is {:.2f} %'.format(sum(predicted == y.flatten())/len(y)*100)) 44 | 45 | u = np.linspace(min(X[:, 0]),max(X[:, 0]), 50) 46 | v = np.linspace(min(X[:, 1]),max(X[:, 1]), 50) 47 | 48 | z = np.zeros((len(u),len(v))) 49 | 50 | for i in range(len(u)): 51 | for j in range(len(v)): 52 | z[i,j] = np.dot(expand_feature(u[i].reshape(1,-1),v[j].reshape(1,-1), power = power),theta) 53 | z = np.transpose(z) 54 | 55 | plt.contour(u,v,z,[0,0.01], cmap = "Reds") 56 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = y.flatten()) 57 | plt.title('Overfitting') 58 | plt.xlabel('X') 59 | plt.ylabel('Y') 60 | plt.savefig(os.path.join(images_dir, 'decision_boundary_overfitting.png')) 61 | plt.clf() 62 | 63 | # underfitting with too much regularization 64 | predicted, theta, costs = logistic_regression_reg(X, y, power = power, alpha = 0.6, lam = 5, num_iters = num_iters) 65 | print('The accuracy is {:.2f} %'.format(sum(predicted == y.flatten())/len(y)*100)) 66 | 67 | z = np.zeros((len(u),len(v))) 68 | for i in range(len(u)): 69 | for j in range(len(v)): 70 | z[i,j] = np.dot(expand_feature(u[i].reshape(1,-1),v[j].reshape(1,-1), power = power),theta) 71 | z = np.transpose(z) 72 | 73 | plt.contour(u,v,z,[0,0.01], cmap = "Reds") 74 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = y.flatten()) 75 | plt.title('Underfitting') 76 | plt.xlabel('X') 77 | plt.ylabel('Y') 78 | plt.savefig(os.path.join(images_dir, 'decision_boundary_underfitting.png')) 79 | plt.clf() 80 | 81 | # proper regularization 82 | predicted, theta, costs = logistic_regression_reg(X, y, power = power, alpha = 0.6, lam = 0.5, num_iters = num_iters) 83 | print('The accuracy is {:.2f} %'.format(sum(predicted == y.flatten())/len(y)*100)) 84 | 85 | z = np.zeros((len(u),len(v))) 86 | for i in range(len(u)): 87 | for j in range(len(v)): 88 | z[i,j] = np.dot(expand_feature(u[i].reshape(1,-1),v[j].reshape(1,-1), power = power),theta) 89 | z = np.transpose(z) 90 | 91 | plt.contour(u,v,z,[0,0.01], cmap = "Reds") 92 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = y.flatten()) 93 | plt.title('Adequate Regularization') 94 | plt.xlabel('X') 95 | plt.ylabel('Y') 96 | plt.savefig(os.path.join(images_dir, 'decision_boundary_regularization.png')) 97 | plt.clf() 98 | -------------------------------------------------------------------------------- /neural_network/images/decision_boundary_nnet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/neural_network/images/decision_boundary_nnet.png -------------------------------------------------------------------------------- /neural_network/neural_network.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import seaborn as sns 4 | import os, sys 5 | 6 | def init_para(D, K, h): 7 | # initialize parameters randomly 8 | W = np.random.normal(0, 0.01, (D, h)) 9 | b = np.zeros((1, h), dtype = float) 10 | W2 = np.random.normal(0, 0.01, (h, K)) 11 | b2 = np.zeros((1, K), dtype = float) 12 | return W, b, W2, b2 13 | 14 | def softmax(scores): 15 | exp_scores = np.exp(scores) 16 | return exp_scores / np.sum(exp_scores, axis = 1).reshape(-1, 1) 17 | 18 | def nnet(X, y, step_size = 0.4, lam = 0.0001, h = 10, num_iters = 1000): 19 | # get dim of input 20 | N, D = X.shape 21 | K = y.shape[1] 22 | 23 | W, b, W2, b2 = init_para(D, K, h) 24 | 25 | # gradient descent loop to update weight and bias 26 | for i in range(num_iters): 27 | # hidden layer, ReLU activation 28 | hidden_layer = np.maximum(0, np.dot(X, W) + np.repeat(b, N, axis = 0)) 29 | 30 | # class score 31 | scores = np.dot(hidden_layer, W2) + np.repeat(b2, N, axis = 0) 32 | 33 | # compute and normalize class probabilities 34 | probs = softmax(scores) 35 | 36 | # compute the loss with regularization 37 | data_loss = np.sum(-np.log(probs) * y) / N 38 | reg_loss = 0.5 * lam * np.sum(W * W) + 0.5 * lam * np.sum(W2 * W2) 39 | loss = data_loss + reg_loss 40 | 41 | # check progress 42 | if i%1000 == 0 or i == num_iters: 43 | print("iteration {}: loss {}".format(i, loss)) 44 | 45 | # compute the gradient on scores 46 | dscores = (probs - y) / N 47 | 48 | # backpropate the gradient to the parameters 49 | dW2 = np.dot(hidden_layer.T, dscores) 50 | db2 = np.sum(dscores, axis = 0) 51 | # next backprop into hidden layer 52 | dhidden = np.dot(dscores, W2.T) 53 | # backprop the ReLU non-linearity 54 | dhidden[hidden_layer <= 0] = 0 55 | # finally into W,b 56 | dW = np.dot(X.T, dhidden) 57 | db = np.sum(dhidden, axis = 0) 58 | 59 | # add regularization gradient contribution 60 | dW2 = dW2 + lam * W2 61 | dW = dW + lam * W 62 | 63 | # update parameter 64 | W = W - step_size * dW 65 | b = b - step_size * db 66 | W2 = W2 - step_size * dW2 67 | b2 = b2 - step_size * db2 68 | return W, b, W2, b2 69 | 70 | def predict(X, para): 71 | W, b, W2, b2 = para 72 | N = X.shape[0] 73 | hidden_layer = np.maximum(0, np.dot(X, W) + np.repeat(b, N, axis = 0)) 74 | scores = np.dot(hidden_layer, W2) + np.repeat(b2, N, axis = 0) 75 | return np.argmax(scores, axis = 1) 76 | 77 | if __name__ == '__main__': 78 | images_dir = os.path.join(sys.path[0], 'images') 79 | if not os.path.exists(images_dir): 80 | os.makedirs(images_dir) 81 | 82 | N = 80 # number of points per class 83 | D = 2 # dimensionality, we use 2D data for easy visulization 84 | K = 4 # number of classes, binary for logistic regression 85 | X = np.zeros((N * K, D), dtype = float) # data matrix (each row = single example, can view as xy coordinates) 86 | y_ = np.zeros(N * K, dtype = int) # class labels for plotting 87 | y = np.zeros((N * K, K), dtype = int) # class labels for training 88 | 89 | 90 | for i in range(K): 91 | r = np.linspace(0.05, 1, N).reshape(-1, 1) # radius 92 | t = np.linspace(i*4.7, (i+1)*4.7, N).reshape(-1, 1) + np.random.normal(0, 0.3, (N, 1)) # theta 93 | 94 | X[i * N:(i + 1) * N] = np.append(r * np.sin(t), r * np.cos(t), axis = 1) 95 | y_[i * N:(i + 1) * N] = i 96 | y[i * N:(i + 1) * N, i] = 1 97 | 98 | num_iters = 10000 99 | para = nnet(X, y, step_size = 0.3, lam = 0.0005, h = 50, num_iters = num_iters) 100 | predicted = predict(X, para) 101 | print('The accuracy is {:.2f} %'.format(sum(predicted == y_)/len(y_)*100)) 102 | 103 | grid_size = 200 104 | u = np.linspace(min(X[:, 0]),max(X[:, 0]), grid_size) 105 | v = np.linspace(min(X[:, 1]),max(X[:, 1]), grid_size) 106 | 107 | gridx, gridy = np.meshgrid(u, v) 108 | grid = np.array([gridx.reshape(-1, ), gridy.reshape(-1, )]).T 109 | z = predict(grid, para).reshape(grid_size, grid_size) 110 | 111 | plt.contourf(u,v,z, alpha = 0.2, levels = K - 1, antialiased = True) 112 | sns.scatterplot(x = X[:, 0], y = X[:, 1], hue = y_, palette = sns.color_palette('deep', K), edgecolor = "none") 113 | plt.title('Decision Boundary') 114 | plt.xlabel('X') 115 | plt.ylabel('Y') 116 | plt.savefig(os.path.join(images_dir, 'decision_boundary_nnet.png')) 117 | plt.clf() 118 | -------------------------------------------------------------------------------- /neural_network/neural_network_mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import seaborn as sns 4 | import os, sys 5 | from neural_network import * 6 | 7 | if __name__ == '__main__': 8 | images_dir = os.path.join(sys.path[0], 'images') 9 | if not os.path.exists(images_dir): 10 | os.makedirs(images_dir) 11 | 12 | # load train and test data 13 | train = np.loadtxt(os.path.join(sys.path[0], 'data/train.csv'), delimiter = ',', skiprows = 1) 14 | test = np.loadtxt(os.path.join(sys.path[0], 'data/test.csv'), delimiter = ',', skiprows = 1) 15 | trainx, trainy_ = train[:, 1:], train[:, 0].flatten() 16 | trainx_norm = trainx/255 17 | 18 | # generate one-hot trainy 19 | trainy = np.zeros((trainx.shape[0], 10), dtype = int) 20 | for i, v in enumerate(trainy_): 21 | trainy[i, int(v)] = 1 22 | 23 | num_iters = 2000 24 | mnist_para = nnet(trainx_norm, trainy, step_size = 0.4, lam = 0.001, h = 10, num_iters = num_iters) 25 | predicted = predict(trainx_norm, mnist_para) 26 | print('The accuracy is {:.2f} %'.format(sum(predicted == trainy_)/len(trainy_)*100)) 27 | -------------------------------------------------------------------------------- /title.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szjunma/ML-Algorithm-with-Python/25f9f14372b32be342c4e36b9e0c7817e9391c5c/title.png --------------------------------------------------------------------------------