├── Decision Tree for Classification And Regression.ipynb ├── GBDT 梯度提升.ipynb ├── GaussianMixtureModel.ipynb ├── HMM_forward.py ├── K-means算法.ipynb ├── KNN.ipynb ├── LSTM_FS.ipynb ├── MCMC-Metropolis-Hastings.py ├── Naive Bayes.ipynb ├── PCA.ipynb ├── PageRank.ipynb ├── README.md ├── SVM SMO.ipynb ├── TempLinkoping2016.txt ├── Viterbi.py ├── Xgboost 算法.ipynb ├── utils ├── .ipynb_checkpoints │ ├── MLP-checkpoint.ipynb │ └── Simple RNN-checkpoint.ipynb ├── MLP.ipynb ├── Simple RNN.ipynb ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── activations.cpython-36.pyc │ ├── layers.cpython-36.pyc │ ├── losses.cpython-36.pyc │ ├── metric.cpython-36.pyc │ ├── neural_networks.cpython-36.pyc │ ├── optimizers.cpython-36.pyc │ └── progress.cpython-36.pyc ├── activations.py ├── layers.py ├── losses.py ├── metric.py ├── neural_networks.py ├── optimizers.py └── progress.py ├── 对数几率回归.ipynb ├── 线性判别分析LDA.ipynb └── 线性回归.ipynb /GBDT 梯度提升.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# 提升树算法" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "提升方法实际上是采用加法模型，即基函数的线性组合与前向分布算法来进行的一类算法，事实证明，该算法具有很好的拟合性能。\n", 15 | "算法如下\n", 16 | "\n", 17 | "1. 初始化 $f_0(x)=0$\n", 18 | "2. 对m=1,2,3,...M求残差（一般为平方误差 如果是分类问题就交叉熵误差）\n", 19 | "3. 拟合残差学习新的回归树\n", 20 | "4. 更新 $f_m(x)=f_{m-1}(x)+T(x;\\theta_m)$\n", 21 | "5. 直到误差小于我们规定的误差停止\n", 22 | "\n", 23 | "\n", 24 | "上面的是普通提升树的算法，而GBDT为梯度提升，什么意思呢，在刚刚算的残差之后，我们使用的简单的损失函数，如果是复杂的损失函数，优化并不简单的，于是提出了一种梯度上升算法，也就是使用损失函数的负梯度在当前模型的值作为回归树问题算法的残差的近似值，来拟合回归树。" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 1, 30 | "metadata": { 31 | "collapsed": true 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "import numpy as np\n", 36 | "import math" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 3, 42 | "metadata": { 43 | "collapsed": true 44 | }, 45 | "outputs": [], 46 | "source": [ 47 | "\n", 48 | "# 计算信息熵\n", 49 | "def calculate_entropy(y):\n", 50 | " log2 = math.log2\n", 51 | " unique_labels = np.unique(y)\n", 52 | " entropy = 0\n", 53 | " for label in unique_labels:\n", 54 | " count = len(y[y == label])\n", 55 | " p = count / len(y)\n", 56 | " entropy += -p * log2(p)\n", 57 | " return entropy\n", 58 | "# 定义树的节点\n", 59 | "class DecisionNode():\n", 60 | " def __init__(self, feature_i=None, threshold=None,\n", 61 | " value=None, true_branch=None, false_branch=None):\n", 62 | " self.feature_i = feature_i \n", 63 | " self.threshold = threshold \n", 64 | " self.value = value \n", 65 | " self.true_branch = true_branch \n", 66 | " self.false_branch = false_branch\n", 67 | "def divide_on_feature(X, feature_i, threshold):\n", 68 | " split_func = None\n", 69 | " if isinstance(threshold, int) or isinstance(threshold, float):\n", 70 | " split_func = lambda sample: sample[feature_i] >= threshold\n", 71 | " else:\n", 72 | " split_func = lambda sample: sample[feature_i] == threshold\n", 73 | "\n", 74 | " X_1 = np.array([sample for sample in X if split_func(sample)])\n", 75 | " X_2 = np.array([sample for sample in X if not split_func(sample)])\n", 76 | "\n", 77 | " return np.array([X_1, X_2])\n", 78 | "# 超类\n", 79 | "class DecisionTree(object):\n", 80 | " def __init__(self, min_samples_split=2, min_impurity=1e-7,\n", 81 | " max_depth=float(\"inf\"), loss=None):\n", 82 | " self.root = None #根节点\n", 83 | " self.min_samples_split = min_samples_split\n", 84 | " self.min_impurity = min_impurity\n", 85 | " self.max_depth = max_depth\n", 86 | " # 计算值 如果是分类问题就是信息增益，回归问题就基尼指数\n", 87 | " self._impurity_calculation = None\n", 88 | " self._leaf_value_calculation = None #计算叶子\n", 89 | " self.one_dim = None\n", 90 | " self.loss = loss\n", 91 | "\n", 92 | " def fit(self, X, y, loss=None):\n", 93 | " self.one_dim = len(np.shape(y)) == 1\n", 94 | " self.root = self._build_tree(X, y)\n", 95 | " self.loss=None\n", 96 | "\n", 97 | " def _build_tree(self, X, y, current_depth=0):\n", 98 | " \"\"\"\n", 99 | " 递归求解树\n", 100 | " \"\"\"\n", 101 | "\n", 102 | " largest_impurity = 0\n", 103 | " best_criteria = None\n", 104 | " best_sets = None\n", 105 | " \n", 106 | " if len(np.shape(y)) == 1:\n", 107 | " y = np.expand_dims(y, axis=1)\n", 108 | "\n", 109 | " Xy = np.concatenate((X, y), axis=1)\n", 110 | "\n", 111 | " n_samples, n_features = np.shape(X)\n", 112 | "\n", 113 | " if n_samples >= self.min_samples_split and current_depth <= self.max_depth:\n", 114 | " # 计算每一个特征的增益值\n", 115 | " for feature_i in range(n_features):\n", 116 | " feature_values = np.expand_dims(X[:, feature_i], axis=1)\n", 117 | " unique_values = np.unique(feature_values)\n", 118 | "\n", 119 | " for threshold in unique_values:\n", 120 | " Xy1, Xy2 = divide_on_feature(Xy, feature_i, threshold)\n", 121 | " \n", 122 | " if len(Xy1) > 0 and len(Xy2) > 0:\n", 123 | " y1 = Xy1[:, n_features:]\n", 124 | " y2 = Xy2[:, n_features:]\n", 125 | "\n", 126 | " # 计算增益值\n", 127 | " impurity = self._impurity_calculation(y, y1, y2)\n", 128 | "\n", 129 | " if impurity > largest_impurity:\n", 130 | " largest_impurity = impurity\n", 131 | " best_criteria = {\"feature_i\": feature_i, \"threshold\": threshold}\n", 132 | " best_sets = {\n", 133 | " \"leftX\": Xy1[:, :n_features], \n", 134 | " \"lefty\": Xy1[:, n_features:], \n", 135 | " \"rightX\": Xy2[:, :n_features], \n", 136 | " \"righty\": Xy2[:, n_features:] \n", 137 | " }\n", 138 | "\n", 139 | " if largest_impurity > self.min_impurity:\n", 140 | " true_branch = self._build_tree(best_sets[\"leftX\"], best_sets[\"lefty\"], current_depth + 1)\n", 141 | " false_branch = self._build_tree(best_sets[\"rightX\"], best_sets[\"righty\"], current_depth + 1)\n", 142 | " return DecisionNode(feature_i=best_criteria[\"feature_i\"], threshold=best_criteria[\n", 143 | " \"threshold\"], true_branch=true_branch, false_branch=false_branch)\n", 144 | " \n", 145 | " # 计算节点的目标值\n", 146 | " leaf_value = self._leaf_value_calculation(y)\n", 147 | " \n", 148 | " \n", 149 | " return DecisionNode(value=leaf_value)\n", 150 | "\n", 151 | "\n", 152 | " def predict_value(self, x, tree=None):\n", 153 | " \"\"\"\n", 154 | " 预测\n", 155 | " \"\"\"\n", 156 | "\n", 157 | " if tree is None:\n", 158 | " tree = self.root\n", 159 | "\n", 160 | " if tree.value is not None:\n", 161 | " return tree.value\n", 162 | "\n", 163 | " feature_value = x[tree.feature_i]\n", 164 | "\n", 165 | " branch = tree.false_branch\n", 166 | " if isinstance(feature_value, int) or isinstance(feature_value, float):\n", 167 | " if feature_value >= tree.threshold:\n", 168 | " branch = tree.true_branch\n", 169 | " elif feature_value == tree.threshold:\n", 170 | " branch = tree.true_branch\n", 171 | "\n", 172 | " return self.predict_value(x, branch)\n", 173 | "\n", 174 | " def predict(self, X):\n", 175 | " y_pred = []\n", 176 | " for x in X:\n", 177 | " y_pred.append(self.predict_value(x))\n", 178 | " return y_pred\n", 179 | "def calculate_variance(X):\n", 180 | " \"\"\" Return the variance of the features in dataset X \"\"\"\n", 181 | " mean = np.ones(np.shape(X)) * X.mean(0)\n", 182 | " n_samples = np.shape(X)[0]\n", 183 | " variance = (1 / n_samples) * np.diag((X - mean).T.dot(X - mean))\n", 184 | " \n", 185 | " return variance\n", 186 | "class RegressionTree(DecisionTree):\n", 187 | " def _calculate_variance_reduction(self, y, y1, y2):\n", 188 | " var_tot = calculate_variance(y)\n", 189 | " var_1 = calculate_variance(y1)\n", 190 | " var_2 = calculate_variance(y2)\n", 191 | " frac_1 = len(y1) / len(y)\n", 192 | " frac_2 = len(y2) / len(y)\n", 193 | "\n", 194 | " # 使用方差缩减\n", 195 | " variance_reduction = var_tot - (frac_1 * var_1 + frac_2 * var_2)\n", 196 | "\n", 197 | " return sum(variance_reduction)\n", 198 | "\n", 199 | " def _mean_of_y(self, y):\n", 200 | " value = np.mean(y, axis=0)\n", 201 | " return value if len(value) > 1 else value[0]\n", 202 | "\n", 203 | " def fit(self, X, y):\n", 204 | " self._impurity_calculation = self._calculate_variance_reduction\n", 205 | " self._leaf_value_calculation = self._mean_of_y\n", 206 | " super(RegressionTree, self).fit(X, y)" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 11, 212 | "metadata": { 213 | "collapsed": true 214 | }, 215 | "outputs": [], 216 | "source": [ 217 | "class GradientBoosting(object):\n", 218 | " def __init__(self, n_estimators, learning_rate, min_samples_split,\n", 219 | " min_impurity, max_depth, regression):\n", 220 | " self.n_estimators = n_estimators\n", 221 | " self.learning_rate = learning_rate\n", 222 | " self.min_samples_split = min_samples_split\n", 223 | " self.min_impurity = min_impurity\n", 224 | " self.max_depth = max_depth\n", 225 | " self.regression = regression\n", 226 | " \n", 227 | " self.loss = SquareLoss()\n", 228 | " if not self.regression:\n", 229 | " self.loss = CrossEntropy()\n", 230 | "\n", 231 | " self.trees = []\n", 232 | " for _ in range(n_estimators):\n", 233 | " tree = RegressionTree(\n", 234 | " min_samples_split=self.min_samples_split,\n", 235 | " min_impurity=min_impurity,\n", 236 | " max_depth=self.max_depth)\n", 237 | " self.trees.append(tree)\n", 238 | "\n", 239 | "\n", 240 | " def fit(self, X, y):\n", 241 | " y_pred = np.full(np.shape(y), np.mean(y, axis=0))\n", 242 | " for i in range(self.n_estimators):\n", 243 | " gradient = self.loss.gradient(y, y_pred)\n", 244 | " self.trees[i].fit(X, gradient)\n", 245 | " update = self.trees[i].predict(X)\n", 246 | " # Update y prediction\n", 247 | " y_pred -= np.multiply(self.learning_rate, update)\n", 248 | "\n", 249 | "\n", 250 | " def predict(self, X):\n", 251 | " y_pred = np.array([])\n", 252 | " for tree in self.trees:\n", 253 | " update = tree.predict(X)\n", 254 | " update = np.multiply(self.learning_rate, update)\n", 255 | " y_pred = -update if not y_pred.any() else y_pred - update\n", 256 | "\n", 257 | " if not self.regression:\n", 258 | " y_pred = np.exp(y_pred) / np.expand_dims(np.sum(np.exp(y_pred), axis=1), axis=1)\n", 259 | " y_pred = np.argmax(y_pred, axis=1)\n", 260 | " return y_pred" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": 32, 266 | "metadata": { 267 | "collapsed": true 268 | }, 269 | "outputs": [], 270 | "source": [ 271 | "class Loss(object):\n", 272 | " def loss(self, y_true, y_pred):\n", 273 | " return NotImplementedError()\n", 274 | "\n", 275 | " def gradient(self, y, y_pred):\n", 276 | " raise NotImplementedError()\n", 277 | "\n", 278 | " def acc(self, y, y_pred):\n", 279 | " return 0\n", 280 | "# 如果是回归模型\n", 281 | "class SquareLoss(Loss):\n", 282 | " def __init__(self): pass\n", 283 | "\n", 284 | " def loss(self, y, y_pred):\n", 285 | " return 0.5 * np.power((y - y_pred), 2)\n", 286 | "\n", 287 | " def gradient(self, y, y_pred):\n", 288 | " return -(y - y_pred)\n", 289 | "# 如果是分类模型\n", 290 | "class CrossEntropy(Loss):\n", 291 | " def __init__(self): pass\n", 292 | "\n", 293 | " def loss(self, y, p):\n", 294 | " # Avoid division by zero\n", 295 | " p = np.clip(p, 1e-15, 1 - 1e-15)\n", 296 | " return - y * np.log(p) - (1 - y) * np.log(1 - p)\n", 297 | "\n", 298 | " def acc(self, y, p):\n", 299 | " return accuracy_score(np.argmax(y, axis=1), np.argmax(p, axis=1))\n", 300 | "\n", 301 | " def gradient(self, y, p):\n", 302 | " # Avoid division by zero\n", 303 | " p = np.clip(p, 1e-15, 1 - 1e-15)\n", 304 | " return - (y / p) + (1 - y) / (1 - p)" 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": 33, 310 | "metadata": { 311 | "collapsed": true 312 | }, 313 | "outputs": [], 314 | "source": [ 315 | "class GradientBoostingRegressor(GradientBoosting):\n", 316 | " def __init__(self, n_estimators=200, learning_rate=0.5, min_samples_split=2,\n", 317 | " min_var_red=1e-7, max_depth=4, debug=False):\n", 318 | " super(GradientBoostingRegressor, self).__init__(n_estimators=n_estimators, \n", 319 | " learning_rate=learning_rate, \n", 320 | " min_samples_split=min_samples_split, \n", 321 | " min_impurity=min_var_red,\n", 322 | " max_depth=max_depth,\n", 323 | " regression=True)" 324 | ] 325 | }, 326 | { 327 | "cell_type": "code", 328 | "execution_count": 13, 329 | "metadata": { 330 | "collapsed": true 331 | }, 332 | "outputs": [], 333 | "source": [ 334 | "import pandas as pd" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 14, 340 | "metadata": { 341 | "collapsed": true 342 | }, 343 | "outputs": [], 344 | "source": [ 345 | "data = pd.read_csv('TempLinkoping2016.txt', sep=\"\\t\")\n", 346 | "\n", 347 | "time = np.atleast_2d(data[\"time\"].as_matrix()).T\n", 348 | "temp = np.atleast_2d(data[\"temp\"].as_matrix()).T\n", 349 | "\n", 350 | "X = time.reshape((-1, 1)) # Time. Fraction of the year [0, 1]\n", 351 | "X = np.insert(X, 0, values=1, axis=1) # Insert bias term\n", 352 | "y = temp[:, 0] " 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": 15, 358 | "metadata": { 359 | "collapsed": true 360 | }, 361 | "outputs": [], 362 | "source": [ 363 | "from sklearn.cross_validation import train_test_split" 364 | ] 365 | }, 366 | { 367 | "cell_type": "code", 368 | "execution_count": 16, 369 | "metadata": { 370 | "collapsed": true 371 | }, 372 | "outputs": [], 373 | "source": [ 374 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5)" 375 | ] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": 19, 380 | "metadata": {}, 381 | "outputs": [], 382 | "source": [ 383 | "class Loss(object):\n", 384 | " def loss(self, y_true, y_pred):\n", 385 | " return NotImplementedError()\n", 386 | "\n", 387 | " def gradient(self, y, y_pred):\n", 388 | " raise NotImplementedError()\n", 389 | "\n", 390 | " def acc(self, y, y_pred):\n", 391 | " return 0\n", 392 | "class SquareLoss(Loss):\n", 393 | " def __init__(self): pass\n", 394 | "\n", 395 | " def loss(self, y, y_pred):\n", 396 | " return 0.5 * np.power((y - y_pred), 2)\n", 397 | "\n", 398 | " def gradient(self, y, y_pred):\n", 399 | " return -(y - y_pred)" 400 | ] 401 | }, 402 | { 403 | "cell_type": "code", 404 | "execution_count": 20, 405 | "metadata": {}, 406 | "outputs": [], 407 | "source": [ 408 | "model = GradientBoostingRegressor()\n", 409 | "model.fit(X_train, y_train)\n", 410 | "y_pred = model.predict(X_test)\n", 411 | "\n", 412 | "y_pred_line = model.predict(X)" 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "execution_count": 21, 418 | "metadata": { 419 | "collapsed": true 420 | }, 421 | "outputs": [], 422 | "source": [ 423 | "from sklearn.metrics import mean_squared_error" 424 | ] 425 | }, 426 | { 427 | "cell_type": "code", 428 | "execution_count": 22, 429 | "metadata": { 430 | "collapsed": true 431 | }, 432 | "outputs": [], 433 | "source": [ 434 | "mse = mean_squared_error(y_test, y_pred)" 435 | ] 436 | }, 437 | { 438 | "cell_type": "code", 439 | "execution_count": 24, 440 | "metadata": {}, 441 | "outputs": [], 442 | "source": [ 443 | "from matplotlib import pyplot as plt\n", 444 | "%matplotlib inline" 445 | ] 446 | }, 447 | { 448 | "cell_type": "code", 449 | "execution_count": 31, 450 | "metadata": {}, 451 | "outputs": [ 452 | { 453 | "data": { 454 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvX+UI2d15/15Rhq1EQ7NDxtjHI9t2tAYBmPSDeMsOdkJ\nHgTIdoF/jEMf4pPdzTlu3qNhMxCOjbvX/PLbbOxdsn43PSGNCb3YcJrMMLCqZDq4mE4mkOyxwwwM\nMOBp7LYzNqxjYwJNguLuac3z/lFVUqlUJZWk0o+W7uecOt0qlaoelapuPc997v1epbVGEARB6H+2\ndLsBgiAIQmcQgy8IgjAgiMEXBEEYEMTgC4IgDAhi8AVBEAYEMfiCIAgDghh8QRCEAUEMviAIwoAg\nBl8QBGFASHa7AV7OOeccffHFF3e7GYIgCJuKY8eOPau1Prfedj1l8C+++GKOHj3a7WYIgiBsKpRS\np6JsJy4dQRCEAUEMviAIwoAgBl8QBGFAEIMvCIIwIIjBFwRBGBDE4AuCIAwIYvAFQRAGBDH4giAI\nA4IYfEHoMOayyZ59V2N+4GowzW43RxggeirTVhD6HXPZZOLPb6Kg15h/HizccRiDA2AY3W6aMABI\nD18QOoi1YlHQawAUUmBdsA6W1eVWCYOCGHxB6CCZkQxpNQRAeh0yP05BJtO+A5om7NkjriMBAKW1\n7nYbSoyPj2sRTxO6jmnave5Mpi2uFnPZxDo8R2YFjJ2T7XPnmCZMTEChAOk0LCyI66hPUUod01qP\n19tOfPiC4MVrJOfn22IkjVEDY7QDhtey7O8B9l/LEoM/4IhLRxC8BBnJzUomY/fswf7bTteRsCkQ\ngy8IXvrJSBqGPULJ5cSdIwDiwxeEatrswxeEuBEfviA0i2GIoRf6EnHpCIIHc9lkz+IezOWYwxgl\nPFLoAcTgC4KDuWwycXCCfd/cx8TBifiMvhv5s2+f/VeMvtAlxOALgoO1YlE4bUfoFE4XsFZiitDx\nRP6YFxbY8/fT8Y8gBCECLRt8pdSFSqm/UUr9QCn1faXU7zvrX6yU+ppS6hHn74tab64gtI/MSIb0\nVjtCJ701TWYkpggdJ/LHHIWJG2Ff+kS8I4gwxI0k+Iijh78B/IHW+jXAlUBOKfUa4EPAktb6lcCS\n81oQehZj1GDhhgVyb8yxcMNCfMlRTnik9c7tFLbaq2IdQQQhbiQhgJYNvtb6Ka31t5z//wV4GLgA\neCfwOWezzwHvavVYwmASZSI1rslWY9RgNjsbayasuWyyJ2kx/DajPIJQQ2QOP94+Q9xPCWRCbMQa\nh6+Uuhj4OrAdeEJr/UJnvQJ+5r72feYW4BaAbdu2jZ06dSq29gibH3citXC6QHprOrDnHWWbbuFv\n294r97L68HEynz6M8d319mnciI7OQBE1Dj+2SVul1NnAQWCv1voX3ve0/VQJfLJorT+ttR7XWo+f\ne+65cTVH6BOqJlIPz9Xfpp2ukgbxt231AZPZRWxjD+3rfUuWrRBALAZfKbUV29h/QWv9ZWf100qp\n8533zweeieNYwmBRJSf86cOVbhDT5OlvfLXiM8NnDbd+4JgmPCsmgk9DJn8CDh+GIfs7tVW+wTBg\ndlaMvVAijigdBfwZ8LDW+o88b5nA7zr//y6Qb/VYwuBhjBos/Pwqcg/BwkGnZ+z2iE0T847dfGXr\nSsVnVp9bbe2gMU54liaCC9tZ+BIYy8D6Olx1VW/1viWiZyCIo4f/ZuBm4C1KqePOkgX+EHirUuoR\nYJfzWhAaxtg5yezfpm1j6e0RWxbWBesUE+Vtk1uSrYdTxjzhaYwazL55BuNJjyjb5CTMzmKO0p7M\n3kbwPODMO3bb9XYlT6AvaVlLR2v9d4AKefuqVvcvCCV/tF/QLJMhc8e9zK+vU0hBgi3c+uZbW5+w\nzWRsLXx3wjMOl0vAd/BO6N77rXvZdckuJscnOz/h7DzgzFGYuGadwrOLzB880lOT30I8iFqmsLkx\nTcwjc1gjkNkVo7FsUDHTXDaxViwyI5nIbdizuId939xXsa4rUUZOD3/Pvy+wb0d5dfbSLJf8HIZP\nPc3qRefFe36FWIkapSMGXxAcmjHa7ueaCQv1fs5L7o05ZrOzDbe/1nHqfq/pacyv/CET152hkIIh\nlURrWGfDjq9Tdu7Awm/vF6Pfg3Q8LFMQNjOtCKc1GxbqTuiOnT9GQtkTESVJh5gmUf3fa/qvp4Pn\nDFZXMR4+w8JByD0EVz33ctvYQ8lhW9BrPRXyKjSOGHyhL2k087aVWP5mNXjMZZO5o3OceOYERV0k\noRLsvXKvPTk9MYF5eB97PnM95v3Tkdvix/+97vq7u4Ifao7ej7EMs3+bZvKy3ymFw7oZNGk1FJ++\nkNAVpACK0NM042bxukrmj89XuFjc/Q2fNczqc6ul/WZGMswfny+5ZfyGrVY73J56I+0McucUddEO\nKbUszAsLTNwAhVSR+ZW7WVje0ZQrxfu9kluSbJyxe+3uQ620T9+ksmEYLCzvwDo8Jz78PkIMvtCz\n1DLctQjqrRujRqCR9e43zGhHaYcxaoS2Lehh4W2jS+lBkwHriT+lkCra30FtVBrnOufMeyzv9xo+\na5h7Hrwn9KHmr/RV6zsJmxMx+ELPEma46xHWWw8yst79hhm4ZtsB4Q8LbxtTiVRlSOYoZFZvY37l\nbgpqI7KbyH+svVfuLY1i3EngHRfsaGpiWugPxOALXSGKq6aemyWMsN66d38uUfbbbDsg/GFRzw1k\n3Dxju1QaMM7+Y93993ezcWaj4kETqddumjDnaBZNTvZGJrAQCxKWKXScRsIYG/LhR4idD/PhB33W\ne2ygbjuC2hqLkmfEnADvsRIqQVEXS+9FDvU0TbjpJlhbs1+nUnDggBj9Hkfi8IWexZ9w1HTcudcQ\nQtNywOb901gH7iLzw6Itf7CwwPTZD5V6yK6hhhCj7yR/TbxwiYJeqzLsUR9agdtNT8Ndd0GxGOl7\nlR5oTzzNPf/0v0suoYoHTa0HyJ49toaQl1zOFmETepaoBl9cOkLHGX7i6VIyDzSpbunVe5+fh507\nq/VvImbITqzcTWGsyPzrYOFgAY7McdcLHyj1kAunC8wdnePIqSPVE7eOQbYyRQpOlqor4+wa2Chu\nlEBf/zJw9922sfd+Lwg12MaoYX/uDybYceEG1qsSZHbvrTT23vPmf4BkMvCZz1T28Nul5il0HInD\nFzqLabK6+JUK9aWm1C39Amdg94Ddv1GMlGky97n3UVBOqGIKrFclsEaocIckt9j9oqo4fdMsGeTM\nY7Z8M8CWInz1h4tc/eFLMT9wdaTkqSpf/2enbT/6xkZ5o0QChofrx+g758ZYhtm/KGI8tFr1nn2g\ngv3A8rbPMGD/fshm7UXcOX2FGHyhs1gWmR8WS8YxrZtUt3QSheydOOqTjRT8cKSVlxJPlFal9BYy\nu28js2uylEiVUAluffOtTI5PVidXeQyysQx7H7SN/ZkErLwYFhMr3PD8Ra4+cF3d5KlA3Xy/Sud1\n18HqailGf99YkYmVu6uTy/znxvvw874HcOJEtQS0YcChQ/Yixr6vEJeO0FkyGYz5eRYOFhx3Q5Pq\nlmEKmlENlCOtvOa5A1738jdg3DwDEBhBU7HuvofggQcwR8F6BWQeV6y+9hWcSVRq828kYfHSMxyp\nkzxVitr57DSZ/AnbLcNG5UbnnQeZTGCMPnjmF8LOjee8mbPvw0o+QeYxMJaju8CEzY1M2gqdp0El\nyna1wbxjNzcZZaOfSqQ4sPtAtGig66/HvLToZMNCuphg787b+O/fuLusQeMj0uS0d5I2lQKlbH+6\nZ8LWvH/anndQGwwlhtj+0u2ceOYEa8U1EirBbS+9jpkT55VlmH2n21w2mfjzm+wJ5nVY+MsUxp3i\nutnMiHia0Lv0Quk9w8C48wBXFbeVVq0X16s1dIJEzCwLikWsV9jGHqCQsGURDrz7INlLs4ydP8bY\n80YY0j5RtFpMT5cnaZNJ+OAHbX+6z01l3DzDwoR9HI3m2FPHWCvak6xFXeTu//slzMN2tS5z+qGq\n4l3WikVB29sXUmDdskuM/YAgLh1hcDEMJkfhiCdOvsIoh0W0OAVSMo8VmH+D08N3PuuPyImcR2Ca\n5Z492HMDq6tVcgelpo8aWCsW64+uV723kbDdTMZyAcv8t6rgpcz7fIlkuybrnyvPMMEcrZ+TIPQm\nYvCFztIL7hwPNTNeg0odugZ4YQHDslh49TDWS1ZDjV9kPRpn1FAimawbaeQXRtNaU9RFe9L3MSCd\nJmM8j/nHnPSEoQ0yj/8ZxvL5jYm9eR585jfuZeIGRUGvheoKNVtXQOgAWuueWcbGxrTQx+TzWqfT\nWoP9N5/vThtyuWjHdtqbH0Xnrk3o/H1T7W2Xe24SCa2nIhwrn9f592d1bjar8/dNVfzv/Y75vNa5\n7IrOp24MPvf1zkkuZ38OdO4daD5aXnKHcpVNOpnX6Zm05qPo9Exa50924TceQICjOoKN7bqR9y5i\n8Pscj+HQYL/uJFNTtjFt4IGTv29Kpz+SrDRgjTw0GqHWfv3veR8QQ0Nap1K1v1fYuQ95COdP5nXu\nUK78fZ1t8pendPpjQ6EGPXcoV/OBILSHqAZfJm2FzlErPrzdeJKkgMqs1RpYL1ktJ2adLmB94eNw\n/fWVs6BxETaZ7bpUKmZePe6mtTVYX6/9vcLO/dxclduqqvrXKKUcB+OaD7Lw86vInZMNdOdkfjpM\nWtue4kaF5oT2Iz58oXPUig9vN5ZVnbUa4YFToZSphsh86dtQPGO/2YCEQ0sEzSU4E8cUCjA0ZPfb\n19fDH6RB59404fDh8jZDQ3acf5DCp+GEk05MYBQKGOk0LEzCqOcYponx3ntYCJJ0EHoCMfhCZwmJ\nOmkL3glir4FMJuHWWyO1o6KAyJEHsS4+BmfszNqoD42W8bbdNeh+Aw71H6T+c29Z5ZEBwFVXgWGQ\nWSZYDjpsEtu7P0fSwVguwrZVuDm+0yDEQBS/T6cW8eH3JxX+4I4dNMA33YLvPX8yX/ZdT6Hzl22J\nNrEaFzXa3sj5DfPN+33/gfusNemez2udzdrzCd2clB9QkElboRfoWtRGrQniJgx/1WTkbLYNjW6c\nRs5v4LbeSJ8ov03QufM+CFIp2/CLse8oUQ2+TNoKbSXIHxwVc9lkz+KeKnGwsPUV7+0YDp6kDJoA\njUCFuFnUZKU6mCZcfTWMj5eXq6OJa5Zo5PwGbWuOwsQ5R9j37KI9QRtwTivwTyybpp0hXChgjsKe\nq9btSd4eyLEQqhEfvtBWmi0PGFYLtlZB8Yr3tqZZ+NO9tjSw169dzw8dQr2ShI1imrB7d6UL3eXw\n4eiqxI2c36BtW6nXW5GQNUpJV2heLbGwbMqEbQ8iPXyhrbiGMvfGXEPl/aoM0eG54PWeHq3/vbkt\nx6vDHFsIDTVGDWazs7EYMv98qZf19UgRo6U2RT2/QdtWjVxqPDCqRlaeh2eFrpBea2gkJ3QOUcsU\nukatFPwgRUeu+Tpzp87lsLqN9Uu/RFonWRi5tSRpbC6b7D6wm/WibUmHEkPs372/2gh2Ud7BrQ/+\n9NPwne9URoq6dLqMbBQphMDavMvADTfAxkZFD7/p2r1C00hNW6GniVLc2/zA1VgnF21dmOVrmUge\noLAxxFBqnatefxOTv8hjPJnG3PvXfPyBHTz7LCRf/0VWfm2itI9m6+W6hhns2ipxGF9/ffBkEl7/\n+sptzjuvxvG6+KAKq0Ns/pfdWN/+EpkVYCiFdcsuMrsmxdh3GKlpK/Qc3p5kPd+xuWxijUBmKYWx\nvM6eRJbCxhAAa+spLvnmVRjkMbmKG/7rGBtuv+XUb5NcPcXGb32o6UxPv399aclWKW7VxlpW2diD\n3bu/8sqI9cF9AmbWk501rEH+f3PZZCK9SOFNMP/GJAsjH2T25hm7rX+8p2cE8oQy4sMXOoI/XX/4\nrOFQ33Fp22cXmbhBYb4/S+a2N5Rd70MbZFJfB8BKZNnQ3n6L4oKn3lvl064V2ePH719fW4vuU69F\nJmMns7o0VB/c8ZebozBxzXr0qBqXIF3/Bgjy/1c8tNUG1ktWm46CEjqDGHyhI/h79KvPrYZONlZs\nq9ewdl2CMbODvf/9IbZfc4S9/+MYxoGbIZcjc9sbSPrGqe+5cbhicrVKG6aOkcxkbGPs4igOtExL\n9cGdyeaKydGoYa5tMsKBujlBUVBCzyAGX+gIQdEgpaiXZSp6n0Hbmssm9/zsLZwY/y3u+dlb7Fjv\n2VmMmR0cPAhjY3DRRTA1BTP2HG6pVz93dK6hXADDsI2xa5jjcOd4991UfXBHSiHz6ixpZQ8TIrus\nGjDCQQMB04Srf+cxdv+/95cfmvdP27o5X9wgdyzBwoWObk43BfKE+kTJzurUIpm2/U0j6fr+bRuV\n3fVmlQ7dOaRTd6Y2nUZ7WEJww1IVEesQhKlRuOvY+q+ad19rn/9bt8eaySy0BhEzbWXSVugYgdWf\nQhKh/Ns2msDldQutFdfIXprlkhddsmmqMIVVV4QGqmi5RFQpDRsIuOs4/XxYeSvp1y2R2W5A2i2l\n5evJd1IgT2gIMfhC2zGXTeaO2jGOk+O+yJIgJcgAKjJdfzqM8ccWZAg1LP4HRNVxe5wmE4LDiWCE\nw36KkgrzWRtc9fYUk+6cy/CO8IdIj5WyFByiDAPqLcBngWeAE551Lwa+Bjzi/H1Rvf2IS6f/yJ/M\n66E7h0qumNSdqWpXhMcFkD+Z19nPZ3X28yFiXg2USeyKSmdMdKsaZJg2WkMeml4oZTlg0GHxtP8F\nvN237kPAktb6lcCS81oYMKwVi7ViOfh8vbheMWlqLpvsSVqY78tgjsJNB25i8dFFFh9dZPeB3dUR\nNQ1MQLrSAdbhOcwPNKhK1mVcL0wuV+nO6cRx/WoUYYW4QpFInZ4lFoOvtf468M++1e8EPuf8/zng\nXXEcS9hcZEYyDCXKweepRKrkf/eHS84dnav5cLB3GD0KxJVn2PfsIhPPW8S8Y/emM/p+Q9tIPkE7\nqdkOidTpWdoZlnme1vop5/9/As5r47GEHsUYNdi/ez/ZS7NkL81yYPeBki/dH5sPhD4cKti5046X\nrNP1tVYsCtp+gBRSYF1QW5XMlStuVKK4UzSaT9DSsWqci7rtaPPwpFceepuRjkzaaq21UipQtEcp\ndQtwC8C2bds60Ryhw4RFlQRNrE6OT4ZP8HpDV9JpW3TGh1ecjLP/C6mXP8X66JdJr0Pmxyl4b3Bv\ns11yCnHSkpRxA9Q7F5Ha4ZWj9r5utW015LGF+rSzh/+0Uup8AOfvM0Ebaa0/rbUe11qPn3vuuW1s\njtANavXGgtL1jVGDQ+85xKH3HIoWwuk9lmOoFhfh2DE49rcv48zB/WT/78dY+Lcsxp3hqa2tyil0\nYnTQiJSx26YwNYVa79U7F5Ha0abs3lYK6sROi3IVbdtXLaLM7EZZgIupjNL5b8CHnP8/BNxdbx8S\npdM/uNE2YQlPU1Nab9um9chIAxXx6kR/+KsaBuUE1dp1KlX+zNBQ9OCSfL5cyhW0VqrB79UAbuTR\n1J88WDNypl752VpBNFHORd0IqFolJlugayUzqxoSYyRSDPuikzVtgQXgKeA08CPg94CXYEfnPAIc\nBl5cbz9i8LtEzJmR3psyKDt2aqraKKdSDRj9sGLePkMVxXDn81qPjdkPnxtvtI10o4Y67EEDWieT\ntvG/6KL4ap5HsQ+17G0UW+zWJG/6odWu0MxGa/C2izgfaDHsq6MGP65FDH4XaMON6ZdB8PfGtvuy\n8uPsBLqGamysvrHK522D7G1Do0bZPZ5/P2FLHEY/qsFutocfG3FLLPRSfP8g9/DjWsTgd4E2DL0r\ndGw+ktDZT45VuXOa7uHHSFDPfPv26J/33qeplN2TV6q2wW9k/1GOW8s+1LK3m1Lupk1uoqaJ8yS2\nuC8x+EI02tRryt83pXPXJnR+NHi/TfnwY6bVHn6Q/fGOMMbGtN6ypfn912v7pjPYrdJLPfweQwy+\nEIn8ybztD31/zFa313pjIXh9+GHG2DXiIyOV20WxP+7+4/Thb0Ziu84G8klXHzH4Ql3yJ/OlKJpA\njZuWdt4fvbGgiWBvT32Q7M/UlO2Sanie42Repz9m6ymlp9D5y8P9d3W1lNpBH/yIUQ2+FEAZFEwT\n8wNXs2ff1aWY+Lmjc6wX7YDr9eJ6KeEpFrolBhMz/ph0FzdcumGdmS4zPQ2ve539t9HPfeITcOKE\n/Tfy500T67PTFRnP0/9uHfNI9bVmLpv1tZTixjQx79jNnsf2bTrpjWYQgz8IOBf1xPMWbV2ZP7+p\nM2npPWgNG81v8Zc7dOmVr+QmfI2Pl5ew5K+mjTYB8gpRzp+TfJXJnyDtPjQ1nHgZ3DD8ANN/XdkA\n63AELaWYMY/M2TWCd9i1goMeRP2EGPxBwLKwLlgv10LVa1grFpPjkyXtmqHEEJPj1VIF/UQzyZ/e\ncocjI7BtW2UZxW5SlVnsLIuL9vp6Rtp9HeUh6H/ARXrgOZnRxjIsHITtvzwblP3WBkXu+vp/xbx/\nutSIzKcshjbKH0+RjFbCsQWsEU+N4JT9uiE6lSEbE2Lw67HJftBAhofJ/OOWUi8rrYZKlZ/2795P\n7o059u/eX1OTJFAiYZOdm2ZVe906tI8+CqdO9Yaxh3B3E9jr/d8vyGhHfQjOzNgPuu3b4cYb4fjx\n8JFE6bIYvrmsmjmUYtvLRkmoRGm7otJMP/gJ2+hbFsb3N9h/ALI/hLEfwa61CyKeiebJ7Jos1QhO\nkmD4siuifzgu+YhO3kdRHP2dWnpu0rYfJh493yF/2Radu2Os4ckwfzr71NKUHXFxeSr43PTIJJg/\nW7Qffk4vYRPKtfIa/BOvjQZT+WUk/MepOsdTD+r8+7OlSdvkx5N6y0echDznb/ojSZ2/b6q04/yo\nPbnbKfmEqaUpnfhYovHjxRGJFtNFiUzaxkDULmEv93Q938F4+Ayz/3xlcE8+YFK3tAufYNUnvvEJ\ney7gmnXMUSrPTZtEsxrF6+5YXISbbrLX98E8cgmvu2lsrLxks/b6oO83MwPf+155lNKodL1l2WJq\nLt6RhGna8wIVt8zqDqxdl5QmbTfObPD2F4yx/RlK7p2C2sB6yaotyZnNYv3WRWU3SwcE0lafW6Wo\ni6XjzR2diya/HIfuf4eLxYjBr0WUH7RHDFwoNb5DyU1z/3TNSV2vOqKXQgqsVwBDQ+X9tnIBx/jg\nDFN87MF55JZw3U1Hj5aXQ4eif79Gg6kyGfvndkml7HXubXDiRPk993IbPmu45MpJb00zefWHmbly\niiFtrxtKDJH56bD9A01Oktn7P2urccbcwfJe36lEiqXHl6LVHIgjEq3DxWKUPRroDcbHx/XRo0e7\n3YxK6hVj3rPHNvYuuZxtUXoJ5zuYO4axXrJauoFcXfG0TrLzkQ0WX1X+SO6NOWaz5e9hLptM/+//\nzInnTpXWJYrw5f1gjGRtK+Mey6tZH/FGMO+fxtr/h2QeOYPxj0Mti9H7Nd2HfLuUGtvN49YcALsk\ngWFU3wbbtzujiNGyfn1yS5Jb33wrM2+ZwVw22X1gN+vFdVIkOfDlLRjfXS9dM+YopTDhK86/gtXn\n7OvWWKap66vud1o2sVYsHv/Z4yw+ulha778P2kIMF6NS6pjWerzedh0pgLKpMYzy7NaePdU/SiYD\n8/PlC7AXy7kZBuaoY+AfswtH7LxoZ9lNozZgyxbS62copCBdTNg9Lso3wvATT7PtxJM8cjGsJSFZ\nhHc9DNarErD7CgzPsVhYaOgCNpdNJh65i8L4GeYvh4WDaxhzcy3dyK67w2+YoPKZND/fH+6dTuLe\nElC+LYaH7cvfvQ1mZpwHwWLZHbhxZoPV51YB201YygFhg7nXgvFdyqPC0QxHTh2hcLpQMsDzx+dZ\neHYnhn8EGcOP59ZiMJfN0nGj1ByIBe8JbTNi8KNQy0I0YeC6QVA5wfTWdLnaVGYvk199AOsX3ybz\nSBHj0bswl3/IRHrR3l4Dl0LqtB1FccXzR7jntacoqA3mn7yHheUd5bmBBi9ga8WikHB8qI6bKI6z\nGNaMIK9Tj/5sPY1/MLd3L6yuVt4G/qpmrgHNjGS491v3loz+0giYo2A8aXeavNerS+F0AWsEDO+T\nJeYOlluUx1qxSpFs/YT48KNQzy+9CRzD/ipFk+OTldWmbp7BeNGVzP7lGYxlMC8tMv3TL5VvOmeC\nbX0rXPIvCVZfP2qPDGh9Yi0zkimFxqXXIfNEslS+sB2VpKTGdjz4b4vV1erbIKiqmbt+1yW7Stut\nJWHu+otKnamgeaP01jSZXZOVfnOIPWDCGDWYzc72nbEH8eFHo0m/dEfw+/9q+ANd90xoz8U04frr\nMS8tMnGDk5CiKRl7gLROsjByK7xpR3kOYGu65dqi5rKJdXiOzAoYO23/i2na0TVuVEgqFR590vDx\nNqkPP8h/3s22tHJbuFIKbnZtKpGqKHLvdSeuPn6SzHYD4+ZyEoR5/zTWgbvI/LBojwwaaEDde6HN\nxH38qD78rsfee5eei8P30iOx5RX4Y3inplqP6Z2a0rmrVUXxEvVhSjHUU0tl5ay6Ze5aJEivvkdF\nNztCvRj4brWpldsi+/lsYFW0igMEXNP5k3md/kiyLMg2Gv3i6HaZxHYcH4nDj5l6bptuxOI7Y2pz\nFPb8+wLmA/+zfkhkvXbOzJD57dtJF+2QuUQRtHOVeCfdoP1D37AQwEGlVgx8t2jVmzk5Plk7BDPE\nnWqtWGWXYsoJHohycbhibl0shN7NQuxi8JvBbzSdsa15eB97PnN9WR+k3WQymJenmLgBW/zpbf9q\nJ0JBsHM6as7Am3awc/RtZH9ljNueGyv71zsVteBgGKVcnJrJRINCPz4Aw3z8JUImXIbPGia5xY45\nSeskmd231b84vGJup51dduCa9suSVLS9w/eU+PBrUPKz/XQY46HV8t3ld1xaFubhfSW/d1onWZg4\n2BHf4J59V7PvWU/c8EMw++T2Ulxcha/wjy3Ytw9z1I6Eybw6i/FHh6q+s983D0TzN25Wx/gmopd8\n+B3DNDEOIFt2AAAc40lEQVSPzGGN2No3UM4hSagEt/3Gbcy8JYLAkSdZwBwF653byfynmbbep/77\nae+Ve7nnwXtKbb/usus47/nntezLlzj8FvH+UPOnYeEwGPPzsHNnxRDTPDKHdRk8vrqFQuqMvVpt\nYK1YHTH4mV2TzP/5EgW9Zke4/DhVYexL3+H4PAs79sI3UkxcYytnzqslFpbNinYGDTcjuW1aDG6X\nZ0U0Ohiy3TOYozBx4giFZwvMHzxSkUNS1MUKN2NNPDkzxpNpjDfPQJvvUf/9ZC6bFW3/ysNfoaiL\n9v3ZYuBDFMSlE0LFD7XVkRBwDb0zxDQvTzHxwiX2PbvI0iu3kHKc3Z0cphmjBgu/vZ/cOVkW/i2L\ncWfZ71FlvF+yinXLriqZZC/+8M3I36MFSYVeV6cQuktYDon7N/I12oWiPP776dXnvLrkzkluSVZo\n+HTCly89/BAqEkZOQ+YxIJGAK66wx9KWhXXZ4xQcd8qa3iD7yiyXvOiSjod6uVmCLqVwtrOGK5Kr\nMiMZGIH5g0dKQ8rhs4ar9tVU4kkLGceNJkLJaGBwMJdNHv/Z46QSKdaL66UcksnxyebCGj1DpKZC\nI5u4+HZetBOwJSLuefAeNs5skFAJ3vXqd7H4yGJns3qjhPJ0aum1sMxS2OH0jVonk8GhYV0M7woi\nUMrYFzo5tTSlkx9Pxt9uvx5xAx+LGk3abxLHQjjea3nozqH46tzm8xWSzZHvgQYvvop78WNDOnv7\nRVUhqHGFNiNhma1TCjv8+Xmw4ZTi8bgr6kYYtJmgoiT+4e/qw8erfPCrz62ycSaeLNkqjhyx9Ygb\n8M00MtLusJqs0EW817KbnGWtWK2V53T8h9bJxXKd3aj3QIMXX8W9qNd4+iencBRESj36Tmf1isGP\nQo1c/G6lYbsTsn4Z1yqZgk8frjK8TfvpQ9pReui0YI2jxnM3K4vQyyULhGCali2uxdwcFApkHqNc\nAU4nS2KBtRsUcPHVuLAq2n/aruVbTEBSK/ZeubcrGb7iw49CDwqkhSVvWCsWe/91O6snjpF5DIzl\n9SqneFwCUUFRQMZnhuzsIK9Gfow081OIOubmxHudemWL3eu94evWNGFpyd73Mix8ZYsd6vnIBsaT\n98DwDnu7sIvLf/FBzQur1P7Dczz+4FdZvNSO4ttQOnpkUcyIwY9Kj8XD+VUIh88aLsf7nj3Ewo9T\ntrEP6Qb7J3qboeqh84vjGG5eRxvzOxr9KUQdc/MSq2yxL1XZKFyI8ZdufYcCfPzj8J3v2O7bsJ6B\n9+Lbs6fuhVVq/wumObJyNwW1UWp/N/R8xKXTLVr0MfjnD1afW63wF1q37Gp7+Jl3yJpgC8Pf+Idy\nxZFeyPt3EHXMzU+U+bKgOa0K/BfCe95Tfp1Kwbe/HThXF0oDF5Zx8wwLEwdL7QcCXbJtJ8rMbqeW\nXovSaRttCDXJ3zdVFpPqYMTQ1NKUTn40USli1YMhNL2ofSfER6SIuaAoMvfCyGYrVfqSSVuMsN5F\n0+SFlTuUqy0a1yBIlE4PE2Vys5ERgGlivPceFr64Qe5YgoULnQmhDsxUrj63ygaVxUvYvr3nHOWb\noGSB0AJ1BcnciZzFRTuSzMW9MCYny731RALe9S6455762YBNXlhxBk40ghj8MNppLOsNBRtNPXUe\nIMYyzP5F0db9cYu67ttn/22T0a+KCvJIOwhCOwi6Nesa0ChFjNzY4C9/2d7Gu70rYBQTXQvpjjIM\n6NTSMy6dMJdLXH6BeglKjhB8fhSdewc6//5s4+31D1GzdfbRAvmTeZ2bzdrtFJ+J0EZqeUNrJjE1\nmt3nLTwA9ut617bfPnTQj0hEl07Xjbx36RWDn39/1ja0rj/a/dEiXDB1M+ei7Cef1/nLUzo9RSlL\nr65P3n9xddDg12vW2JjW556r9chIw0m4glCBvyhOLtdAIZ6oBjio8k696jvtKEbUAGLwmyR/Ml9O\nuZ5C5y9PlS+UOj9+pImjCPvRWuvcrK8S0Gy2sd6Ct5cSpXfSBvL5siKFd+mFSk3C5qTKrv7Jg/HL\nm3gP4i61jHY+r/X27ZXb+1+3uVRbVIMvPnwf1opVTrlOYYc3GkakEKxIlWwihnJldnkqAW1sITP7\nV5U+/XpzDG71kFzO/tsFn7pllaPcvPRQxKbQYVotSu+X4Vi96P74q0d5DzI1VTu82Z1vO3GivC6d\nhle/GpLJ8uteiQWO8lTo1NIzPfywHoN3SBgwPJxamqrolXvrv1YepM7Q0nk/P32jzl2zpexa8rpn\nmhguTk3ZHY+pkGbFjfTwBS/tqMkbl4Bh0+52/4jdvcHc+zOR6MgNh7h0Aoj4qzbrh48ltta770Si\n2lomk9X++QjDxampyo900uiLD3+wca+Bs8+uvpzj8HTkT+Z19vPZptU0W0qLCfpwRLdtnPSMwQfe\nDiwDjwIfqrVtWw1+nMlOIT9oLL0N377zl20pTyBv2WJb6ia+S5CLURDaTdgoL6yH7waYjY2Vl3qd\nhFbvu5btc1B0Toc1vKMa/LZq6SilEsA+4K3Aj4BvKqVMrfUPYj9YvcIEcQmqmCY8/rgtDra2VuGf\niyRKVq+dnkIi5uUpJq47Q0GdYX48wd6XX8fqtlUy/+d+jHPPhXPOgQ9/ONL3MIxKN6OEyQudIGge\n5+yz4Td/07513vc++MAHYHTUri30yU9WyN2UWFoKn4oKmjuruvdq3Hct1O6x8Ys79aDYYokoT4Vm\nF+DXgQc8r28Hbg/bvukefsRQx5afut59pFKN+yeixvc7r/2ROomPBUgYNOCbiduH7+2NiatGCCKo\nhz81Ve1ixBnABo0E6vW86/bwI9z7jfrw4ypcEhf0gksHuBH4jOf1zcCsb5tbgKPA0W3btjX3baOO\nyVpNhGh17BcYRBx+MXovZLdCVWl+4B3x+2YaOT31clNEu0ZwcX3427aVOxt+F2M9o18vsriWAQ7M\nq2nl+zThQgpsX4w3yaYx+N6lnT38WJ7IQckVEX+w/Mm8zn5yTGd/x4m6iTjB47Z7ammqfJE12cNv\n5Ku1kpvSBRemsMkI6uGnUvb6KD78esnqpe3C8mpaoNHgjMAHRMw3SVSD3249/B8DF3pe/6qzLl7q\n+MyqCnU0q13hPc7wsC2uFKGqhrlssvvAbtaL63ApLI0k2D+yF8Pd3nUgJpP2fr2H9OjW77hghz0/\n8H+exth6EqYMW7cmBhqd4shk4DOfqfS3ujVPRH9eqId72X7+87B1q+3Dn5yMXtBm9+6yEneFf9/n\nqw/KqzFavBj9tSjqCZ8FzjFYdOUmaXfi1TeBVyqlLlFKpYB3A+1R8aqhWhcpIarR46yuYl5YYM87\nwLywtna2tWLZxt5hTRWxfnHcTpwC2LvXVujb2LAfIiEZKcaoQWYkg/XvzsP80kxsxh6iS3u7iTNz\nc/AHfwDZLIyN2X/dm64V/Xl3/+Pj9tJsgo7Q+8zMwKlT8OijcOhQdHtnWWVjD3anw7IIFB2sElXb\nNdlyuxsVPgsUdutWkYYow4BWFiAL/BBYAaZrbduusMy4kjMq9nnflE5PO1o30+j8feGulfzJvE7d\nmSoNAYc+lrSHlu5wLmJcfTu+R8X+I+SDRU2cacY9GTQvEMV/KwwW+bx97VVdHzXCpTs6wRpw8Q+E\nD7/RpZ1x+HH/6M348UrJIe+3DXx+FJ2dQGd//9zKB0DIBRB30YTQtob4R4P89nHmlITNC8R9HGHz\nE3iNxukX9xjjhmzH1FQ5YbKDE1hi8NtMS71tRw0zNU1lr7+OvHC7e/hO0yp62cmkPXE2MmJny3qj\nKOKWSJAevhBEvfDfCuM/9WDrvWbPgyN/eao86fuRZM2RfGAMaod6KmLwO0ArowZ/jH3UHnvQMUPC\n+Ju65mv1st2lnRIJ/kxLie8fbKKE/wa6d1rBcxPk3uG7R69NhB/Af/MkamwbM2Lwe5wqv/6dETTv\ng/YTECkaJQfNNaojI1pfdFE5ujOsl+1dRJZB6BT1pOnb4mb09/Dv8CU81srzcW8+tyZuhxCD3wXc\nBJNzzrH/1nu4tyr6pHWwWF+ti9/fI/IuXqPvPhCCdFA6eB0LA07LPfxmh7teH/59Uzp3baIyf6bG\n5/Lvz9oV4DqYhSsGP4B2ztYHue+SyfaP6Brt4ddy2QT13F3j7x8JCEKnaMiH7zf2bZjErblZB+bZ\ngohq8NudeNUzxJZ8FUKQSNTGhhMfPGrWFlQrNdIsF0uOmIUSlHO2Y0e4blMmA/feWxnH7N1X0DpJ\nmhK6Sb1r0DDAwEm4IgM4G8eZAWh49ul97SOSkFs3ifJU6NTSzh5+u0Mag3r4iWQxegm2tsw+hR8q\nzIcvCJuOgJ58/mRe5+4Y0/nLtsTXw49Y01p6+D1Ao+nQjWIYcPAgfGDqZ6w8vg7DT5C46m6Ov6BA\n4ZkIT/yw9ME2dK+l1y70Fb6evHlkjonjSxQSa8xfBwtswbhub2sXvWVhXljAegVkHitgTE/b6337\njCSR3kUGxuB34ocwDLCSd7Dvm/sAWAd4Zoz01nT9B43f1+IK0wiCUBufoL119tOV+jkXn8FYXW3p\nEOaOYSZeCIWtMP8GWDh4AmNiIlBDy6t/1WsMVBFzY9RgNjvb1h8jM5IhrYYASK/D5P/6HgsX7q2v\nu2EYcOCALUrjFaYRBKE23qLje/eSyX+PtNNvShRh+EyyqvPUaDF16yWrFLba/xdSYL2C8rxARPw6\nUV3Riori9+nUstnDMl3i1t8WBCEiThja1FvQyTscrauPDVUlKvqzyRsqozhNtBBN7+dr5LfEkbFO\nRB/+QPXwo2Ium+xZ3IO53Nyj19g5yezfpjGW6awSniAMOo4K5eoQbCTsVQW9VqGQa1mVst4bG7C4\nWBLYDKRCIXN0CmNXrqYkuh//Mb2srzc0UGiJgfHhRyWW8M1ma1rWq3crCEJtnHsvc2SOebVEQa9V\nzZ0F1XKASg9N0G1Y4Zu/ubFmhR0TIJXqXJ9Q2aOB3mB8fFwfPXq06c+byxHj3WuwZ3FPadIVIPfG\nHLPZ2abbFBlXy9utpNxA70EQhGpq2QM35eXpp+HECdsQp9N2aQq3rlHct6H3mC7nnRe98EstlFLH\ntNbj9bbrmx5+XIlV7Q7fDEXKRAlCrNSKlvGGJnsH1u28DXshHLpvfPhxVbVqtJpNbMRQAcc07SJa\nUiFKEKLjLZbXrUJUnaJvDH5gGbEmccM3WTZiMaCRJoG9oWXOOLIRAx5Q3U0QhAYJuA37qyMVJZSn\nU0urYZlxiqPFobvkqmEO3TnUcKp1o8cPqe4WZxU1QRg4WrUD9YTf4oJBDMuMM7EqyJfXCO6cwuKj\ni6wVnay/BlxNjR4/aCgqvX5BaI1W7IBpwk032SGfx47Zf2+6qbv3YV8Z/Dhp1ZfnnVNwacTV1Ojx\ng4airT60BGHQacUOBMXeuxJZ3WLgDX6Yfz3IgDaCd04hlUiRvTTb0CRwmC+xVjq4d/IJ+n8CShDa\nTVQ7MP2ph3jdtUeY/tRDpXWZjC2J5aXbEll9FYcfRM1YXE8oZ3prOvaonDjyAtyQseFh+OQnyz2G\nVAo++EFYXa2dpyW5XILQXqY/9RCf+P3tcPr5sPWXTP1/J5j5f3YAlbH3ccXcBxE1Dr//DL7Hwpmj\nlA26TrIwcivGzTOlTbuWZBURby5WMlldYMVdJ3lagtA9XnftEU785c7S6+3XHOF7f7EzdPt2ENXg\n95dLxzdLaR2eK8fmqw3m/vj77Ln6sZI7JM5Qznbg9cFvbMAWz6+1ZUv5ASD+eUHoHkb2ebD1l/aL\nrb+0X/cofZNpC1TNUmZWYP4FSQpqg9SJa1k6+gXW9POZP+L2iNuvkd+KW8cn883evXD8uP3eFVdU\npoCLf14QuoPtvnkIc/HfMLLPK7lzepG+cumY0w9h3fVtMsVFjPQSLCxgrj6EdeAuHv/G/2Dx5+8r\nbZvL2ROcVfuIwe/u3VercwS1fPDinxcEAQZRS8eEiXt2UCjuYD75H1nYexzD2IGBgTG8A7P4FEeW\nNiisJUN7xHEXOo+joHEt/Y1e0OYQBGHz0Dc+/ApvzsYQ1qpnWGUYGIcmWdifJJd9jIWdc3aVe/8+\nYtLjcen1OQJBELpLp2Ub+sbgR4k5NzCZPfI6jMX3BqaeNmqg68bFd0uITRCEnqcbmfB949KJVHOk\njvZpI4XOTRN27y7XHF9aCi5DW0ui1Y3RhfbF5wqC0Jt0QxG9bww+RPBp+8NeAoYBUSvOW1bZ2EM5\nZbqR4lZRHhiCIPQnEcxR7PSNSycSreoleMhk7GxXl0ZTpsMeGIIgbG6i+uVjNEeR6auwzE7TikvG\n38MfGpIeviBsdrpVqXTgwjJbosmA9lbCIg0DDhwQH74gbBaimIler1QqBt95JJsXFrCe+FMyq7dV\n6O20E4mjF4TNgbfnPj8f3nPvhl++EQbLhx+EZWFeWGDiBtg3VmRi5e7apQgFQRg4otaW6IZfvhHE\n4GcyWK9KUHAmYAtqo+WEK0EQeoc4kpsaqS3hr0vRS7Rk8JVSu5VS31dKnVFKjfveu10p9ahSalkp\n9bbWmtkgDf7CmV+5gnQxAUhGrCBsVoJu+7iSm3q95x6VVn34J4DrgTnvSqXUa4B3A68FXg4cVkq9\nSmtdbPF49YnqbPNsaxQKLFyeYu4/XAEvPa/tTRQEIV7Cbvs4J1H7Yc6tpR6+1vphrfVywFvvBL6o\ntV7TWj8OPAq8qZVjRcb/C8/NRdt2bZ0jq99h8dFFJg5OiB9fEDYRYT52KfNZSbt8+BcAT3pe/8hZ\n1378hSQPHw4fx3muButVCQrKrigSh3CaIAidI8ywt8sV02nRs7ioa/CVUoeVUicClnfG0QCl1C1K\nqaNKqaM/+clPWt+hYcBVV5Vfr69HmlLP7L4tdmVLV1zt0kvhpS+1/4YJrQmC0Dx+ww5lYUOIdxK1\nG6JnsaG1bnkBjgDjnte3A7d7Xj8A/Hq9/YyNjelYyOe1Tqe1BvtvPh/tYyfzOncop/Mno21frwmp\nlN0E/zI0FLlJgiA0iP/ei/t+y+Uq7+dcLr59NwtwVEew1e1y6ZjAu5VSQ0qpS4BXAv/QpmNV0+Q4\nzhg1mM3OxiJj7NfK8SK6OYLQPtqtU7WZ5wVaDcu8Tin1I+DXgUNKqQcAtNbfB/YDPwC+CuR0JyJ0\nvHQ5GNYvrualUaE1QRCi06qwYT02c4imiKe1EVdcbXkZfvELeMELYHRUdHMEod0MWq2JqOJpYvAF\nQRA2OVENfn9LK2zW2ClBEIQ20L8Gf1PHTgmCIMRP/xr8qPJ2giAIA0L/GvzNHDslCILQBvq3AIob\nO9VEJStBEIR+pH8NPvSHvJ0gCEJM9K9LRxAEQahADL4gCMKAMDAGX0LyBUEYdAbC4EtIviAIwoAY\nfAnJFwRBGBCDLyH5giAI/R6W6SAh+YIgCANi8KEyJH/QpFMFQRBggAy+i2nC7t3lijhLD5xm/23f\nwpjZ0d2GCYIgtJmB8OF7qSp/VtyKdde3JXRHEIS+Z+AMflX5M54jU1yU0B1BEPqegTP4hgEHDkB2\n7J/Ibvkr9nMTRnpJQncEQeh7Bs6HD+4E7svAPA3WNsgsYGIwdzU8/TT8/OewsQHveQ/MzHS7tYIg\nCPEwkAa/hBO6Y5pw002wtlb59ic+Yf8Voy8IQj8wcC6dICyr2ti7yFyuIAj9ghh8bPf90FDwexKj\nLwhCvzDYLh0Hw4D9++1kLPHhC4LQrwyswTeXTawVi+FTN7P6/R1kMnDoULdbJQiC0D4G0uBP//U0\nd//93Wz84B1wcDuchvl5W29HXDiCIPQr/evDD6l4Yi6b3PV3d7FxZgNW3gqnnw+IbLIgCP1Pfxr8\nGhVPrBWLoi7aL0a+Blt/CYhssiAI/U9/GvwaFU8yIxnSW21x/MRli9z44S+Ty4k7RxCE/qc/ffiZ\njO2ULxSquu7GqMHCDQtYKxaZkQzGqFh5QRAGA6W17nYbSoyPj+ujR4/GszPTlIongiAMBEqpY1rr\n8Xrb9WcPHyorngiCIAh96sMXBEEQquh7gx8SnSkIgjBw9LXBrxGdKQiCMHD0tcGvEZ0pCIIwcPS1\nwc9k7KhMkMQqQRCE/o3SwQ7SWViQ6ExBEARo0eArpf4bcC2wDqwA/1Fr/XPnvduB3wOKwH/WWj/Q\nYlubQqIzBUEQbFp16XwN2K61vhz4IXA7gFLqNcC7gdcCbwf+RCmVaPFYgiAIQgu0ZPC11pbWesN5\n+SDwq87/7wS+qLVe01o/DjwKvKmVYwmCIAitEeek7X8C/sr5/wLgSc97P3LWCYIgCF2irg9fKXUY\neFnAW9Na67yzzTSwAXyh0QYopW4BbgHYtm1box8XBEEQIlLX4Gutd9V6Xyn1H4BrgKt0WYntx8CF\nns1+1VkXtP9PA58GWzytfpMFQRCEZmjJpaOUejtwK2BorQuet0zg3UqpIaXUJcArgX9o5ViCIAhC\na7Qahz8LDAFfU0oBPKi1fq/W+vtKqf3AD7BdPTmt3TJTgiAIQjdoyeBrrS+t8d4MMNPK/gVBEIT4\n6KkCKEqpnwCnmvz4OcCzMTanXUg742MztBE2Rzs3QxtB2hnGRVrrc+tt1FMGvxWUUkejVHzpNtLO\n+NgMbYTN0c7N0EaQdrZKX4unCYIgCGXE4AuCIAwI/WTwP93tBkRE2hkfm6GNsDnauRnaCNLOlugb\nH74gCIJQm37q4QuCIAg16AuDr5R6u1JqWSn1qFLqQ91uj4tS6h+VUt9TSh1XSh111r1YKfU1pdQj\nzt8XdaFdn1VKPaOUOuFZF9oupdTtzrldVkq9rcvt/KhS6sfOOT2ulMp2s51KqQuVUn+jlPqBUur7\nSqnfd9b31Pms0c6eOZ9KqbOUUv+glPqO08aPOet77VyGtbNnzmUoWutNvQAJ7OIrrwBSwHeA13S7\nXU7b/hE4x7fubuBDzv8fAu7qQrt+E/g14ES9dgGvcc7pEHCJc64TXWznR4EPBmzblXYC5wO/5vz/\nK9h1IV7Ta+ezRjt75nwCCjjb+X8r8BBwZQ+ey7B29sy5DFv6oYf/JuBRrfVjWut14IvYevy9yjuB\nzzn/fw54V6cboLX+OvDPvtVh7epabYOQdobRlXZqrZ/SWn/L+f9fgIexpcB76nzWaGcYHW+ntvlX\n5+VWZ9H03rkMa2cYPVMfpB8Mfi9r72vgsFLqmCMDDXCe1vop5/9/As7rTtOqCGtXL57f9ymlvuu4\nfNzhfdfbqZS6GHgDdo+vZ8+nr53QQ+dTKZVQSh0HngG+prXuyXMZ0k7ooXMZRD8Y/F7mN7TWVwDv\nAHJKqd/0vqnt8V7PhUn1arscPoXtvrsCeAr4ZHebY6OUOhs4COzVWv/C+14vnc+AdvbU+dRaF517\n5leBNymltvve74lzGdLOnjqXQfSDwY+svd9ptNY/dv4+A3wFexj3tFLqfADn7zPda2EFYe3qqfOr\ntX7audnOAPdSHhp3rZ1Kqa3YRvQLWusvO6t77nwGtbMXz6fTrp8Df4NdE7vnzmVQO3v1XHrpB4P/\nTeCVSqlLlFIp7OLpZpfbhFLq+UqpX3H/BzLACey2/a6z2e8C+e60sIqwdvVUbQP3xne4DvucQpfa\nqZRSwJ8BD2ut/8jzVk+dz7B29tL5VEqdq5R6ofP/84C3AifpvXMZ2M5eOpehdGOmOO4FyGJHHaxg\nl17shTa9Antm/jvA9912AS8BloBHgMPAi7vQtgXsIedpbH/i79VqFzDtnNtl4B1dbuf9wPeA72Lf\nSOd3s53Ab2C7GL4LHHeWbK+dzxrt7JnzCVwOfNtpywngw876XjuXYe3smXMZtkimrSAIwoDQDy4d\nQRAEIQJi8AVBEAYEMfiCIAgDghh8QRCEAUEMviAIwoAgBl8QBGFAEIMvCIIwIIjBFwRBGBD+f5KG\nQdp+aHAwAAAAAElFTkSuQmCC\n", 455 | "text/plain": [ 456 | "" 457 | ] 458 | }, 459 | "metadata": {}, 460 | "output_type": "display_data" 461 | } 462 | ], 463 | "source": [ 464 | "m1 = plt.scatter(366 * X_train[:, 1],y_train, c='r',s=10)\n", 465 | "m2 = plt.scatter(366 * X_test[:, 1], y_test,c='g',s=10)\n", 466 | "m3 = plt.scatter(366 * X_test[:, 1], y_pred,c='b',s=10)" 467 | ] 468 | }, 469 | { 470 | "cell_type": "markdown", 471 | "metadata": {}, 472 | "source": [ 473 | "### 分类模型" 474 | ] 475 | }, 476 | { 477 | "cell_type": "code", 478 | "execution_count": 34, 479 | "metadata": { 480 | "collapsed": true 481 | }, 482 | "outputs": [], 483 | "source": [ 484 | "class GradientBoostingClassifier(GradientBoosting):\n", 485 | " def __init__(self, n_estimators=200, learning_rate=.5, min_samples_split=2,\n", 486 | " min_info_gain=1e-7, max_depth=2, debug=False):\n", 487 | " super(GradientBoostingClassifier, self).__init__(n_estimators=n_estimators, \n", 488 | " learning_rate=learning_rate, \n", 489 | " min_samples_split=min_samples_split, \n", 490 | " min_impurity=min_info_gain,\n", 491 | " max_depth=max_depth,\n", 492 | " regression=False)\n", 493 | "\n", 494 | " def fit(self, X, y):\n", 495 | " y = to_categorical(y)\n", 496 | " super(GradientBoostingClassifier, self).fit(X, y)" 497 | ] 498 | }, 499 | { 500 | "cell_type": "code", 501 | "execution_count": 35, 502 | "metadata": { 503 | "collapsed": true 504 | }, 505 | "outputs": [], 506 | "source": [ 507 | "def to_categorical(x, n_col=None):\n", 508 | " \"\"\" One-hot encoding of nominal values \"\"\"\n", 509 | " if not n_col:\n", 510 | " n_col = np.amax(x) + 1\n", 511 | " one_hot = np.zeros((x.shape[0], n_col))\n", 512 | " one_hot[np.arange(x.shape[0]), x] = 1\n", 513 | " return one_hot" 514 | ] 515 | }, 516 | { 517 | "cell_type": "code", 518 | "execution_count": 38, 519 | "metadata": {}, 520 | "outputs": [ 521 | { 522 | "name": "stderr", 523 | "output_type": "stream", 524 | "text": [ 525 | "/Users/haxu/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:42: RuntimeWarning: overflow encountered in exp\n", 526 | "/Users/haxu/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:42: RuntimeWarning: invalid value encountered in true_divide\n" 527 | ] 528 | } 529 | ], 530 | "source": [ 531 | "from sklearn import datasets\n", 532 | "from sklearn.metrics import accuracy_score\n", 533 | "data = datasets.load_iris()\n", 534 | "X = data.data\n", 535 | "y = data.target\n", 536 | "\n", 537 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)\n", 538 | "\n", 539 | "clf = GradientBoostingClassifier()\n", 540 | "clf.fit(X_train, y_train)\n", 541 | "y_pred = clf.predict(X_test)\n", 542 | "accuracy = accuracy_score(y_test, y_pred)" 543 | ] 544 | }, 545 | { 546 | "cell_type": "code", 547 | "execution_count": 39, 548 | "metadata": {}, 549 | "outputs": [ 550 | { 551 | "data": { 552 | "text/plain": [ 553 | "0.93333333333333335" 554 | ] 555 | }, 556 | "execution_count": 39, 557 | "metadata": {}, 558 | "output_type": "execute_result" 559 | } 560 | ], 561 | "source": [ 562 | "accuracy" 563 | ] 564 | }, 565 | { 566 | "cell_type": "code", 567 | "execution_count": null, 568 | "metadata": { 569 | "collapsed": true 570 | }, 571 | "outputs": [], 572 | "source": [] 573 | } 574 | ], 575 | "metadata": { 576 | "kernelspec": { 577 | "display_name": "Python 3", 578 | "language": "python", 579 | "name": "python3" 580 | }, 581 | "language_info": { 582 | "codemirror_mode": { 583 | "name": "ipython", 584 | "version": 3 585 | }, 586 | "file_extension": ".py", 587 | "mimetype": "text/x-python", 588 | "name": "python", 589 | "nbconvert_exporter": "python", 590 | "pygments_lexer": "ipython3", 591 | "version": "3.6.1" 592 | } 593 | }, 594 | "nbformat": 4, 595 | "nbformat_minor": 2 596 | } 597 | -------------------------------------------------------------------------------- /GaussianMixtureModel.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# GMM EM算法" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 3, 13 | "metadata": { 14 | "collapsed": true 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import math\n", 19 | "import numpy as np\n", 20 | "from utils.metric import calculate_covariance_matrix,euclidean_distance" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 51, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "class GaussianMixtureModel():\n", 32 | " def __init__(self,k=3,max_iterations=400,tolerance=1e-8):\n", 33 | " self.k = k\n", 34 | " self.parameters = []\n", 35 | " self.max_iterations = max_iterations\n", 36 | " self.tolerance = tolerance\n", 37 | " self.responsibilities = []\n", 38 | " self.sample_assignments = None\n", 39 | " self.responsibility = None\n", 40 | " def _init_random_gaussians(self, X):\n", 41 | " n_samples = np.shape(X)[0]\n", 42 | " self.priors = (1./self.k) * np.ones(self.k)\n", 43 | " for i in range(self.k):\n", 44 | " params = {}\n", 45 | " params['mean'] = X[np.random.choice(range(n_samples))]\n", 46 | " params[\"cov\"] = calculate_covariance_matrix(X)\n", 47 | " self.parameters.append(params)\n", 48 | " \n", 49 | " # 计算多元高斯分布\n", 50 | " def multivariate_gaussian(self, X, params):\n", 51 | " n_features = np.shape(X)[1]\n", 52 | " mean = params[\"mean\"]\n", 53 | " covar = params[\"cov\"]\n", 54 | " determinant = np.linalg.det(covar)\n", 55 | " likelihoods = np.zeros(np.shape(X)[0])\n", 56 | " for i, sample in enumerate(X):\n", 57 | " d = n_features\n", 58 | " coeff = (1./(math.pow(2*math.pi,d/2)*math.sqrt(determinant)))\n", 59 | " exponent = math.exp(-0.5 * (sample - mean).T.dot(np.linalg.pinv(covar)).dot((sample - mean)))\n", 60 | " likelihoods[i] = coeff * exponent\n", 61 | " \n", 62 | " return likelihoods\n", 63 | " def _get_likelihoods(self, X):\n", 64 | " n_samples = np.shape(X)[0]\n", 65 | " likelihoods = np.zeros((n_samples, self.k))\n", 66 | " for i in range(self.k):\n", 67 | " likelihoods[:,i] = self.multivariate_gaussian(X,self.parameters[i])\n", 68 | " return likelihoods\n", 69 | " # E步\n", 70 | " def _expectation(self, X):\n", 71 | " weighted_likelihoods = self._get_likelihoods(X) * self.priors \n", 72 | " sum_likelihoods = np.expand_dims(np.sum(weighted_likelihoods, axis=1), axis=1)\n", 73 | " self.responsibility = weighted_likelihoods / sum_likelihoods\n", 74 | " self.sample_assignments = self.responsibility.argmax(axis=1)\n", 75 | " self.responsibilities.append(np.max(self.responsibility, axis=1))\n", 76 | " # M步\n", 77 | " def _maximization(self, X):\n", 78 | " for i in range(self.k):\n", 79 | " resp = np.expand_dims(self.responsibility[:, i], axis=1)\n", 80 | " mean = (resp * X).sum(axis=0) / resp.sum()\n", 81 | " covariance = (X - mean).T.dot((X - mean) * resp) / resp.sum()\n", 82 | " self.parameters[i][\"mean\"], self.parameters[i][\"cov\"] = mean, covariance\n", 83 | " n_samples = np.shape(X)[0]\n", 84 | " self.priors = self.responsibility.sum(axis=0) / n_samples\n", 85 | " \n", 86 | " def _converged(self, X):\n", 87 | " if len(self.responsibilities) < 2:\n", 88 | " return False\n", 89 | " diff = np.linalg.norm(\n", 90 | " self.responsibilities[-1] - self.responsibilities[-2])\n", 91 | " return diff <= self.tolerance\n", 92 | " def predict(self,X):\n", 93 | " self._init_random_gaussians(X)\n", 94 | " for _ in range(self.max_iterations):\n", 95 | " self._expectation(X)\n", 96 | " self._maximization(X)\n", 97 | " if self._converged(X):\n", 98 | " break\n", 99 | " self._expectation(X)\n", 100 | " return self.sample_assignments" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 68, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "X = np.array([[ 0.697 ,0.46 ],\n", 110 | " [ 0.774 ,0.376],\n", 111 | " [ 0.634 ,0.264],\n", 112 | " [ 0.608 ,0.318],\n", 113 | " [ 0.556 ,0.215],\n", 114 | " [ 0.403 ,0.237],\n", 115 | " [ 0.481 ,0.149],\n", 116 | " [ 0.437 ,0.211],\n", 117 | " [ 0.666 ,0.091],\n", 118 | " [ 0.243 ,0.267],\n", 119 | " [ 0.245 ,0.057],\n", 120 | " [ 0.343 ,0.099],\n", 121 | " [ 0.639 ,0.161],\n", 122 | " [ 0.657 ,0.198],\n", 123 | " [ 0.36 ,0.37 ],\n", 124 | " [ 0.593 ,0.042],\n", 125 | " [ 0.719 ,0.103],\n", 126 | " [ 0.359 ,0.188],\n", 127 | " [ 0.339 ,0.241],\n", 128 | " [ 0.282 ,0.257],\n", 129 | " [ 0.748 ,0.232],\n", 130 | " [ 0.714 ,0.346],\n", 131 | " [ 0.483 ,0.312],\n", 132 | " [ 0.478 ,0.437],\n", 133 | " [ 0.525 ,0.369],\n", 134 | " [ 0.751 ,0.489],\n", 135 | " [ 0.532 ,0.472],\n", 136 | " [ 0.473 ,0.376],\n", 137 | " [ 0.725 ,0.445],\n", 138 | " [ 0.446 ,0.459]])\n", 139 | "clf = GaussianMixtureModel(k=3,max_iterations=10000,tolerance=1e-15)\n", 140 | "res = clf.predict(X)" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 69, 146 | "metadata": { 147 | "collapsed": true 148 | }, 149 | "outputs": [], 150 | "source": [ 151 | "from matplotlib import pyplot as plt\n", 152 | "%matplotlib inline" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": 70, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "k0 = X[res==0]\n", 162 | "k1 = X[res==1]\n", 163 | "k2 = X[res==2]" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": 71, 169 | "metadata": {}, 170 | "outputs": [ 171 | { 172 | "data": { 173 | "text/plain": [ 174 | "" 175 | ] 176 | }, 177 | "execution_count": 71, 178 | "metadata": {}, 179 | "output_type": "execute_result" 180 | }, 181 | { 182 | "data": { 183 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD8CAYAAACSCdTiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFClJREFUeJzt3W+MVNd9xvHn6QYXQhtoy6ZrA6mx4sQl7iahEyJZlZoK\nBeNShN0gglM1alLJopLjTaVacSoVrZwXaZRKzhI7sZBlJa+KVo6D7ICLJaI2bd1KDNhZGydUlDRh\nN4uyjmuirJYa8K8vZgZml/1zZ3dm7twz34+EdufMZeZ39y4Pd8499xxHhAAAafqVvAsAALQOIQ8A\nCSPkASBhhDwAJIyQB4CEEfIAkDBCHgASRsgDQMIIeQBI2NvyeuM1a9bEzTffnNfbA0AhnThx4rWI\n6M26fW4hf/PNN6tcLuf19gBQSLZ/3Mj2dNcAQMIIeQBIGCEPAAkj5AEgYYQ8ACSMkAeAhGUKedvb\nbJ+2fcb2Q7M8/xHbF2y/VP2zr/mlAgAateA4eds9kh6T9FFJo5KO234mIl6dsem/RsSftKBGAMAi\nZTmT3yzpTEScjYg3JR2UtLO1ZQEAmiFLyK+VdK7u8Wi1baY7bI/Yfs72+2Z7Idv32S7bLk9MTCyi\nXABAI5p14fWkpHdFRL+kr0o6NNtGEXEgIkoRUertzTz1AgBgkbLMXTMmaX3d43XVtqsi4hd13x+x\n/TXbayLiteaUCQCd4dCLY/ry0dP66RtTumn1Cj1453t19wdn69zoDFnO5I9LutX2Bts3SNoj6Zn6\nDWz32Xb1+83V1/15s4sFgDwdenFMn3/6ZY29MaWQNPbGlD7/9Ms69OLYgn83LwuGfERclnS/pKOS\nfiBpOCJO2d5re291s12SXrH9fUn7Je2JiGhV0QCQhy8fPa2pS1emtU1duqIvHz2dU0ULyzTVcEQc\nkXRkRtvjdd8/KunR5pYGAJ3lp29MNdTeCbjjFQAyumn1iobaOwEhDwAZPXjne7ViWc+0thXLevTg\nne/NqaKF5bYyFAAUTW0UTZFG1xDywBIVbUgdlubuD64t1PEl5IElqA2pq424qA2pk1SoIEC66JMH\nlqCIQ+rQXQh5YAmKOKQO3YWQB5agiEPq0F0IeWAJijikDt2FC6/IVdFHphRxSF0Kiv57006EPHKT\nysiUog2pK7pUfm/ahe4a5IaRKVgMfm8aQ8gjN4xMwWLwe9MYQh65YWQKFoPfm8YQ8sgNI1OwGPze\nNIYLr8gNI1OwGM34vemm0TnOawGnUqkU5XI5l/cG0L1mjs6RKp8Evvinv1eIoLd9IiJKWbenuwZA\nV+m20Tl01yAJ3fTxG0vTbaNzOJNH4dU+fo+9MaXQtZtjDr04lndp6EDdNjqHkEfhddvHbyxNt43O\nobsGhddtH7+xNN02qouQR+HdtHqFxmYJ9FQ/fmPpumm+IbprUHjd9vEbaARn8ii8bvv4DTSCkEcS\nuunjN9AIumsAIGGEPAAkjJAHgIQR8gCQMEIeABJGyANAwgh5AEgYIQ8ACcsU8ra32T5t+4zth+bZ\n7kO2L9ve1bwSAQCLtWDI2+6R9JikuyRtlHSv7Y1zbPclSc83u0gAwOJkOZPfLOlMRJyNiDclHZS0\nc5btPiPpW5J+1sT6AABLkCXk10o6V/d4tNp2le21ku6R9PX5Xsj2fbbLtssTExON1goAaFCzLrx+\nRdLnIuKt+TaKiAMRUYqIUm9vb5PeGgAwlyyzUI5JWl/3eF21rV5J0kHbkrRG0h/bvhwRh5pSJQBg\nUbKE/HFJt9reoEq475H0ifoNImJD7Xvb35D0HQIeAPK3YMhHxGXb90s6KqlH0pMRccr23urzj7e4\nRgDAImVaNCQijkg6MqNt1nCPiL9YelkAgGbgjlcASBghj0I7fPawtj61Vf3f7NfWp7bq8NnDeZcE\nXG9kWHrkdmlwdeXryHDb3po1XlFYh88e1uALg7p45aIkaXxyXIMvDEqStt+yPcfKgDojw9KzD0iX\npiqPL5yrPJak/t0tf3vO5FFYQyeHrgZ8zcUrFzV0ciinioBZHHv4WsDXXJqqtLcBIV9w3dxdcX7y\nfEPtzdLNP3MswoXRxtqbjJAvsFp3xfjkuEJxtbuiW0Knb2VfQ+3N0O0/cyzCqnWNtTcZIV9g3d5d\nMbBpQMt7lk9rW96zXAObBlr2nt3+M8cibNknLVsxvW3Zikp7G3DhtcDy6q7oFLWLq0Mnh3R+8rz6\nVvZpYNNASy+6dvvPHItQu7h67OFKF82qdZWAb8NFV4mQL7S+lX0anxyftb1bbL9le1tH0vAzx6L0\n725bqM9Ed02B5dFd0e34maNoOJMvsDy6K7odP3MUjSMilzculUpRLpdzeW8AKCrbJyKilHV7umsA\nIGGEPAAkjJAHkK4cJwbrFFx4BZCmnCcG6xScyQNIU84Tg3UKQh5AmnKeGKxTEPIA0pTzxGCdgpAH\nkKacJwbrFIQ8gDT175Z27JdWrZfkytcd+7vqoqvE6BoAKctxYrBOwZk8ACSMkAeAhBHyAJAwQh4A\nEkbIA13k8NnD2vrUVvV/s19bn9rKAuRdgNE1QJc4fPawBl8YvLoQ+fjkuAZfGJQkFj1JGGfyQJcY\nOjl0NeBrLl65qKGTQzlVhHYg5JEcuiRmd37yfEPtSAMhj6TUuiTGJ8cViqtdEgS91Leyr6F2pIGQ\nR1LokpjbwKYBLe9ZPq1tec9yDWwayKkitAMXXpEUuiTmVru4OnRySOcnz6tvZZ8GNg1w0TVxmULe\n9jZJQ5J6JD0REX8/4/mdkr4g6S1JlyV9NiL+rcm1AgvqW9mn8cnxWdtRCXpCvbss2F1ju0fSY5Lu\nkrRR0r22N87Y7Jik90fEByR9WtITzS4UyIIuCWC6LGfymyWdiYizkmT7oKSdkl6tbRARv6zbfqWk\naGaRQFZ0SQDTZQn5tZLO1T0elfThmRvZvkfSFyW9UxL/opAbuiSAa5o2uiYivh0Rt0m6W5X++evY\nvs922XZ5YmKiWW8NAJhDlpAfk7S+7vG6atusIuJ7km6xvWaW5w5ERCkiSr29vQ0XCwBoTJaQPy7p\nVtsbbN8gaY+kZ+o3sP1u265+v0nSr0r6ebOLBQA0ZsE++Yi4bPt+SUdVGUL5ZEScsr23+vzjkj4m\n6ZO2L0makvTxiODiKwDkzHllcalUinK5nMt7A0BR2T4REaWs2zOtAQAkjJAHgIQR8gCQMEIeABJG\nyANAwgh5AEhYIUOe5d0AIJvCLRrCivNATkaGpWMPSxdGpVXrpC37pP7deVeFBRTuTJ7l3YAcjAxL\nzz4gXTgnKSpfn32g0o6OVriQZ3k3IAfHHpYuTU1vuzRVaUdHK1zIt3LFefr6gTlcGG2sHR2jcCHf\nquXdan3945PjCsXVvn6CHlClD76RdnSMwoX89lu2a/COQd248kZZ1o0rb9TgHYNLvuhKXz8wjy37\npGUrprctW1FpR0cr3OgaqTXLu9HXD8yjNoqG0TWFU8iQb4W+lX0anxyftR2AKoFOqBdO4bprWqVV\nff0AkCfO5Ktq3T9DJ4d0fvK8+lb2aWDTADdYASg0Qr5OK/r6G3X47GH+owHQNIR8B2HKBgDNRp98\nB+nEYZzcIAYUG2fyHaTThnHyyQIoPs7kO0grp2xYjE78ZAHkZmRYeuR2aXB15WtBJmcj5DtIpw3j\n7LRPFkBuCjwLJyHfQVo1ZcNiddonCyA3BZ6Fkz75DtMJwzhrBjYNTOuTl7hBDF2qwLNwEvKYEzeI\nAVWr1lW7amZp73CEPObVSZ8sgNxs2Vfpg6/vsinILJz0yQPAQvp3Szv2S6vWS3Ll6479hZiwjTN5\nICdMYVEwBZ2Fk5AHcsCNZmgXumvQNkyRcA03mqFdOJNHW3DmOh03mqFdOJNHW3DmOh03mqFdCHm0\nBWeu03XaFBZIV6aQt73N9mnbZ2w/NMvzf2Z7xPbLtl+w/f7ml4oi48x1uk6bwgLpWrBP3naPpMck\nfVTSqKTjtp+JiFfrNvuRpD+MiP+1fZekA5I+3IqCUUxMkXA9bjRDO2S58LpZ0pmIOCtJtg9K2inp\nashHxAt12/+npM6/1xdtxRQJXWpkuDKJ14XRyhQAW/YVcqx5kWUJ+bWS6idtGNX8Z+l/Kem5pRSF\nNHHm2mVq0/PWpgKoTc8rEfRt1NQLr7b/SJWQ/9wcz99nu2y7PDEx0cy3BtBpCjw9b0qyhPyYpPV1\nj9dV26ax3S/pCUk7I+Lns71QRByIiFJElHp7exdTL4CiKPD0vCnJEvLHJd1qe4PtGyTtkfRM/Qa2\n3yXpaUl/HhH/1fwyMRvuIEVHm2sa3gJMz5uSBUM+Ii5Lul/SUUk/kDQcEads77W9t7rZPkm/Jelr\ntl+yXW5ZxZB07Q7S8clxheLqHaQEPTrGln2V6XjrFWR63pQ4InJ541KpFOUy/xcs1tantmp8cvy6\n9htX3qjndz2fQ0XALBhd03S2T0REKev2zF1TUNxBikIo6PS8KWFag4LiDlIAWRDyBcXcJwCyoLum\noLiDFEAWhHyBcQcpgIXQXQMACSPkASBhhDwAJIyQB9C4kWHpkdulwdWVryPDeVeEOXDhFUBjmEK4\nUDiTB9AYphAuFEIeQGOYQrhQCHkAjWEK4UIh5AE0himEC4WQR1JYSKUN+ndLO/ZLq9ZLcuXrjv1c\ndO1QjK5BMmoLqVy8clGSri6kIonpH5qNKYQLgzN5JGPo5NDVgK+5eOWihk4O5VQRkD9CHslgIRXg\neoQ8ksFCKsD1CHkkg4VUgOtx4RXJYCEV4HqEPJLCQirAdHTXAEDCCHkASBghDwAJI+QBIGGEPAAk\njJAHgIQR8gCQMEIeABJGyANAwgh5AEgYIQ8ACSPkAXSHkWHpkdulwdWVryPDeVfUFplC3vY226dt\nn7H90CzP32b7P2z/n+2/aX6ZALAEI8PSsw9IF85JisrXZx/oiqBfMORt90h6TNJdkjZKutf2xhmb\nvS7pAUn/0PQKuxQLUgNNdOxh6dLU9LZLU5X2xGU5k98s6UxEnI2INyUdlLSzfoOI+FlEHJd0qQU1\ndp3agtTjk+MKxdUFqQl6YJEujDbWnpAsIb9W0rm6x6PVNrQIC1IDTbZqXWPtCWnrhVfb99ku2y5P\nTEy0860LhQWpkbQ8LoBu2SctWzG9bdmKSnvisoT8mKT1dY/XVdsaFhEHIqIUEaXe3t7FvERXYEFq\nJCuvC6D9u6Ud+6VV6yW58nXH/kp74rKE/HFJt9reYPsGSXskPdPasrobC1IjWXleAO3fLf31K9Lg\nG5WvXRDwUoY1XiPisu37JR2V1CPpyYg4ZXtv9fnHbfdJKkt6h6S3bH9W0saI+EULa08WC1IjWV18\nATQvmRbyjogjko7MaHu87vvzqnTjoElYkBpJWrWu2lUzSztagjteAbRPF18AzQshD6B9uvgCaF4y\nddcAQNP07ybU24gzeQBIGCEPAAkj5AEgYYQ8ACSMkAeAhBHyAJAwQh4AElbYkGflJABYWCFvhqqt\nnFRbWKO2cpIk5nsBgDqFPJNn5SQAyKaQIc/KSQCQTSFDnpWTACCbQoY8KycBQDaFvPDKyknALEaG\nK8voXRitLMKxZR+zPaKYIS+xchIwTW2B7Nr6qbUFsiWCvssVsrsGwAx5LpCNjkbIAylggWzMgZAH\nUjDXQtgskN31CHkgBSyQjTkQ8kAKWCAbcyjs6BoAM7BANmbBmTwAJIyQB4CEEfIAkDBCHgASRsgD\nQMIIeQBImCMinze2JyT9OJc3b641kl7Lu4gWSn3/pPT3MfX9k7prH38nInqz/qXcQj4VtssRUcq7\njlZJff+k9Pcx9f2T2Mf50F0DAAkj5AEgYYT80h3Iu4AWS33/pPT3MfX9k9jHOdEnDwAJ40weABJG\nyGdge5vt07bP2H5olud32h6x/ZLtsu0/yKPOpVhoH+u2+5Dty7Z3tbO+pcpwDD9i+0L1GL5ku3AT\nsWc5htX9fMn2Kdv/0u4alyrDcXyw7hi+YvuK7d/Mo9bFyLB/q2w/a/v71WP4qQVfNCL4M88fST2S\n/lvSLZJukPR9SRtnbPNrutb11S/ph3nX3ex9rNvuu5KOSNqVd91NPoYfkfSdvGtt8T6ulvSqpHdV\nH78z77qbvY8ztt8h6bt5193kY/i3kr5U/b5X0uuSbpjvdTmTX9hmSWci4mxEvCnpoKSd9RtExC+j\n+lOXtFJS0S50LLiPVZ+R9C1JP2tncU2Qdf+KLMs+fkLS0xHxE0mKiNSP472S/rEtlTVHlv0LSb9u\n26qcXL4u6fJ8L0rIL2ytpHN1j0erbdPYvsf2DyUdlvTpNtXWLAvuo+21ku6R9PU21tUsmY6hpDuq\n3W7P2X5fe0prmiz7+B5Jv2H7n22fsP3JtlXXHFmPo2y/XdI2VU5KiiLL/j0q6Xcl/VTSy5IGIuKt\n+V6UkG+SiPh2RNwm6W5JX8i7nhb4iqTPLfQLVWAnVenG6Jf0VUmHcq6nFd4m6fclbZd0p6S/s/2e\nfEtqmR2S/j0iXs+7kCa7U9JLkm6S9AFJj9p+x3x/gZBf2Jik9XWP11XbZhUR35N0i+01rS6sibLs\nY0nSQdv/I2mXpK/Zvrs95S3ZgvsXEb+IiF9Wvz8iaVmCx3BU0tGImIyI1yR9T9L721RfMzTyb3GP\nitVVI2Xbv0+p0uUWEXFG0o8k3Tbvq+Z9saHT/6hy9nNW0gZduxjyvhnbvFvXLrxuqh4Y5117M/dx\nxvbfULEuvGY5hn11x3CzpJ+kdgxV+Zh/rLrt2yW9Iun2vGtv5j5Wt1ulSl/1yrxrbsEx/Lqkwer3\nv13NmjXzvS4LeS8gIi7bvl/SUVWufj8ZEads760+/7ikj0n6pO1LkqYkfTyqR6EIMu5jYWXcv12S\n/sr2ZVWO4Z7UjmFE/MD2P0kakfSWpCci4pX8qm5MA7+n90h6PiImcyp1UTLu3xckfcP2y5KsShfq\nvLNvcscrACSMPnkASBghDwAJI+QBIGGEPAAkjJAHgIQR8gCQMEIeABJGyANAwv4fbpok1PJKebMA\nAAAASUVORK5CYII=\n", 184 | "text/plain": [ 185 | "" 186 | ] 187 | }, 188 | "metadata": {}, 189 | "output_type": "display_data" 190 | } 191 | ], 192 | "source": [ 193 | "plt.scatter(k0[:,0],k0[:,1])\n", 194 | "plt.scatter(k1[:,0],k1[:,1])\n", 195 | "plt.scatter(k2[:,0],k2[:,1])" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "metadata": { 202 | "collapsed": true 203 | }, 204 | "outputs": [], 205 | "source": [] 206 | } 207 | ], 208 | "metadata": { 209 | "kernelspec": { 210 | "display_name": "Python 3", 211 | "language": "python", 212 | "name": "python3" 213 | }, 214 | "language_info": { 215 | "codemirror_mode": { 216 | "name": "ipython", 217 | "version": 3 218 | }, 219 | "file_extension": ".py", 220 | "mimetype": "text/x-python", 221 | "name": "python", 222 | "nbconvert_exporter": "python", 223 | "pygments_lexer": "ipython3", 224 | "version": "3.6.1" 225 | } 226 | }, 227 | "nbformat": 4, 228 | "nbformat_minor": 2 229 | } 230 | -------------------------------------------------------------------------------- /HMM_forward.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | ### 维比特算法 4 | 5 | # 红 白 红 6 | A = np.array([[0.5,0.2,0.3], 7 | [0.3,0.5,0.2], 8 | [0.2,0.3,0.5]]) 9 | 10 | B = np.array([[0.5,0.5], 11 | [0.4,0.6], 12 | [0.7,0.3]]) 13 | 14 | pi = np.array([[0.2,0.4,0.4]]).T 15 | 16 | 17 | a_1 = [pi[i]*B[i,0] for i in range(3)] 18 | 19 | a_2 = [sum([a_1[i]*A[i,j] for i in range(3)])*B[j,1] for j in range(3)] 20 | 21 | a_3 = [sum([a_2[i]*A[i,j] for i in range(3)])*B[j,0] for j in range(3)] 22 | 23 | 24 | 25 | ## 最终 26 | 27 | res = sum(a_3) 28 | 29 | print(res) -------------------------------------------------------------------------------- /K-means算法.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# K-means算法" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "### K-means算法是一类典型的聚类算法，该算法的思想是最小化平方误差\n", 15 | "\n", 16 | "过程\n", 17 | "\n", 18 | "1. 从D中随机选择k个样本作为初始均值向量\n", 19 | "2. repeat\n", 20 | " 1. 令 C_i = 0\n", 21 | " 2. for j=1,2,...m\n", 22 | " 计算 $d_{ji}$ = ${||x_j-\\mu_i||}_2$\n", 23 | " \n", 24 | " 选择最近的均值向量$x_j$,放入C_i\n", 25 | " 3. 计算新的均值向量\n", 26 | " $\\hat{\\mu} = \\frac{1}{|C_i|}\\sum{x}$\n", 27 | " \n", 28 | " 更新当前均值向量\n", 29 | " 4. 结束\n", 30 | " \n", 31 | "### 这里根据周志华老师的西瓜数据集进行聚类操作" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 45, 37 | "metadata": { 38 | "collapsed": true 39 | }, 40 | "outputs": [], 41 | "source": [ 42 | "import random\n", 43 | "import numpy as np\n", 44 | "import matplotlib.pyplot as plt\n", 45 | "from collections import defaultdict\n", 46 | "from utils.metric import euclidean_distance\n", 47 | "\n", 48 | "%matplotlib inline" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 46, 54 | "metadata": { 55 | "collapsed": true 56 | }, 57 | "outputs": [], 58 | "source": [ 59 | "watermelon = np.array([[ 0.697 ,0.46 ],\n", 60 | " [ 0.774 ,0.376],\n", 61 | " [ 0.634 ,0.264],\n", 62 | " [ 0.608 ,0.318],\n", 63 | " [ 0.556 ,0.215],\n", 64 | " [ 0.403 ,0.237],\n", 65 | " [ 0.481 ,0.149],\n", 66 | " [ 0.437 ,0.211],\n", 67 | " [ 0.666 ,0.091],\n", 68 | " [ 0.243 ,0.267],\n", 69 | " [ 0.245 ,0.057],\n", 70 | " [ 0.343 ,0.099],\n", 71 | " [ 0.639 ,0.161],\n", 72 | " [ 0.657 ,0.198],\n", 73 | " [ 0.36 ,0.37 ],\n", 74 | " [ 0.593 ,0.042],\n", 75 | " [ 0.719 ,0.103],\n", 76 | " [ 0.359 ,0.188],\n", 77 | " [ 0.339 ,0.241],\n", 78 | " [ 0.282 ,0.257],\n", 79 | " [ 0.748 ,0.232],\n", 80 | " [ 0.714 ,0.346],\n", 81 | " [ 0.483 ,0.312],\n", 82 | " [ 0.478 ,0.437],\n", 83 | " [ 0.525 ,0.369],\n", 84 | " [ 0.751 ,0.489],\n", 85 | " [ 0.532 ,0.472],\n", 86 | " [ 0.473 ,0.376],\n", 87 | " [ 0.725 ,0.445],\n", 88 | " [ 0.446 ,0.459]])" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 50, 94 | "metadata": { 95 | "collapsed": true 96 | }, 97 | "outputs": [], 98 | "source": [ 99 | "def kmeans(data,k=3):\n", 100 | " m = data.shape[0]\n", 101 | " index = random.sample(range(m),k)\n", 102 | " mu = data[index] #随机选择初始均值向量\n", 103 | "\n", 104 | "\n", 105 | " while True:\n", 106 | "\n", 107 | " C = defaultdict(list)\n", 108 | "\n", 109 | " for j in range(0,m):\n", 110 | " dij = [euclidean_distance(data[j],mu[i]) for i in range(k)]\n", 111 | " lambda_j = np.argmin(dij) #选择最小的值得下标\n", 112 | "\n", 113 | " C[lambda_j].append(data[j].tolist())\n", 114 | "\n", 115 | " new_mu = [np.mean(C[i],axis=0).tolist() for i in range(k)]\n", 116 | "\n", 117 | " if (distance(np.array(new_mu),np.array(mu))>1e-9):\n", 118 | " mu = new_mu\n", 119 | " else:\n", 120 | " break\n", 121 | "\n", 122 | " return C,mu" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 57, 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [ 131 | "k = 2\n", 132 | "res,mu = kmeans(watermelon,k)" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 58, 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "data": { 142 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFGZJREFUeJzt3WGMXFd5h/Hn7eJUCy3etjZFWTuNEcFpSAyhS5BIpdJG\nyKE0shOiEKiKCpWiVAopSLVwKtWKwocE+UNoSCCKIsS3RpYwq4SYLpKtlhZayRsc1knAlRsK9kYV\nDqmNQKtm7bz9MDP2eLPeubOemTv3zvOTrN05e73z3r3OP3fPOfecyEwkSfXya2UXIEnqPcNdkmrI\ncJekGjLcJamGDHdJqiHDXZJqyHCXpBoy3CWphgx3SaqhN5T1xuvWrcvLL7+8rLeXpEp65plnXs7M\n9Z2OKy3cL7/8cmZnZ8t6e0mqpIj4SZHj7JaRpBoy3CWphgx3Saohw12Sashwl6QaMtwlqYYKhXtE\n3BgRRyLiaETsXObrH4iIUxHxbPPPrt6XKkkqquM894gYAx4BPggcBw5GxJOZ+cKSQ/81M/+sDzVK\nkrpU5M79OuBoZr6Yma8CTwDb+luWJOliFAn3SeBY2+vjzbal3h8RcxHxrYh453LfKCLuiIjZiJg9\nceLEKsqVJBXRqwHV7wOXZeYW4EvA9HIHZeZjmTmVmVPr13dcGkGStEpF1paZBza2vd7QbDsrM3/R\n9vm+iPhyRKzLzJd7U6YklWf60Dy7Z47w0skFLp0YZ8fWzWy/drkOjOFR5M79IHBFRGyKiEuA24En\n2w+IiLdGRDQ/v675fX/e62IladCmD81zz97DzJ9cIIH5kwvcs/cw04fmO/7dMnUM98w8DdwFzAA/\nBPZk5vMRcWdE3Nk87FbguYj4AfAQcHtmZr+KlqRB2T1zhIXFM+e1LSyeYffMkZIqKqbQkr+ZuQ/Y\nt6Tt0bbPHwYe7m1pklS+l04udNU+LHxCVZJWcOnEeFftw8Jwl6QV7Ni6mfE1Y+e1ja8ZY8fWzSVV\nVExpOzFJUhW0ZsVUbbaM4S6t1twe2H8fnDoOazfADbtgy21lV6U+2H7t5NCH+VKGu7Qac3vgqbth\nsTmodupY4zUY8BoK9rlLq7H/vnPB3rK40GiXhoDhLq3GqePdtUsDZrhLq7F2Q3ft0oAZ7irP3B54\n8Gq4d6LxcW5P2RUVd8MuWLNknvOa8Ua7+mb60DzXP3CATTuf5voHDgz9EgBlckBV5aj6gGSrRmfL\nDExrjZfWUgCtNV6Ays1kGYQoawmYqampnJ2dLeW9NQQevLoR6Eut3QiffW7w9WjoXf/AAeaXeeR/\ncmKc7+78kxIqKkdEPJOZU52Os1tG5XBAUl2q6hovZTHcVQ4HJNWlqq7xUhbDXeVwQFJdquoaL2Vx\nQFXlcEBSXbrYNV6quJvSxXBAVVLtLZ1pA427/vtvuaZyAe+AqiQ1VXU3pYthuEuqvVGcaWO4q/qq\n/KSrBmIUZ9oY7qq21pOup44Bee5JVwNebUZxpo3hrmpz6V0VsP3aSe6/5RomJ8YJGk+1VnEwtRtO\nhVS1+aSrCqribkoXwzt3VZtPukrLMtxVbT7pKi3LcFe1bbkNbnqosZok0fh400M+6aqRZ5+7qm/L\nbYa5tIR37pJUQ4a7JNWQ4S5JNWS4S1INGe6SVEOGuyTVUKFwj4gbI+JIRByNiJ0rHPfeiDgdEbf2\nrkRJUrc6hntEjAGPAB8CrgI+FhFXXeC4LwDf7nWRkqTuFLlzvw44mpkvZuarwBPAtmWO+zTwdeBn\nPaxPkrQKRcJ9EjjW9vp4s+2siJgEbga+0rvSJEmr1asB1S8Cn8vM11Y6KCLuiIjZiJg9ceJEj95a\nkrRUkbVl5oGNba83NNvaTQFPRATAOuBPI+J0Zk63H5SZjwGPAUxNTeVqi5YkraxIuB8EroiITTRC\n/Xbg4+0HZOam1ucR8TXgm0uDXZI0OB3DPTNPR8RdwAwwBnw1M5+PiDubX3+0zzVKkrpUaMnfzNwH\n7FvStmyoZ+ZfXnxZkqSL4ROqklRDhrsk1ZA7Mama5vbA/vvg1PHGZtg37HI3Jg2V6UPz7J45wksn\nF7h0YpwdWzez/drJzn+xRwx3Vc/cHnjqblhcaLw+dazxGgx4DYXpQ/Pcs/cwC4tnAJg/ucA9ew8D\nDCzg7ZZR9ey/71ywtywuNNqlIbB75sjZYG9ZWDzD7pkjA6vBcK+quT3w4NVw70Tj49yesisanFPH\nu2vvlVH+masrL51c6Kq9Hwz3Kmp1S5w6BuS5bolRCZu1G7pr74VR/5mrK5dOjHfV3g+GexWNerfE\nDbtgzZL/SNaMN9r7ZdR/5urKjq2bGV8zdl7b+JoxdmzdPLAaHFCtorK6JYZFa9B0kLNlRv1nrq60\nBk2dLaPurN3Q7B5Ypn1UbLltsDNj/JmrS9uvnRxomC9lt0wVldEtMer8matiDPcq2nIb3PQQrN0I\nROPjTQ85x7uf/JmrYiKznGXVp6amcnZ2tpT3lqSqiohnMnOq03HeuUtSDRnuklRDzpaRVCtlL9g1\nLAx3SbUxDAt2DQu7ZSTVxjAs2DUsDHdJtTEMC3YNC8NdUm0Mw4Jdw8Jwl1Qbw7Bg17BwQFVSbQzD\ngl3DwnCXVCtlL9g1LOyWkaQaMtwlqYYMd0mqIfvcpRHgI/mjx3CXas5H8keT3TJSzflI/mgy3FUf\nc3vgwavh3onGx7k9ZVc0FHwkfzQZ7qqHuT3w1N3NTayz8fGpuw14fCR/VBnuqof998HikjvRxYVG\n+4jzkfzR5ICq6uHU8e7aR4iP5I+mQuEeETcC/wCMAY9n5gNLvr4N+DzwGnAa+Exm/luPa5UubO2G\nZpfMMu3ykfwR1LFbJiLGgEeADwFXAR+LiKuWHLYfeFdmvhv4FPB4rwuVVnTDLlizpA95zXijXRpB\nRfrcrwOOZuaLmfkq8ASwrf2AzPxlZmbz5ZuARBqkLbfBTQ/B2o1AND7e9FCjXRpBRbplJoH233eP\nA+9belBE3AzcD7wF+HBPqpO6seU2w1xq6tlsmcz8RmZeCWyn0f/+OhFxR0TMRsTsiRMnevXWkqQl\nioT7PLCx7fWGZtuyMvM7wNsiYt0yX3ssM6cyc2r9+vVdFytJKqZIuB8EroiITRFxCXA78GT7ARHx\n9oiI5ufvAX4d+Hmvi5UkFdOxzz0zT0fEXcAMjamQX83M5yPizubXHwU+AnwiIhaBBeCjbQOskqQB\ni7IyeGpqKmdnZ0t5b0mqqoh4JjOnOh3n8gOSVEOGuyTVkOEuSTVkuEtSDRnuklRD1Qx3d9yRpBVV\nbz331o47rY0ZWjvugOuKSFJT9cJ9pR13DHepp6YPzbvJR0VVL9zdcUcaiOlD89yz9zALi2cAmD+5\nwD17DwMY8BVQvT73C+2s4447Uk/tnjlyNthbFhbPsHvmSEkVqRvVC/d+7rjjQK101ksnF7pq13Cp\nXrj3a8ed1kDtqWNAnhuoNeA1oi6dGO+qXcOlen3u0J8ddxyolc6zY+vm8/rcAcbXjLFj6+YSq1JR\n1Qz3fnCgVjpPa9DU2TLVZLi3rN3Q7JJZpl0aUduvnTTMK6p6fe790s+BWkkaMMO9pV8Dtd1wto6k\nHrFbpl0/BmqLclkFST3knfuwWGm2Tln8TUKqLO/ch8WwzdbxNwmp0rxzHxbDtqzCMP4mIQ3Y9KF5\nrn/gAJt2Ps31Dxxg+tB82SUVZrgPi2GbrTNsv0lIA9ZaOG3+5ALJuYXTqhLwhvuwGIbZOu2G7TcJ\nacCqvnCafe7DpMzZOkvdsOv8Pndw3r9GStUXTvPOXcsbtt8kpAGr+sJp3rnrwobpNwlpwKq+cJrh\nLknLqPrCaYa7NEDuSVotVV44zXCXBsQ9STVIDqhqMFzKoPJT61Qt3rmr/1zKAKj+1DpVi3fu6j+X\nMgCqP7VO1WK4q/9cygBoTK0bXzN2XluVptapWgqFe0TcGBFHIuJoROxc5ut/HhFzEXE4Ir4XEe/q\nfamqLJcyABqDpvffcg2TE+MEMDkxzv23XONgqvqiY597RIwBjwAfBI4DByPiycx8oe2wHwN/lJn/\nGxEfAh4D3tePglVBLmVwVpWn1hXldM/hUGRA9TrgaGa+CBARTwDbgLPhnpnfazv+P4DRuiXTylqD\npvvva3TFrN3QCPYRGkwdFU73HB5Fwn0SONb2+jgr35X/FfCt5b4QEXcAdwBcdtllBUtULbiUwUhY\nabqn4T5YPR1QjYg/phHun1vu65n5WGZOZebU+vXre/nWkoaA0z2HR5Fwnwc2tr3e0Gw7T0RsAR4H\ntmXmz3tTni7Ih4I0hJzuOTyKhPtB4IqI2BQRlwC3A0+2HxARlwF7gb/IzP/sfZk6T+uhoFPHgDz3\nUJABr5I53XN4dAz3zDwN3AXMAD8E9mTm8xFxZ0Tc2TxsF/A7wJcj4tmImO1bxfKhIA0tp3sOj8jM\nUt54amoqZ2f9f8Cq3DsBLHfdAu49OehqJA1QRDyTmVOdjvMJ1SryoSBJHRjuVXTDrsZDQO1G9KEg\nScsz3KvI/U0ldeCSv1XlQ0GSVuCduyTVkOEuSTVkuEtSDRnuklRDDqhKKsR12qvFcJfUkeu0V4/d\nMpI6Wmmddg0nw11SR67TXj2Gu6SOXKe9egx31YObl/SV67RXjwOqqr7W5iWtNe5bm5eASzT0SGvQ\n1Nky1eF67qq+B69u7kq1xNqN8NnnBl+P1Eeu567Rcep4d+3SCDDcVX1uXiK9juGu6nPzEul1DHdV\nn5uXSK/jbBnVg5uXSOfxzl2Sashwl6QaMtwlqYYMd0mqIcNdkmrIcJekGjLcJamGDHdJqiHDXZJq\nyHCXpBoy3CWphgqFe0TcGBFHIuJoROxc5utXRsS/R8T/RcTf9r5MSere9KF5rn/gAJt2Ps31Dxxg\n+tB82SUNTMeFwyJiDHgE+CBwHDgYEU9m5gtth70C3A1s70uVo2ZuD+y/r7HZxNoNjaVrXRRL6sr0\noXnu2XuYhcUzAMyfXOCevYcBRmJ7wCJ37tcBRzPzxcx8FXgC2NZ+QGb+LDMPAot9qHG0tPYDPXUM\nyHP7gbrhs9SV3TNHzgZ7y8LiGXbPHCmposEqEu6TQPsGlcebbeqH/fed2+i5ZXGh0S6psJdOLnTV\nXjcDHVCNiDsiYjYiZk+cODHIt64O9wOVeuLSifGu2uumSLjPAxvbXm9otnUtMx/LzKnMnFq/fv1q\nvkX9uR+oaqiMgc0dWzczvmbsvLbxNWPs2Lq57+89DIqE+0HgiojYFBGXALcDT/a3rBHmfqCqmdbA\n5vzJBZJzA5v9Dvjt105y/y3XMDkxTgCTE+Pcf8s1IzGYCgVmy2Tm6Yi4C5gBxoCvZubzEXFn8+uP\nRsRbgVngzcBrEfEZ4KrM/EUfa6+n1qwYZ8uoJlYa2Ox30G6/dnJkwnypQnuoZuY+YN+StkfbPv8f\nGt016gX3A1WNjPrAZll8QlVSX436wGZZDHdJfTXqA5tlKdQtI0mr1erz3j1zhJdOLnDpxDg7tm4e\n2b7wQTHcJfXdKA9slsVuGUmqIcNdkmrIcJekGjLcJamGDHdJqiHDXZJqqLrhPrcHHrwa7p1ofHQz\nC0k6q5rz3Fu7FbU2tWjtVgSuySJJVPXO3d2KJGlF1Qx3dyuSpBVVM9zdrUiSVlTNcHe3IklaUTUH\nVN2tSDpr+tC8Ky7qdaoZ7uBuRRLn9idtbWPX2p8UMOBHXDW7ZSQBK+9PqtFmuEsV5v6kuhDDXaow\n9yfVhRjuUoW5P6kupLoDqpLcn1QXZLhLFef+pFqO3TKSVEOGuyTVkOEuSTVkuEtSDRnuklRDhrsk\n1ZDhLkk1ZLhLUg0Z7pJUQ4a7JNWQ4S5JNRSZWc4bR5wAflLKm/fOOuDlsovoI8+v2jy/arvQ+f1e\nZq7v9JdLC/c6iIjZzJwqu45+8fyqzfOrtos9P7tlJKmGDHdJqiHD/eI8VnYBfeb5VZvnV20XdX72\nuUtSDXnnLkk1ZLh3EBE3RsSRiDgaETuX+fq2iJiLiGcjYjYi/rCMOler0/m1HffeiDgdEbcOsr5e\nKHANPxARp5rX8NmI2FVGnatV5Bo2z/HZiHg+Iv5l0DVejALXb0fbtXsuIs5ExG+XUetqFDi/tRHx\nVET8oHn9PlnoG2emfy7wBxgD/gt4G3AJ8APgqiXH/Abnure2AD8qu+5enl/bcQeAfcCtZdfdh2v4\nAeCbZdfax/ObAF4ALmu+fkvZdffy/JYcfxNwoOy6e3z9/g74QvPz9cArwCWdvrd37iu7DjiamS9m\n5qvAE8C29gMy85fZ/KkDbwKqNIjR8fyaPg18HfjZIIvrkaLnWFVFzu/jwN7M/ClAZlbpOnZ7/T4G\n/ONAKuuNIueXwG9GRNC4mXwFON3pGxvuK5sEjrW9Pt5sO09E3BwRPwKeBj41oNp6oeP5RcQkcDPw\nlQHW1UuFriHw/mb32rci4p2DKa0nipzfO4Dfioh/johnIuITA6vu4hW9fkTEG4EbadyIVEWR83sY\n+H3gJeAw8DeZ+Vqnb2y490BmfiMzrwS2A58vu54e+yLwuSL/mCrs+zS6LLYAXwKmS66n194A/AHw\nYWAr8PcR8Y5yS+qLm4DvZuYrZRfSY1uBZ4FLgXcDD0fEmzv9JcN9ZfPAxrbXG5pty8rM7wBvi4h1\n/S6sR4qc3xTwRET8N3Ar8OWI2D6Y8nqi4zlm5i8y85fNz/cBa2p2DY8DM5n5q8x8GfgO8K4B1Xex\nuvlv8Haq1SUDxc7vkzS61TIzjwI/Bq7s+J3LHlAY5j807nheBDZxbrDjnUuOeTvnBlTf07wwUXbt\nvTq/Jcd/jeoNqBa5hm9tu4bXAT+t0zWk8Sv9/uaxbwSeA64uu/ZenV/zuLU0+qLfVHbNfbh+XwHu\nbX7+u82MWdfpe7+hY/qPsMw8HRF3ATM0RrW/mpnPR8Sdza8/CnwE+ERELAILwEezeRWGXcHzq7SC\n53gr8NcRcZrGNby9TtcwM38YEf8EzAGvAY9n5nPlVV1cF/9Gbwa+nZm/KqnUVSl4fp8HvhYRh4Gg\n0U3acTVMn1CVpBqyz12Sashwl6QaMtwlqYYMd0mqIcNdkmrIcJekGjLcJamGDHdJqqH/B0AwnxpB\nPtPcAAAAAElFTkSuQmCC\n", 143 | "text/plain": [ 144 | "" 145 | ] 146 | }, 147 | "metadata": {}, 148 | "output_type": "display_data" 149 | } 150 | ], 151 | "source": [ 152 | "for i in range(k):\n", 153 | " res_i = np.array(res[i])\n", 154 | " plt.scatter(res_i[:,0],res_i[:,1])\n", 155 | "plt.show()" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": { 162 | "collapsed": true 163 | }, 164 | "outputs": [], 165 | "source": [] 166 | } 167 | ], 168 | "metadata": { 169 | "kernelspec": { 170 | "display_name": "Python 3", 171 | "language": "python", 172 | "name": "python3" 173 | }, 174 | "language_info": { 175 | "codemirror_mode": { 176 | "name": "ipython", 177 | "version": 3 178 | }, 179 | "file_extension": ".py", 180 | "mimetype": "text/x-python", 181 | "name": "python", 182 | "nbconvert_exporter": "python", 183 | "pygments_lexer": "ipython3", 184 | "version": "3.6.1" 185 | } 186 | }, 187 | "nbformat": 4, 188 | "nbformat_minor": 2 189 | } 190 | -------------------------------------------------------------------------------- /KNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# KNN-K近邻法\n", 8 | "是一种基本的分类与回归算法，K近邻法输入为实例的特征向量，对应于空间的点，输出为实例的类别" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 6, 14 | "metadata": { 15 | "collapsed": true 16 | }, 17 | "outputs": [], 18 | "source": [ 19 | "import numpy as np\n", 20 | "from sklearn import datasets\n", 21 | "from utils.metric import euclidean_distance,accuracy_score\n", 22 | "from utils.progress import normalize,train_test_split" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 10, 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "class KNN(object):\n", 34 | " def __init__(self,k=5):\n", 35 | " self.k = 5\n", 36 | " def _vote(self,neighbours):\n", 37 | " counts = np.bincount(neighbours[:, 1].astype('int'))\n", 38 | " return counts.argmax()\n", 39 | " def predict(self, X_test, X_train, y_train):\n", 40 | " y_pred = np.empty(X_test.shape[0])\n", 41 | " # 对每一个test进行循环\n", 42 | " for i,test in enumerate(X_test):\n", 43 | " neighbours = np.empty((X_train.shape[0],2))\n", 44 | " # 对每一个train进行计算\n", 45 | " for j, train in enumerate(X_train):\n", 46 | " dis = euclidean_distance(train,test)\n", 47 | " label = y_train[j]\n", 48 | " neighbours[j] = [dis,label]\n", 49 | " k_nearest_neighbors = neighbours[neighbours[:,0].argsort()][:self.k]\n", 50 | " label = self._vote(k_nearest_neighbors)\n", 51 | " y_pred[i] = label\n", 52 | " return y_pred" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 11, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "data = datasets.load_iris()\n", 62 | "X = normalize(data.data)\n", 63 | "y = data.target\n", 64 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 12, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "clf = KNN(k=5)\n", 74 | "y_pred = clf.predict(X_test, X_train, y_train)\n", 75 | "accuracy = accuracy_score(y_test, y_pred)" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 13, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "data": { 85 | "text/plain": [ 86 | "0.97959183673469385" 87 | ] 88 | }, 89 | "execution_count": 13, 90 | "metadata": {}, 91 | "output_type": "execute_result" 92 | } 93 | ], 94 | "source": [ 95 | "accuracy" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "#### 参考\n", 103 | "1. 《统计学习方法》\n", 104 | "2. 《机器学习》（周志华）\n", 105 | "3. [blog](https://www.cnblogs.com/ybjourney/p/4702562.html)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": { 112 | "collapsed": true 113 | }, 114 | "outputs": [], 115 | "source": [] 116 | } 117 | ], 118 | "metadata": { 119 | "kernelspec": { 120 | "display_name": "Python 3", 121 | "language": "python", 122 | "name": "python3" 123 | }, 124 | "language_info": { 125 | "codemirror_mode": { 126 | "name": "ipython", 127 | "version": 3 128 | }, 129 | "file_extension": ".py", 130 | "mimetype": "text/x-python", 131 | "name": "python", 132 | "nbconvert_exporter": "python", 133 | "pygments_lexer": "ipython3", 134 | "version": "3.6.1" 135 | } 136 | }, 137 | "nbformat": 4, 138 | "nbformat_minor": 2 139 | } 140 | -------------------------------------------------------------------------------- /LSTM_FS.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# LSTM" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "##### LSTM是RNN的一种变种，在于RNN随着时间的推移，一些记忆会消失，于是在LSTM中，添加了遗忘门、输入门以及输出门来防止遗忘，其实遗忘的好坏是sigmoid函数来学习的" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": { 21 | "collapsed": true 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "import numpy as np\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "from tensorflow.examples.tutorials.mnist import input_data\n", 28 | "%matplotlib inline" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "metadata": {}, 35 | "outputs": [ 36 | { 37 | "name": "stdout", 38 | "output_type": "stream", 39 | "text": [ 40 | "Extracting data/train-images-idx3-ubyte.gz\n", 41 | "Extracting data/train-labels-idx1-ubyte.gz\n", 42 | "Extracting data/t10k-images-idx3-ubyte.gz\n", 43 | "Extracting data/t10k-labels-idx1-ubyte.gz\n" 44 | ] 45 | } 46 | ], 47 | "source": [ 48 | "mnist = input_data.read_data_sets('data/', one_hot=True)\n", 49 | "INPUT = 28\n", 50 | "HIDDEN = 128\n", 51 | "OUTPUT = 10\n", 52 | "\n", 53 | "INPUT += HIDDEN\n", 54 | "\n", 55 | "ALPHA = 0.001\n", 56 | "BATCH_NUM = 64\n", 57 | "\n", 58 | "ITER_NUM = 1000\n", 59 | "LOG_ITER = ITER_NUM // 10\n", 60 | "PLOT_ITER = ITER_NUM // 200" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 3, 66 | "metadata": { 67 | "collapsed": true 68 | }, 69 | "outputs": [], 70 | "source": [ 71 | "def softmax(arr):\n", 72 | " c = np.clip(arr, -700, 700) # 防止溢出 \n", 73 | " e = np.exp(c)\n", 74 | " return e / np.sum(e, axis=1, keepdims=True)\n", 75 | "\n", 76 | "# 交叉熵\n", 77 | "def cross_entropy(out, label):\n", 78 | " entropy = label * np.log(out + 1e-6)\n", 79 | " return -np.sum(entropy, axis=1, keepdims=True)\n", 80 | "\n", 81 | "def sigmoid(arr):\n", 82 | " c = np.clip(arr, -700, 700)\n", 83 | " return 1 / (1 + np.exp(-c))\n", 84 | "\n", 85 | "def deriv_sigmoid(out):\n", 86 | " return out * (1 - out)\n", 87 | "\n", 88 | "def tanh(arr):\n", 89 | " c = np.clip(arr, -350, 350)\n", 90 | " return 2 / (1 + np.exp(-2 * c)) - 1\n", 91 | "\n", 92 | "def deriv_tanh(out):\n", 93 | " return 1 - np.square(out)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 4, 99 | "metadata": { 100 | "collapsed": true 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "errors = []\n", 105 | "\n", 106 | "wf = np.random.randn(INPUT, HIDDEN) / np.sqrt(INPUT / 2)\n", 107 | "wi = np.random.randn(INPUT, HIDDEN) / np.sqrt(INPUT / 2)\n", 108 | "wc = np.random.randn(INPUT, HIDDEN) / np.sqrt(INPUT / 2)\n", 109 | "wo = np.random.randn(INPUT, HIDDEN) / np.sqrt(INPUT / 2)\n", 110 | "wy = np.random.randn(HIDDEN, OUTPUT) / np.sqrt(HIDDEN / 2)\n", 111 | "\n", 112 | "bf = np.zeros(HIDDEN)\n", 113 | "bi = np.zeros(HIDDEN)\n", 114 | "bc = np.zeros(HIDDEN)\n", 115 | "bo = np.zeros(HIDDEN)\n", 116 | "by = np.zeros(OUTPUT)\n", 117 | "\n", 118 | "dwf = np.zeros_like(wf)\n", 119 | "dwi = np.zeros_like(wi)\n", 120 | "dwc = np.zeros_like(wc)\n", 121 | "dwo = np.zeros_like(wo)\n", 122 | "dwy = np.zeros_like(wy)\n", 123 | "\n", 124 | "dbf = np.zeros_like(bf)\n", 125 | "dbi = np.zeros_like(bi)\n", 126 | "dbc = np.zeros_like(bc)\n", 127 | "dbo = np.zeros_like(bo)\n", 128 | "dby = np.zeros_like(by)" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "![image.png](fig1.png)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 5, 141 | "metadata": { 142 | "collapsed": true 143 | }, 144 | "outputs": [], 145 | "source": [ 146 | "def LSTM_Cell(input_val):\n", 147 | " batch_num = input_val.shape[1]\n", 148 | " \n", 149 | " caches = []\n", 150 | " states = []\n", 151 | " # 初始状态 c 和 h_prev都是为0\n", 152 | " states.append([np.zeros([batch_num, HIDDEN]), np.zeros([batch_num, HIDDEN])])\n", 153 | " \n", 154 | " for x in input_val:\n", 155 | " c_prev, h_prev = states[-1]\n", 156 | " x = np.column_stack([x, h_prev])\n", 157 | " hf = sigmoid(np.dot(x, wf) + bf)\n", 158 | " hi = sigmoid(np.dot(x, wi) + bi)\n", 159 | " ho = sigmoid(np.dot(x, wo) + bo)\n", 160 | " hc = tanh(np.dot(x, wc) + bc)\n", 161 | " \n", 162 | " c = hf * c_prev + hi * hc\n", 163 | " h = ho * tanh(c)\n", 164 | " \n", 165 | " states.append([c, h])\n", 166 | " caches.append([x, hf, hi, ho, hc])\n", 167 | " return caches, states" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 6, 173 | "metadata": { 174 | "collapsed": true 175 | }, 176 | "outputs": [], 177 | "source": [ 178 | "def predict(img):\n", 179 | " input_val = np.reshape(img, [28, 1, 28])\n", 180 | " \n", 181 | " caches, states = LSTM_Cell(input_val)\n", 182 | " c, h = states[-1]\n", 183 | " \n", 184 | " pred = softmax(np.dot(h, wy) + by)\n", 185 | " label = np.argmax(pred)\n", 186 | " \n", 187 | " return label" 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "metadata": {}, 193 | "source": [ 194 | "![image.png](fig2.png)" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 7, 200 | "metadata": {}, 201 | "outputs": [ 202 | { 203 | "name": "stdout", 204 | "output_type": "stream", 205 | "text": [ 206 | "iter 0\n", 207 | "entropy 148.230102882\n", 208 | "----------\n", 209 | "iter 100\n", 210 | "entropy 130.411356399\n", 211 | "----------\n", 212 | "iter 200\n", 213 | "entropy 26.9214080264\n", 214 | "----------\n", 215 | "iter 300\n", 216 | "entropy 35.5380413727\n", 217 | "----------\n", 218 | "iter 400\n", 219 | "entropy 19.3635887288\n", 220 | "----------\n", 221 | "iter 500\n", 222 | "entropy 16.4149703994\n", 223 | "----------\n", 224 | "iter 600\n", 225 | "entropy 5.40962387843\n", 226 | "----------\n", 227 | "iter 700\n", 228 | "entropy 9.29227429211\n", 229 | "----------\n", 230 | "iter 800\n", 231 | "entropy 7.63320640489\n", 232 | "----------\n", 233 | "iter 900\n", 234 | "entropy 19.3816912508\n", 235 | "----------\n", 236 | "iter 1000\n", 237 | "entropy 13.2405430479\n", 238 | "----------\n" 239 | ] 240 | } 241 | ], 242 | "source": [ 243 | "for i in range(ITER_NUM+1):\n", 244 | " X, Y = mnist.train.next_batch(BATCH_NUM)\n", 245 | " Xt = np.transpose(np.reshape(X, [-1, 28, 28]), [1, 0, 2])\n", 246 | " \n", 247 | " caches, states = LSTM_Cell(Xt) # 每次的lstm单元迭代\n", 248 | " c, h = states[-1] # 获取前一个的状态\n", 249 | " \n", 250 | " out = np.dot(h, wy) + by #最后结果的输出\n", 251 | " pred = softmax(out)\n", 252 | " entropy = cross_entropy(pred, Y) # 计算交叉熵\n", 253 | " \n", 254 | " # BPTT\n", 255 | " dout = pred - Y #计算误差\n", 256 | " dwy = np.dot(h.T, dout) #对wy的偏导\n", 257 | " dby = np.sum(dout, axis=0) #对by的偏导\n", 258 | " \n", 259 | " dc_next = np.zeros_like(c) #存储下一个节点的dc\n", 260 | " dh_next = np.zeros_like(h) #存储下一个节点的dh\n", 261 | " \n", 262 | " # 28为时间 通过28来反向传播\n", 263 | " for t in range(Xt.shape[0]):\n", 264 | " c, h = states[-t-1] #当前时间点的状态\n", 265 | " c_prev, h_prev = states[-t-2] #前一个时间点的状态\n", 266 | "\n", 267 | " x, hf, hi, ho, hc = caches[-t-1] #当前时间点的lstm_cell内的变量的值\n", 268 | " \n", 269 | " # 当前状态c的值\n", 270 | " tc = tanh(c)\n", 271 | " dh = np.dot(dout, wy.T) + dh_next\n", 272 | " \n", 273 | " \n", 274 | " # 根据前面的图 \n", 275 | " dc = dh * ho * deriv_tanh(tc)\n", 276 | " dc = dc + dc_next\n", 277 | " \n", 278 | " dho = dh * tc \n", 279 | " dho = dho * deriv_sigmoid(ho)\n", 280 | " \n", 281 | " dhf = dc * c_prev \n", 282 | " dhf = dhf * deriv_sigmoid(hf)\n", 283 | " \n", 284 | " dhi = dc * hc \n", 285 | " dhi = dhi * deriv_sigmoid(hi)\n", 286 | " \n", 287 | " dhc = dc * hi \n", 288 | " dhc = dhc * deriv_tanh(hc)\n", 289 | " \n", 290 | " dwf += np.dot(x.T, dhf)\n", 291 | " dbf += np.sum(dhf, axis=0)\n", 292 | " dXf = np.dot(dhf, wf.T)\n", 293 | " \n", 294 | " dwi += np.dot(x.T, dhi)\n", 295 | " dbi += np.sum(dhi, axis=0)\n", 296 | " dXi = np.dot(dhi, wi.T)\n", 297 | " \n", 298 | " dwo += np.dot(x.T, dho)\n", 299 | " dbo += np.sum(dho, axis=0)\n", 300 | " dXo = np.dot(dho, wo.T)\n", 301 | " \n", 302 | " dwc += np.dot(x.T, dhc)\n", 303 | " dbc += np.sum(dhc, axis=0)\n", 304 | " dXc = np.dot(dhc, wc.T)\n", 305 | "\n", 306 | " dX = dXf + dXi + dXo + dXc\n", 307 | " \n", 308 | " dc_next = hf * dc\n", 309 | " dh_next = dX[:, -HIDDEN:]\n", 310 | " \n", 311 | " # 更新lstm中的权重\n", 312 | " wf -= ALPHA * dwf\n", 313 | " wi -= ALPHA * dwi\n", 314 | " wc -= ALPHA * dwc\n", 315 | " wo -= ALPHA * dwo\n", 316 | " wy -= ALPHA * dwy\n", 317 | " \n", 318 | " bf -= ALPHA * dbf\n", 319 | " bi -= ALPHA * dbi\n", 320 | " bc -= ALPHA * dbc\n", 321 | " bo -= ALPHA * dbo\n", 322 | " by -= ALPHA * dby\n", 323 | " \n", 324 | " # 每次完成后初始化\n", 325 | " dwf = 0\n", 326 | " dwi = 0\n", 327 | " dwc = 0\n", 328 | " dwo = 0\n", 329 | " dwy = 0\n", 330 | " \n", 331 | " dbf = 0\n", 332 | " dbi = 0\n", 333 | " dbc = 0\n", 334 | " dbo = 0\n", 335 | " dby = 0\n", 336 | " \n", 337 | " # 打印输出日记\n", 338 | " if i % PLOT_ITER == 0:\n", 339 | " errors.append(np.sum(entropy))\n", 340 | " \n", 341 | " if i % LOG_ITER == 0:\n", 342 | " print('iter', i)\n", 343 | " print('entropy', np.sum(entropy))\n", 344 | " print('----------')" 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 8, 350 | "metadata": {}, 351 | "outputs": [ 352 | { 353 | "data": { 354 | "text/plain": [ 355 | "[]" 356 | ] 357 | }, 358 | "execution_count": 8, 359 | "metadata": {}, 360 | "output_type": "execute_result" 361 | }, 362 | { 363 | "data": { 364 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsvXl4nFd59/85s6/aR7IWb3LsxHbi\nxI5JTEISSAIkFEjY2rCU0EJTKNCF9m2hC/TXt+2PpS3QlgJpCYSl7JSklIYlBJKQ1U6cYMf7LluS\ntUuj2WfO+8ezzDOj0WKtY+v+XJcvjZ55NHP0jPw99/O973MfpbVGEARBuHBxLfUABEEQhIVFhF4Q\nBOECR4ReEAThAkeEXhAE4QJHhF4QBOECR4ReEAThAmdaoVdK3aOUOquU2lN2/P1KqQNKqb1KqY87\njn9IKXXYfO6VCzFoQRAEYeZ4ZnDOl4B/Bb5sHVBKvQy4DdiitU4rpZrN45uAO4DNQBvwU6XUBq11\nfr4HLgiCIMyMaSN6rfXDwGDZ4fcAH9Vap81zzprHbwO+obVOa62PAYeBq+ZxvIIgCMI5MpOIvhIb\ngOuUUn8HpIA/0Vo/DbQDTzjO6zKPTUlTU5Nes2bNLIciCIKwPNm1a1e/1jo23XmzFXoPUA/sAF4E\nfEsp1QmoCudW7LGglLoLuAtg1apV7Ny5c5ZDEQRBWJ4opU7M5LzZVt10Ad/TBk8BBaDJPL7ScV4H\ncKbSC2it79Zab9dab4/Fpp2QBEEQhFkyW6H/PnAjgFJqA+AD+oH7gTuUUn6l1FpgPfDUfAxUEARB\nmB3TWjdKqa8DLwWalFJdwEeAe4B7zJLLDHCnNtpg7lVKfQt4AcgB75WKG0EQhKVFVUOb4u3bt2vx\n6AVBEM4NpdQurfX26c6TlbGCIAgXOCL0giAIFzgi9IIgCBc4F5zQP3tyiOe7hpd6GIIgCFXDBSf0\nf/n9Pfzt/+xb6mEIgiBUDbNdGVuVaK05MZCgKeJb6qEIgiBUDReU0A+OZ4incxSqoGRUEAShWrig\nrJsTgwkAEpk84+ncEo9GEAShOrighP7kQMJ+fHYsvYQjEQRBqB7Oa6HvGUnxgW/uZmg8A8AJp9CP\npnj6+CCPHe5fquEJgiBUBee1R7/71DA/eL6bx44M8LnfvJITg+P2c33xNP/xyDFyhQI/eP91SzhK\nQRCEpeW8juhvuXQF3/u9a3Ap+OB3n+fkQIINLREAzo6mOdIXp08sHEEQljnntdADXNpey7tfuo79\nPWM83zXClo46vG7Fvu5RxlI5BuIZqqFxmyAIwlJx3gs9wGu2tOF1KzL5AmsaQ8Qifp44NgBArqAZ\nSWaXeISCIAhLxwUh9PVhHzde0gzAqsYwsaifU4NJ+/n+uNg3giAsXy4IoQd4y9WrcSnY1FpDLBoo\nea4/nlmiUQmCICw953XVjZMbNsR45q9eTl3IR3ONHwC/x0U6V5CIXhCEZc20Eb1S6h6l1Flz28Dy\n5/5EKaWVUk3m90op9c9KqcNKqeeVUtsWYtCTURcyetzEIobQb1tVD0C/VN4IgrCMmYl18yXglvKD\nSqmVwMuBk47Dt2JsCL4euAv47NyHeO5YEf2Vq+txKRgYF+tGEITly7RCr7V+GBis8NQngT8FnLWL\ntwFf1gZPAHVKqdZ5Gek5YEX061siNIT9Yt0IgrCsmVUyVin1WuC01vq5sqfagVOO77vMY4vKttX1\nvPTiGC9e10hTxCfJWEEQljXnnIxVSoWAvwBeUenpCscqrlZSSt2FYe+watWqcx3GlDRF/Hzpt66y\nH0tELwjCcmY2Ef06YC3wnFLqONABPKOUWoERwa90nNsBnKn0Ilrru7XW27XW22Ox2CyGMTMaIz4R\nekEQljXnLPRa619prZu11mu01mswxH2b1roHuB94u1l9swMY0Vp3z++Qz42miJ8BsW4EQVjGzKS8\n8uvA48DFSqkupdQ7pzj9h8BR4DDw78Dvzcso50BTxE8ikyeRkY1IBEFYnkzr0Wut3zzN82scjzXw\n3rkPa/5oNPeP/fwvjtJSE+AtV89vPkAQBKHauWBaIEyGVWr56QcP8e+PHF3i0QiCICw+F7zQr2oM\nAVAT8DCWki6WgiAsPy54oV8Xi7DrL2/mzVevYjQpPr0gCMuPC17oARojfmoCXjL5AqlsfqmHIwiC\nsKgsC6EHqAl6ARgV+0YQhGXG8hH6gFFg5LRvHtjTw3ha7BxBEC5slo/Ql0X0x/rHefdXd/G9Z0/P\n6vVS2Ty5fGHexicIgrBQLB+hNyP6sZQRwR85Gwfg1GBiVq/3xs89xj8/eGh+BicIgrCALCOhNyN6\nc6Pw4wPjwOyF/sxwitPDqfkZnCAIwgKyfIS+zLqxhX5odkKfzRXIinUjCMJ5wPIRejuiN6yb4/2G\nwHcNJe1zzgwnefhg34xeL1sQoRcE4fxg2Qh9wOvC41IlyViA4UTWXjF798NHec9Xd83o9bJ5LUIv\nCMJ5wbIReqUUNUEvY6ksqWyeMyNJ1jdHADg1aET1Z8dSjGfyFAoV90qxKRQ0+YImk5/6PEEQhGpg\n2Qg9GJU3o8kcpwYTaA3XrTc2POkyffr+MaNvfSo39erZbMGI5LM5iegFQah+lpfQB72MprK2bXPd\n+iYATpk+vbUTVSIzjdCbkXxGrBtBEM4DlpfQB7yMJrN2xc22VfWEfO5iRG8KfXI6oTcjefHoBUE4\nH1hWQh8NeBhL5TjWn6Au5KU25GVlfYhTg0nSuTyj5mKq5DSNzyzrJiPWjSAI5wHLSuhrAoZ1s79n\nlA0tUQA66oN0DSVK9pWdNqI3rRuJ6AVBOB+YyZ6x9yilziql9jiOfUIptV8p9bxS6r+UUnWO5z6k\nlDqslDqglHrlQg18NtQEPQwnsuzvHmNzWw0AKxtCnBxM0DeWts9LZvP0jaUnrakvWjdSdSMIQvUz\nk4j+S8AtZcd+Alyqtd4CHAQ+BKCU2gTcAWw2f+bflFLueRvtHKkJeEnnCiSzeS5tqwVgXSxMIpNn\n75lR+7xkJs9XnzjBO774FPEK3S1zBfHoBUE4f5hW6LXWDwODZcd+rLW2FPAJoMN8fBvwDa11Wmt9\nDDgMXDWP450T0UBxL/TN7UZE3xkzaumfOjZgP5fM5hlJZiloONQ7NuF1MrmJ1k0uX+AXM1xVKwiC\nsJjMh0f/28D/mo/bgVOO57rMYxNQSt2llNqplNrZ17c4Amn1u/F5XKwzBb4zFgbgqWPFuSyRydtd\nLg9WEHpL4J3J2J/tP8ud9zxll24KgiBUC3MSeqXUXwA54GvWoQqnVTSytdZ3a623a623x2KxuQxj\nxlj9bi5ZEcXrNn71FTUBQj43Z0aKnSiT2by9IcmBnviE17GE3unRWxODbGQiCEK1MWuhV0rdCbwa\neKvW2lK8LmCl47QO4Mzshze/WBH9ZtOfB6M1wtomI6pvivgBSGXyjGemiugnWjfWatq0lFwKglBl\nzErolVK3AH8GvFZr7ezzez9wh1LKr5RaC6wHnpr7MOeHupAl9DUlxy2ffmVDECi1bg5MYd3kCtru\ni5POSoJWEITqZCbllV8HHgcuVkp1KaXeCfwrEAV+opTarZT6HIDWei/wLeAF4AHgvVrrqYvSF5H1\nzRE++vrLeP220rRBpxnRt0QD+DyuEuumbyzN4Him5HynmFuLp6xIXhZRCYJQbXimO0Fr/eYKh78w\nxfl/B/zdXAa1UCiluOOqVROOWwnZpqiPoNdNMpNjPJ2jOern7Fiag71j7OhstM93evPZvMbvgbRp\n3UhELwhCtbGsVsZOhlWB0xTxG0KfzTOWznHl6noADvSU2jclEX1OInpBEKobEXrgouYIm9tq2L66\ngZDPTSJjWDfrYhGaIn52nRgqOd8p9FYHy5TZH0c6WgqCUG1Ma90sBwJeN//z+9fZj4cSGQoawn4P\n161v4uGDfRQKGpfLqB7NOaybTFlEL20RBEGoNiSiLyPkc9t9byIBQ+gHxjO80F1skeCM2q3o3qq6\nEetGEIRqQ4S+jKDPTb/ZyTLid/OSi4zNSR451G+fU+LRmxG8JGMFQahWROjLCHrddjll2OehuSbA\nJSuiPHKo2KYhV1J1I8lYQRCqGxH6MoK+YrPNiN9IYbzkoiZ2Hh8q9riRZKwgCOcRIvRlBL0OoTe7\nXbbVBcnkC/YiKimvFAThfEKEvgxnRB82I3rrmLXFYGWPXlogCIJQnYjQl1ES0VtCbx5LmZU1FT16\ny7qRiF4QhCpDhL6MUAWPPuA1LpO1l2wljz4jEb0gCFWKCH0ZATN6V6oo+taxytZNeTJWFkwJglBd\niNCXEfIZUXzY50EpYyVs0boxxFzKKwVBOJ8QoS8j6DMuiWXbGMdKhT6TL2B2QyCbk2SsIAjVjQh9\nGVb0Hva7JxwrWjfajvwzdkQvyVhBEKoTEfoygqaAOyN626PPWNZNwfbvs/kC+YKuuL2gIAhCNSBC\nX0Yxop8o9ClHMtZ6PpMr2NE8yMpYQRCqj5lsJXiPUuqsUmqP41iDUuonSqlD5td687hSSv2zUuqw\nUup5pdS2hRz8QmBF6pU8+qSjssYZ0VudK0GsG0EQqo+ZRPRfAm4pO/ZB4EGt9XrgQfN7gFsxNgRf\nD9wFfHZ+hrl4WNF7iXXjMS5TccFUwY78M3ltJ2KN70XoBUGoLqYVeq31w8Bg2eHbgHvNx/cCtzuO\nf1kbPAHUKaVa52uwi4EVqTutG4/bhdetSurofR7jWDZfat2IRy8IQrUxW4++RWvdDWB+bTaPtwOn\nHOd1mcfOG6xI3WpoZhHwuh0rYzVetwuf20U2V7AjfSiWWwqCIFQL852MVRWOVVQ+pdRdSqmdSqmd\nfX19lU5ZEkJ+NwGvi+aov+R40Ot2LJgq4HUrvB5XSUQf9rnFuhEEoeqY7Z6xvUqpVq11t2nNnDWP\ndwErHed1AGcqvYDW+m7gboDt27dXTRjs97j54e9fR1tdsOR40Ocuqbrxul143a4Sjz4a8EoyVhCE\nqmO2Ef39wJ3m4zuB+xzH325W3+wARiyL53yiMxaxk7IWQa+7ZMGUbd04qm4iAY9E9IIgVB3TRvRK\nqa8DLwWalFJdwEeAjwLfUkq9EzgJvMk8/YfAq4DDQAL4rQUY85Lg97pJOjYA97jVhGRsxO+hP55e\nymEKgiBMYFqh11q/eZKnbqpwrgbeO9dBVSNBr4uUtTK2UMBnWjfZfDEZGw14xLoRBKHqkJWxM6SS\ndeN1u8jktB3R1wS8Ul4pCELVIUI/Q0qSsZZ1Y1fdFCP6bF5j3NgIgiBUByL0MyTgcUT0pnXjszz6\nbNGjB1kdKwhCdSFCP0MCJeWVReumNKL32s8LgiBUCyL0MyRorowtFDT5gjarblxkHCtjrdW0kpAV\nBKGaEKGfIUGvm1SuYNsypQum8nhcym6fIAlZQRCqCRH6GRL0uckXtN3vxud24fMo27oJeN143UYH\nCInoBUGoJkToZ4jfbFU8msoC2NaNtWDK73HhM8+RZKwgCNWECP0MsTYfGUvlACZ0r/R7jO9BInpB\nEKoLEfoZYvnvo0kjove5XXg9xaZmfq8bryn04tELglBNiNDPEFvozYje41aOpmal1o0IvSAI1YQI\n/QyxulmOmR691+3cYcqwbqyIPi3WjSAIVYQI/QwJlEX0pQum8vg9bkdELwumBEGoHkToZ4iVjLU9\neo9VdaNJZgv4vZKMFQShOhGhnyFBb2nVjcdV9OTH07myiF6EXhCE6kGEfoYEJ/HoAYYTGdOjlwVT\ngiBUHyL0MyTgLV0wZVk3AP3xDDs6G+zvZcGUIAjVxJyEXin1R0qpvUqpPUqpryulAkqptUqpJ5VS\nh5RS31RK+eZrsEtJwPboi9aNJexrGkP8xotW2atnJaIXBKGamLXQK6Xagd8HtmutLwXcwB3Ax4BP\naq3XA0PAO+djoEuNbd2ki9ZNyBT///PKS/A5yivFoxcEoZqYq3XjAYJKKQ8QArqBG4HvmM/fC9w+\nx/eoCrxuFx6XspOxPo/ilZtX8B9v386rLlthHhOhFwSh+pi10GutTwP/AJzEEPgRYBcwrLXOmad1\nAe1zHWS1EA146B1NAYZ1E/Z7uHlTC0oZSVivlFcKglCFzMW6qQduA9YCbUAYuLXCqRVXDyml7lJK\n7VRK7ezr65vtMBaVt+1YbW8y4vVMvHR21Y0smBIEoYqYi3VzM3BMa92ntc4C3wOuAepMKwegAzhT\n6Ye11ndrrbdrrbfHYrE5DGPx+L2XXkR7XRAoiroTpYz+NxLRC4JQTcxF6E8CO5RSIWV4FzcBLwAP\nAW80z7kTuG9uQ6wegj43f//6y9jR2UB9qHIxkc/jEo9eEISqwjP9KZXRWj+plPoO8AyQA54F7gb+\nB/iGUupvzWNfmI+BVgs3bIhxw4bJ70C8biURvSAIVcWshR5Aa/0R4CNlh48CV83ldc9nrEZngiAI\n1YKsjJ1nfB7x6AVBqC5E6OcZn9slLRAEQagqROjnGZ/HRSKTX+phCIIg2IjQzzNbV9Xz6OF++uPp\npR6KIAgCIEI/77zrurVk8wW+/NjxKc/rj6e54m9+zK4Tg4szMEEQli0i9PPMuliEmze28OUnTpDI\n5CY972DPGMOJLM+dGlnE0QmCsBwRoV8A3nL1KoYTWXafGp70nNPDSQC7d44gCMJCIUK/AHQ2hQE4\nPZSc9Jwzw4bA94jQC4KwwIjQLwCttUGUgq4phP70cAKAnpGFE/p4OkdSKoAEYdkjQr8A+DwuWqIB\n256phBXRL6R18+6v7OIj9+9ZsNcXBOH8YE4tEITJaa8P0jWUmPT5M+Yk0DOaQmtt97SfT86MJFmA\nlxUE4TxDIvoFoqM+WBLR3/3wET75k4MAaK05PZzE73GRyhbsfWjnm2QmTyor1o0gLHdE6BeI9rog\n3cMp8gVjE5L/3dPDj/b2ADAwniGdK7CloxZYuIRsMpsnKUIvCMseEfoFoqM+RK6gbQ9+aDzDaNLY\nWNyqxtm2qh4whP7McNKeFOYLI6KXvjuCsNwRoV8g2uuNnags+2ZwPMOoubG45c9vW20I/RNHB7j+\n4w/ZEf98UCho0rmCVN0IgiBCv1BYWw52DSXI5guMpnLE0zly+YIt/ltX1QHwn0+eJFfQdM9jqWUq\nZwh8OidCLwjLHRH6BaLDiuiHkgwlMvbxsVSO08NJwj43sYifhrCPEdPSSaSnTso+eXSAgRk2S7Mi\nebFuBEEQoV8gAl43TREfXUNJhsaz9vHRVJbu4RStdUGUUqyoCdjPxafojVMoaH7zC09x7+MnZvT+\nVhJWkrGCIMxJ6JVSdUqp7yil9iul9imlXqyUalBK/UQpdcj8Wj9fgz3faK8PcXo4ycB4MQofTeYY\nGE/TFDE2F19Rawi9z+NifIqIfiyVI5MvEE/NrBTTKqvMF7RsbSgIy5y5RvSfBh7QWl8CXA7sAz4I\nPKi1Xg88aH6/LOmoCxrWTVlEPzCeoTHsB+BVl7XyjmvW0Bz1k0hPHn1b9s5MPfdkpijuEtULwvJm\n1kKvlKoBrge+AKC1zmith4HbgHvN0+4Fbp/rIM9X2uuDdE2I6LMMjmdoCBsR/Ruv7OCvX7uZiN9D\nfIqIvij0M4vOneIui6YEYXkzl4i+E+gDvqiUelYp9R9KqTDQorXuBjC/Nlf6YaXUXUqpnUqpnX19\nfXMYRvXSUR8kkytwsHfMPjYwnmE4kaXRtG4sQj4341N49MNJI6E7G6FPS0JWEJY1cxF6D7AN+KzW\neiswzjnYNFrru7XW27XW22Ox2ByGUb1YJZa/6hrB6zaazpwYGAegMVwq9GG/h/GZWDczjM6TjklD\nrBtBWN7MRei7gC6t9ZPm99/BEP5epVQrgPn17NyGeP5iLZra1zNGW10Ql4Jj/UajswbTo7eI+D1T\nJmMtoU+JdSMIwjkya6HXWvcAp5RSF5uHbgJeAO4H7jSP3QncN6cRnsdYEX0mV6Ah7KMm6OW4GdE3\nVIzoZ+DRzziiL04IUksvCMububYpfj/wNaWUDzgK/BbG5PEtpdQ7gZPAm+b4Huct0YCX2qCXkWSW\nxrCPgXiGkwNGRN9U5tGHfe4FS8aKdSMIy5s5Cb3WejewvcJTN83ldS8kOuqDjCSz1Id81Aa9nBy0\nrJuJEX0iky/pTZ8vaG751MP84c0b7IZoMxX61DK1brqGEtSFfET8stWCIFjIytgFxrJvDOvGEB+l\noC40UehzZiMyi7FUlkNn4zx9fHAWdfTLU+jf8NnH+NzPjyz1MAShqhChX2CshGxD2EdNwAtAfciH\n21W69VPY5wYo8enHzFWw3SNJhhOWRy/J2MlI5/L0jqbpn2E/IEFYLojQLzAd9SEA6h1CX15aCUZE\nD5BwROJWXf2Z4dS5R/TZPB5zMlkuyVhrBfJymdgEYaaI0C8wlnXT6LBuyv15wPaUnQnZuCOiL1bd\nzNCjz+Rte2i5JGOtFcjLZWIThJkiQr/AvLizkduuaOPK1fXFiD4yUehDptCXWDfm4/54xrYjKiVj\nc/kChbLdqZLZPPUh4/2WS4Q7OG6sHk5JD35BKEGEfoGpDXn59B1bqQsZdfQwWURvevQO68bZqTKV\nLeBSkMkXyBc0P9vfa5dqvu7fHuNTDx4C4DMPHea/nztDMpsn5Pfgc7uWTURvC/0y+X0FYaaI0C8i\ntbbQ+yc8F64Q0ZfX1TdFjJ/L5Ar8/td38/mHj5AvaF7oHuVoXxyArz5xgv969jTJTJ6g10XA61o2\nvW4G4pbQL4/fVxBmigj9ImJ59OWLpQDCvokefflK2eYaQ+iT2TzxdI4zw0kG4mnyBW1X6IylcvSO\npkhm8wS9bgJe97KJcCWiF4TKiNAvIpZHX8m6satuKpRXWrREjU1KrK0Ju0dS9IymzHOz5AuaeDpH\n72jaiOh9htAvF+tmYPzcOnwKwnJBlg8uIpd11HLX9Z1ct35it85wJY8+nSNsivXAeMaO6K3ItWc0\nRY+5oXg8nbPvBgbG03hcioDXTXBZRfRW1c3y+H0FYaaI0C8ifo+bP3/VxorP+dwuPC41obwyEvDQ\nHA0wMJ4hZkb0lhc9nMhywkzIjqVyjKWMEkytoXcsZVo3rmXjWYt1IwiVEeumSlBKGf1uypKxYb+H\nVnNf2RYzoresG4Ddp4YBQ+idk4TW2B79crNulsvEJggzRYS+ijC2Eyy1bqJ+D23moqtmM6K3IleA\nZ08O2eeOJLI4sTz68tbG3SNJTpnN1S4knHX0WutpzhaE5YNYN1VEyOdmPJ3j0UP9XL6ylnjasG5e\nvaUVKJZnOoX+jOnRA3Zi1iJgWjflEf2H79vL4HiG777nmoX6VRadXL7AcCKLz+0iky+QyRfwe9xL\nPSxBqApE6KuIsN/Ds6eGeGBvD3/16k3EUzmaIiG2r2lg+5oGnjNtGqfQOzkzXCr0QTsZW2plDI1n\nON4/vjC/xBIxZN7NtNYFODGQIJUVoRcEC7FuqoiI30PvqFE50jWUMCJ6v9d+3u81Pq6BMqFvjhre\n/ZnhJIC9P61l3ZQnJxOZPAPjmUVNWvaNpRlOVJ6g5gNr8murNWyume7EJQjLARH6KsIqsQToHk4x\nlsrarREAAmaEOmSKmtUwbX1LBCgK/ZrGMDB5MtYSeOv8yUhkclz/8Yd4cF/vrH8ni/f+5zP81X17\n5/w6k2E1NLPyGZKQFYQicxZ6pZRbKfWsUuoH5vdrlVJPKqUOKaW+aW4zKMwAa3VsXchL90jS9ugt\nrIjeil47Y4agr2+OAnB6OInXrVjZYLRGLiZjS0UvaQt9qdVTzv6eMU4OJthzenSuvxpnR1PTTixz\nYdCe/IyEtTQ2E4Qi8xHR/wGwz/H9x4BPaq3XA0PAO+fhPZYFWzpquWpNAy/f2MLR/nEKmlLrxozo\nB8cz+DwuVpmCflGzEdF3j6SIBry01BhiZ9XRW43QLKye96eHp668OdAzBpSWc86WZDZvt1peCGzr\nxo7oRegFwWJOQq+U6gB+DfgP83sF3Ah8xzzlXuD2ubzHcuId167lW+9+MW11Qbv9QUlE7zE+rmQ2\nT9jntuvrLaEfSWaJBjy2Zx/0GclYKBU+K6I/PU1Ebwn9uQq01rpkK0MwJpeFFHprEdkK85qIdSMI\nReYa0X8K+FPA+l/VCAxrra2VO11Ae6UfVErdpZTaqZTa2dfXN8dhXFhYAg6UePSW0INRoXPTxhZe\nt7WdTW019vFowFMW0ZcKfb6gyZi9YKazUmYb0T98qJ+t//fHJVv6JTN5RhLZBatvHxzPUBfy2j2D\nJKIXhCKzFnql1KuBs1rrXc7DFU6t+D9ba3231nq71np7LDax98typtW0H6DUuvGYbRLA8PM3ttbw\nyd+4gqjfY+9BG/V7ubqzgRetqWd1Y4iAt3gX4PwKUwu91poDvYbQDyfOLRI/OTBOKluwSzgzuQK5\ngiaTLyxYpD04nqEh7LMT1iL0glBkLhH9tcBrlVLHgW9gWDafAuqUUpbf0AGcmdMIlyGlEX3pUgcr\nqndW6CiliJoWTzTgYV0swrfffQ3RgNcR0RsC67RUphL6vnja9r3PtSzSmky6zcVczvccTi5MieXA\neJrGsM+e2FKL1MFSa80nf3LQvvsRhGpk1kKvtf6Q1rpDa70GuAP4mdb6rcBDwBvN0+4E7pvzKJcZ\nTqGPBsqE3hTucNkEUBR6b8nxcuvGEt2miI8zI6kJWxBaWMJ1UXOE4XP01q1kr9VZM5Et9uBZKJ/e\njugr5CQWkkQmz6cfPMT/PC/xjFC9LEQd/Z8BH1BKHcbw7L+wAO9xQRMNeO1IvjyiD5gRfchXuurT\nsnjKJwarZNPaxMSKttfFImRyhQmLryxeOGOUVO7obGAkmS2p2pmO8og+4Yjoy/vxzBeG0PvtEtTF\nWjBlNZIbLds7QBCqiXkReq31z7XWrzYfH9VaX6W1vkhr/SatdXq6nxcmYkX15ZH79BF96XGrP44l\nRLbQN5cusrIoFDT/59vP8dEH9rMuFqazKYLWMJrM8uihfnux1lRYdw09o8mS74FzvjuYCYWCZiiR\nNa2bUqtqoXHu7CUI1YqsjK1SrDLBCdaN5dH7So/XTCL01vaFlmWSyBiCtN4U+lNDpbX0xwbG+fau\nLl53RTtfv2sH9WGvfd7b73lPGdEaAAAgAElEQVSSrz15YtqxW8JeMaJfAKEfNXfXWopkrBXRW3sB\nCEI1IkJfpbTVBvG4VElJJTiTseURvbfkq4W1feGoKbCWAF6ywijJPNpX2tysf8y4AXv9tg6aowHq\ngsbC5ue6Riho7F48U5Ew36PXFnqHR78A1o1lPzVGfHjdCpdavJWx42mJ6IXqR7pXVimv39ZOS40f\nYw1aEdu6meDRV47ore9HU1ZEbwhgY8RHW22Ao33xkvP7zYVHTVFD4OtCxkTxvNk501kbPxlWRN87\nZmxcnlzgiN6qDmoI+1BKmY3cFtm6SUtEL1QvIvRVytWdjVzd2Tjh+OQRfeWqG4/bRcTvsQXWEt2g\n101nLMLRsnbFlpA3RYzVtXUhQ/Cf7xopeX4qrPfIFzT98fSCWzfWqlhr0/VKHTsXirhE9MJ5gFg3\n5xlWvxtnHT04rZuJc3dt0Mto0hAiSwCDPjedsTBH+8YZGs/whs8+xv6eUfrjaVwK6k2Brzcj+kNn\njXJLS1SnIpHNY67fonskZVs5YZ97QjI2my+Qzc8t+nZG9GBUJi1WRB8375RGF7C9gyDMFRH68wxr\nQdBkEX1NBaGPBjyOZKwjom8KE0/n+ObOU+w6McTjRwboj6dpCPvtlbY1AS9KgVVd2TeDiD6VydNe\nb6zu7RlJkjQ9+ta64ISI/s+++zy/97Vnpn/NbJ4bPlG5ZfKg2aK4JKJfLI/evJ5jqZxsXyhULSL0\n5xl2RF9WdfOSi5q47Yo2VjWEJ/xMTdBre/RWeWXA67ZLLL/yuFFJ0zWUpG8sQ1Ok2Fna5VJ2iSYY\ngjadLZLI5ljbVOyoaU0uK2oCjJStst3fPcaRsjxBJc4MJzkxkOBXp0cmPDcwniHi99jXxl9hn9yF\nwrJscgUtjdSEqkWE/jzDWhBUvmBqTVOYT9+xFZ9n4kdqWDdFofd7XLhdis6YIcanzVr6rqEE/fE0\nMbP7pYVl41gTwGRbGVokM3na6wL4PC56RlIkM3l8HhcNYd+EiL4vnp5RJU6fWQ1Uqe+OtSrWIuBd\nROvGkYQtL7H81tOn2Htm4sQkCIuNCP15xmTJ2KmoCTiEPpMnaE4SrTUB2wryuhWnBpP0x9N2ItbC\niui3rqoHpk/IJjN5gl4PK2oCdkQf8rmpDXpLPPp8QTMQTzOSnL6r5VlT6Ct10iwX+uBiJmMdSdjy\n1bEfuX8vX/zl8UUZhyBMhQj9eUZgkpWxU1Eb9BZXxmbydo96l0vZFssrNq3g1FCCvrF0iXUDxYTs\nNlPop0rIaq1JZA1hj0X9DIwbVTchr5u6kDHhWP11BsczFLRhe4xnphZmK6KvdDcxEM/QWBLRL55H\nH08X38cZ0WdyBZLZPKcGp97cRRAWAxH684xK3SunoyboIZ7OkcsXSGSLET3A1WsbuPaiRi5fWctY\nKkc6V5gQ0VvWzdZVdcDUCdl0roDWRlVPY9jHQNzYhDxoRvQFDXEzOWuJN0zfIfNsBevmWP84n/zJ\nQbpHkktq3Vito50llpbodw0t3PaJgjBTROjPM2oCXjwuRdTvnf5kx8+AmUh1RPQAf/3azXz1nVfT\nUR+yj5ULfSzqJ+B1cWl7LVC0bn60t4dXfPIX9kYmUFqn3xjxm3X0OUI+j20BWZ782bHiDlfT1df3\nVbBuvr3zFJ9+8BBDiazdMgKMTdQXs47e2uhlrIKN0z2SnHP5qCDMFVkwdZ7x6y9aydZVdSVR+XQU\nG5tlSZq2ihOlFB31xc1OypOxv3N9J6+8dAURv4ewz21bN08eHeRgb5xj/eNcvMLYoNyq6gn53DRF\nfAyOZxhPFyN6MMR6ZUOoJKKfLCF7ajBBW13QnhScEf1QIktD2Mcnf+MKLu+otY/7F3FlbDyVo7U2\nwOnhZIl1Yz0uaOgeTrGqMTTZSwjCgiMR/XlGxO+xk6IzpcaKpJNZEpm87fM7mSqib4r4bX/eitLB\nqNKB4mIqcNTp+9w0RfwUtFHVE/K5bbE7Zq7GdVpAlSL6o31xXvoPP+f7z562J4V4OmffQYwmDaG/\nYUPMXsELhnWzeG2K8/bdRElEnyw+Lm8cN1NGklnuuPtxTg6Izy/MDRH6ZYAd0SeNGvhgBaGvD3nt\nSN/qc1OJpojPFnqrLPPw2WIdfKl1Y7xO94gh9J1NETwuZW9qUuLRO4S+ZySF1prv7z5DvqB55uQQ\nfWNp2wu3/PzhZIa64EQL61yTsSPJLH/0zd3Tlo1WIp7OsqImgFKlydhRx+OuWQr9wd4xnjg6yMOH\nSvdUzuQKvOvenfaeAYIwHSL0ywCrVfFk1g0U7RuloCE0udA3Rvy2dWMJ/SGn0NvWjYfGsHFnUNAQ\n9HrweVx0xsIc7C0KfUuNcY5lyew9M8I1H32Qzz98lPt3nwaMPjuDiQxrm4zFYEPmucOJbMliLouA\nx002r2e8WcozJ4f4r2dP84uDZ2d0vkXW3AO3JmhsFDNaEtEXhf7U4OwSstbEcaysH1HXUIKf7uvl\noQPnNl5h+SJCvwywkrGWdTOZv99RH6Ih5MPjnvzPosm0buLpnC3Oh3uLQm+1JA76XCVlmtbkcvGK\nGvY7IvrVDWF8bpdt3Xz1iRMUNHz8gf0cH0gQi/rZc2YErWFDi5EHsBKyw4kstaFKEb25b+wM7Rtr\n4jrQU/w9Tg0m2N9jRMw/faGXf/3ZoQk/Z7Uojvg91AS8ZclY4/epD3lnbd1Yr3e8TOitCWWpK3q0\n1pNuRSlUF7MWeqXUSqXUQ0qpfUqpvUqpPzCPNyilfqKUOmR+PTdDWZh3itZNltQkHj3AHS9aybuu\n65zytZqjfgbGM3Z74/a6IEf74+TMyhK7aZrXQ6PD67eFviVC11CSeDpHXzxNrMZPTdDLSDLDaCrL\n9589w8s3tVAb9OJzu3jPDeuw1lLZQm9aLCPJrN0v34n1XpUWV1ViwLSiDvUWcw3/9wcv8Ltf2QXA\nlx47zj/8+CC7TgyV/JwlxBG/h2jAU5aMzeFScPGKaEVBPjOc5P1ff9aeLCphCXp5RG9Niqen2Nx9\nMfjoA/t5w+cem9NrJDK5kqotYWGYS0SfA/5Ya70R2AG8Vym1Cfgg8KDWej3woPm9sISEfG7cLsVI\ncnLrBuAVm1fwnpeum/K1LmuvRWv43z09ANxwcYxsXnPSXBjkTMbWBb12c7SgI6IHw3/uG00Ti/ip\nC3kZSWb5/rOnSWbzvP/Gi/j8b27nE2/awg5Hq2arsmcokSWbLxBP5+x++U6uuagJgPt2T9ywu5Kd\nY+UcDjiE/sRAghMDCUaSWTuy//gD+0tW8I6bdy+RgCX0pdZNNOBlZX2o4qKphw6c5b+fO8PzXSOM\nprL8/Q/3TbgDsSaOk4OJkhJNS+hn6/3PF48dHmD3qeGS/QbOlTd+9nE+9sD+OY1DmslNz6yFXmvd\nrbV+xnw8BuwD2oHbgHvN0+4Fbp/rIIW5oZTRmGwgniFX0BWTsTNl22rjBu1+U0RfdnEzUPTpLaEP\n+dy4XMpeyFSM6A2xfu7UMGPpHLGo32iNkMjykxd6ubglypaOOq5a28BtV7RzUXMEn2klbWgxVvEO\nJTK22FUS+nWxCC+5qImvPnHCvtMAeOrYIJs+/AC9o6mS8y3rpmsoyXja6EJpRcuPHOqjP57h4pYo\nTx4b5PGjA/bPxUsiem/J5iOjqRw1QQ8rG0KcHUtPEHFrYjw1lOCh/We5++GjPHVssOQcZ8M0512B\n5f+fHkoumcjl8gUO9I6hNTNqSjcZx/rHecJxTS0+8M3dfPR/p58ADp8d49qP/owfPD9xUheKzItH\nr5RaA2wFngRatNbdYEwGQPN8vIcwN2oCHnrNWvTJrJuZ0BD20RkLc3o4ic/tYkdnA1C0PZz97gG7\nNUHQ7LbZUR8k5HPz0AGjkiQW9VNnCv2+7lG2OOrhAXweF+tNge+oDxHwuhhOZOz8QKVkLMDbX7ya\n7pEUP91XTFg+fXyQdK4wwfPud1TbHDobZySZtTcU+d4zRkL4T155MQDPnSo2KRtLTx3R1wS89t68\nL3SXVsh0mQnarsEEx/sN0e8eMY7tOT2C1rqkj45zzNYkl84V7B3B5sKJgXHe/ZVdJZVCkzGaytI3\nluZY/7htuRx03AlNxmceOsxz5i5lFqlsnmQ2z8HesRL7ZjiR4b7nzvCjvT1TvqbWmg/ft5czIyk+\n9L1f2dfPyXd2dfHrn3982tdxLt67EJmz0CulIsB3gT/UWs+43kspdZdSaqdSamdfX9/0PyDMiZqg\n17YQQr65rZPbbkb1bXUBogEvG1oi/Gy/IajOfvdQrMl39tfZ1FrDwweNz7w56qc25OXEwDj98QyX\ntNZMeL8rVtaxosbohtkQ8jE4nmUkaQhc3SQVQjdtbKEp4ueBPd32MWsyGipbnDUQT9MZMyp6DvaM\nlUTPvzDHeeXqeqIBDz0OMbGEOGp69CPJLE8eHWA0lWU0ZQj99jXGRLjz+CAHe8d491d2kczkHRF9\nkhMDhoifGU7xzMkhXv0vj7LzxBBjqay9z4BzJzBnRc982DdffvwED+zt4ecHpv9/+OHv7+FNn3uM\nvY7SzoO9U0f0A/E0n/jRAb7+1MmS49Zknc3rksni5wf6yBc0x/rH7Qm3Ej94vpvHjgxw1/Wd5PKa\nV//zo7zkYz9j5/HindEjh/p46tjglIn57z1zmpd87KFJ23D8w48O8IVHj1V8LpXN8+H79pSUClcj\ncxJ6pZQXQ+S/prX+nnm4VynVaj7fClSsAdNa36213q613h6LxeYyDGEGvLizkSPmRuBB39zm9ytN\nobcWWb3pypU8c3KYg71jJLN5vG6F17RbrFp6Z17g02/eyj++6XI+8ppNXLOuidqg125qttH04Z38\n6S2X8I27dgCGsDsj+kp19ABul2Jja7REIC17qTxJOxDPsHVlPX6Pi4O9RaFvivjIFzQtNX4awj7a\naoN0jxQjPyuRGjatm+FElt+4+wk+9/MjjKVyRAMeYlE/a5vCPHVsiC89dpwH9vbw7MmhotAPJjhm\nC32Sg2ZF0umhJGOpHKsaQtQEPBzrL4qpM/J2JmS11nziR/vPqTVyvqD57+cM2+Oxw/3Tnv9C9yjH\nBxL851Mn8bmNctlD00T01jaU5RbPcLL4OTjXBPzUsbnM/u7JY8dvPn2KzqYwf3bLJXzmrVu5am0D\nXUNJdjqS5tZ1HphijcQvj/STyRUqlsFqrfnakyfsa1TOr06P8OXHT3CfWQpcrcyl6kYBXwD2aa3/\nyfHU/cCd5uM7gftmPzxhvnjndWvthmhz8egBrlxtRKntdUbbhNdta8frVnzz6VMky6p67IjeIfTt\ndUHecGUHv3XtWnweV0nlTKWIvjboZY1ZQ18f9jLkFPoKHr3F2qYwx/rG0dqoqbcWdjkXRmmtGRhP\n01zjZ31LhAO9Y7Z43nRJizEmM4G8ojZAj8Pfjzusm1+7rJU3X7WS1toAB3vjhnVjTkLbV9ez88Qg\nD5gJ7EcP99v2y6khI+kLxiYtxweKwmRNFmtjkZLKm5GksUgLSkss++JpPvPQEb67a+ai88TRAc6O\npakJePjlkamFPl/Q9vieOjbIRc0RNrXWlCSxK1EU+lLLbGi8OGFZk1MmV+AXB/q4br2RUC+3vCy0\n1rzQPcqL1jTgdiluvKSFz77tSkI+d0l0ba0qHpiiEd9u01LqGZ1o35wdSzOUyE5a4WS919PHBys+\nXy3MJbS7FvhN4Eal1G7z36uAjwIvV0odAl5ufi8sMc3RAG++ahVQ9Mtny7pYmOs3xLjhYuNOrCni\n5+WbWvjeM12MJrMl0bsd0U8xuVhibUXOU2FE9Fl7JW2l8kqLzqYwY2YZ5+mhJGnTBx5yCP1oKkc2\nr2kM+9jSUcezJ4c5OTBO0OvmWlNsNpqTT2ttgDPDRTGwPPmwz8Ol7bX8/6/fwpaOWo71x41krLl+\n4UVrGxhOZO0J5oe/6rZft3c0bR83dtEyxHBwPM1oyqjcWRcLc8hhj4wks7TXB6kJeDjtEHprwnBG\n/9Nx3+7TRPwe3vuyizg1mJyyrfKZ4SSZXMHe3GZTWw0bWqJ2Ensynu8aNn+nTMkka1kl0YCHPWZE\nv+vEEGPpHG/bsZr6kJe9pysLfc9oisHxDJvbSwODWLTYoiOeztmRvLO19r2PHbcrqUYSWY6aE1Al\nod9nTjR9Y2nSFVZbW0K/8/hQVVf/zKXq5lGttdJab9FaX2H++6HWekBrfZPWer35tbqnumXE771s\nHbdf0caW9trpT54CpRRf/u2reNVlrfax12xpYyiR5cljgyV3DE3m6tip8gJWQtWKnKeiIeRjMJFh\nJJFBqcqboVusNXfQOtY3XtKPx+nRW6IQi/q5dl0T8XSOH+3tpaM+yNaVdXhcyraqVtQG6I+nyeQK\naK158tgAzdHi/rpg7PR1cjBBPJ2zVyS/yPTpQz43N17SbEfF164rlo6ubAhyZiRpPzfoiOg3rqjh\n7FjajkpHkzlqAh466kMlHr2VsC2vu5+MnpEU9z93hl+7rJUbLzFqJn45hX1jve5bzIBhY2uNXQnl\nbIPhRGvNc10jdqO8I31xUtk8hYK2J+tr1jWyr3vUuOsy7Z0rVtaxqa1m0ojemgA2ld0BNkX8tvg6\newRZn/PZsRQfuX8vX3/SyBc811VMEPeOVBL64t9N93CliN84NjCese9YBuJpe52JRTqXX9KJQFbG\nLiOaowE+dcdW6qeJmmfDi9YaYnZ6OFlyx/DidY287OKYneyshLW69ZLWif58OasaQgwnshzui1MT\n8OJyiGw5nabdc7R/3PbnVzWESjx6K9JrDPt5sSm8PaMp2uuDrGwI8fiHbuLmjYYIttUaVlXvaIof\n7e3liaODvO/Giya8ZzZv/IeOmhH9msYQ7XVBbrl0BdvXFNcPXnNRUeiv6WwilS3YfvdAPMOYmdDd\n1GaImSU6I0mj9UNHfbDEUrAi+lNDM2uN/MmfHKRQgPfdeBEXNUdojvp57MjEUscP37eH93x1ly1e\nv3N9J+956Tpec3mrvYhtzyR5ge6RFP3xNLdf0QYYnvtN//gL/uVnh+3P4dqLmkiYCequoQQ+t4tY\nxM/mtloO9I4xksza6x9S2Tzj6RwvdI+iVPFuyyLmFPrB4oRnVSc9fczw78+Yom5VAtUEPBPKbgE7\n8gfjjqYcZw8mKwn88QcO8PrPPmYngEeSWa76uwe5fxKffzEQoRfmhaaI3xZzp3WzsiHEF3/rqil3\nxLJKMMujs0pYm588cqh/Sn8eoK0uiM/j4lj/OId647TU+FndGDJbJ+f41E8P2hFxY8RHQ9hnj8Fq\n2xyL+jHSUdhdKruGkvz9D/exoSViR7cWaxqLE1qNebehlOK/3nsNf3v7pWxpN8ZfF/KyqbV4Z7Vj\nnTFR5kxBGxjPEE+bEX2rJfSm3WAK/fqWCEf7xu1I/7hp++QLusSCyeQK/GhvT0n54YGeMb696xRv\n27GalQ0hlFJsXVXHngqbrz904Cw/2tvDMyeHifo9tNUG+LNbLqE5GmBtU5i1TeFJk5WWP3/Lpa34\nPS6++NhxTg8neaF7hOFEFr/HxWZzIjvWH6drKEl7fdCuzsrkClz+//2Y9/3nMwB84Fu7ed2//ZLn\nu4ZZ2xie8HcVi/rtrqhWItbjUvY1euqYMZFZ12L3qWHWxcKsjUUqWjf7u8fsEtmuSYT+4hVRGsM+\nnjKF/tjAOMOJLD9+wUgqP35kgJFk1m7mtxSI0AvzxlWmRXGuyd7L2mv51G9cwa2Xtk577qXttXhc\nirFUbtKKGwu3S7G2MczRvjh7z4ywvjlKfcjHUCLDz/af5VM/PWTv6WrlEq41o+z2uon941tNof/R\n3h5ODiZ478sumtAXaK3jzqXGMb7maICQz8Olpqe8qiFEc9SPz+NiRU3A3tIRjHLNU4MJCtqwphrC\nPlpq/OzrHqVQ0EbpZtDLay9vJ1fQ9grgEwMJ2wazShP/45Gj3PCJh/jdr+zi4w8cAAwb4Y++uZu6\nkK/kjmRTay3HBsZL/PaxVJZTg0kK2sgtrI2F7YkPjEns9VvbeeLoYEV//5mTQ3jdis1tNXTGIrYf\nfno4ydB4hvqQz54cj/Un6BpK2pPszZta+N0bOrm8o5Ynjw2itWbn8SEO9sb56b6zbGybGBjEon6G\nE1kyuYJ9PVrrArZX/6S5KM2yYZ7rGuGKlfWsqPFPiOjTuTxH+uK87JJmlKIkH2LRF0/THPWzbXW9\nndS1JpFv7zwFwKOHjbLVSltw7useLVnUt1CI0AvzhuVFn8umKGCIxe1b2+0k31QEvG7byqidosum\nxdqmMA8f7Gd/zxgv39RCQ9jYDMWK9n5lRrBWx85rzfYJKxuCE16r1awy+v7u0ygF16+fWBYci/iJ\nmFGmlYx1UhfysaElwoaWKC6X0TF0TVOItrriDllXrKqzt0607J+NrYZfHc/k0NrIa1y8IsqWjlq+\ns6sLrTXH+8e5foMxpv09Y9z66Yf52//Zx+rGEFtX1fH4kQG01vzTTw7yQvcoH3vDlpLk9+a2GrQu\ntSucUWiuoO0Ook5u39oOwD2/PMZ3d3WV1KM/cqif7asbCHjdrDMnQaWMNQPDySx1IS8NYR/RgIfj\n/eOcHkrYQh/xe/jQrRt53dZ2Bscz7Ose4+xYGq9b2eMtx6ryGhhPc3IwwerGEI1hI0E7nMiwv2eM\naMDDwHiGs6OGrXTxiggragL0lHn0R86OkytoLmuvpTnqn9S6iUX9rItF6BpMkssX6BlJEfS6efRw\nP2eGk/zysHEX0V9W+XN6OMmvf/5x/uYHL0x43flGhF6YN65aO7uI/lzZutK0P6aJ6AE6Y2Ey+QKX\nrIjy1qtXUR/yMZbK2ZElGB0mrcj8+vUxPvHGLdy8sWXCa0X8HqJ+D8OJLJe111bMdSilWNNk3A1Y\nydhyvvquq/nwazYB8Le3X8oHb91IU9iP161wKUpWB1vJ5k2tNRw+G6ffnACsu4U3XtnBC92jPHq4\nn7F0jq0r66gLefniL49xajDJZ96yjW/c9WLesK2DntEUz3eN8MVHj/PGKzt4+abS39GaQPeeGSWR\nyZHI5Gy76GVmhVUloV/ZEGJHZwNf/OVx/vjbz/Hlx08Ahgju6x7lJWb1kuXn/9plrQyOZ+geSVIX\n8qKUYm1TmH3do/THMyWb4EDRh//eM10AfOjWjcSifl5iTspOrKRv35gh9CsbQuYeChm7xcStl64A\nsPMRnU0RWmoDjKZyJX17rAnvkhVR2uuMfMjJgYR951IoaPrjGZqjAVY1hMjkC+w9M0o2r3nbjlUo\nDKvJSmI7V2AXCpo//tZuCgXN70zTSHA+EKEX5o2O+iDrmyMVo+H5xNphazqPHmBzWy0uBX9z26V4\n3C4awsbPPNc1bLdRdnbZdLkUb9q+ctI2EZZPf20FkbGwbJhKET0YNo713DXrmrhiZR0ul2JFbYD2\n+iAraovXzxnR5wra7qBp/fxrL28j6HXzV9/fA8CaphBrm8L0xzOsi4V51WWGqFnN4T583x4y+QLv\nuGbNhHG11gbsksa3/PuTvOOep9nXM0Zt0MvbX2ycvy4WmfBzAB95zWb+/FWX0F4XtBc/WRU81p3P\n23as5nNvu9KeRA/2xO2N59c0hm3rw7mtJWBbNN83Larbt7bz9F/czJaOugnjsIS+ZyTF6aEkqxuM\niH4gnmbniSF8bpdtEVrj64yFaYkan6vTpz90No7XrVjTFKatLsjJwQRv/cITfOBbuwFj4V2+oIlF\n/axqMCYnq2/PVWsb+dCtG3niqDG5XNZea0/SAN95posnjg7ykdduZmXDwm8zKXvGCvOGUor/fv9L\n7FWxC4WVkJ2sz42TWy9dwS8/eCOtpnhaLROO9MW5/Yp29nWP0lwTmOolSmitC3LobJzrphR6I+qt\nmcH4nFzWXovH5bKT01CM6C2b4lFTnKzfvS7k4/03XWT776sbjeTosyeHefuL19h++rpYmFjUz3Nd\nI2xoiVS0PZRSbG6r5QfPn7FXKh/ui3PJiigvvTjGv75lK6/YtKLi2De21rCxtYbdp4bZY5Y+PnKo\nn/qQ136vhrCPWy5dYS8uyuQL9uexpilsJ6LLI/qagFFh1DWUnHathSX0jxzqJ1fQbGiJckiNMTie\nYdeJIS5tr2G1uaXlLw/343EpVjaE7BXPPSMp+/M71DvG2qYwXreL9vogP3jeWP8wPJ41++MUS3Ot\n17RyAK21AW7e2MyB3jGePTnEjs4Gvvz4CbTWKKV44qhRmvumKzsm/V3mE4nohXkl4HWX1JUvBKsa\nQvzRzRt4zeVt057rcilb5AFbJLQ2XueLv/UiPv6GLTN+7476IEGv2+7iWYm3Xb2Kf3zT5TOaiJz8\ny5u38cnfuKJEyKzKnbVNhlD/zGzS5rSF3vWSTtbFwriUMb4rV9cTi/p5/bZ2+xyllB3Vv35bR0lC\n1cmmthrGM3laaoxcw+B4ho2tNSilePWWtmnzKJtaazg5mGA0leWRQ31cc1HThBLYtrri52HdlXU6\nLKGV9RPvCK1qqPJyynKsu7T/NXscXd3ZQGPYT66g2X1qmK2r6u2/hzMjKVY1hPC6XbSYk70zIXuw\nN856025qd4x5LJ2jZzRll3HGon5aawO4XYqnTaFvqwuilOITb9zCj//oBpoiftK5gj2BHugZ4+IV\n0Uk/h/lGhF4471BK8Qc3r7c933Oh3pHAXdUQorU2aNsxM+H3b1zP137n6ik7gDbXBHjDLCI1t0vh\ndqmyiN4QQqUU16xrtDtmOicRn8fFZ992JR9/4+X4PW7eevVqHv/gjfbPWty8sZmwz83tV7QzGVb0\n/TvXdfKm7cbvsHEG6xssLCH+5lOnODuWtttYO2lxLDKrN4XeanHh87gmbE4PxfzBdELv97ipCXjo\nj2fsz9eqqMoXNFtX1Rl7JVgTjJkgtv4GLOsmmclzaijBhmbjd7esmV83r8mh3nhR6CN+PG4X7XVB\nxtI5/B6X/XspZXym1uaAyx4AAAnLSURBVO/UP5YmX9AcOmvcKS0WYt0Iy4r6cFH8rNvtc2FFbeCc\nJobZ4IzoI4468WvXNdmllOV3CxtaoiUTX6XtIF97eRuv3LxiyknqFZtW8OevuoS37VjNwHiGg71j\ndiXPTLAE+d9+fhif28UrNk9ManvcRknp6eGkbd2sNUssO+qCFRfBbZxhRA9GhD2aytnFAc6JY5uZ\n32mtDTKcyNJp5hwifg8Rv4dux4b3xvaVxvPXrY9x729fxabWGr61s4uDvWO21WTZRasaQpwcTNBu\nRvNOrMlmYDxNXmsyucKsApXZIhG9sKwoiehnIfSLQV3Ih1JGhO9cfGatpHUpo7/OuaKUmnYvgqDP\nzV3XryPgddNeF+Rr79pRYn1Nx4oaI6E7lMhyw8WxSRPSVjmp9XnUhrzUh7y0V7BtAG7YEOOPX76B\nl1eohirHEt6rTaG3RLalxm+vhbC+Oi2jbavr+cHz3cTTObttsrUXgtuluGFDjFjUyBFYEX3I57YX\nbVl/T611EwMBa7LpG8vYJaszafkxX4jQC8uKgNdNyOcm6HUTq2ARVANul6I+5CPi95REhh31IaNt\ncXDq1g9LiVLKjrqnyqFYnrezcur9N67nrVevrnh+wOvm/Tetn9EajZhZQWPlJCyR3bqy3r6ettA7\nqog+8PINDIxn+MIjx+yKm9WNE8tJ1zdHOHTWqOlvjhb/hix7p9LE6KzvP9AzhlLFSWQxEOtGWHZU\nEtFqoyHsq7hZxmsvbytpxFWNbFtVz6+6Rrjpksk3l7MSsvUOof/tl6ydl/e/clUdPSPFFbb1IR+d\nTeGSdQNWZY+zB9MVK+u4ZfMKPv/wEWMntaZIxQqyDS1RvvtMFwd6xuwOrlAU+rYK1p5lx/WbEf2a\nxvCcdno7V0TohWWH1cCrmmkI+0q2JrSwtjSsZt5340W8dceqKfsbbW6rJeRzn1Np60x5x7Vrece1\nxUnD7VL87E9eWnLOW65exea2mgmJ37989UY+9L0cjx0ZsBOv5axviZDI5Al63Xzo1o32cVvo6yZG\n9D6Py9i3eTzNwd4xe+/kxUKEXlh2/Pvbt1PFwTwA776hk2Rm4XugLAQBr3taX/9Vl63g+g1NEyqD\nFovaoLdikrmjPsRX3nk1o6ksAU/liNuqTPrjV2woWey0ua2Gv7ltM7+2pXLPpqaIj71nRjk+MM5t\nU1Q+LQQi9MKyYyY9dZaaGy+ZPul4PqOUWjKRnwmTJZHBsKb++30vsRvUWSil7BXElWiM+Hnq2CAu\nBa/bKkIvCIJQtSiluKzj3DfvsZL/r9y8YtErvqo/tBEEQbgAsMo837UITczKWbCIXil1C/BpwA38\nh9Za9o4VBGHZ8qYrV9JSE7C3plxMFkTolVJu4DMYm4N3AU8rpe7XWi9842VBEIQq5LKO2llZPvPB\nQlk3VwGHtdZHtdYZ4BvAbQv0XoIgCMIULJTQtwOnHN93mcdslFJ3KaV2KqV29vX1LdAwBEEQhIUS\n+kpVyrrkG63v1lpv11pvj8Vm3jRJEARBODcWSui7gJWO7zuAytvEC4IgCAvKQgn908B6pdRapZQP\nuAO4f4HeSxAEQZiCBam60VrnlFLvA36EUV55j9Z670K8lyAIgjA1C1ZHr7X+IfDDhXp9QRAEYWbI\nylhBEIQLHKW1nv6shR6EUn3AiVn+eBPQP4/DmS+qdVxQvWOTcZ0bMq5z40Ic12qt9bRli1Uh9HNB\nKbVTa719qcdRTrWOC6p3bDKuc0PGdW4s53GJdSMIgnCBI0IvCIJwgXMhCP3dSz2ASajWcUH1jk3G\ndW7IuM6NZTuu896jFwRBEKbmQojoBUEQhCk4r4VeKXWLUuqAUuqwUuqDSziOlUqph5RS+5RSe5VS\nf2Ae/2ul1Gml1G7z36uWYGzHlVK/Mt9/p3msQSn1E6XUIfProu6EoJS62HFNdiulRpVSf7gU10sp\ndY9S6qxSao/jWMXrowz+2fx7e14ptW2Rx/UJpdR+873/SylVZx5fo5RKOq7b5xZ5XJN+bkqpD5nX\n64BS6pWLPK5vOsZ0XCm12zy+mNdrMm1Y3L8xrfV5+Q+jtcIRoBPwAc8Bm5ZoLK3ANvNxFDgIbAL+\nGviTJb5Ox4GmsmMfBz5oPv4g8LEl/hx7gNVLcb2A64FtwJ7prg/wKuB/Mbqz7gCeXORxvQLwmI8/\n5hjXGud5S3C9Kn5u5v+B5wA/sNb8/+perHGVPf+PwIeX4HpNpg2L+jd2Pkf0VbO5ida6W2v9jPl4\nDNhHWf/9KuM24F7z8b3A7Us4lpuAI1rr2S6YmxNa64eBwbLDk12f24Ava4MngDqlVOtijUtr/WOt\ndc789gmMrrCLyiTXazJuA76htU5rrY8BhzH+3y7quJRSCvh14OsL8d5TMYU2LOrf2Pks9NNubrIU\nKKXWAFuBJ81D7zNvwe5ZbIvERAM/VkrtUkrdZR5r0Vp3g/GHCDQvwbgs7qD0P+BSXy+Y/PpU09/c\nb2NEfhZrlVLPKqV+oZS6bgnGU+lzq5brdR3Qq7U+5Di26NerTBsW9W/sfBb6aTc3WWyUUhHgu8Af\naq1Hgc8C64ArgG6M28fF5lqt9TbgVuC9Sqnrl2AMFVFGC+vXAt82D1XD9ZqKqvibU0r9BZADvmYe\n6gZWaa23Ah8A/lMpVbOIQ5rsc6uK6wW8mdJgYtGvVwVtmPTUCsfmfM3OZ6Gvqs1NlFJejA/ya1rr\n7wForXu11nmtdQH4dxbotnUqtNZnzK9ngf8yx9Br3Q6aX88u9rhMbgWe0Vr3mmNc8utlMtn1WfK/\nOaXUncCrgbdq09Q1rZEB8/EuDC98w2KNaYrPrRqulwd4PfBN69hiX69K2sAi/42dz0JfNZubmB7g\nF4B9Wut/chx3emuvA/aU/+wCjyuslIpajzGSeXswrtOd5ml3Avct5rgclERaS329HEx2fe4H3m5W\nRuwARqzb78VAKXUL8GfAa7XWCcfxmFLKbT7uBNYDRxdxXJN9bvcDdyil/Eqptea4nlqscZncDOzX\nWndZBxbzek2mDSz239hiZJ4X6h9Ghvogxoz8F/+vfTs2QRiI4jD+1TaCVrbOYOkCClaOYOMOziFY\nCm5g7wpiFAvR3soJbCzuAiIErE58fD8IgSPF4+X4J3ckP6xjSFpeHYFDPsbABjjl8S3QK1xXn/TV\nQwWc6x4BXWAHXPO584OetYAH0H4bK94v0oPmDjxJb1Ozpv6QltXLPN9OwKBwXTfS/m09x1b52mm+\nvxWwByaF62q8b8Ai9+sCjErWlcfXwPzj2pL9asqGonPMP2MlKbh/3rqRJH3BoJek4Ax6SQrOoJek\n4Ax6SQrOoJek4Ax6SQrOoJek4F7SU1YOsFy9fgAAAABJRU5ErkJggg==\n", 365 | "text/plain": [ 366 | "" 367 | ] 368 | }, 369 | "metadata": {}, 370 | "output_type": "display_data" 371 | } 372 | ], 373 | "source": [ 374 | "plt.plot(errors)" 375 | ] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": 9, 380 | "metadata": {}, 381 | "outputs": [ 382 | { 383 | "name": "stdout", 384 | "output_type": "stream", 385 | "text": [ 386 | "prediction : 3\n" 387 | ] 388 | }, 389 | { 390 | "data": { 391 | "text/plain": [ 392 | "" 393 | ] 394 | }, 395 | "execution_count": 9, 396 | "metadata": {}, 397 | "output_type": "execute_result" 398 | }, 399 | { 400 | "data": { 401 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADfFJREFUeJzt3W2MFfUVx/HfUeob2ygrAqtVlxpo\nbEjc1o02sWloKg02JGhMibyAbVq7TSxJMZgUMbEm2mAK9OEVBgIWQrEVH7Gp2rppao3GsJIGsVRr\nmm27ZXmKxNoAIbKnL3a2Wda9/9m9d+6dYc/3k5D7cO7cObnwY+be/8z8zd0FIJ4Lym4AQDkIPxAU\n4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoKa1cmVmxuGEQJO5u03kdQ1t+c1skZm9Y2bvmdmaRt4L\nQGtZvcf2m9mFkt6VtFDSgKS9kpa5+18Sy7DlB5qsFVv+GyW95+5/d/czkn4laUkD7weghRoJ/5WS\n/jXq8UD23DnMrMfM+sysr4F1AShYIz/4jbdr8bHdenffLGmzxG4/UCWNbPkHJF016vGnJR1qrB0A\nrdJI+PdKmmtmc8zsIkl3StpTTFsAmq3u3X53/8jMVkp6SdKFkra5+9uFdQagqeoe6qtrZXznB5qu\nJQf5ADh/EX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQ\nhB8IivADQRF+ICjCDwRF+IGgCD8QVEun6EZ9rrjiimR91apVNWvd3d3JZWfMmJGsX3BBevswNDSU\nrD/55JM1ay+++GJy2RdeeCFZP3z4cLKONLb8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxBUQ7P0mlm/\npA8lnZX0kbt35byeWXrrsGbNmmT94Ycfbtq6zdITvrZyluexrr/++mT97bdjzhg/0Vl6izjI5yvu\nfryA9wHQQuz2A0E1Gn6X9Dsze9PMeopoCEBrNLrbf7O7HzKzmZJ+b2Z/dfdXRr8g+0+B/xiAimlo\ny+/uh7Lbo5KekXTjOK/Z7O5deT8GAmitusNvZheb2adG7kv6mqQDRTUGoLka2e2fJemZbChomqRd\n7p4+RxNAZTQ0zj/plTHOP67p06cn6/v370/W29vba9Zef/315LIbNmxI1k+ePJmsL1q0KFlfvnx5\nzVpbW1ty2TwffPBBsj5//vyatcHBwYbWXWUTHednqA8IivADQRF+ICjCDwRF+IGgCD8QFEN9FTB7\n9uxkfWBgoO737uzsTNYPHGjucVkdHR01axs3bkwuu2TJkmQ973Tj1GXLd+7cmVz2fMZQH4Akwg8E\nRfiBoAg/EBThB4Ii/EBQhB8Iiim6K+DYsWPJ+n333Zesr1u3rsh2CtXf31+z9uyzzyaXzRvnR2PY\n8gNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIzzV8DZs2eT9fXr1yfru3fvrllbsWJFctlmn8+fsnr1\n6mS90enBT58+PemeImHLDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANB5V6338y2SVos6ai7z8+ea5P0\na0kdkvolLXX3E7kr47r9U87VV1+drN911101a3fffXdy2UsvvTRZzzv+Ie86CFNVkdft/4WksZOw\nr5HU6+5zJfVmjwGcR3LD7+6vSHp/zNNLJG3P7m+XdFvBfQFosnq/889y90FJym5nFtcSgFZo+rH9\nZtYjqafZ6wEwOfVu+Y+YWbskZbdHa73Q3Te7e5e7d9W5LgBNUG/490gamQK1W9JzxbQDoFVyw29m\nj0t6XdJnzWzAzL4t6RFJC83sb5IWZo8BnEdyx/kLXRnj/Oedzs7OZP35559P1tvb22vWBgcHk8s+\n9thjyfoDDzyQrEdV5Dg/gCmI8ANBEX4gKMIPBEX4gaAIPxAUl+6e4hYvXpys33LLLcn6smXLkvXL\nLrssWe/t7a1Zyzvldt++fck6GsOWHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCYpy/AhYsWJCsL1o0\n9uLJ50pNw93W1pZcdtq09D+BU6dOJet5xwm89tprNWtnzpxJLovmYssPBEX4gaAIPxAU4QeCIvxA\nUIQfCIrwA0Fx6e4WyLu8dd4590NDQ0W2MyknT55M1jdt2lT3e+/YsSNZP378eLJ++PDhutc9lXHp\nbgBJhB8IivADQRF+ICjCDwRF+IGgCD8QVO44v5ltk7RY0lF3n58996Ck70g6lr1srbv/NndlQcf5\nDx06lKzPmjUrWW/lsRhjmaWHjMvsbcuWLcn62rVra9ZOnDhRdDuVUeQ4/y8kjXc1iZ+6e2f2Jzf4\nAKolN/zu/oqk91vQC4AWauQ7/0oz229m28xsemEdAWiJesO/SdK1kjolDUraWOuFZtZjZn1m1lfn\nugA0QV3hd/cj7n7W3YckbZF0Y+K1m929y9276m0SQPHqCr+ZtY96eLukA8W0A6BVci/dbWaPS1og\naYaZDUj6oaQFZtYpySX1S/puE3sE0AScz98Cq1evTtYXLlyYrDfz76ijoyNZnzdvXrJe5WMQuru7\na9Z27txZdDuVwfn8AJIIPxAU4QeCIvxAUIQfCIrwA0Ex1Bfc5Zdfnqy3t7cn641cVnz37t3J+ty5\nc5N1hvrGx1AfgCTCDwRF+IGgCD8QFOEHgiL8QFCEHwgq93x+TG3Hjh1rqJ7n3nvvrVmbM2dOQ++N\nxrDlB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgGOcvwPLly5P1lStXJus33XRTke0Uavbs2cn6unXr\nkvUVK1YU2c45tm/fnqxP5XP2i8CWHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCyr1uv5ldJWmHpNmS\nhiRtdvefm1mbpF9L6pDUL2mpu5/Iea8ped3+Xbt2JetLly5N1qdNK+9wi5kzZybrvb29yfp1111X\nZDvnyLuWQN6cAlEVed3+jyStdvfrJH1R0vfM7HOS1kjqdfe5knqzxwDOE7nhd/dBd9+X3f9Q0kFJ\nV0paImnkEKvtkm5rVpMAijep7/xm1iHp85LekDTL3Qel4f8gJKX3HwFUyoS/bJrZJyU9JWmVu/8n\nb560Ucv1SOqprz0AzTKhLb+ZfULDwf+luz+dPX3EzNqzeruko+Mt6+6b3b3L3buKaBhAMXLDb8Ob\n+K2SDrr7T0aV9kgamQa1W9JzxbcHoFkmstt/s6Tlkt4ysz9nz62V9IikJ8zs25L+KekbzWmx+hod\n7rr//vuT9b179ybrd9xxR83aNddck1w2rz5v3rxkvZEp3rdu3Zqsr1+/vu73Rr7c8Lv7q5JqfcH/\narHtAGgVjvADgiL8QFCEHwiK8ANBEX4gKMIPBJV7Sm+hK5uip/Tec889yXreeHXeodKt/DsaK6+3\nV199NVnfsGFDzdrLL7+cXPbUqVPJOsZX5Cm9AKYgwg8ERfiBoAg/EBThB4Ii/EBQhB8Iiim6C/Do\no48m65dcckmyfuuttybrN9xww6R7mqi+vr5k/aGHHkrW8y7tffr06Un3hNZgyw8ERfiBoAg/EBTh\nB4Ii/EBQhB8IivADQXE+PzDFcD4/gCTCDwRF+IGgCD8QFOEHgiL8QFCEHwgqN/xmdpWZ/cHMDprZ\n22b2/ez5B83s32b25+zP15vfLoCi5B7kY2btktrdfZ+ZfUrSm5Juk7RU0n/dvfasDB9/Lw7yAZps\nogf55F7Jx90HJQ1m9z80s4OSrmysPQBlm9R3fjPrkPR5SW9kT600s/1mts3MptdYpsfM+swsfb0o\nAC014WP7zeyTkv4o6Ufu/rSZzZJ0XJJLekjDXw2+lfMe7PYDTTbR3f4Jhd/MPiHpN5JecvefjFPv\nkPQbd5+f8z6EH2iywk7sseFpWrdKOjg6+NkPgSNul3Rgsk0CKM9Efu3/kqQ/SXpL0lD29FpJyyR1\nani3v1/Sd7MfB1PvxZYfaLJCd/uLQviB5uN8fgBJhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8\nQFCEHwiK8ANBEX4gKMIPBEX4gaByL+BZsOOS/jHq8YzsuSqqam9V7Uuit3oV2ds1E31hS8/n/9jK\nzfrcvau0BhKq2ltV+5LorV5l9cZuPxAU4QeCKjv8m0tef0pVe6tqXxK91auU3kr9zg+gPGVv+QGU\npJTwm9kiM3vHzN4zszVl9FCLmfWb2VvZzMOlTjGWTYN21MwOjHquzcx+b2Z/y27HnSatpN4qMXNz\nYmbpUj+7qs143fLdfjO7UNK7khZKGpC0V9Iyd/9LSxupwcz6JXW5e+ljwmb2ZUn/lbRjZDYkM/ux\npPfd/ZHsP87p7v6DivT2oCY5c3OTeqs1s/Q3VeJnV+SM10UoY8t/o6T33P3v7n5G0q8kLSmhj8pz\n91ckvT/m6SWStmf3t2v4H0/L1eitEtx90N33Zfc/lDQys3Spn12ir1KUEf4rJf1r1OMBVWvKb5f0\nOzN708x6ym5mHLNGZkbKbmeW3M9YuTM3t9KYmaUr89nVM+N10coI/3iziVRpyOFmd/+CpFslfS/b\nvcXEbJJ0rYancRuUtLHMZrKZpZ+StMrd/1NmL6ON01cpn1sZ4R+QdNWox5+WdKiEPsbl7oey26OS\nntHw15QqOTIySWp2e7Tkfv7P3Y+4+1l3H5K0RSV+dtnM0k9J+qW7P509XfpnN15fZX1uZYR/r6S5\nZjbHzC6SdKekPSX08TFmdnH2Q4zM7GJJX1P1Zh/eI6k7u98t6bkSezlHVWZurjWztEr+7Ko243Up\nB/lkQxk/k3ShpG3u/qOWNzEOM/uMhrf20vAZj7vK7M3MHpe0QMNnfR2R9ENJz0p6QtLVkv4p6Rvu\n3vIf3mr0tkCTnLm5Sb3Vmln6DZX42RU543Uh/XCEHxATR/gBQRF+ICjCDwRF+IGgCD8QFOEHgiL8\nQFCEHwjqfxu5Rxnzc+/XAAAAAElFTkSuQmCC\n", 402 | "text/plain": [ 403 | "" 404 | ] 405 | }, 406 | "metadata": {}, 407 | "output_type": "display_data" 408 | } 409 | ], 410 | "source": [ 411 | "i = np.random.randint(55000)\n", 412 | "img = mnist.train.images[i]\n", 413 | "img = np.reshape(img, [28, 28])\n", 414 | "\n", 415 | "pred = predict(img)\n", 416 | "print('prediction :', pred)\n", 417 | "\n", 418 | "plt.imshow(img, cmap='gray')" 419 | ] 420 | }, 421 | { 422 | "cell_type": "code", 423 | "execution_count": 10, 424 | "metadata": { 425 | "collapsed": true 426 | }, 427 | "outputs": [], 428 | "source": [ 429 | "res = 0\n", 430 | "for img,labels in zip(mnist.test.images,mnist.test.labels):\n", 431 | " pred = predict(img)\n", 432 | " res += (pred==labels.argmax())" 433 | ] 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": 11, 438 | "metadata": {}, 439 | "outputs": [ 440 | { 441 | "data": { 442 | "text/plain": [ 443 | "0.95599999999999996" 444 | ] 445 | }, 446 | "execution_count": 11, 447 | "metadata": {}, 448 | "output_type": "execute_result" 449 | } 450 | ], 451 | "source": [ 452 | "res / mnist.test.num_examples" 453 | ] 454 | }, 455 | { 456 | "cell_type": "code", 457 | "execution_count": null, 458 | "metadata": { 459 | "collapsed": true 460 | }, 461 | "outputs": [], 462 | "source": [] 463 | } 464 | ], 465 | "metadata": { 466 | "kernelspec": { 467 | "display_name": "Python 3", 468 | "language": "python", 469 | "name": "python3" 470 | }, 471 | "language_info": { 472 | "codemirror_mode": { 473 | "name": "ipython", 474 | "version": 3 475 | }, 476 | "file_extension": ".py", 477 | "mimetype": "text/x-python", 478 | "name": "python", 479 | "nbconvert_exporter": "python", 480 | "pygments_lexer": "ipython3", 481 | "version": "3.6.1" 482 | } 483 | }, 484 | "nbformat": 4, 485 | "nbformat_minor": 2 486 | } 487 | -------------------------------------------------------------------------------- /MCMC-Metropolis-Hastings.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.special as ss 3 | import scipy.stats as st 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def circle(x, y): 8 | return (x+2)**2 + (y-2)**2/100 - 3**2 9 | 10 | 11 | mus = np.array([5, 5]) 12 | sigmas = np.array([[1, .9], [.9, 1]]) 13 | def pgauss(x, y): 14 | return st.multivariate_normal.pdf([x, y], mean=mus, cov=sigmas) 15 | 16 | def HM(p,iter=100000): 17 | x,y = 0.,0. 18 | samples = np.zeros((iter,2)) 19 | for i in range(iter): 20 | x_star, y_star = np.array([x, y]) + np.random.normal(size=2) 21 | if np.random.rand() < p(x_star,y_star)/p(x,y): 22 | x, y = x_star, y_star 23 | samples[i] = np.array([x,y]) 24 | return samples 25 | 26 | if __name__ == '__main__': 27 | samples = HM(pgauss, iter=10000) 28 | plt.scatter(samples[:,0],samples[:,1]) 29 | plt.show() 30 | 31 | -------------------------------------------------------------------------------- /Naive Bayes.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Naive bayes" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "朴素贝叶斯是基于贝叶斯定理与特征条件的独立假设的分类方法，对于给定的训练数据集，首先基于特征条件独立假设学习输入\\输出的联合概率分布，然后基于此模型，对给定的输入x，利用贝叶斯定理求后验概率最大的输出y。" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "metadata": { 21 | "collapsed": true 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "import math\n", 26 | "import numpy as np\n", 27 | "from utils.metric import accuracy_score\n", 28 | "from utils.progress import normalize,train_test_split" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "> 这里使用高斯函数来作为概率密度的计算，因为在实际生活中，很多情况的特征值是连续的" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 12, 41 | "metadata": { 42 | "collapsed": true 43 | }, 44 | "outputs": [], 45 | "source": [ 46 | "class NaiveBayes(): \n", 47 | " \"\"\"The Gaussian Naive Bayes classifier. \"\"\"\n", 48 | " def fit(self, X, y):\n", 49 | " \n", 50 | " \"\"\"\n", 51 | " X [shape,features]\n", 52 | " y [shape,label]\n", 53 | " \"\"\"\n", 54 | " \n", 55 | " self.X, self.y = X, y\n", 56 | " self.classes = np.unique(y)\n", 57 | " self.parameters = []\n", 58 | " # 计算每一个类别的每一个特征的方差和均值\n", 59 | " for i, c in enumerate(self.classes):\n", 60 | " X_where_c = X[np.where(y == c)]\n", 61 | " self.parameters.append([])\n", 62 | " # 计算每一个特征\n", 63 | " for j in range(X.shape[1]):\n", 64 | " col = X_where_c[:, j] #列\n", 65 | " parameters = {\"mean\": col.mean(), \"var\": col.var()} #求方差 与 均值\n", 66 | " self.parameters[i].append(parameters)\n", 67 | " \n", 68 | " def _calculate_likelihood(self, mean, var, x):\n", 69 | " \"\"\" 计算高斯概率密度 输入均值 和 方差\"\"\"\n", 70 | " eps = 1e-4 # Added in denominator to prevent division by zero\n", 71 | " coeff = 1.0 / math.sqrt(2.0 * math.pi * var + eps)\n", 72 | " exponent = math.exp(-(math.pow(x - mean, 2) / (2 * var + eps)))\n", 73 | " return coeff * exponent\n", 74 | " def _calculate_prior(self, c):\n", 75 | " \"\"\" 计算先验概率 \"\"\"\n", 76 | " X_where_c = self.X[np.where(self.y == c)]\n", 77 | " n_class_instances = X_where_c.shape[0]\n", 78 | " n_total_instances = self.X.shape[0]\n", 79 | " return n_class_instances / n_total_instances\n", 80 | " \n", 81 | " def _classify(self, sample):\n", 82 | " posteriors = []\n", 83 | " for i, c in enumerate(self.classes):\n", 84 | " # 计算每一个类别的先验概率 p(y=c)=?\n", 85 | " posterior = self._calculate_prior(c)\n", 86 | " \n", 87 | " for j, params in enumerate(self.parameters[i]):\n", 88 | " # 提取每一个类别下的特征值的方差 以及 均值\n", 89 | " sample_feature = sample[j]\n", 90 | " # 计算高斯密度\n", 91 | " likelihood = self._calculate_likelihood(params[\"mean\"], params[\"var\"], sample_feature)\n", 92 | " posterior *= likelihood\n", 93 | " posteriors.append(posterior)\n", 94 | " # 求最大概率对应的类别\n", 95 | " index_of_max = np.argmax(posteriors)\n", 96 | " return self.classes[index_of_max]\n", 97 | " def predict(self, X):\n", 98 | " y_pred = []\n", 99 | " for sample in X:\n", 100 | " y = self._classify(sample)\n", 101 | " y_pred.append(y)\n", 102 | " return y_pred" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 13, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "from sklearn import datasets\n", 112 | "data = datasets.load_digits()\n", 113 | "X = normalize(data.data)\n", 114 | "y = data.target\n", 115 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 14, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "clf = NaiveBayes()\n", 125 | "clf.fit(X_train, y_train)\n", 126 | "y_pred = clf.predict(X_test)" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 15, 132 | "metadata": {}, 133 | "outputs": [ 134 | { 135 | "data": { 136 | "text/plain": [ 137 | "0.90250696378830086" 138 | ] 139 | }, 140 | "execution_count": 15, 141 | "metadata": {}, 142 | "output_type": "execute_result" 143 | } 144 | ], 145 | "source": [ 146 | "accuracy_score(y_pred,y_test)" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "metadata": {}, 152 | "source": [ 153 | "### 实际过程中，朴素贝叶斯用的还是蛮多的，因为计算简单，快。\n", 154 | "\n", 155 | "参考\n", 156 | "\n", 157 | "1. 李航《统计学习方法》\n", 158 | "2. 周志华《机器学习》\n", 159 | "3. [ML_from_scratch](https://github.com/eriklindernoren/ML-From-Scratch)" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": null, 165 | "metadata": { 166 | "collapsed": true 167 | }, 168 | "outputs": [], 169 | "source": [] 170 | } 171 | ], 172 | "metadata": { 173 | "kernelspec": { 174 | "display_name": "Python 3", 175 | "language": "python", 176 | "name": "python3" 177 | }, 178 | "language_info": { 179 | "codemirror_mode": { 180 | "name": "ipython", 181 | "version": 3 182 | }, 183 | "file_extension": ".py", 184 | "mimetype": "text/x-python", 185 | "name": "python", 186 | "nbconvert_exporter": "python", 187 | "pygments_lexer": "ipython3", 188 | "version": "3.6.1" 189 | } 190 | }, 191 | "nbformat": 4, 192 | "nbformat_minor": 2 193 | } 194 | -------------------------------------------------------------------------------- /PageRank.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# PageRank\n", 8 | "\n", 9 | "> PageRank 是网页排名的一种算法，也是google搜索的核心算法。属于图数据上的无监督学习方法。PageRank是有向图上的定义的一个随机游走模型，即一阶马尔科夫链，描述每一个节点在稳定状态下的概率，该概率就是每一个节点的PageRank值。\n", 10 | "\n", 11 | "这里定义一个简单的有向图的模型。" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import torch" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 29, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "# 定义每一个节点的转移矩阵\n", 30 | "M = torch.tensor([[0, 1/2, 1, 0],\n", 31 | " [1/3, 0, 0, 1/2],\n", 32 | " [1/3, 0, 0, 1/2],\n", 33 | " [1/3, 1/2, 0, 0],\n", 34 | "])" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "> 由于是概率图，因此需要满足和为1的性质。" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 30, 47 | "metadata": {}, 48 | "outputs": [ 49 | { 50 | "data": { 51 | "text/plain": [ 52 | "tensor([1., 1., 1., 1.])" 53 | ] 54 | }, 55 | "execution_count": 30, 56 | "metadata": {}, 57 | "output_type": "execute_result" 58 | } 59 | ], 60 | "source": [ 61 | "torch.sum(M, dim=0)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 31, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "def PageRank(R0, M, iters):\n", 71 | " for _ in range(iters):\n", 72 | " R0 = torch.matmul(M, R0)\n", 73 | " return R0" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 32, 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "data": { 83 | "text/plain": [ 84 | "tensor([[0.3333],\n", 85 | " [0.2222],\n", 86 | " [0.2222],\n", 87 | " [0.2222]])" 88 | ] 89 | }, 90 | "execution_count": 32, 91 | "metadata": {}, 92 | "output_type": "execute_result" 93 | } 94 | ], 95 | "source": [ 96 | "# 开始的时候，设置每一个节点的概率一样。\n", 97 | "R0 = torch.ones((4,1)).fill_(1/4)\n", 98 | "PageRank(R0, M, 1000000)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "> 我们得到了每一个节点的稳定值." 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "## PageRank的一般定义\n", 113 | "\n", 114 | "由于有些网页无法调到其他网页，因此基本的PageRank算法是不适用的。这里需要加上一个平滑项。\n", 115 | "\n", 116 | "$ R = dMR + \\frac{1-d}{n} 1$\n", 117 | "\n", 118 | "其中d为阻尼因子，$1$是分量为1的n维向量。" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 38, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "def PageRank(R0, d, M, iters):\n", 128 | " dM = d * M\n", 129 | " smooth = (1-d) * torch.ones((len(M),1)).fill_(1/len(M))\n", 130 | " \n", 131 | " for _ in range(iters):\n", 132 | " R0 = torch.matmul(dM, R0) + smooth\n", 133 | " return R0" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 39, 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "data": { 143 | "text/plain": [ 144 | "tensor([[0.1014],\n", 145 | " [0.1284],\n", 146 | " [0.6419],\n", 147 | " [0.1284]])" 148 | ] 149 | }, 150 | "execution_count": 39, 151 | "metadata": {}, 152 | "output_type": "execute_result" 153 | } 154 | ], 155 | "source": [ 156 | "M = torch.tensor([[0, 1/2, 0, 0],\n", 157 | " [1/3, 0, 0, 1/2],\n", 158 | " [1/3, 0, 1, 1/2],\n", 159 | " [1/3, 1/2, 0, 0],\n", 160 | "])\n", 161 | "\n", 162 | "R0 = torch.ones((4,1)).fill_(1/4)\n", 163 | "PageRank(R0, 0.8, M, 100000)" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "# 幂法来计算PageRank\n", 171 | "\n", 172 | "思想是使用特征值与特征向量计算。" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 84, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "M = torch.tensor([[0, 0, 1],\n", 182 | " [1/2, 0, 0],\n", 183 | " [1/2, 1, 0],\n", 184 | "])\n", 185 | "\n", 186 | "x = torch.ones((3,1))\n", 187 | "\n", 188 | "A = 0.85 * M + (1-0.85) / 3 * torch.ones((3,3))\n", 189 | "\n", 190 | "# 这里采用的范数是无穷范数,即分量中绝对值的最大值。\n", 191 | "def get_norm(A):\n", 192 | " A = torch.abs(A)\n", 193 | " return torch.max(A)" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": 72, 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [ 202 | "y = A @ x\n", 203 | "x = y / get_norm(y)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": 81, 209 | "metadata": {}, 210 | "outputs": [ 211 | { 212 | "name": "stdout", 213 | "output_type": "stream", 214 | "text": [ 215 | "tensor([[0.9758],\n", 216 | " [0.5405],\n", 217 | " [1.0000]])\n" 218 | ] 219 | } 220 | ], 221 | "source": [ 222 | "for _ in range(100):\n", 223 | " t = x\n", 224 | " y = A @ x\n", 225 | " x = y / get_norm(y)\n", 226 | " \n", 227 | " if torch.norm(x-t, 2) < 0.0000001:\n", 228 | " break\n", 229 | "print(x)" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": 83, 235 | "metadata": {}, 236 | "outputs": [ 237 | { 238 | "name": "stdout", 239 | "output_type": "stream", 240 | "text": [ 241 | "tensor([[0.3878],\n", 242 | " [0.2148],\n", 243 | " [0.3974]])\n" 244 | ] 245 | } 246 | ], 247 | "source": [ 248 | "print(x / torch.sum(x))" 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "metadata": {}, 254 | "source": [ 255 | "## 逆矩阵" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 65, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "M = torch.tensor([[0, 0, 1],\n", 265 | " [1/2, 0, 0],\n", 266 | " [1/2, 1, 0],\n", 267 | "])" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 70, 273 | "metadata": {}, 274 | "outputs": [ 275 | { 276 | "data": { 277 | "text/plain": [ 278 | "tensor([[0.3878],\n", 279 | " [0.2148],\n", 280 | " [0.3974]])" 281 | ] 282 | }, 283 | "execution_count": 70, 284 | "metadata": {}, 285 | "output_type": "execute_result" 286 | } 287 | ], 288 | "source": [ 289 | "torch.inverse(torch.eye(3) - 0.85*M) @ ((1-0.85) / 3 * torch.ones((3,1)))" 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "execution_count": null, 295 | "metadata": {}, 296 | "outputs": [], 297 | "source": [] 298 | } 299 | ], 300 | "metadata": { 301 | "kernelspec": { 302 | "display_name": "Python 3", 303 | "language": "python", 304 | "name": "python3" 305 | }, 306 | "language_info": { 307 | "codemirror_mode": { 308 | "name": "ipython", 309 | "version": 3 310 | }, 311 | "file_extension": ".py", 312 | "mimetype": "text/x-python", 313 | "name": "python", 314 | "nbconvert_exporter": "python", 315 | "pygments_lexer": "ipython3", 316 | "version": "3.6.4" 317 | } 318 | }, 319 | "nbformat": 4, 320 | "nbformat_minor": 2 321 | } 322 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ML-From-Scratch 2 | 机器学习算法 基于西瓜书以及《统计学习方法》，当然包括DL 3 | 4 | 5 | ### 监督学习算法 6 | 7 | 1. ~~[linear regression](https://github.com/HadXu/ML-From-Scratch/blob/master/线性回归.ipynb)~~ 8 | 2. ~~[logistic regression](https://github.com/HadXu/ML-From-Scratch/blob/master/对数几率回归.ipynb)~~ 9 | 3. ~~[Linear Discriminant Analysis](https://github.com/HadXu/ML-From-Scratch/blob/master/线性判别分析LDA.ipynb)~~ 10 | 4. ~~[Decision Tree](https://github.com/HadXu/ML-From-Scratch/blob/master/Decision%20Tree%20for%20Classification%20And%20Regression.ipynb)~~ 11 | 5. ~~[GBDT](https://github.com/HadXu/ML-From-Scratch/blob/master/GBDT%20梯度提升.ipynb)~~ 12 | 6. ~~[Xgboost](https://github.com/HadXu/ML-From-Scratch/blob/master/Xgboost%20算法.ipynb)~~ 13 | 7. ~~[KNN](https://github.com/HadXu/ML-From-Scratch/blob/master/KNN.ipynb)~~ 14 | 8. ~~[Naive bayes](https://github.com/HadXu/ML-From-Scratch/blob/master/Naive%20Bayes.ipynb)~~ 15 | 9. ~~[SVM](https://github.com/HadXu/ML-From-Scratch/blob/master/SVM%20SMO.ipynb)~~ 16 | 10. ~~[EM(GMM)](https://github.com/HadXu/ML-From-Scratch/blob/master/GaussianMixtureModel.ipynb)~~ 17 | 11. ~~[MLP](https://github.com/HadXu/ML-From-Scratch/blob/master/utils/MLP.ipynb)~~ 18 | 12. ~~HMM [Forward算法](https://github.com/HadXu/ML-From-Scratch/blob/master/HMM_forward.py) [Viterbi算法](https://github.com/HadXu/ML-From-Scratch/blob/master/Viterbi.py)~~ 19 | 13. ~~[RNN](https://github.com/HadXu/ML-From-Scratch/blob/master/utils/Simple%20RNN.ipynb)~~ 20 | 14. ~~[LSTM](https://github.com/HadXu/ML-From-Scratch/blob/master/LSTM_FS.ipynb)~~ 21 | 15. ~~[MCMC-HM算法]()~~ 22 | 23 | ### 非监督学习算法 24 | 25 | 1. ~~[K-means](https://github.com/HadXu/ML-From-Scratch/blob/master/K-means算法.ipynb)~~ 26 | 2. ~~[PCA](https://github.com/HadXu/ML-From-Scratch/blob/master/PCA.ipynb)~~ 27 | 28 | 29 | ### 强化学习 30 | 31 | 1. Q-learning 32 | 33 | -------------------------------------------------------------------------------- /TempLinkoping2016.txt: -------------------------------------------------------------------------------- 1 | time temp 2 | 0.00273224 0.1 3 | 0.005464481 -4.5 4 | 0.008196721 -6.3 5 | 0.010928962 -9.6 6 | 0.013661202 -9.9 7 | 0.016393443 -17.1 8 | 0.019125683 -11.6 9 | 0.021857923 -6.2 10 | 0.024590164 -6.4 11 | 0.027322404 -0.5 12 | 0.030054645 0.5 13 | 0.032786885 -2.4 14 | 0.035519126 -7.5 15 | 0.038251366 -16.8 16 | 0.040983607 -16.6 17 | 0.043715847 -14.6 18 | 0.046448087 -9.6 19 | 0.049180328 -5.8 20 | 0.051912568 -8.6 21 | 0.054644809 -9.0 22 | 0.057377049 -9.7 23 | 0.06010929 -6.9 24 | 0.06284153 -3.9 25 | 0.06557377 1.4 26 | 0.068306011 1.9 27 | 0.071038251 4.3 28 | 0.073770492 6.9 29 | 0.076502732 4.3 30 | 0.079234973 5.9 31 | 0.081967213 3.8 32 | 0.084699454 1.5 33 | 0.087431694 0.1 34 | 0.090163934 4.6 35 | 0.092896175 0.8 36 | 0.095628415 -0.5 37 | 0.098360656 -1.0 38 | 0.101092896 4.2 39 | 0.103825137 6.6 40 | 0.106557377 4.8 41 | 0.109289617 4.7 42 | 0.112021858 1.3 43 | 0.114754098 0.9 44 | 0.117486339 -2.8 45 | 0.120218579 -3.3 46 | 0.12295082 -5.3 47 | 0.12568306 -6.8 48 | 0.128415301 -5.1 49 | 0.131147541 -2.6 50 | 0.133879781 -0.5 51 | 0.136612022 -0.5 52 | 0.139344262 0.1 53 | 0.142076503 1.7 54 | 0.144808743 2.4 55 | 0.147540984 -0.9 56 | 0.150273224 -1.3 57 | 0.153005464 -1.4 58 | 0.155737705 -0.1 59 | 0.158469945 -0.7 60 | 0.161202186 -2.6 61 | 0.163934426 -4.1 62 | 0.166666667 -2.7 63 | 0.169398907 0.7 64 | 0.172131148 2.0 65 | 0.174863388 1.7 66 | 0.177595628 0.9 67 | 0.180327869 0.3 68 | 0.183060109 0.9 69 | 0.18579235 1.1 70 | 0.18852459 0.1 71 | 0.191256831 -0.9 72 | 0.193989071 0.2 73 | 0.196721311 0.1 74 | 0.199453552 1.0 75 | 0.202185792 3.4 76 | 0.204918033 5.2 77 | 0.207650273 4.9 78 | 0.210382514 4.9 79 | 0.213114754 2.2 80 | 0.215846995 2.9 81 | 0.218579235 5.3 82 | 0.221311475 3.7 83 | 0.224043716 3.4 84 | 0.226775956 2.1 85 | 0.229508197 1.8 86 | 0.232240437 4.3 87 | 0.234972678 7.0 88 | 0.237704918 7.7 89 | 0.240437158 6.2 90 | 0.243169399 7.5 91 | 0.245901639 4.9 92 | 0.24863388 4.4 93 | 0.25136612 3.8 94 | 0.254098361 6.4 95 | 0.256830601 8.0 96 | 0.259562842 7.9 97 | 0.262295082 8.9 98 | 0.265027322 6.6 99 | 0.267759563 6.5 100 | 0.270491803 5.8 101 | 0.273224044 5.6 102 | 0.275956284 4.7 103 | 0.278688525 5.5 104 | 0.281420765 5.5 105 | 0.284153005 5.8 106 | 0.286885246 5.3 107 | 0.289617486 6.9 108 | 0.292349727 5.9 109 | 0.295081967 6.1 110 | 0.297814208 6.6 111 | 0.300546448 6.7 112 | 0.303278689 6.5 113 | 0.306010929 7.0 114 | 0.308743169 5.8 115 | 0.31147541 3.0 116 | 0.31420765 2.5 117 | 0.316939891 2.4 118 | 0.319672131 4.3 119 | 0.322404372 2.8 120 | 0.325136612 3.6 121 | 0.327868852 6.8 122 | 0.330601093 9.1 123 | 0.333333333 8.4 124 | 0.336065574 9.3 125 | 0.338797814 13.3 126 | 0.341530055 10.6 127 | 0.344262295 10.5 128 | 0.346994536 11.8 129 | 0.349726776 14.7 130 | 0.352459016 16.2 131 | 0.355191257 16.4 132 | 0.357923497 16.9 133 | 0.360655738 12.3 134 | 0.363387978 10.2 135 | 0.366120219 11.2 136 | 0.368852459 6.1 137 | 0.371584699 6.4 138 | 0.37431694 6.1 139 | 0.37704918 10.4 140 | 0.379781421 10.3 141 | 0.382513661 11.9 142 | 0.385245902 12.9 143 | 0.387978142 12.5 144 | 0.390710383 17.5 145 | 0.393442623 19.9 146 | 0.396174863 19.3 147 | 0.398907104 11.4 148 | 0.401639344 9.7 149 | 0.404371585 10.7 150 | 0.407103825 13.0 151 | 0.409836066 12.4 152 | 0.412568306 16.3 153 | 0.415300546 19.2 154 | 0.418032787 19.2 155 | 0.420765027 19.8 156 | 0.423497268 19.5 157 | 0.426229508 16.6 158 | 0.428961749 13.0 159 | 0.431693989 12.6 160 | 0.43442623 17.6 161 | 0.43715847 13.7 162 | 0.43989071 11.3 163 | 0.442622951 10.2 164 | 0.445355191 10.2 165 | 0.448087432 11.6 166 | 0.450819672 14.2 167 | 0.453551913 14.4 168 | 0.456284153 17.4 169 | 0.459016393 13.1 170 | 0.461748634 17.4 171 | 0.464480874 15.9 172 | 0.467213115 15.9 173 | 0.469945355 15.5 174 | 0.472677596 16.4 175 | 0.475409836 16.7 176 | 0.478142077 18.2 177 | 0.480874317 20.9 178 | 0.483606557 22.2 179 | 0.486338798 19.1 180 | 0.489071038 16.3 181 | 0.491803279 16.6 182 | 0.494535519 15.1 183 | 0.49726776 14.5 184 | 0.5 17.4 185 | 0.50273224 16.5 186 | 0.505464481 13.7 187 | 0.508196721 14.0 188 | 0.510928962 14.2 189 | 0.513661202 15.6 190 | 0.516393443 15.7 191 | 0.519125683 15.6 192 | 0.521857923 16.2 193 | 0.524590164 16.3 194 | 0.527322404 18.3 195 | 0.530054645 16.6 196 | 0.532786885 16.1 197 | 0.535519126 15.9 198 | 0.538251366 16.0 199 | 0.540983607 15.9 200 | 0.543715847 16.0 201 | 0.546448087 15.7 202 | 0.549180328 17.2 203 | 0.551912568 19.9 204 | 0.554644809 21.0 205 | 0.557377049 19.4 206 | 0.56010929 20.4 207 | 0.56284153 23.1 208 | 0.56557377 23.0 209 | 0.568306011 19.9 210 | 0.571038251 17.6 211 | 0.573770492 18.8 212 | 0.576502732 17.8 213 | 0.579234973 18.6 214 | 0.581967213 16.4 215 | 0.584699454 15.2 216 | 0.587431694 15.3 217 | 0.590163934 16.0 218 | 0.592896175 18.0 219 | 0.595628415 17.7 220 | 0.598360656 16.0 221 | 0.601092896 16.4 222 | 0.603825137 16.7 223 | 0.606557377 14.3 224 | 0.609289617 12.2 225 | 0.612021858 10.0 226 | 0.614754098 12.0 227 | 0.617486339 16.2 228 | 0.620218579 15.9 229 | 0.62295082 14.5 230 | 0.62568306 15.3 231 | 0.628415301 13.3 232 | 0.631147541 14.5 233 | 0.633879781 15.5 234 | 0.636612022 15.3 235 | 0.639344262 17.3 236 | 0.642076503 15.3 237 | 0.644808743 16.4 238 | 0.647540984 17.0 239 | 0.650273224 20.2 240 | 0.653005464 22.4 241 | 0.655737705 18.1 242 | 0.658469945 11.6 243 | 0.661202186 14.6 244 | 0.663934426 13.5 245 | 0.666666667 17.9 246 | 0.669398907 16.4 247 | 0.672131148 15.5 248 | 0.674863388 15.9 249 | 0.677595628 14.1 250 | 0.680327869 13.2 251 | 0.683060109 14.5 252 | 0.68579235 19.0 253 | 0.68852459 18.3 254 | 0.691256831 18.8 255 | 0.693989071 16.8 256 | 0.696721311 16.8 257 | 0.699453552 14.3 258 | 0.702185792 18.4 259 | 0.704918033 18.3 260 | 0.707650273 18.4 261 | 0.710382514 14.9 262 | 0.713114754 11.4 263 | 0.715846995 12.6 264 | 0.718579235 14.0 265 | 0.721311475 14.8 266 | 0.724043716 9.9 267 | 0.726775956 11.4 268 | 0.729508197 12.9 269 | 0.732240437 12.1 270 | 0.734972678 12.8 271 | 0.737704918 13.5 272 | 0.740437158 12.9 273 | 0.743169399 14.0 274 | 0.745901639 14.6 275 | 0.74863388 12.0 276 | 0.75136612 10.5 277 | 0.754098361 9.5 278 | 0.756830601 7.6 279 | 0.759562842 6.4 280 | 0.762295082 7.0 281 | 0.765027322 8.1 282 | 0.767759563 8.1 283 | 0.770491803 7.6 284 | 0.773224044 7.4 285 | 0.775956284 7.2 286 | 0.778688525 7.0 287 | 0.781420765 6.4 288 | 0.784153005 5.8 289 | 0.786885246 5.5 290 | 0.789617486 6.4 291 | 0.792349727 7.3 292 | 0.795081967 7.4 293 | 0.797814208 7.8 294 | 0.800546448 7.9 295 | 0.803278689 6.9 296 | 0.806010929 6.1 297 | 0.808743169 3.7 298 | 0.81147541 5.3 299 | 0.81420765 6.1 300 | 0.816939891 4.3 301 | 0.819672131 3.3 302 | 0.822404372 8.8 303 | 0.825136612 9.8 304 | 0.827868852 6.4 305 | 0.830601093 4.6 306 | 0.833333333 5.2 307 | 0.836065574 5.5 308 | 0.838797814 1.4 309 | 0.841530055 0.5 310 | 0.844262295 -2.6 311 | 0.846994536 2.4 312 | 0.849726776 -0.8 313 | 0.852459016 -3.3 314 | 0.855191257 -2.8 315 | 0.857923497 -3.5 316 | 0.860655738 -2.8 317 | 0.863387978 -2.2 318 | 0.866120219 -0.3 319 | 0.868852459 0.0 320 | 0.871584699 2.3 321 | 0.87431694 4.9 322 | 0.87704918 3.1 323 | 0.879781421 3.6 324 | 0.882513661 5.2 325 | 0.885245902 3.8 326 | 0.887978142 3.2 327 | 0.890710383 7.7 328 | 0.893442623 7.8 329 | 0.896174863 6.9 330 | 0.898907104 2.7 331 | 0.901639344 2.8 332 | 0.904371585 6.6 333 | 0.907103825 1.9 334 | 0.909836066 -1.4 335 | 0.912568306 2.2 336 | 0.915300546 1.9 337 | 0.918032787 -1.3 338 | 0.920765027 -1.6 339 | 0.923497268 -3.2 340 | 0.926229508 -2.7 341 | 0.928961749 3.7 342 | 0.931693989 -3.2 343 | 0.93442623 -0.2 344 | 0.93715847 9.3 345 | 0.93989071 7.1 346 | 0.942622951 3.2 347 | 0.945355191 1.1 348 | 0.948087432 -6.0 349 | 0.950819672 1.7 350 | 0.953551913 -1.3 351 | 0.956284153 -2.2 352 | 0.959016393 -1.2 353 | 0.961748634 1.0 354 | 0.964480874 1.7 355 | 0.967213115 3.7 356 | 0.969945355 4.7 357 | 0.972677596 -0.3 358 | 0.975409836 3.5 359 | 0.978142077 3.4 360 | 0.980874317 3.9 361 | 0.983606557 4.5 362 | 0.986338798 5.3 363 | 0.989071038 2.7 364 | 0.991803279 -0.4 365 | 0.994535519 4.3 366 | 0.99726776 7.0 367 | 1 9.3 -------------------------------------------------------------------------------- /Viterbi.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | ### 维比特算法 4 | 5 | # 红 白 红 6 | A = np.array([[0.5,0.2,0.3], 7 | [0.3,0.5,0.2], 8 | [0.2,0.3,0.5]]) 9 | 10 | B = np.array([[0.5,0.5], 11 | [0.4,0.6], 12 | [0.7,0.3]]) 13 | 14 | pi = np.array([[0.2,0.4,0.4]]).T 15 | 16 | 17 | theta_1 = [pi[i]*B[i,0] for i in range(3)] 18 | theta_1_index = np.argmax([pi[i]*B[i,0] for i in range(3)]) 19 | 20 | 21 | print(theta_1_index) 22 | 23 | theta_2 = [max([theta_1[j]*A[j,i]*B[i,1] for j in range(3)]) for i in range(3)] 24 | theta_2_index = [np.argmax([theta_1[j]*A[j,i]*B[i,1] for j in range(3)]) for i in range(3)] 25 | 26 | 27 | theta_3 = [max([theta_2[j]*A[j,i]*B[i,0] for j in range(3)]) for i in range(3)] 28 | theta_3_index = [np.argmax([theta_2[j]*A[j,i]*B[i,0] for j in range(3)]) for i in range(3)] 29 | 30 | 31 | theta_4 = [max([theta_3[j]*A[j,i]*B[i,1] for j in range(3)]) for i in range(3)] 32 | theta_4_index = [np.argmax([theta_3[j]*A[j,i]*B[i,1] for j in range(3)]) for i in range(3)] 33 | 34 | 35 | ## 那么最优路径的解为 36 | 37 | p4 = np.argmax(theta_4_index) 38 | p3 = theta_4_index[p4] 39 | p2 = theta_3_index[p3] 40 | p1 = theta_2_index[p2] 41 | print(p1,p2,p3,p4) 42 | 43 | 44 | -------------------------------------------------------------------------------- /Xgboost 算法.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Xgboost" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "### 在之前的算法中，梯度提升树都是使用一阶导求解来拟合残差，xgboost提出了一种使用二阶导，也就是海森矩阵来拟合\n", 15 | "\n", 16 | "1. [知乎资料](https://www.zhihu.com/question/41354392)\n", 17 | "2. [52cs](http://www.52cs.org/?p=429)\n", 18 | "3. [xgboost](http://www.kdd.org/kdd2016/papers/files/rfp0697-chenAemb.pdf)" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": { 25 | "collapsed": true 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "import numpy as np\n", 30 | "from sklearn import datasets\n", 31 | "from sklearn.metrics import accuracy_score,mean_squared_error" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 15, 37 | "metadata": { 38 | "collapsed": true 39 | }, 40 | "outputs": [], 41 | "source": [ 42 | "# 定义Sigmoid函数用来做对数几率回归\n", 43 | "\n", 44 | "class Sigmoid():\n", 45 | " def __call__(self, x):\n", 46 | " return 1 / (1 + np.exp(-x))\n", 47 | "\n", 48 | " def gradient(self, x):\n", 49 | " return self.__call__(x) * (1 - self.__call__(x))\n", 50 | "\n", 51 | "class LogisticLoss():\n", 52 | " def __init__(self):\n", 53 | " sigmoid = Sigmoid()\n", 54 | " self.log_func = sigmoid\n", 55 | " self.log_grad = sigmoid.gradient\n", 56 | "\n", 57 | " def loss(self, y, y_pred):\n", 58 | " y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)\n", 59 | " p = self.log_func(y_pred)\n", 60 | " return y * np.log(p) + (1 - y) * np.log(1 - p)\n", 61 | "\n", 62 | " # gradient w.r.t y_pred\n", 63 | " def gradient(self, y, y_pred):\n", 64 | " p = self.log_func(y_pred)\n", 65 | " return -(y - p)\n", 66 | "\n", 67 | " # w.r.t y_pred\n", 68 | " def hess(self, y, y_pred):\n", 69 | " p = self.log_func(y_pred)\n", 70 | " return p * (1 - p)\n", 71 | "def to_categorical(x, n_col=None):\n", 72 | " \"\"\" One-hot encoding of nominal values \"\"\"\n", 73 | " if not n_col:\n", 74 | " n_col = np.amax(x) + 1\n", 75 | " one_hot = np.zeros((x.shape[0], n_col))\n", 76 | " one_hot[np.arange(x.shape[0]), x] = 1\n", 77 | " return one_hot" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 16, 83 | "metadata": { 84 | "collapsed": true 85 | }, 86 | "outputs": [], 87 | "source": [ 88 | "def divide_on_feature(X, feature_i, threshold):\n", 89 | " \"\"\" Divide dataset based on if sample value on feature index is larger than\n", 90 | " the given threshold \"\"\"\n", 91 | " split_func = None\n", 92 | " if isinstance(threshold, int) or isinstance(threshold, float):\n", 93 | " split_func = lambda sample: sample[feature_i] >= threshold\n", 94 | " else:\n", 95 | " split_func = lambda sample: sample[feature_i] == threshold\n", 96 | "\n", 97 | " X_1 = np.array([sample for sample in X if split_func(sample)])\n", 98 | " X_2 = np.array([sample for sample in X if not split_func(sample)])\n", 99 | "\n", 100 | " return np.array([X_1, X_2])\n", 101 | "class DecisionNode():\n", 102 | " \"\"\"Class that represents a decision node or leaf in the decision tree\n", 103 | " Parameters:\n", 104 | " -----------\n", 105 | " feature_i: int\n", 106 | " Feature index which we want to use as the threshold measure.\n", 107 | " threshold: float\n", 108 | " The value that we will compare feature values at feature_i against to \n", 109 | " determine the prediction.\n", 110 | " value: float\n", 111 | " The class prediction if classification tree, or float value if regression tree.\n", 112 | " true_branch: DecisionNode\n", 113 | " Next decision node for samples where features value met the threshold.\n", 114 | " false_branch: DecisionNode\n", 115 | " Next decision node for samples where features value did not meet the threshold.\n", 116 | " \"\"\"\n", 117 | " def __init__(self, feature_i=None, threshold=None,\n", 118 | " value=None, true_branch=None, false_branch=None):\n", 119 | " self.feature_i = feature_i # Index for the feature that is tested\n", 120 | " self.threshold = threshold # Threshold value for feature\n", 121 | " self.value = value # Value if the node is a leaf in the tree\n", 122 | " self.true_branch = true_branch # 'Left' subtree\n", 123 | " self.false_branch = false_branch # 'Right' subtree\n", 124 | "\n", 125 | "\n", 126 | "# Super class of RegressionTree and ClassificationTree\n", 127 | "class DecisionTree(object):\n", 128 | " \"\"\"Super class of RegressionTree and ClassificationTree.\n", 129 | " Parameters:\n", 130 | " -----------\n", 131 | " min_samples_split: int\n", 132 | " The minimum number of samples needed to make a split when building a tree.\n", 133 | " min_impurity: float\n", 134 | " The minimum impurity required to split the tree further. \n", 135 | " max_depth: int\n", 136 | " The maximum depth of a tree.\n", 137 | " loss: function\n", 138 | " Loss function that is used for Gradient Boosting models to calculate impurity.\n", 139 | " \"\"\"\n", 140 | " def __init__(self, min_samples_split=2, min_impurity=1e-7,\n", 141 | " max_depth=float(\"inf\"), loss=None):\n", 142 | " self.root = None # Root node in dec. tree\n", 143 | " # Minimum n of samples to justify split\n", 144 | " self.min_samples_split = min_samples_split\n", 145 | " # The minimum impurity to justify split\n", 146 | " self.min_impurity = min_impurity\n", 147 | " # The maximum depth to grow the tree to\n", 148 | " self.max_depth = max_depth\n", 149 | " # Function to calculate impurity (classif.=>info gain, regr=>variance reduct.)\n", 150 | " self._impurity_calculation = None\n", 151 | " # Function to determine prediction of y at leaf\n", 152 | " self._leaf_value_calculation = None\n", 153 | " # If y is one-hot encoded (multi-dim) or not (one-dim)\n", 154 | " self.one_dim = None\n", 155 | " # If Gradient Boost\n", 156 | " self.loss = loss\n", 157 | "\n", 158 | " def fit(self, X, y, loss=None):\n", 159 | " \"\"\" Build decision tree \"\"\"\n", 160 | " self.one_dim = len(np.shape(y)) == 1\n", 161 | " self.root = self._build_tree(X, y)\n", 162 | " self.loss=None\n", 163 | "\n", 164 | " def _build_tree(self, X, y, current_depth=0):\n", 165 | " \"\"\" Recursive method which builds out the decision tree and splits X and respective y\n", 166 | " on the feature of X which (based on impurity) best separates the data\"\"\"\n", 167 | "\n", 168 | " largest_impurity = 0\n", 169 | " best_criteria = None # Feature index and threshold\n", 170 | " best_sets = None # Subsets of the data\n", 171 | "\n", 172 | " # Check if expansion of y is needed\n", 173 | " if len(np.shape(y)) == 1:\n", 174 | " y = np.expand_dims(y, axis=1)\n", 175 | "\n", 176 | " # Add y as last column of X\n", 177 | " Xy = np.concatenate((X, y), axis=1)\n", 178 | "\n", 179 | " n_samples, n_features = np.shape(X)\n", 180 | "\n", 181 | " if n_samples >= self.min_samples_split and current_depth <= self.max_depth:\n", 182 | " # Calculate the impurity for each feature\n", 183 | " for feature_i in range(n_features):\n", 184 | " # All values of feature_i\n", 185 | " feature_values = np.expand_dims(X[:, feature_i], axis=1)\n", 186 | " unique_values = np.unique(feature_values)\n", 187 | "\n", 188 | " # Iterate through all unique values of feature column i and\n", 189 | " # calculate the impurity\n", 190 | " for threshold in unique_values:\n", 191 | " # Divide X and y depending on if the feature value of X at index feature_i\n", 192 | " # meets the threshold\n", 193 | " Xy1, Xy2 = divide_on_feature(Xy, feature_i, threshold)\n", 194 | " \n", 195 | " if len(Xy1) > 0 and len(Xy2) > 0:\n", 196 | " # Select the y-values of the two sets\n", 197 | " y1 = Xy1[:, n_features:]\n", 198 | " y2 = Xy2[:, n_features:]\n", 199 | "\n", 200 | " # Calculate impurity\n", 201 | " impurity = self._impurity_calculation(y, y1, y2)\n", 202 | "\n", 203 | " # If this threshold resulted in a higher information gain than previously\n", 204 | " # recorded save the threshold value and the feature\n", 205 | " # index\n", 206 | " if impurity > largest_impurity:\n", 207 | " largest_impurity = impurity\n", 208 | " best_criteria = {\"feature_i\": feature_i, \"threshold\": threshold}\n", 209 | " best_sets = {\n", 210 | " \"leftX\": Xy1[:, :n_features], # X of left subtree\n", 211 | " \"lefty\": Xy1[:, n_features:], # y of left subtree\n", 212 | " \"rightX\": Xy2[:, :n_features], # X of right subtree\n", 213 | " \"righty\": Xy2[:, n_features:] # y of right subtree\n", 214 | " }\n", 215 | "\n", 216 | " if largest_impurity > self.min_impurity:\n", 217 | " # Build subtrees for the right and left branches\n", 218 | " true_branch = self._build_tree(best_sets[\"leftX\"], best_sets[\"lefty\"], current_depth + 1)\n", 219 | " false_branch = self._build_tree(best_sets[\"rightX\"], best_sets[\"righty\"], current_depth + 1)\n", 220 | " return DecisionNode(feature_i=best_criteria[\"feature_i\"], threshold=best_criteria[\n", 221 | " \"threshold\"], true_branch=true_branch, false_branch=false_branch)\n", 222 | "\n", 223 | " # We're at leaf => determine value\n", 224 | " leaf_value = self._leaf_value_calculation(y)\n", 225 | "\n", 226 | " return DecisionNode(value=leaf_value)\n", 227 | "\n", 228 | "\n", 229 | " def predict_value(self, x, tree=None):\n", 230 | " \"\"\" Do a recursive search down the tree and make a prediction of the data sample by the\n", 231 | " value of the leaf that we end up at \"\"\"\n", 232 | "\n", 233 | " if tree is None:\n", 234 | " tree = self.root\n", 235 | "\n", 236 | " # If we have a value (i.e we're at a leaf) => return value as the prediction\n", 237 | " if tree.value is not None:\n", 238 | " return tree.value\n", 239 | "\n", 240 | " # Choose the feature that we will test\n", 241 | " feature_value = x[tree.feature_i]\n", 242 | "\n", 243 | " # Determine if we will follow left or right branch\n", 244 | " branch = tree.false_branch\n", 245 | " if isinstance(feature_value, int) or isinstance(feature_value, float):\n", 246 | " if feature_value >= tree.threshold:\n", 247 | " branch = tree.true_branch\n", 248 | " elif feature_value == tree.threshold:\n", 249 | " branch = tree.true_branch\n", 250 | "\n", 251 | " # Test subtree\n", 252 | " return self.predict_value(x, branch)\n", 253 | "\n", 254 | " def predict(self, X):\n", 255 | " \"\"\" Classify samples one by one and return the set of labels \"\"\"\n", 256 | " y_pred = []\n", 257 | " for x in X:\n", 258 | " y_pred.append(self.predict_value(x))\n", 259 | " return y_pred\n", 260 | "\n", 261 | " def print_tree(self, tree=None, indent=\" \"):\n", 262 | " \"\"\" Recursively print the decision tree \"\"\"\n", 263 | " if not tree:\n", 264 | " tree = self.root\n", 265 | "\n", 266 | " # If we're at leaf => print the label\n", 267 | " if tree.value is not None:\n", 268 | " print (tree.value)\n", 269 | " # Go deeper down the tree\n", 270 | " else:\n", 271 | " # Print test\n", 272 | " print (\"%s:%s? \" % (tree.feature_i, tree.threshold))\n", 273 | " # Print the true scenario\n", 274 | " print (\"%sT->\" % (indent), end=\"\")\n", 275 | " self.print_tree(tree.true_branch, indent + indent)\n", 276 | " # Print the false scenario\n", 277 | " print (\"%sF->\" % (indent), end=\"\")\n", 278 | " self.print_tree(tree.false_branch, indent + indent)\n", 279 | "\n", 280 | "\n", 281 | "\n", 282 | "class XGBoostRegressionTree(DecisionTree):\n", 283 | " \"\"\"\n", 284 | " Regression tree for XGBoost\n", 285 | " - Reference -\n", 286 | " http://xgboost.readthedocs.io/en/latest/model.html\n", 287 | " \"\"\"\n", 288 | "\n", 289 | " def _split(self, y):\n", 290 | " \"\"\" y contains y_true in left half of the middle column and \n", 291 | " y_pred in the right half. Split and return the two matrices \"\"\"\n", 292 | " col = int(np.shape(y)[1]/2)\n", 293 | " y, y_pred = y[:, :col], y[:, col:]\n", 294 | " return y, y_pred\n", 295 | "\n", 296 | " def _gain(self, y, y_pred):\n", 297 | " nominator = np.power((y * self.loss.gradient(y, y_pred)).sum(), 2)\n", 298 | " denominator = self.loss.hess(y, y_pred).sum()\n", 299 | " return 0.5 * (nominator / denominator)\n", 300 | "\n", 301 | " def _gain_by_taylor(self, y, y1, y2):\n", 302 | " # Split\n", 303 | " y, y_pred = self._split(y)\n", 304 | " y1, y1_pred = self._split(y1)\n", 305 | " y2, y2_pred = self._split(y2)\n", 306 | "\n", 307 | " true_gain = self._gain(y1, y1_pred)\n", 308 | " false_gain = self._gain(y2, y2_pred)\n", 309 | " gain = self._gain(y, y_pred)\n", 310 | " return true_gain + false_gain - gain\n", 311 | "\n", 312 | " def _approximate_update(self, y):\n", 313 | " # y split into y, y_pred\n", 314 | " y, y_pred = self._split(y)\n", 315 | " # Newton's Method\n", 316 | " gradient = np.sum(y * self.loss.gradient(y, y_pred), axis=0)\n", 317 | " hessian = np.sum(self.loss.hess(y, y_pred), axis=0)\n", 318 | " update_approximation = gradient / hessian \n", 319 | "\n", 320 | " return update_approximation\n", 321 | "\n", 322 | " def fit(self, X, y):\n", 323 | " self._impurity_calculation = self._gain_by_taylor\n", 324 | " self._leaf_value_calculation = self._approximate_update\n", 325 | " super(XGBoostRegressionTree, self).fit(X, y)" 326 | ] 327 | }, 328 | { 329 | "cell_type": "code", 330 | "execution_count": 25, 331 | "metadata": { 332 | "collapsed": true 333 | }, 334 | "outputs": [], 335 | "source": [ 336 | "class XGBoost(object):\n", 337 | " \"\"\"The XGBoost classifier.\n", 338 | " Reference: http://xgboost.readthedocs.io/en/latest/model.html\n", 339 | " Parameters:\n", 340 | " -----------\n", 341 | " n_estimators: int\n", 342 | " The number of classification trees that are used.\n", 343 | " learning_rate: float\n", 344 | " The step length that will be taken when following the negative gradient during\n", 345 | " training.\n", 346 | " min_samples_split: int\n", 347 | " The minimum number of samples needed to make a split when building a tree.\n", 348 | " min_impurity: float\n", 349 | " The minimum impurity required to split the tree further. \n", 350 | " max_depth: int\n", 351 | " The maximum depth of a tree.\n", 352 | " \"\"\"\n", 353 | " def __init__(self, n_estimators=200, learning_rate=0.001, min_samples_split=2,\n", 354 | " min_impurity=1e-7, max_depth=2):\n", 355 | " self.n_estimators = n_estimators # Number of trees\n", 356 | " self.learning_rate = learning_rate # Step size for weight update\n", 357 | " self.min_samples_split = min_samples_split # The minimum n of sampels to justify split\n", 358 | " self.min_impurity = min_impurity # Minimum variance reduction to continue\n", 359 | " self.max_depth = max_depth # Maximum depth for tree\n", 360 | "\n", 361 | " \n", 362 | " # Log loss for classification\n", 363 | " self.loss = LogisticLoss()\n", 364 | "\n", 365 | " # Initialize regression trees\n", 366 | " self.trees = []\n", 367 | " for _ in range(n_estimators):\n", 368 | " tree = XGBoostRegressionTree(\n", 369 | " min_samples_split=self.min_samples_split,\n", 370 | " min_impurity=min_impurity,\n", 371 | " max_depth=self.max_depth,\n", 372 | " loss=self.loss)\n", 373 | "\n", 374 | " self.trees.append(tree)\n", 375 | "\n", 376 | " def fit(self, X, y):\n", 377 | " y = to_categorical(y)\n", 378 | "\n", 379 | " y_pred = np.zeros(np.shape(y))\n", 380 | " for i in range(self.n_estimators):\n", 381 | " tree = self.trees[i]\n", 382 | " y_and_pred = np.concatenate((y, y_pred), axis=1)\n", 383 | " tree.fit(X, y_and_pred)\n", 384 | " update_pred = tree.predict(X)\n", 385 | "\n", 386 | " y_pred -= np.multiply(self.learning_rate, update_pred)\n", 387 | "\n", 388 | " def predict(self, X):\n", 389 | " y_pred = None\n", 390 | " # Make predictions\n", 391 | " for tree in self.trees:\n", 392 | " # Estimate gradient and update prediction\n", 393 | " update_pred = tree.predict(X)\n", 394 | " if y_pred is None:\n", 395 | " y_pred = np.zeros_like(update_pred)\n", 396 | " y_pred -= np.multiply(self.learning_rate, update_pred)\n", 397 | "\n", 398 | " # Turn into probability distribution (Softmax)\n", 399 | " y_pred = np.exp(y_pred) / np.sum(np.exp(y_pred), axis=1, keepdims=True)\n", 400 | " # Set label to the value that maximizes probability\n", 401 | " y_pred = np.argmax(y_pred, axis=1)\n", 402 | " return y_pred" 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": 26, 408 | "metadata": { 409 | "collapsed": true 410 | }, 411 | "outputs": [], 412 | "source": [ 413 | "data = datasets.load_iris()\n", 414 | "X = data.data\n", 415 | "y = data.target" 416 | ] 417 | }, 418 | { 419 | "cell_type": "code", 420 | "execution_count": 27, 421 | "metadata": { 422 | "collapsed": true 423 | }, 424 | "outputs": [], 425 | "source": [ 426 | "from sklearn.cross_validation import train_test_split" 427 | ] 428 | }, 429 | { 430 | "cell_type": "code", 431 | "execution_count": 41, 432 | "metadata": {}, 433 | "outputs": [], 434 | "source": [ 435 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5) " 436 | ] 437 | }, 438 | { 439 | "cell_type": "code", 440 | "execution_count": 42, 441 | "metadata": {}, 442 | "outputs": [], 443 | "source": [ 444 | "clf = XGBoost()\n", 445 | "clf.fit(X_train, y_train)\n", 446 | "y_pred = clf.predict(X_test)" 447 | ] 448 | }, 449 | { 450 | "cell_type": "code", 451 | "execution_count": 43, 452 | "metadata": {}, 453 | "outputs": [], 454 | "source": [ 455 | "accuracy = accuracy_score(y_test,y_pred)" 456 | ] 457 | }, 458 | { 459 | "cell_type": "code", 460 | "execution_count": 44, 461 | "metadata": {}, 462 | "outputs": [ 463 | { 464 | "data": { 465 | "text/plain": [ 466 | "0.94666666666666666" 467 | ] 468 | }, 469 | "execution_count": 44, 470 | "metadata": {}, 471 | "output_type": "execute_result" 472 | } 473 | ], 474 | "source": [ 475 | "accuracy" 476 | ] 477 | }, 478 | { 479 | "cell_type": "code", 480 | "execution_count": null, 481 | "metadata": { 482 | "collapsed": true 483 | }, 484 | "outputs": [], 485 | "source": [] 486 | } 487 | ], 488 | "metadata": { 489 | "kernelspec": { 490 | "display_name": "Python 3", 491 | "language": "python", 492 | "name": "python3" 493 | }, 494 | "language_info": { 495 | "codemirror_mode": { 496 | "name": "ipython", 497 | "version": 3 498 | }, 499 | "file_extension": ".py", 500 | "mimetype": "text/x-python", 501 | "name": "python", 502 | "nbconvert_exporter": "python", 503 | "pygments_lexer": "ipython3", 504 | "version": "3.6.1" 505 | } 506 | }, 507 | "nbformat": 4, 508 | "nbformat_minor": 2 509 | } 510 | -------------------------------------------------------------------------------- /utils/.ipynb_checkpoints/Simple RNN-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# RNN" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 17, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np\n", 17 | "from neural_networks import NeuralNetwork\n", 18 | "from layers import RNN,Activation\n", 19 | "from losses import CrossEntropy\n", 20 | "from optimizers import StochasticGradientDescent\n", 21 | "from progress import to_categorical,train_test_split\n", 22 | "from metric import accuracy_score" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 18, 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "optimizer = StochasticGradientDescent()\n", 34 | "\n", 35 | "def gen_mult_ser(nums):\n", 36 | " \"\"\" Method which generates multiplication series \"\"\"\n", 37 | " X = np.zeros([nums, 10, 61], dtype=float)\n", 38 | " y = np.zeros([nums, 10, 61], dtype=float)\n", 39 | " for i in range(nums):\n", 40 | " start = np.random.randint(2, 7)\n", 41 | " mult_ser = np.linspace(start, start*10, num=10, dtype=int)\n", 42 | " X[i] = to_categorical(mult_ser, n_col=61)\n", 43 | " y[i] = np.roll(X[i], -1, axis=0)\n", 44 | " y[:, -1, 1] = 1 # Mark endpoint as 1\n", 45 | " return X, y\n", 46 | "def gen_num_seq(nums):\n", 47 | " \"\"\" Method which generates sequence of numbers \"\"\"\n", 48 | " X = np.zeros([nums, 10, 20], dtype=float)\n", 49 | " y = np.zeros([nums, 10, 20], dtype=float)\n", 50 | " for i in range(nums):\n", 51 | " start = np.random.randint(0, 10)\n", 52 | " num_seq = np.arange(start, start+10)\n", 53 | " X[i] = to_categorical(num_seq, n_col=20)\n", 54 | " y[i] = np.roll(X[i], -1, axis=0)\n", 55 | " y[:, -1, 1] = 1 # Mark endpoint as 1\n", 56 | " return X, y\n", 57 | "X, y = gen_mult_ser(3000)\n", 58 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 19, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "clf = NeuralNetwork(optimizer=optimizer,\n", 68 | " loss=CrossEntropy)\n", 69 | "clf.add(RNN(10, activation=\"tanh\", bptt_trunc=5, input_shape=(10, 61)))\n", 70 | "clf.add(Activation('softmax'))" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 20, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "Number Series Problem:\n", 83 | "X = [3 6 9 12 15 18 21 24 27 30]\n", 84 | "y = [6 9 12 15 18 21 24 27 30 1]\n", 85 | "\n" 86 | ] 87 | } 88 | ], 89 | "source": [ 90 | "tmp_X = np.argmax(X_train[0], axis=1)\n", 91 | "tmp_y = np.argmax(y_train[0], axis=1)\n", 92 | "print (\"Number Series Problem:\")\n", 93 | "print (\"X = [\" + \" \".join(tmp_X.astype(\"str\")) + \"]\")\n", 94 | "print (\"y = [\" + \" \".join(tmp_y.astype(\"str\")) + \"]\")\n", 95 | "print ()" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 21, 101 | "metadata": {}, 102 | "outputs": [ 103 | { 104 | "name": "stderr", 105 | "output_type": "stream", 106 | "text": [ 107 | "Training: 100% [------------------------------------------------] Time: 0:01:21\n" 108 | ] 109 | } 110 | ], 111 | "source": [ 112 | "train_err, _ = clf.fit(X_train, y_train, n_epochs=500, batch_size=512)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 22, 118 | "metadata": {}, 119 | "outputs": [ 120 | { 121 | "name": "stdout", 122 | "output_type": "stream", 123 | "text": [ 124 | "\n", 125 | "Results:\n", 126 | "X = [5 10 15 20 25 30 35 40 45 50]\n", 127 | "y_true = [10 15 20 25 30 35 40 45 50 1]\n", 128 | "y_pred = [10 15 27 25 30 35 40 45 50 1]\n", 129 | "\n", 130 | "X = [3 6 9 12 15 18 21 24 27 30]\n", 131 | "y_true = [6 9 12 15 18 21 24 27 30 1]\n", 132 | "y_pred = [6 9 12 15 18 21 24 27 32 3]\n", 133 | "\n", 134 | "X = [5 10 15 20 25 30 35 40 45 50]\n", 135 | "y_true = [10 15 20 25 30 35 40 45 50 1]\n", 136 | "y_pred = [10 15 27 25 30 35 40 45 50 1]\n", 137 | "\n", 138 | "X = [3 6 9 12 15 18 21 24 27 30]\n", 139 | "y_true = [6 9 12 15 18 21 24 27 30 1]\n", 140 | "y_pred = [6 9 12 15 18 21 24 27 32 3]\n", 141 | "\n", 142 | "X = [3 6 9 12 15 18 21 24 27 30]\n", 143 | "y_true = [6 9 12 15 18 21 24 27 30 1]\n", 144 | "y_pred = [6 9 12 15 18 21 24 27 32 3]\n", 145 | "\n" 146 | ] 147 | } 148 | ], 149 | "source": [ 150 | "y_pred = np.argmax(clf.predict(X_test), axis=2)\n", 151 | "y_test = np.argmax(y_test, axis=2)\n", 152 | "\n", 153 | "print ()\n", 154 | "print (\"Results:\")\n", 155 | "for i in range(5):\n", 156 | " # Print a problem instance and the correct solution\n", 157 | " tmp_X = np.argmax(X_test[i], axis=1)\n", 158 | " tmp_y1 = y_test[i]\n", 159 | " tmp_y2 = y_pred[i]\n", 160 | " print (\"X = [\" + \" \".join(tmp_X.astype(\"str\")) + \"]\")\n", 161 | " print (\"y_true = [\" + \" \".join(tmp_y1.astype(\"str\")) + \"]\")\n", 162 | " print (\"y_pred = [\" + \" \".join(tmp_y2.astype(\"str\")) + \"]\")\n", 163 | " print ()" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": 23, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "accuracy = np.mean(accuracy_score(y_test, y_pred))" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 24, 178 | "metadata": {}, 179 | "outputs": [ 180 | { 181 | "data": { 182 | "text/plain": [ 183 | "0.65825" 184 | ] 185 | }, 186 | "execution_count": 24, 187 | "metadata": {}, 188 | "output_type": "execute_result" 189 | } 190 | ], 191 | "source": [ 192 | "accuracy" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "metadata": { 199 | "collapsed": true 200 | }, 201 | "outputs": [], 202 | "source": [] 203 | } 204 | ], 205 | "metadata": { 206 | "kernelspec": { 207 | "display_name": "Python 3", 208 | "language": "python", 209 | "name": "python3" 210 | }, 211 | "language_info": { 212 | "codemirror_mode": { 213 | "name": "ipython", 214 | "version": 3 215 | }, 216 | "file_extension": ".py", 217 | "mimetype": "text/x-python", 218 | "name": "python", 219 | "nbconvert_exporter": "python", 220 | "pygments_lexer": "ipython3", 221 | "version": "3.6.1" 222 | } 223 | }, 224 | "nbformat": 4, 225 | "nbformat_minor": 2 226 | } 227 | -------------------------------------------------------------------------------- /utils/Simple RNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# RNN" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 17, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np\n", 17 | "from neural_networks import NeuralNetwork\n", 18 | "from layers import RNN,Activation\n", 19 | "from losses import CrossEntropy\n", 20 | "from optimizers import StochasticGradientDescent\n", 21 | "from progress import to_categorical,train_test_split\n", 22 | "from metric import accuracy_score" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 18, 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "optimizer = StochasticGradientDescent()\n", 34 | "\n", 35 | "def gen_mult_ser(nums):\n", 36 | " \"\"\" Method which generates multiplication series \"\"\"\n", 37 | " X = np.zeros([nums, 10, 61], dtype=float)\n", 38 | " y = np.zeros([nums, 10, 61], dtype=float)\n", 39 | " for i in range(nums):\n", 40 | " start = np.random.randint(2, 7)\n", 41 | " mult_ser = np.linspace(start, start*10, num=10, dtype=int)\n", 42 | " X[i] = to_categorical(mult_ser, n_col=61)\n", 43 | " y[i] = np.roll(X[i], -1, axis=0)\n", 44 | " y[:, -1, 1] = 1 # Mark endpoint as 1\n", 45 | " return X, y\n", 46 | "def gen_num_seq(nums):\n", 47 | " \"\"\" Method which generates sequence of numbers \"\"\"\n", 48 | " X = np.zeros([nums, 10, 20], dtype=float)\n", 49 | " y = np.zeros([nums, 10, 20], dtype=float)\n", 50 | " for i in range(nums):\n", 51 | " start = np.random.randint(0, 10)\n", 52 | " num_seq = np.arange(start, start+10)\n", 53 | " X[i] = to_categorical(num_seq, n_col=20)\n", 54 | " y[i] = np.roll(X[i], -1, axis=0)\n", 55 | " y[:, -1, 1] = 1 # Mark endpoint as 1\n", 56 | " return X, y\n", 57 | "X, y = gen_mult_ser(3000)\n", 58 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 19, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "clf = NeuralNetwork(optimizer=optimizer,\n", 68 | " loss=CrossEntropy)\n", 69 | "clf.add(RNN(10, activation=\"tanh\", bptt_trunc=5, input_shape=(10, 61)))\n", 70 | "clf.add(Activation('softmax'))" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 20, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "Number Series Problem:\n", 83 | "X = [3 6 9 12 15 18 21 24 27 30]\n", 84 | "y = [6 9 12 15 18 21 24 27 30 1]\n", 85 | "\n" 86 | ] 87 | } 88 | ], 89 | "source": [ 90 | "tmp_X = np.argmax(X_train[0], axis=1)\n", 91 | "tmp_y = np.argmax(y_train[0], axis=1)\n", 92 | "print (\"Number Series Problem:\")\n", 93 | "print (\"X = [\" + \" \".join(tmp_X.astype(\"str\")) + \"]\")\n", 94 | "print (\"y = [\" + \" \".join(tmp_y.astype(\"str\")) + \"]\")\n", 95 | "print ()" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 21, 101 | "metadata": {}, 102 | "outputs": [ 103 | { 104 | "name": "stderr", 105 | "output_type": "stream", 106 | "text": [ 107 | "Training: 100% [------------------------------------------------] Time: 0:01:21\n" 108 | ] 109 | } 110 | ], 111 | "source": [ 112 | "train_err, _ = clf.fit(X_train, y_train, n_epochs=500, batch_size=512)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 22, 118 | "metadata": {}, 119 | "outputs": [ 120 | { 121 | "name": "stdout", 122 | "output_type": "stream", 123 | "text": [ 124 | "\n", 125 | "Results:\n", 126 | "X = [5 10 15 20 25 30 35 40 45 50]\n", 127 | "y_true = [10 15 20 25 30 35 40 45 50 1]\n", 128 | "y_pred = [10 15 27 25 30 35 40 45 50 1]\n", 129 | "\n", 130 | "X = [3 6 9 12 15 18 21 24 27 30]\n", 131 | "y_true = [6 9 12 15 18 21 24 27 30 1]\n", 132 | "y_pred = [6 9 12 15 18 21 24 27 32 3]\n", 133 | "\n", 134 | "X = [5 10 15 20 25 30 35 40 45 50]\n", 135 | "y_true = [10 15 20 25 30 35 40 45 50 1]\n", 136 | "y_pred = [10 15 27 25 30 35 40 45 50 1]\n", 137 | "\n", 138 | "X = [3 6 9 12 15 18 21 24 27 30]\n", 139 | "y_true = [6 9 12 15 18 21 24 27 30 1]\n", 140 | "y_pred = [6 9 12 15 18 21 24 27 32 3]\n", 141 | "\n", 142 | "X = [3 6 9 12 15 18 21 24 27 30]\n", 143 | "y_true = [6 9 12 15 18 21 24 27 30 1]\n", 144 | "y_pred = [6 9 12 15 18 21 24 27 32 3]\n", 145 | "\n" 146 | ] 147 | } 148 | ], 149 | "source": [ 150 | "y_pred = np.argmax(clf.predict(X_test), axis=2)\n", 151 | "y_test = np.argmax(y_test, axis=2)\n", 152 | "\n", 153 | "print ()\n", 154 | "print (\"Results:\")\n", 155 | "for i in range(5):\n", 156 | " # Print a problem instance and the correct solution\n", 157 | " tmp_X = np.argmax(X_test[i], axis=1)\n", 158 | " tmp_y1 = y_test[i]\n", 159 | " tmp_y2 = y_pred[i]\n", 160 | " print (\"X = [\" + \" \".join(tmp_X.astype(\"str\")) + \"]\")\n", 161 | " print (\"y_true = [\" + \" \".join(tmp_y1.astype(\"str\")) + \"]\")\n", 162 | " print (\"y_pred = [\" + \" \".join(tmp_y2.astype(\"str\")) + \"]\")\n", 163 | " print ()" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": 23, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "accuracy = np.mean(accuracy_score(y_test, y_pred))" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 24, 178 | "metadata": {}, 179 | "outputs": [ 180 | { 181 | "data": { 182 | "text/plain": [ 183 | "0.65825" 184 | ] 185 | }, 186 | "execution_count": 24, 187 | "metadata": {}, 188 | "output_type": "execute_result" 189 | } 190 | ], 191 | "source": [ 192 | "accuracy" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "metadata": { 199 | "collapsed": true 200 | }, 201 | "outputs": [], 202 | "source": [] 203 | } 204 | ], 205 | "metadata": { 206 | "kernelspec": { 207 | "display_name": "Python 3", 208 | "language": "python", 209 | "name": "python3" 210 | }, 211 | "language_info": { 212 | "codemirror_mode": { 213 | "name": "ipython", 214 | "version": 3 215 | }, 216 | "file_extension": ".py", 217 | "mimetype": "text/x-python", 218 | "name": "python", 219 | "nbconvert_exporter": "python", 220 | "pygments_lexer": "ipython3", 221 | "version": "3.6.1" 222 | } 223 | }, 224 | "nbformat": 4, 225 | "nbformat_minor": 2 226 | } 227 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HadXu/ML-From-Scratch/5ddb5e9f832cffcdfe99125c1264420c2421c4c1/utils/__init__.py -------------------------------------------------------------------------------- /utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HadXu/ML-From-Scratch/5ddb5e9f832cffcdfe99125c1264420c2421c4c1/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/activations.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HadXu/ML-From-Scratch/5ddb5e9f832cffcdfe99125c1264420c2421c4c1/utils/__pycache__/activations.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/layers.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HadXu/ML-From-Scratch/5ddb5e9f832cffcdfe99125c1264420c2421c4c1/utils/__pycache__/layers.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/losses.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HadXu/ML-From-Scratch/5ddb5e9f832cffcdfe99125c1264420c2421c4c1/utils/__pycache__/losses.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/metric.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HadXu/ML-From-Scratch/5ddb5e9f832cffcdfe99125c1264420c2421c4c1/utils/__pycache__/metric.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/neural_networks.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HadXu/ML-From-Scratch/5ddb5e9f832cffcdfe99125c1264420c2421c4c1/utils/__pycache__/neural_networks.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/optimizers.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HadXu/ML-From-Scratch/5ddb5e9f832cffcdfe99125c1264420c2421c4c1/utils/__pycache__/optimizers.cpython-36.pyc -------------------------------------------------------------------------------- /utils/__pycache__/progress.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HadXu/ML-From-Scratch/5ddb5e9f832cffcdfe99125c1264420c2421c4c1/utils/__pycache__/progress.cpython-36.pyc -------------------------------------------------------------------------------- /utils/activations.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # 各种激活函数 4 | 5 | class Sigmoid(): 6 | def __call__(self, x): 7 | return 1 / (1 + np.exp(-x)) 8 | 9 | def gradient(self, x): 10 | return self.__call__(x) * (1 - self.__call__(x)) 11 | 12 | class Softmax(): 13 | def __call__(self, x): 14 | e_x = np.exp(x - np.max(x, axis=-1, keepdims=True)) 15 | return e_x / np.sum(e_x, axis=-1, keepdims=True) 16 | 17 | def gradient(self, x): 18 | p = self.__call__(x) 19 | return p * (1 - p) 20 | 21 | class TanH(): 22 | def __call__(self, x): 23 | return np.tanh(x) 24 | 25 | def gradient(self, x): 26 | return 1 - np.power(self.__call__(x), 2) 27 | 28 | class ReLU(): 29 | def __call__(self, x): 30 | return np.where(x >= 0, x, 0) 31 | 32 | def gradient(self, x): 33 | return np.where(x >= 0, 1, 0) 34 | 35 | class LeakyReLU(): 36 | def __init__(self, alpha=0.2): 37 | self.alpha = alpha 38 | 39 | def __call__(self, x): 40 | return np.where(x >= 0, x, self.alpha * x) 41 | 42 | def gradient(self, x): 43 | return np.where(x >= 0, 1, self.alpha) 44 | 45 | class ELU(): 46 | def __init__(self, alpha=0.1): 47 | self.alpha = alpha 48 | 49 | def __call__(self, x): 50 | return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1)) 51 | 52 | def gradient(self, x): 53 | return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha) 54 | 55 | class SELU(): 56 | def __init__(self): 57 | self.alpha = 1.6732632423543772848170429916717 58 | self.scale = 1.0507009873554804934193349852946 59 | 60 | def __call__(self, x): 61 | return self.scale * np.where(x >= 0.0, x, self.alpha*(np.exp(x)-1)) 62 | 63 | def gradient(self, x): 64 | return self.scale * np.where(x >= 0.0, 1, self.alpha * np.exp(x)) 65 | 66 | class SoftPlus(): 67 | def __call__(self, x): 68 | return np.log(1 + np.exp(x)) 69 | 70 | def gradient(self, x): 71 | return 1 / (1 + np.exp(-x)) 72 | -------------------------------------------------------------------------------- /utils/layers.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import math 3 | import numpy as np 4 | from activations import * 5 | 6 | 7 | class Layer(object): 8 | def set_input_shape(self, shape): 9 | self.input_shape = shape 10 | 11 | def layer_name(self): 12 | return self.__class__.__name__ 13 | 14 | def parameters(self): 15 | return 0 16 | 17 | def forward_pass(self, X, training): 18 | raise NotImplementedError() 19 | 20 | def backward_pass(self, accum_grad): 21 | raise NotImplementedError() 22 | 23 | def output_shape(self): 24 | raise NotImplementedError() 25 | 26 | 27 | class Dense(Layer): 28 | """ 29 | 全连接网络 30 | """ 31 | 32 | def __init__(self, n_units, input_shape=None): 33 | self.layer_input = None 34 | self.input_shape = input_shape 35 | self.n_units = n_units 36 | self.trainable = True 37 | self.W = None 38 | self.w0 = None 39 | 40 | def initialize(self, optimizer): 41 | limit = 1 / math.sqrt(self.input_shape[0]) 42 | self.W = np.random.uniform(-limit, limit, (self.input_shape[0], self.n_units)) 43 | self.w0 = np.zeros((1, self.n_units)) 44 | self.W_opt = copy.copy(optimizer) 45 | self.w0_opt = copy.copy(optimizer) 46 | 47 | def parameters(self): 48 | return np.prod(self.W.shape) + np.prod(self.w0.shape) 49 | 50 | def forward_pass(self, X, training=True): 51 | self.layer_input = X 52 | return X.dot(self.W) + self.w0 53 | 54 | def backward_pass(self, accum_grad): 55 | W = self.W 56 | 57 | if self.trainable: 58 | grad_w = self.layer_input.T.dot(accum_grad) 59 | grad_w0 = np.sum(accum_grad, axis=0, keepdims=True) 60 | 61 | self.W = self.W_opt.update(self.W, grad_w) 62 | self.w0 = self.w0_opt.update(self.w0, grad_w0) 63 | 64 | accum_grad = accum_grad.dot(W.T) 65 | return accum_grad 66 | 67 | def output_shape(self): 68 | return (self.n_units,) 69 | 70 | 71 | class Dropout(Layer): 72 | def __init__(self, p=0.2): 73 | self.p = p 74 | self._mask = None 75 | self.input_shape = None 76 | self.n_units = None 77 | self.pass_through = True 78 | self.trainable = True 79 | 80 | def forward_pass(self, X, training=True): 81 | c = (1 - self.p) 82 | if training: 83 | self._mask = np.random.uniform(size=X.shape) > self.p 84 | c = self._mask 85 | return X * c 86 | 87 | def backward_pass(self, accum_grad): 88 | return accum_grad * self._mask 89 | 90 | def output_shape(self): 91 | return self.input_shape 92 | 93 | 94 | class BatchNormalization(Layer): 95 | """Batch normalization. 96 | """ 97 | 98 | def __init__(self, momentum=0.99): 99 | self.momentum = momentum 100 | self.trainable = True 101 | self.eps = 0.01 102 | self.running_mean = None 103 | self.running_var = None 104 | 105 | def initialize(self, optimizer): 106 | self.gamma = np.ones(self.input_shape) 107 | self.beta = np.zeros(self.input_shape) 108 | self.gamma_opt = copy.copy(optimizer) 109 | self.beta_opt = copy.copy(optimizer) 110 | 111 | def parameters(self): 112 | return np.prod(self.gamma.shape) + np.prod(self.beta.shape) 113 | 114 | def forward_pass(self, X, training=True): 115 | if self.running_mean is None: 116 | self.running_mean = np.mean(X, axis=0) 117 | self.running_var = np.var(X, axis=0) 118 | 119 | if training: 120 | mean = np.mean(X, axis=0) 121 | var = np.var(X, axis=0) 122 | self.X_centered = X - mean 123 | self.stddev_inv = 1 / np.sqrt(var + self.eps) 124 | self.running_mean = self.momentum * self.running_mean + (1 - self.momentum) * mean 125 | self.running_var = self.momentum * self.running_var + (1 - self.momentum) * var 126 | else: 127 | mean = self.running_mean 128 | var = self.running_var 129 | 130 | X_norm = (X - mean) / np.sqrt(var + self.eps) 131 | output = self.gamma * X_norm + self.beta 132 | 133 | return output 134 | 135 | def backward_pass(self, accum_grad): 136 | 137 | gamma = self.gamma 138 | 139 | if self.trainable: 140 | X_norm = self.X_centered * self.stddev_inv 141 | grad_gamma = np.sum(accum_grad * X_norm, axis=0) 142 | grad_beta = np.sum(accum_grad, axis=0) 143 | 144 | self.gamma = self.gamma_opt.update(self.gamma, grad_gamma) 145 | self.beta = self.beta_opt.update(self.beta, grad_beta) 146 | 147 | batch_size = accum_grad.shape[0] 148 | 149 | accum_grad = (1 / batch_size) * gamma * self.stddev_inv * ( 150 | batch_size * accum_grad 151 | - np.sum(accum_grad, axis=0) 152 | - self.X_centered * self.stddev_inv ** 2 * np.sum(accum_grad * self.X_centered, axis=0) 153 | ) 154 | 155 | return accum_grad 156 | 157 | def output_shape(self): 158 | return self.input_shape 159 | 160 | 161 | class Flatten(Layer): 162 | """ 张开 """ 163 | 164 | def __init__(self, input_shape=None): 165 | self.prev_shape = None 166 | self.trainable = True 167 | self.input_shape = input_shape 168 | 169 | def forward_pass(self, X, training=True): 170 | self.prev_shape = X.shape 171 | return X.reshape((X.shape[0], -1)) 172 | 173 | def backward_pass(self, accum_grad): 174 | return accum_grad.reshape(self.prev_shape) 175 | 176 | def output_shape(self): 177 | return (np.prod(self.input_shape),) 178 | 179 | 180 | activation_functions = { 181 | 'relu': ReLU, 182 | 'sigmoid': Sigmoid, 183 | 'selu': SELU, 184 | 'elu': ELU, 185 | 'softmax': Softmax, 186 | 'leaky_relu': LeakyReLU, 187 | 'tanh': TanH, 188 | 'softplus': SoftPlus 189 | } 190 | 191 | 192 | class Activation(Layer): 193 | def __init__(self, name): 194 | self.activation_name = name 195 | self.activation_func = activation_functions[name]() 196 | self.trainable = True 197 | 198 | def layer_name(self): 199 | return "Activation (%s)" % (self.activation_func.__class__.__name__) 200 | 201 | def forward_pass(self, X, training=True): 202 | self.layer_input = X 203 | return self.activation_func(X) 204 | 205 | def backward_pass(self, accum_grad): 206 | return accum_grad * self.activation_func.gradient(self.layer_input) 207 | 208 | def output_shape(self): 209 | return self.input_shape 210 | 211 | 212 | class RNN(Layer): 213 | """ 214 | 215 | """ 216 | 217 | def __init__(self, n_units, activation='tanh', bptt_trunc=5, input_shape=None): 218 | self.input_shape = input_shape 219 | self.n_units = n_units 220 | self.activation = activation_functions[activation]() 221 | self.trainable = True 222 | self.bptt_trunc = 5 223 | self.W = None # for hidden 224 | self.V = None # for output 225 | self.U = None # for input 226 | 227 | def initialize(self, optimizer): 228 | timesteps, input_dim = self.input_shape 229 | limit = 1 / math.sqrt(input_dim) 230 | self.U = np.random.uniform(-limit, limit, (self.n_units, input_dim)) 231 | limit = 1 / math.sqrt(self.n_units) 232 | self.V = np.random.uniform(-limit, limit, (input_dim, self.n_units)) 233 | self.W = np.random.uniform(-limit, limit, (self.n_units, self.n_units)) 234 | 235 | # Weight optimizers 236 | self.U_opt = copy.copy(optimizer) 237 | self.V_opt = copy.copy(optimizer) 238 | self.W_opt = copy.copy(optimizer) 239 | 240 | 241 | def parameters(self): 242 | return np.prod(self.W.shape) + np.prod(self.U.shape) + np.prod(self.V.shape) 243 | 244 | 245 | def forward_pass(self, X, trainable=True): 246 | self.layer_input = X 247 | batch_size, timesteps, input_dim = X.shape 248 | self.state_input = np.zeros((batch_size, timesteps, self.n_units)) 249 | self.states = np.zeros((batch_size, timesteps + 1, self.n_units)) 250 | self.outputs = np.zeros((batch_size, timesteps, input_dim)) 251 | 252 | self.states[:, -1] = np.zeros((batch_size, self.n_units)) 253 | 254 | for t in range(timesteps): 255 | self.state_input[:, t] = X[:, t].dot(self.U.T) + self.states[:, t - 1].dot(self.W.T) 256 | self.states[:, t] = self.activation(self.state_input[:, t]) 257 | self.outputs[:, t] = self.states[:, t].dot(self.V.T) 258 | return self.outputs 259 | 260 | 261 | def backward_pass(self, accum_grad): 262 | _, timesteps, _ = accum_grad.shape 263 | 264 | grad_U = np.zeros_like(self.U) 265 | grad_V = np.zeros_like(self.V) 266 | grad_W = np.zeros_like(self.W) 267 | 268 | accum_grad_next = np.zeros_like(accum_grad) 269 | 270 | for t in reversed(range(timesteps)): 271 | grad_V += accum_grad[:, t].T.dot(self.states[:, t]) 272 | grad_wrt_state = accum_grad[:, t].dot(self.V) * self.activation.gradient(self.state_input[:, t]) 273 | accum_grad_next[:, t] = grad_wrt_state.dot(self.U) 274 | for t_ in reversed(np.arange(max(0, t - self.bptt_trunc), t + 1)): 275 | grad_U += grad_wrt_state.T.dot(self.layer_input[:, t_]) 276 | grad_W += grad_wrt_state.T.dot(self.states[:, t_ - 1]) 277 | grad_wrt_state = grad_wrt_state.dot(self.W) * self.activation.gradient(self.state_input[:, t_ - 1]) 278 | 279 | self.U = self.U_opt.update(self.U, grad_U) 280 | self.V = self.V_opt.update(self.V, grad_V) 281 | self.W = self.W_opt.update(self.W, grad_W) 282 | 283 | return accum_grad_next 284 | 285 | 286 | def output_shape(self): 287 | return self.input_shape 288 | -------------------------------------------------------------------------------- /utils/losses.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from activations import Sigmoid 3 | from metric import accuracy_score 4 | 5 | class Loss(object): 6 | def loss(self, y_true, y_pred): 7 | return NotImplementedError() 8 | 9 | def gradient(self, y, y_pred): 10 | raise NotImplementedError() 11 | 12 | def acc(self, y, y_pred): 13 | return 0 14 | 15 | class SquareLoss(Loss): 16 | def __init__(self): pass 17 | 18 | def loss(self, y, y_pred): 19 | return 0.5 * np.power((y - y_pred), 2) 20 | 21 | def gradient(self, y, y_pred): 22 | return -(y - y_pred) 23 | 24 | class CrossEntropy(Loss): 25 | def __init__(self): pass 26 | 27 | def loss(self, y, p): 28 | # Avoid division by zero 29 | p = np.clip(p, 1e-15, 1 - 1e-15) 30 | return - y * np.log(p) - (1 - y) * np.log(1 - p) 31 | 32 | def acc(self, y, p): 33 | return accuracy_score(np.argmax(y, axis=1), np.argmax(p, axis=1)) 34 | 35 | def gradient(self, y, p): 36 | # Avoid division by zero 37 | p = np.clip(p, 1e-15, 1 - 1e-15) 38 | return - (y / p) + (1 - y) / (1 - p) -------------------------------------------------------------------------------- /utils/metric.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | 4 | 5 | def calculate_entropy(y): 6 | """ Calculate the entropy of label array y """ 7 | log2 = lambda x: math.log(x) / math.log(2) 8 | unique_labels = np.unique(y) 9 | entropy = 0 10 | for label in unique_labels: 11 | count = len(y[y == label]) 12 | p = count / len(y) 13 | entropy += -p * log2(p) 14 | return entropy 15 | 16 | def mean_squared_error(y_true, y_pred): 17 | """ Returns the mean squared error between y_true and y_pred """ 18 | mse = np.mean(np.power(y_true - y_pred, 2)) 19 | return mse 20 | 21 | def calculate_variance(X): 22 | """ Return the variance of the features in dataset X """ 23 | mean = np.ones(np.shape(X)) * X.mean(0) 24 | n_samples = np.shape(X)[0] 25 | variance = (1 / n_samples) * np.diag((X - mean).T.dot(X - mean)) 26 | 27 | return variance 28 | 29 | def calculate_std_dev(X): 30 | """ Calculate the standard deviations of the features in dataset X """ 31 | std_dev = np.sqrt(calculate_variance(X)) 32 | return std_dev 33 | 34 | def euclidean_distance(x1, x2): 35 | """ Calculates the l2 distance between two vectors """ 36 | distance = 0 37 | # Squared distance between each coordinate 38 | for i in range(len(x1)): 39 | distance += pow((x1[i] - x2[i]), 2) 40 | return math.sqrt(distance) 41 | 42 | def accuracy_score(y_true, y_pred): 43 | """ Compare y_true to y_pred and return the accuracy """ 44 | accuracy = np.sum(y_true == y_pred, axis=0) / len(y_true) 45 | return accuracy 46 | 47 | 48 | def calculate_covariance_matrix(X, Y=None): 49 | """ Calculate the covariance matrix for the dataset X """ 50 | if Y is None: 51 | Y = X 52 | n_samples = np.shape(X)[0] 53 | covariance_matrix = (1 / (n_samples-1)) * (X - X.mean(axis=0)).T.dot(Y - Y.mean(axis=0)) 54 | 55 | return np.array(covariance_matrix, dtype=float) 56 | 57 | 58 | def calculate_correlation_matrix(X, Y=None): 59 | """ Calculate the correlation matrix for the dataset X """ 60 | if Y is None: 61 | Y = X 62 | n_samples = np.shape(X)[0] 63 | covariance = (1 / n_samples) * (X - X.mean(0)).T.dot(Y - Y.mean(0)) 64 | std_dev_X = np.expand_dims(calculate_std_dev(X), 1) 65 | std_dev_y = np.expand_dims(calculate_std_dev(Y), 1) 66 | correlation_matrix = np.divide(covariance, std_dev_X.dot(std_dev_y.T)) 67 | 68 | return np.array(correlation_matrix, dtype=float) 69 | 70 | 71 | def main(): 72 | X = np.array([[0, 0, 0], [1, 0, 1], [1, 0, 0], [1, 1, 0]]) 73 | print(calculate_covariance_matrix(X)) 74 | print(np.cov(X.T)) 75 | 76 | 77 | if __name__ == '__main__': 78 | main() -------------------------------------------------------------------------------- /utils/neural_networks.py: -------------------------------------------------------------------------------- 1 | import progressbar 2 | import numpy as np 3 | from progress import batch_iterator 4 | 5 | 6 | 7 | 8 | bar_widgets = [ 9 | 'Training: ', progressbar.Percentage(), ' ', progressbar.Bar(marker="-", left="[", right="]"), 10 | ' ', progressbar.ETA() 11 | ] 12 | 13 | # 神经网络的结构 14 | # 包括MLP CNN RNN 等。 15 | 16 | class NeuralNetwork(): 17 | def __init__(self, optimizer, loss, validation_data=None): 18 | self.optimizer = optimizer 19 | self.layers = [] 20 | self.errors = {"training": [], "validation": []} 21 | self.loss_function = loss() 22 | self.progressbar = progressbar.ProgressBar(widgets=bar_widgets) 23 | 24 | self.val_set = None 25 | if validation_data: 26 | X, y = validation_data 27 | self.val_set = {"X": X, "y": y} 28 | 29 | def set_trainable(self, trainable): 30 | for layer in self.layers: 31 | layer.trainable = trainable 32 | 33 | def add(self, layer): 34 | """添加层""" 35 | if self.layers: 36 | layer.set_input_shape(shape=self.layers[-1].output_shape()) 37 | 38 | if hasattr(layer, 'initialize'): 39 | layer.initialize(optimizer=self.optimizer) 40 | 41 | self.layers.append(layer) 42 | 43 | def test_on_batch(self, X, y): 44 | y_pred = self._forward_pass(X, training=False) 45 | loss = np.mean(self.loss_function.loss(y, y_pred)) 46 | acc = self.loss_function.acc(y, y_pred) 47 | 48 | return loss, acc 49 | 50 | def train_on_batch(self, X, y): 51 | y_pred = self._forward_pass(X) 52 | loss = np.mean(self.loss_function.loss(y, y_pred)) 53 | acc = self.loss_function.acc(y, y_pred) 54 | loss_grad = self.loss_function.gradient(y, y_pred) 55 | self._backward_pass(loss_grad=loss_grad) 56 | 57 | return loss, acc 58 | 59 | def fit(self, X, y, n_epochs, batch_size): 60 | """ 训练 """ 61 | for _ in self.progressbar(range(n_epochs)): 62 | 63 | batch_error = [] 64 | for X_batch, y_batch in batch_iterator(X, y, batch_size=batch_size): 65 | loss, _ = self.train_on_batch(X_batch, y_batch) 66 | batch_error.append(loss) 67 | 68 | self.errors["training"].append(np.mean(batch_error)) 69 | 70 | if self.val_set is not None: 71 | val_loss, _ = self.test_on_batch(self.val_set["X"], self.val_set["y"]) 72 | self.errors["validation"].append(val_loss) 73 | 74 | return self.errors["training"], self.errors["validation"] 75 | 76 | def _forward_pass(self, X, training=True): 77 | """ 前向传播 """ 78 | layer_output = X 79 | for layer in self.layers: 80 | layer_output = layer.forward_pass(layer_output, training) 81 | 82 | return layer_output 83 | 84 | def _backward_pass(self, loss_grad): 85 | """ 反向传播 """ 86 | for layer in reversed(self.layers): 87 | loss_grad = layer.backward_pass(loss_grad) 88 | 89 | 90 | def predict(self, X): 91 | """ 预测 """ 92 | return self._forward_pass(X, training=False) 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /utils/optimizers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class StochasticGradientDescent(): 4 | def __init__(self, learning_rate=0.01, momentum=0): 5 | self.learning_rate = learning_rate 6 | self.momentum = momentum 7 | self.w_updt = None 8 | 9 | def update(self, w, grad_wrt_w): 10 | if self.w_updt is None: 11 | self.w_updt = np.zeros(np.shape(w)) 12 | self.w_updt = self.momentum * self.w_updt + (1 - self.momentum) * grad_wrt_w 13 | return w - self.learning_rate * self.w_updt 14 | 15 | 16 | -------------------------------------------------------------------------------- /utils/progress.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | 4 | 5 | def shuffle_data(X, y, seed=None): 6 | """ Random shuffle of the samples in X and y """ 7 | if seed: 8 | np.random.seed(seed) 9 | idx = np.arange(X.shape[0]) 10 | np.random.shuffle(idx) 11 | return X[idx], y[idx] 12 | 13 | def batch_iterator(X, y=None, batch_size=64): 14 | """ Simple batch generator """ 15 | n_samples = X.shape[0] 16 | for i in np.arange(0, n_samples, batch_size): 17 | begin, end = i, min(i+batch_size, n_samples) 18 | if y is not None: 19 | yield X[begin:end], y[begin:end] 20 | else: 21 | yield X[begin:end] 22 | 23 | def divide_on_feature(X, feature_i, threshold): 24 | """ Divide dataset based on if sample value on feature index is larger than 25 | the given threshold """ 26 | split_func = None 27 | if isinstance(threshold, int) or isinstance(threshold, float): 28 | split_func = lambda sample: sample[feature_i] >= threshold 29 | else: 30 | split_func = lambda sample: sample[feature_i] == threshold 31 | 32 | X_1 = np.array([sample for sample in X if split_func(sample)]) 33 | X_2 = np.array([sample for sample in X if not split_func(sample)]) 34 | 35 | return np.array([X_1, X_2]) 36 | 37 | def normalize(X, axis=-1, order=2): 38 | """ Normalize the dataset X """ 39 | l2 = np.atleast_1d(np.linalg.norm(X, order, axis)) 40 | l2[l2 == 0] = 1 41 | return X / np.expand_dims(l2, axis) 42 | 43 | def standardize(X): 44 | """ Standardize the dataset X """ 45 | X_std = X 46 | mean = X.mean(axis=0) 47 | std = X.std(axis=0) 48 | for col in range(np.shape(X)[1]): 49 | if std[col]: 50 | X_std[:, col] = (X_std[:, col] - mean[col]) / std[col] 51 | # X_std = (X - X.mean(axis=0)) / X.std(axis=0) 52 | return X_std 53 | 54 | def train_test_split(X, y, test_size=0.5, shuffle=True, seed=None): 55 | """ Split the data into train and test sets """ 56 | if shuffle: 57 | X, y = shuffle_data(X, y, seed) 58 | # Split the training data from test data in the ratio specified in 59 | # test_size 60 | split_i = len(y) - int(len(y) // (1 / test_size)) 61 | X_train, X_test = X[:split_i], X[split_i:] 62 | y_train, y_test = y[:split_i], y[split_i:] 63 | 64 | return X_train, X_test, y_train, y_test 65 | 66 | def to_categorical(x, n_col=None): 67 | """ One-hot encoding of nominal values """ 68 | if not n_col: 69 | n_col = np.amax(x) + 1 70 | one_hot = np.zeros((x.shape[0], n_col)) 71 | one_hot[np.arange(x.shape[0]), x] = 1 72 | return one_hot 73 | 74 | def to_nominal(x): 75 | """ Conversion from one-hot encoding to nominal """ 76 | return np.argmax(x, axis=1) 77 | 78 | 79 | def make_diagonal(x): 80 | """ Converts a vector into an diagonal matrix """ 81 | m = np.zeros((len(x), len(x))) 82 | for i in range(len(m[0])): 83 | m[i, i] = x[i] 84 | return m 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /线性判别分析LDA.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# 线性判别分析" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## 线性判别分析就是将样例投影到一条直线上，使得同类的样例投影点尽可能小，不同类投影点尽可能远离" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": { 21 | "collapsed": true 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "import os \n", 26 | "import sys \n", 27 | "import numpy as np \n", 28 | "import operator \n", 29 | "import matplotlib.pyplot as plt\n", 30 | "\n", 31 | "%matplotlib inline" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 14, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "group1 = np.random.random((8,2))*5+20\n", 41 | "group2 = np.random.random((8,2))*5+2" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 30, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "X1 = group1\n", 51 | "y1 = np.ones((8,1))\n", 52 | "X0 = group2\n", 53 | "y0 = np.zeros((8,1))" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 32, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "text/plain": [ 64 | "" 65 | ] 66 | }, 67 | "execution_count": 32, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | }, 71 | { 72 | "data": { 73 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADipJREFUeJzt3V+InfWdx/HPJyQXG5UQmyEEceYgLHuz7kY4eFMpLmlL\nFbrqjSBTsVAYL4ro1VacC+3FgHRrt70SRpRm4axQ0G5btizYILjC4nImZE00F8KSCQ0xGQ1EJb1o\nzXcvzjPMZDwn5+9znjPf5/2C4Zz5nefM+ebHwye/83ue5/c4IgQA2P32VF0AAGAyCHQASIJAB4Ak\nCHQASIJAB4AkCHQASIJAB4AkCHQASKJvoNu+0/bbtj+0/YHtp4v2F2xfsH2q+Hmw/HIBAL2435Wi\nto9IOhIRJ23fJmlN0sOSHpX0RUT8dNAPO3ToUDQajTHKBYD6WVtb+yQi5vptt7ffBhFxUdLF4vnn\nts9KumOUohqNhtrt9ihvBYDasr0+yHZDzaHbbki6R9J7RdNTtt+3/Zrtg0NVCACYqIED3fatkt6Q\n9ExEfCbpZUl3STqqzgj+pR7vW7Ldtt3e2NiYQMkAgG4GCnTb+9QJ81ZEvClJEXEpIr6MiOuSXpF0\nb7f3RsRqRDQjojk313cKCAAwokHOcrGkVyWdjYifbWs/sm2zRySdmXx5AIBB9T0oKunrkh6XdNr2\nqaLtOUmP2T4qKSSdk/RkKRUCAAbSd4QeEe9GhCPi7yLiaPHz+4h4PCLuLtr/sTgbBgAG12pJjYa0\nZ0/nsdWquqJdbZAROgBMXqslLS1J1651fl9f7/wuSYuL1dW1i3HpP4BqLC9vhfmma9c67RgJgQ6g\nGufPD9eOvgh0ANWYnx+uHX0R6ACqsbIi7d9/Y9v+/Z12jIRAB1CNxUVpdVVaWJDszuPqKgdEx8BZ\nLgCqs7hIgE8QI3QAGMYMnzvPCB0ABjXj584zQgeAQc34ufMEOgAMOo0y4+fOE+gA6m1zGmV9XYrY\nmkbpFuozfu48gQ6g3oaZRhn23PkpH0Al0AHU2zDTKMOcOz/MyH9CHBGl/fGdms1mcJNoADOl0eiE\n7U4LC9K5czPxd22vRUSz33aM0AHUW1lLEFRwAJVAB1BvZS1BUMEBVAIdABYXO9Mg1693HidxkVAF\ni48R6ABQhgoWH+PSfwAoy5QXH2OEDgBJEOgAkASBDgAzvCTuMJhDB1BvM74k7jAYoQOotxlfEncY\nBDqAeut2eb40M0viDoNAB1BfrVbnHPFuZmRJ3GEQ6ADqa3m5sxLiTnapV3SWhUAHUF+9plUidt0B\nUYlAB1BnvaZVFhamW8eEEOgA6quCBbTKRKADqK8KFtAqExcWAai3KS+gVSZG6ACQBIEOAEkQ6ACQ\nBIEOYHea1RUSK6yrb6DbvtP227Y/tP2B7aeL9tttv2X7o+LxYPnlAoC2VkhcX+9cBLS5QmLVoV5x\nXY5ul71u38A+IulIRJy0fZukNUkPS/q+pCsR8aLtZyUdjIgf3exvNZvNaLfbk6kcQH01Gt0X1VpY\n6NzkuSol1WV7LSKa/bbrO0KPiIsRcbJ4/rmks5LukPSQpOPFZsfVCXkAKF+vS/arXiGx4rqGmkO3\n3ZB0j6T3JB2OiIvFSx9LOjzRygCgl16X7Fe9QmLFdQ0c6LZvlfSGpGci4rPtr0Vn3qbr3I3tJdtt\n2+2NjY2xigUASbN7yX7FdQ0U6Lb3qRPmrYh4s2i+VMyvb86zX+723ohYjYhmRDTn5uYmUTOAupvV\nS/YrrmuQg6JWZ478SkQ8s639nyV9uu2g6O0R8U83+1scFAWA4Q16UHSQtVy+LulxSadtnyranpP0\noqRf2f6BpHVJj45aLABgfH0DPSLeldTjHk06NtlyAACj4kpRAEiCQAeAJAh0AEiCQAeAJAh0AEiC\nQAeAJAh0AEiCQAeAJAh0AEiCQAeAJAh0AEiCQAeAJAh0AEiCQAeAJAh0AEiCQAeAJAh0AEiCQAeA\nJAh0AEiCQAeAJAh0AEiCQAeAJAh0AEiCQAeAJAh0AEiCQAeAJAh0AEiCQAeAJAh0AEiCQAeAJAh0\nAEiCQAeAJAh0AEiCQAeAJAh0AEiCQAeAJAh0AEiib6Dbfs32ZdtntrW9YPuC7VPFz4PllgkA6GeQ\nEfovJX2nS/u/RMTR4uf3ky0LADCsvoEeEe9IujKFWgAAYxhnDv0p2+8XUzIHJ1YRAGAkowb6y5Lu\nknRU0kVJL/Xa0PaS7bbt9sbGxogfBwDoZ6RAj4hLEfFlRFyX9Iqke2+y7WpENCOiOTc3N2qdAIA+\nRgp020e2/fqIpDO9tgUATMfefhvYfl3S/ZIO2f6jpOcl3W/7qKSQdE7SkyXWCAAYQN9Aj4jHujS/\nWkItAIAxcKUoACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRB\noANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANA\nEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6\nACRBoANAEn0D3fZrti/bPrOt7Xbbb9n+qHg8WG6ZAIB+Bhmh/1LSd3a0PSvpRET8taQTxe8AgAr1\nDfSIeEfSlR3ND0k6Xjw/LunhCdcFABjSqHPohyPiYvH8Y0mHe21oe8l223Z7Y2NjxI8DAPQz9kHR\niAhJcZPXVyOiGRHNubm5cT8OANDDqIF+yfYRSSoeL0+uJADAKEYN9N9KeqJ4/oSk30ymHADAqAY5\nbfF1Sf8t6W9s/9H2DyS9KOlbtj+S9M3idwBAhfb22yAiHuvx0rEJ1wIAGANXigJAEgQ6ACRBoANA\nEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6\nACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgQ6ACRBoANAEgR6D63T\nLTV+3tCeH+9R4+cNtU63qi4JAG6qloHeL6xbp1ta+t2S1q+uKxRav7qupd8tEeoAZlrtAn2QsF4+\nsaxrf752w/uu/fmalk8sl14b3woAjKp2gT5IWJ+/er7re3u1j6t1uqVDPzmk7735Pb4VABhZ7QJ9\nkLCePzDfdZte7ePY/Mbw6Z8+/cpr0/hWACCP2gX6IGG9cmxF+/ftv+H1/fv2a+XYysTr6faNYbuy\nvhUAyKd2gT5IWC/evajV765q4cCCLGvhwIJWv7uqxbsXJ15Pv8Au41sBgJz2Vl3AtG2G8vKJZZ2/\nel7zB+a1cmzlK2G9ePdiKQG+0/yBea1fXe/6WlnfCgDk5IiY2oc1m81ot9tT+7zdYHMOfee0y9f+\n6mv6xQO/mMp/KgBmm+21iGj22652I/RZM+g3BgDoZ6wRuu1zkj6X9KWkv/T7H4QROgAMb5oj9H+I\niE8m8HcAAGOo3VkuAJDVuIEekv5ge832UrcNbC/Zbttub2xsjPlxAIBexg30+yLiqKQHJP3Q9jd2\nbhARqxHRjIjm3NzcmB8HAOhlrECPiAvF42VJv5Z07ySKAgAMb+RAt32L7ds2n0v6tqQzkyoMADCc\ncUbohyW9a/t/Jf2PpP+IiP+cTFk3qnJZWZa0BbBbjHzaYkT8n6S/n2AtXe28knJzWVlJpV98U+Vn\nA8CwZv60xapuNlH1ZwPAsGY+0Kd9s4lZ+WwAGNbMB/qkbzYxzJz4NG90AQDjmvlAn+TNJoa9+fM0\nb3QBAOOa+UCf5M0mhp0Tn+aNLgBgXLVaD33Pj/co9NV/r2Vdf/56BRUBQH+DrrY48yP0SWJOHEBm\ntQp05sQBZFarQGdOHEBmtZpDB4DdiDl0AKgZAl0swAUgh0ncU3RXYwEuAFnUfoTOAlwAsqh9oLMA\nF4Asah/oXGwEIIvaBzoXGwHIovaBzsVGALLgwiIAmHFcWAQANUOgA0ASBDoAJEGgA0ASBDoAJEGg\nA0ASUz1t0faGpPWpfWC5Dkn6pOoiZgj9sYW+uBH9sWXUvliIiLl+G0010DOx3R7kvNC6oD+20Bc3\noj+2lN0XTLkAQBIEOgAkQaCPbrXqAmYM/bGFvrgR/bGl1L5gDh0AkmCEDgBJEOgjsH3O9mnbp2zX\nbvlI26/Zvmz7zLa2222/Zfuj4vFglTVOS4++eMH2hWL/OGX7wSprnBbbd9p+2/aHtj+w/XTRXtd9\no1d/lLZ/MOUyAtvnJDUjopbn1tr+hqQvJP1rRPxt0fYTSVci4kXbz0o6GBE/qrLOaejRFy9I+iIi\nflplbdNm+4ikIxFx0vZtktYkPSzp+6rnvtGrPx5VSfsHI3QMLSLekXRlR/NDko4Xz4+rs+Om16Mv\naikiLkbEyeL555LOSrpD9d03evVHaQj00YSkP9hes71UdTEz4nBEXCyefyzpcJXFzICnbL9fTMnU\nYophO9sNSfdIek/sGzv7Qypp/yDQR3NfRByV9ICkHxZfu1GIzjxenefyXpZ0l6Sjki5KeqnacqbL\n9q2S3pD0TER8tv21Ou4bXfqjtP2DQB9BRFwoHi9L+rWke6utaCZcKuYMN+cOL1dcT2Ui4lJEfBkR\n1yW9ohrtH7b3qRNerYh4s2iu7b7RrT/K3D8I9CHZvqU4wCHbt0j6tqQzN39XLfxW0hPF8yck/abC\nWiq1GV6FR1ST/cO2Jb0q6WxE/GzbS7XcN3r1R5n7B2e5DMn2XeqMyiVpr6R/i4iVCkuaOtuvS7pf\nnZXjLkl6XtK/S/qVpHl1VtR8NCLSHyzs0Rf3q/N1OiSdk/TktjnktGzfJ+m/JJ2WdL1ofk6deeM6\n7hu9+uMxlbR/EOgAkARTLgCQBIEOAEkQ6ACQBIEOAEkQ6ACQBIEOAEkQ6ACQBIEOAEn8P4MAUrty\njmIAAAAAAElFTkSuQmCC\n", 74 | "text/plain": [ 75 | "" 76 | ] 77 | }, 78 | "metadata": {}, 79 | "output_type": "display_data" 80 | } 81 | ], 82 | "source": [ 83 | "plt.scatter(X1[:,0],X1[:,1],c='r')\n", 84 | "plt.scatter(X0[:,0],X0[:,1],c='g')" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 34, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "mean1=np.array([np.mean(X1[:,0]),np.mean(X1[:,1])])\n", 94 | "mean0=np.array([np.mean(X0[:,0]),np.mean(X0[:,1])])" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": {}, 100 | "source": [ 101 | "### 根据周志华西瓜书的61页进行求解" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 91, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "from numpy import mat\n", 111 | "m1=np.shape(X1)[0]\n", 112 | "sw=np.zeros(shape=(2,2))\n", 113 | "for i in range(m1): \n", 114 | " xsmean=mat(X1[i,:]-mean1) \n", 115 | " sw+=xsmean.transpose()*xsmean\n", 116 | "m0=np.shape(X0)[0] \n", 117 | "for i in range(m0): \n", 118 | " xsmean=mat(X0[i,:]-mean0) \n", 119 | " sw+=xsmean.transpose()*xsmean\n", 120 | "w=(mean0-mean1)*(mat(sw).I)" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 92, 126 | "metadata": {}, 127 | "outputs": [ 128 | { 129 | "data": { 130 | "text/plain": [ 131 | "matrix([[-0.5934389 , -0.57983378]])" 132 | ] 133 | }, 134 | "execution_count": 92, 135 | "metadata": {}, 136 | "output_type": "execute_result" 137 | } 138 | ], 139 | "source": [ 140 | "w" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": 94, 146 | "metadata": {}, 147 | "outputs": [ 148 | { 149 | "data": { 150 | "text/plain": [ 151 | "[]" 152 | ] 153 | }, 154 | "execution_count": 94, 155 | "metadata": {}, 156 | "output_type": "execute_result" 157 | }, 158 | { 159 | "data": { 160 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFEhJREFUeJzt3X2MHHd9x/HP9x7s5ELq2vjpHHJ3kDh+SM4C1bgQAthx\nHowdZKhaROpWoSAdSLQQqZVIcFXSShb0CUUqpdQtUVN0JEWF1oFzcGIXSghJyJmmPtsRiZP4HMfn\npziYBDexb/fbP3bWt3fevd27nd3Znd/7JaHbnV3vzLDKe/d+85s5c3cBANKvJekNAADUB8EHgEAQ\nfAAIBMEHgEAQfAAIBMEHgEAQfAAIBMEHgEAQfAAIRFvSG1Bo7ty53tPTk/RmAEBT2b1790l3n1fu\neQ0V/J6eHg0ODia9GQDQVMxsuJLnMaQDAIEg+AAQCIIPAIEg+AAQCIIPAIEg+ACaS3+/1NMjtbTk\nfvb3J71FTaOhpmUCwKT6+6W+PunMmdz94eHcfUnatCm57WoSfMMH0Dw2bx6Lfd6ZM7nlKIvgA2ge\nhw5NbTnGIfgAmkdX19SWJ6VBjzMQfADNY8sWqaNj/LKOjtzyeqgk5PnjDMPDkvvYcYYGiD7BB9A8\nNm2Stm6Vursls9zPrVvrc8C20pBP5ThDnX8TMHev6QqmYuXKlc7F0wA0pJ6eXOQn6u6WDh4cu9/S\nkvtAmMhMymbH7k+ccSTlfluZxgeYme1295Xlnsc3fACoRKUHjCs9zpDAjCOCDwCVqDTklR5nSGDG\nEcEHgEps2SK1t49f1t5+YcgrPc6QwIwjgg8AlXj0UencufHLzIo/d9Om3Lh+Npv7WWxMPoEZRwQf\nAMrp75e+9rULl589O/0x9wRmHDFLBwDKKTVDR7pw9k0CmKUDAHGZ7EBqo53lOwmCDwDllIq6Wf3O\n8o1B1cE3s8vN7Admtt/M9pnZZ6Plc8zsYTN7Nvo5u/rNBYBpquas1mIHWM2kT31q6mPuCV5nJ45v\n+KOS/tjdl0t6l6RPm9lySXdI2uXuiyXtiu4DQP1Ve32bYgdYv/EN6atfre92VCn2g7Zmtk3SV6L/\nrXb3ETPrlPRDd18y2b/loC2Amqj0sghNuh2JHLQ1sx5J75D0hKQF7j4SPXRU0oI41wUAFWuU6+gn\nvB2xBd/M3iTp25Jud/dfFj7muV8jiv4qYWZ9ZjZoZoMnTpyIa3MAYEyjXEc/4e2IJfhm1q5c7Pvd\n/TvR4mPRUI6in8eL/Vt33+ruK9195bx58+LYHAAYL+nr6DfIdsQxS8ckfV3S0+7+5YKHHpB0W3T7\nNknbql0XAExLktfRb6DtqPqgrZldJ+kRSUOS8qebfV65cfxvSeqSNCzpI+5+arLX4qAtAExdpQdt\n26pdkbv/WFKJKwhpbbWvDwCIB2faAkAgCD4ABILgA0AgCD4ABILgA0AgCD4ABILgA0AgCD4ABILg\nA0AgCD4ABILgA0AgCD4ABILgA0AgCD4ABILgA0AgCD4ABILgA0AgCD4ABILgA0AgCD4ABILgA0Ag\nCD4ABILgA0AgCD4ABILgA0AgCD4ABILgA0AgCD4ABILgA0AgCD4ABILgA0AgYgm+md1jZsfNbG/B\nsjlm9rCZPRv9nB3HugAA0xPXN/x/kbRuwrI7JO1y98WSdkX3AQAJiSX47v4jSacmLN4o6d7o9r2S\nPhTHugAA01PLMfwF7j4S3T4qaUEN1wUAKKMuB23d3SV5scfMrM/MBs1s8MSJE/XYHAAIUi2Df8zM\nOiUp+nm82JPcfau7r3T3lfPmzavh5gBA2GoZ/Ack3Rbdvk3SthquCwBQRlzTMu+T9JikJWZ22Mw+\nIelLkm40s2cl3RDdBwAkpC2OF3H3W0s8tDaO1wcAVI8zbQEgEAQfAAJB8AEgEAQfAAJB8AEgEAQf\nAAJB8AEgEAQ/Yf1D/eq5u0ctf96inrt71D/UP6XHAaBSsZx4henpH+pX33f7dObcGUnS8Olh9X23\nT5K0qXdT2cfj2obNuzZr+PSwWq1VGc+oe1a3tqzdEts6ADQGy13IsjGsXLnSBwcHk96Muum5u0fD\np4cvWN49q1sHbz9Y9vFqTfxAKdTR3qGtH9xK9IEmYGa73X1luecxpJOgQ6cPTbq83OPV2rxrc9HY\nS9KZc2e0edfmWNYDoDEQ/AR1zeqadHm5x6tV7oMjrg8WAI2B4Cdoy9ot6mjvGLeso71DW9Zuqejx\napX74IjrgwVAYyD4CdrUu0lbP7hV3bO6ZTJ1z+oeN25e7vFqFftAyYvzgwVAY+CgbZPIz6Y5dPqQ\numZ1xTaLhlk6QPOr9KAtwZ+GWsV3svVNnE3DLBoAeczSqZF8fIdPD8vl5+fGT+eEqEpPqio2m4ZZ\nNACmiuBPUVzxncoHR62nZwIIA8GforjiO5UPjlpPzwQQBoI/RXHFdyofHLWengkgDAR/iuKKb6kP\niBZruWBMv9bTMwGEgVk60xDHLJ3JrmOTx0wcAJVgWmYTKPzgaLEWZTxzwXPiulAagPRiWmYT2NS7\nSQdvP6jsF7LKerboc5iJAyAuBL9BMBMHQK0R/AbBTBwAtUbwGwQzcQDUGgdtAaDJcdAWADAOwQeA\nQBB8AAhEzYNvZuvM7OdmdsDM7qj1+gAAxdU0+GbWKunvJX1A0nJJt5rZ8lquEwBQXK2/4a+SdMDd\nn3f3s5Lul7Qx7pVks65HD5zUaKb42aoAAKmtxq9/maQXC+4flvSbca9kcPgVbfrnJzS7o13rrlmo\n9b2devfb3qy2Vg5RAEBerYNflpn1SeqTpK6u6V1GYMVbZulrv/cbGhga0banjui+n76o2R3tuvnq\nhdqwgvgDgFTjE6/M7N2S7nL3m6P7d0qSu3+x2PPjOPHq9XMZ/fDnJ7R9aEQ7nz6mM2czxB9AqjXE\n5ZHNrE3SM5LWSnpJ0pOSftfd9xV7ftxn2k4W//W9nXr3FW9WO/EH0OQqDX5Nh3TcfdTM/lDSDkmt\nku4pFftauKi9VeuuWah11ywcF//v/u8R3f/ki8QfQFCCvJbO6+cy+u9nTmhgz4h2PX1Mvzqb0a93\ntGsd8QfQhBpiSGeqkrh4Wqn437w8GvMn/gAaHMGfhnz8tw+NaOd+4g+gORD8Kk0W//UrOnUt8QfQ\nIAh+jIg/gEZG8Gvk9XMZ/eiZExog/gAaBMGvg1Lxv2n5Am1YsYj4A6gLgl9n+fjnTvI6rtfeGCX+\nAOqC4Cdosviv7+3Ue66cS/wBxIbgNwjiD6DWCH4Dev1cRo88e1IDe46cj/+si9t189XEH8D0EfwG\nVyr+uTF/4g+gcgS/ieTjv31oRA/vP0b8AUwJwW9ShfHfuf+YXi2I//oVnXrPFXM1o434AxhD8FOA\n+AOoBMFPmTdGM3rkmZPnT/Ii/gDyCH6KFYv/r13UlvtjLsQfCA7BD0Q+/vkDvvn43xT9DV/iD6Rf\nQ/yJQ9TezLZW3bB8gW5YvmBc/HfsPap/3314LP7RPH/iD4SL4KcI8QcwGYKfUhPj/+NnT2pgD/EH\nQkbwAzCzrVVrly3Q2mUF8eebPxAcgh+YSuJ/4/KFumUF8QfShuAHrFT8H9p/VN/+2Vj8N6xYqOuu\nnEf8gSZH8CGJ+AMhIPi4wMT4P3rgpL63Zyz+l17UppuIP9B0CD4mNbOtVdcvXaDrl47Ff2DPUeIP\nNCHOtMW0TIz/q6+P6tKL2nTj8gW6ZUUn8QfqiEsroG6IP5Asgo9EnB3NjhvzL4z/ht5OXbd4rma2\ntSa9mUCqEHwkjvgD9UHw0VDy8R8YGtFD+47ql8QfiE1dgm9mvyPpLknLJK1y98GCx+6U9AlJGUmf\ncfcd5V6P4IehZPyX5f6GL/EHpqZel0feK+m3JP3jhJUvl/RRSVdLWiRpp5ld5e6ZKteHFJjR1qI1\nS+drzdL5Ovvh3nHx/87/vKRLZ0bf/Ik/EKuqgu/uT0uSmU18aKOk+939DUkvmNkBSaskPVbN+pA+\nF8T/udxVPSfGf31vp957FfEHqlGrE68uk/R4wf3D0bILmFmfpD5J6urqqtHmoBnMaGvRmiXztWYJ\n8QdqoWzwzWynpIVFHtrs7tuq3QB33yppq5Qbw6/29ZAOxeK/fc+IdhB/YNrKBt/db5jG674k6fKC\n+2+JlgFTVhj/LSXif0M024f4A6XFMi3TzH4o6U/ys3TM7GpJ31Ru3H6RpF2SFpc7aMssHUzF2dHs\n+fg/tP+YTv/fOeKPINVrWuaHJf2dpHmSfiHpKXe/OXpss6SPSxqVdLu7P1ju9Qg+puvsaFY/yY/5\nT4j/+t5OvXfxXF3UTvyRTpx4hWARf4SG4AMai//2oRHt2JeL/5sKD/gSf6QAwQcmOJfJneE7Mf43\nLJuvDSsWEX80LYIPTIL4I00IPlChc5msfvLcyxrYc+SC+K/v7dT7rppH/NHQCD4wDcQfzYjgA1XK\nx3/7nhHt2H9UvzhD/NGYCD4QI+KPRkbwgRopFf+1y+ZrA/FHAgg+UAfnMlk99tzLGigS//W9nXo/\n8UcdEHygzvLx3z40ou/vI/6oH4IPJKhY/C+Z0Xr+8g7EH3Ei+ECDKIz/jn1H9QrxR8wIPtCASsV/\nbfQH3Ik/poPgAw3uXCarx5+PDvgSf1SB4ANNJB//7UMj+v7e8fFf39up1UuIP0oj+ECTIv6YKoIP\npADxRyUIPpAyo5msHisS/+uX5f6GL/EPF8EHUmw0k9Xjz5/SwNAR4g+CD4RiLP652T6nfnW2IP4L\ntXrJfOKfcgQfCFCx+Hfkp3oS/9Qi+EDgiH84CD6A80YzWT3xwil9b8/4+F+/dL5uWdFJ/JscwQdQ\nFPFPH4IPoKx8/AeiqZ6F8c/N9pmvi2cQ/0ZH8AFMCfFvXgQfwLQVxn/H3qN6mfg3NIIPIBajmax+\n+sIpfW9C/Ncsna9biH9DIPgAYlcs/he3t+r6ZcQ/SQQfQE3l458f8y+M/4beTq0h/nVTl+Cb2V9L\n+qCks5Kek/QH7v6L6LE7JX1CUkbSZ9x9R7nXI/hAcyqM/459R3XyNeJfT/UK/k2S/svdR83sLyXJ\n3T9nZssl3SdplaRFknZKusrdM5O9HsEHmh/xr79Kg99WzUrc/aGCu49L+u3o9kZJ97v7G5JeMLMD\nysX/sWrWB6DxtbW26Nor5+raK+fqLzZeoydeGPszjgN7RnLxXzpfG1YQ/3qrKvgTfFzSv0W3L1Pu\nAyDvcLQMQEBaW0zXXjFX115RJP5DxL/eygbfzHZKWljkoc3uvi16zmZJo5L6p7oBZtYnqU+Surq6\npvrPATSJYvHP/zGXwviv7+3UmqXz1DEjzu+jkGKYpWNmH5P0SUlr3f1MtOxOSXL3L0b3d0i6y90n\nHdJhDB8ITybr4+J/fsyf+FesXgdt10n6sqT3u/uJguVXS/qmxg7a7pK0mIO2ACZD/KenXsE/IGmm\npJejRY+7+6eixzYrN64/Kul2d3+w3OsRfAB5maxHs32OEP8yOPEKQGqUiv+apfO0oXdR8PEn+ABS\nKR//7UMjenDvUZ187Q1d1N5y/pv/9UvnBxd/gg8g9Yh/DsEHEJSQ40/wAQQrk3U9efCUBvaEEX+C\nDwAai//2oRFtHxqL/5oluTN80xB/gg8AE6Q1/gQfACZRGP8H9x7ViVfH4p8f9rlkZnPEn+ADQIWa\nPf4EHwCmIZN1DR7MXc+/WeJP8AGgSvn4bx8a0faC+K++amzMvxHiT/ABIEbF4j+zbfwB36TiT/AB\noEYK4//g3qM6XhD/9Ss6tbbO8Sf4AFAHjRB/gg8AdZbJunYPv6KBPUfqGn+CDwAJymZdg8OvRCd5\njZyP/+ol87RhxaJY40/wAaBB1Dr+BB8AGlCp+P/+u7r1p7csn9ZrVhr85CeQAkBAWlpMq946R6ve\nOkd/dsty7T70igb2jOiy2RfXfN0EHwAS0tJiemfPHL2zZ0591leXtQAAEkfwASAQBB8AAkHwASAQ\nBB8AAkHwASAQBB8AAkHwASAQDXVpBTM7IWm4ipeYK+lkTJvTLNjnMLDPYZjuPne7+7xyT2qo4FfL\nzAYruZ5EmrDPYWCfw1DrfWZIBwACQfABIBBpC/7WpDcgAexzGNjnMNR0n1M1hg8AKC1t3/ABACWk\nIvhmts7Mfm5mB8zsjqS3px7M7KCZDZnZU2aW2j8TZmb3mNlxM9tbsGyOmT1sZs9GP2cnuY1xK7HP\nd5nZS9H7/ZSZrU9yG+NkZpeb2Q/MbL+Z7TOzz0bL0/4+l9rvmr3XTT+kY2atkp6RdKOkw5KelHSr\nu+9PdMNqzMwOSlrp7qmep2xm75P0mqR/dfdromV/JemUu38p+oCf7e6fS3I741Rin++S9Jq7/02S\n21YLZtYpqdPdf2Zml0raLelDkj6mdL/Ppfb7I6rRe52Gb/irJB1w9+fd/ayk+yVtTHibEBN3/5Gk\nUxMWb5R0b3T7XuX+I0mNEvucWu4+4u4/i26/KulpSZcp/e9zqf2umTQE/zJJLxbcP6wa/5/WIFzS\nTjPbbWZ9SW9MnS1w95Ho9lFJC5LcmDr6IzPbEw35pGp4I8/MeiS9Q9ITCuh9nrDfUo3e6zQEP1TX\nufvbJX1A0qejYYDgeG5MsrnHJSvzD5LeJuntkkYk/W2ymxM/M3uTpG9Lut3df1n4WJrf5yL7XbP3\nOg3Bf0nS5QX33xItSzV3fyn6eVzSfyg3tBWKY9H4Z34c9HjC21Nz7n7M3TPunpX0T0rZ+21m7cpF\nr9/dvxMtTv37XGy/a/lepyH4T0pabGZvNbMZkj4q6YGEt6mmzOyS6CCPzOwSSTdJ2jv5v0qVByTd\nFt2+TdK2BLelLvLhi3xYKXq/zcwkfV3S0+7+5YKHUv0+l9rvWr7XTT9LR5KiaUt3S2qVdI+7b0l4\nk2rKzN6m3Ld6SWqT9M207rOZ3SdptXJXETwm6QuS/lPStyR1KXd11Y+4e2oOcpbY59XK/Yrvkg5K\n+mTB+HZTM7PrJD0iaUhSNlr8eeXGs9P8Ppfa71tVo/c6FcEHAJSXhiEdAEAFCD4ABILgA0AgCD4A\nBILgA0AgCD4ABILgA0AgCD4ABOL/AUbbqB4NN3cqAAAAAElFTkSuQmCC\n", 161 | "text/plain": [ 162 | "" 163 | ] 164 | }, 165 | "metadata": {}, 166 | "output_type": "display_data" 167 | } 168 | ], 169 | "source": [ 170 | "plt.scatter(X1[:,0],X1[:,1],c='r')\n", 171 | "plt.scatter(X0[:,0],X0[:,1],c='g')\n", 172 | "x=np.arange(0,25,0.1)\n", 173 | "y=np.array((-w[0,0]*x)/w[0,1])\n", 174 | "plt.plot(x,y)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "metadata": { 181 | "collapsed": true 182 | }, 183 | "outputs": [], 184 | "source": [] 185 | } 186 | ], 187 | "metadata": { 188 | "kernelspec": { 189 | "display_name": "Python 3", 190 | "language": "python", 191 | "name": "python3" 192 | }, 193 | "language_info": { 194 | "codemirror_mode": { 195 | "name": "ipython", 196 | "version": 3 197 | }, 198 | "file_extension": ".py", 199 | "mimetype": "text/x-python", 200 | "name": "python", 201 | "nbconvert_exporter": "python", 202 | "pygments_lexer": "ipython3", 203 | "version": "3.6.1" 204 | } 205 | }, 206 | "nbformat": 4, 207 | "nbformat_minor": 2 208 | } 209 | --------------------------------------------------------------------------------