├── .DS_Store ├── .ipynb_checkpoints ├── Source_Separation-checkpoint.ipynb └── stft_tests-checkpoint.ipynb ├── Baseline_Using_FastICA.ipynb ├── Final_Report.pdf ├── HW5_Project_Part.ipynb ├── Mini_batch_K_Means.ipynb ├── Online_Algorithm.ipynb ├── Output Files ├── CML_Both_Separation_1 │ ├── sm-1.wav │ ├── sm-10.wav │ ├── sm-2.wav │ ├── sm-3.wav │ ├── sm-4.wav │ ├── sm-5.wav │ ├── sm-6.wav │ ├── sm-7.wav │ ├── sm-8.wav │ └── sm-9.wav ├── Mary_Separation │ ├── is-1.wav │ ├── is-2.wav │ └── is-3.wav └── Mozart_Separation │ ├── sm-1.wav │ ├── sm-10.wav │ ├── sm-2.wav │ ├── sm-3.wav │ ├── sm-4.wav │ ├── sm-5.wav │ ├── sm-6.wav │ ├── sm-7.wav │ ├── sm-8.wav │ └── sm-9.wav ├── README.md ├── Source_Separation.ipynb ├── Source_Separation.py ├── divergence_benchmark.py ├── istft.py ├── mini_batch.py ├── nmf.py ├── objective_function.png ├── presentation.pdf ├── stft.py └── stft_tests.ipynb /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/.DS_Store -------------------------------------------------------------------------------- /Final_Report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Final_Report.pdf -------------------------------------------------------------------------------- /HW5_Project_Part.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 4, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "\n", 13 | "def interval_projection(mu):\n", 14 | " if(mu < 0):\n", 15 | " mu = 0\n", 16 | " elif(mu >= 1):\n", 17 | " mu = 1\n", 18 | " return mu\n", 19 | "\n", 20 | "# projects vector z onto all vectors living in [0,1]^f\n", 21 | "# z_in is the ith column of the matrix C in hottopixx\n", 22 | "def project_column(z_in, i):\n", 23 | " \n", 24 | " f = len(z_in)\n", 25 | " \n", 26 | " z = np.zeros(f)\n", 27 | " \n", 28 | " # have to sort the remaining elements of z for the algorithm\n", 29 | " # and keep track of where each index went\n", 30 | " # the [::-1] makes sure this is the decreasing order\n", 31 | " ind = np.argsort(z_in)[::-1]\n", 32 | " # but regardless of the sorting, the diagonal element has to be first\n", 33 | " ind = ind[ ind != i ]\n", 34 | " ind = np.insert(ind,0,i)\n", 35 | " z = z_in[ind]\n", 36 | " \n", 37 | " # initialize projection vector x\n", 38 | " x = np.zeros(f) \n", 39 | " \n", 40 | " mu = z[0]\n", 41 | " \n", 42 | " kc = 0\n", 43 | " for k in range(1,f):\n", 44 | " if(z[k] <= interval_projection(mu)):\n", 45 | " kc = k - 1\n", 46 | " break\n", 47 | " else:\n", 48 | " mu = (k-1)/k * mu + 1/k * z[k]\n", 49 | "\n", 50 | " x[0] = interval_projection(mu)\n", 51 | " for k in range(0,kc):\n", 52 | " x[k] = interval_projection(mu)\n", 53 | " for k in range(kc+1, f):\n", 54 | " # assign the positive part of z[k]\n", 55 | " x[k] = z[k] if (z[k] > 0) else 0\n", 56 | "\n", 57 | " # reverse the permutations of the original elements\n", 58 | " ind = np.argsort(ind)\n", 59 | " x = x[ind]\n", 60 | "\n", 61 | " return x\n", 62 | " \n", 63 | "# helper function for hottopixx below\n", 64 | "def project_on_phi(C):\n", 65 | " (f,f) = C.shape\n", 66 | " mu = np.zeros(f)\n", 67 | " for i in range(f):\n", 68 | " z = C[:,i]\n", 69 | " C[:,i] = project_column(z, i)\n", 70 | " return C" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 5, 76 | "metadata": { 77 | "collapsed": true 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "# step 11 in hottopixx\n", 82 | "def extract_col(C):\n", 83 | " (f,f) = C.shape\n", 84 | " I = []\n", 85 | " num_ones = 0\n", 86 | " for i in range(f):\n", 87 | " if(C[i,i] == 1):\n", 88 | " I.append(i)\n", 89 | " num_ones += 1\n", 90 | " return I, num_ones" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 6, 96 | "metadata": { 97 | "collapsed": false 98 | }, 99 | "outputs": [], 100 | "source": [ 101 | "import math\n", 102 | "\n", 103 | "# X is the non-negative data matrix, f by n\n", 104 | "# expects X's rows to sum to 1\n", 105 | "# r is the dimension of the NMF, i.e. X = FW where F is f by r, W is r by n\n", 106 | "\n", 107 | "def hottopixx(X,r):\n", 108 | " \n", 109 | " # get dimensions of data matrix (Careful: X is features times samples, different than usual)\n", 110 | " (f,n) = X.shape\n", 111 | " \n", 112 | " # primal and dual stepsizes\n", 113 | " sp = 1e-1\n", 114 | " sd = 1e-2\n", 115 | "\n", 116 | " # initialize cost vector randomly (does this have to be a positive vector?)\n", 117 | " p = np.random.rand(f)\n", 118 | "\n", 119 | " # 'one' vector\n", 120 | " one = np.ones(f)\n", 121 | " \n", 122 | " # mu vector, mu[i] is the number of non-zero entries (in X or C?)\n", 123 | " mu = np.ones(f)\n", 124 | " \n", 125 | " # C is f by f\n", 126 | " C = np.zeros((f,f))\n", 127 | " \n", 128 | " beta = 0\n", 129 | " \n", 130 | " # number of epochs\n", 131 | " N_ep = 1000\n", 132 | " \n", 133 | " for t in range(N_ep):\n", 134 | " \n", 135 | " # do an epoch\n", 136 | " for i in range(n):\n", 137 | " \n", 138 | " # choose random column of data matrix\n", 139 | " k = math.floor(np.random.rand() * n)\n", 140 | " Xk = X[:,k].reshape(f,1)\n", 141 | " # term in the sign function (see paper)\n", 142 | " term1 = np.sign(Xk - np.dot(C,Xk))\n", 143 | " # incremental gradient descent step, (potentially add the mu factor here)\n", 144 | " C = C + sp * np.dot(term1, Xk.T) - sp * np.diag((beta * one - p))\n", 145 | " # project C onto Phi_0\n", 146 | " C = project_on_phi(C)\n", 147 | " # update beta\n", 148 | " beta = beta + sd * (np.trace(C) - r)\n", 149 | " \n", 150 | " # now, we have selected the most salient features of the data matrix.\n", 151 | " # Let's extract and return them:\n", 152 | " I, num_ones = extract_col(C)\n", 153 | " F = C[:,I]\n", 154 | " W = X[I,:]\n", 155 | " #print(np.diag(C))\n", 156 | " print('number of hottopixx found: ' + str(num_ones))\n", 157 | " return F, W" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 7, 163 | "metadata": { 164 | "collapsed": true 165 | }, 166 | "outputs": [], 167 | "source": [ 168 | "# normalizes non-negative X row-wise\n", 169 | "def row_normalize(X):\n", 170 | " for i in range(X.shape[0]):\n", 171 | " row_sum = X[i].sum()\n", 172 | " X[i] = X[i]/row_sum\n", 173 | " return X" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 8, 179 | "metadata": { 180 | "collapsed": true 181 | }, 182 | "outputs": [], 183 | "source": [ 184 | "import wave\n", 185 | "import matplotlib.mlab as mlab\n", 186 | "from sklearn import preprocessing\n", 187 | "\n", 188 | "# computes power and l2 normalized spectrogram from .wav file\n", 189 | "def wav_to_spect(filename):\n", 190 | " # import sample soundfile and get it ready for kmeans\n", 191 | " w = wave.open(filename, \"rb\")\n", 192 | "\n", 193 | " waveParams = w.getparams()\n", 194 | " s = w.readframes(waveParams[3])\n", 195 | " w.close()\n", 196 | " waveArray = np.fromstring(s, np.int16)\n", 197 | " \n", 198 | " spectrum, freq, bins = mlab.specgram(waveArray, NFFT=256, Fs = waveParams[2], sides = 'onesided')\n", 199 | "\n", 200 | " (n,m) = spectrum.shape\n", 201 | " \n", 202 | " # power normalization \n", 203 | " for i in range(n):\n", 204 | " for j in range(m):\n", 205 | " spectrum[i,j] = np.sqrt(spectrum[i,j])\n", 206 | "\n", 207 | " # l2 normalization\n", 208 | " preprocessing.normalize(spectrum, norm = 'l2', axis = 1, copy = False)\n", 209 | "\n", 210 | " return spectrum" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 13, 216 | "metadata": { 217 | "collapsed": false 218 | }, 219 | "outputs": [], 220 | "source": [ 221 | "X = np.random.rand(3,5)\n", 222 | "X = row_normalize(X)" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 14, 228 | "metadata": { 229 | "collapsed": true 230 | }, 231 | "outputs": [], 232 | "source": [ 233 | "np.savetxt(\"X.tsv\", X, delimiter = '\\t', newline = '\\n' )" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": 589, 239 | "metadata": { 240 | "collapsed": false 241 | }, 242 | "outputs": [ 243 | { 244 | "name": "stdout", 245 | "output_type": "stream", 246 | "text": [ 247 | "[ 0.06167466 1. 1. 0.06359649 0.06418499 1.\n", 248 | " 0.07823604 0.03292089 0.07389511 0.07453211 0.06379164 0.08358108\n", 249 | " 1. 0.06355391 1. 0.08277986 1. 0.03974574\n", 250 | " 0.06682618 0.05145228 0.08304687 0.08475537 0.05972625 0.07325193\n", 251 | " 0.06452291 0.05369627 0.05778829 0.04318103 0.06019314 0.04510099\n", 252 | " 0.0733751 0.04514393 0.08536761 0.0661271 1. 0.05689076\n", 253 | " 0.08751247 0.07093722 1. 0.05663606 0.08934751 0.04460133\n", 254 | " 1. 0.09130883 1. 1. 0.06352689 0.04287782\n", 255 | " 0.0489692 1. 1. 0.06530096 0.05465004 0.04663404\n", 256 | " 1. 0.07980866 0.05031084 1. 0.08526564 0.04893781\n", 257 | " 1. 1. 0.07856928 1. 0.108563 0.05148659\n", 258 | " 0.09608442 0.11188698 0.03195004 0.05259868 0.05904399 0.05928603\n", 259 | " 0.0577429 0.05711509 0.04087195 1. 0.08822425 1. 0.0718375\n", 260 | " 1. 0.08690466 0.05761321 0.08229948 0.06753397 0.06743956\n", 261 | " 1. 0.06471683 1. 0.05668456 0.07216813 0.10389689\n", 262 | " 1. 1. 0.06599365 0.06180278 0.06877502 0.05937596\n", 263 | " 0.08023195 0.06639781 0.0658842 ]\n", 264 | "25\n", 265 | "18.73723292350769\n" 266 | ] 267 | } 268 | ], 269 | "source": [ 270 | "import time\n", 271 | "start = time.time()\n", 272 | "F,W = hottopixx(X,30)\n", 273 | "end = time.time()\n", 274 | "print(end-start)" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": 590, 280 | "metadata": { 281 | "collapsed": false 282 | }, 283 | "outputs": [ 284 | { 285 | "data": { 286 | "text/plain": [ 287 | "array([[ 4.12319812e-01, 2.37305169e-01, 2.88670031e-01, ...,\n", 288 | " 1.61894630e-02, 0.00000000e+00, 5.04335185e-02],\n", 289 | " [ 1.00000000e+00, 0.00000000e+00, 0.00000000e+00, ...,\n", 290 | " 3.79001124e-03, 0.00000000e+00, 6.34406126e-03],\n", 291 | " [ 0.00000000e+00, 1.00000000e+00, 0.00000000e+00, ...,\n", 292 | " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00],\n", 293 | " ..., \n", 294 | " [ 8.76829628e-02, 0.00000000e+00, 2.19733878e-01, ...,\n", 295 | " 1.59791647e-01, 1.86722366e-04, 0.00000000e+00],\n", 296 | " [ 1.20495260e-01, 0.00000000e+00, 4.73769258e-02, ...,\n", 297 | " 0.00000000e+00, 7.62885322e-01, 4.46973523e-01],\n", 298 | " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, ...,\n", 299 | " 0.00000000e+00, 0.00000000e+00, 3.42076642e-01]])" 300 | ] 301 | }, 302 | "execution_count": 590, 303 | "metadata": {}, 304 | "output_type": "execute_result" 305 | } 306 | ], 307 | "source": [ 308 | "F" 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": 9, 314 | "metadata": { 315 | "collapsed": false 316 | }, 317 | "outputs": [ 318 | { 319 | "name": "stdout", 320 | "output_type": "stream", 321 | "text": [ 322 | "(129, 41131)\n" 323 | ] 324 | } 325 | ], 326 | "source": [ 327 | "spect = wav_to_spect('train.wav')\n", 328 | "print(spect.shape)" 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "execution_count": 10, 334 | "metadata": { 335 | "collapsed": true 336 | }, 337 | "outputs": [], 338 | "source": [ 339 | "np.savetxt(\"spect.tsv\", spect, delimiter = '\\t', newline = '\\n' )" 340 | ] 341 | }, 342 | { 343 | "cell_type": "code", 344 | "execution_count": 592, 345 | "metadata": { 346 | "collapsed": false 347 | }, 348 | "outputs": [ 349 | { 350 | "name": "stdout", 351 | "output_type": "stream", 352 | "text": [ 353 | "(129, 1029)\n" 354 | ] 355 | } 356 | ], 357 | "source": [ 358 | "# to speed up the computation, lets downsample for now (Cédric Févotte samples at 11kHz, while this recording is at 44.1kHz)\n", 359 | "spect2 = spect[:,0:spect.shape[1]:40]\n", 360 | "print(spect2.shape)\n", 361 | "spect2 = row_normalize(spect2)" 362 | ] 363 | }, 364 | { 365 | "cell_type": "code", 366 | "execution_count": 606, 367 | "metadata": { 368 | "collapsed": false 369 | }, 370 | "outputs": [ 371 | { 372 | "name": "stdout", 373 | "output_type": "stream", 374 | "text": [ 375 | "[ 1. 0.17031393 0.1494879 1. 1. 0.09968434\n", 376 | " 1. 1. 0.11151226 0.1213821 0.10989163 1.\n", 377 | " 0.10466324 1. 1. 0.09844468 1. 0.13635795\n", 378 | " 0.10905873 1. 1. 1. 0.08168864 0.10030063\n", 379 | " 0.09869733 1. 0.05097285 1. 1. 0.07212907\n", 380 | " 0.05288313 0.09532182 0.05921396 1. 1. 0.04236489\n", 381 | " 0.05284964 1. 1. 1. 1. 1. 1.\n", 382 | " 1. 1. 1. 0.05279188 1. 0.06143447\n", 383 | " 0.08760213 0.08204593 0.09656134 0.12138481 1. 0.09901281\n", 384 | " 1. 1. 0.16466243 0.13591287 0.16288352 1.\n", 385 | " 0.15609421 1. 1. 1. 1. 0.14896122\n", 386 | " 0.17628856 0.12628027 0.17100116 1. 0.09777623 1. 1.\n", 387 | " 1. 1. 0.11836214 1. 1. 1. 1.\n", 388 | " 1. 0.06496446 1. 0.06077733 0.03903926 1.\n", 389 | " 0.06253592 0.10359999 0.0769135 0.06206032 0.05961127 1. 1.\n", 390 | " 0.05036091 0.08182608 1. 0.05339818 1. 1.\n", 391 | " 0.06449717 0.05630494 1. 1. 0.08328078 1.\n", 392 | " 0.11101694 0.15423266 1. 1. 0.10119046 0.12545826\n", 393 | " 1. 0.12413739 0.15535741 0.12020291 0.12716419 0.11429355\n", 394 | " 1. 1. 1. 0.10073175 1. 1. 0.086015\n", 395 | " 0.13037756 0.19425896 0.19048184 0.20363396]\n", 396 | "63\n", 397 | "172.8562891483307\n" 398 | ] 399 | } 400 | ], 401 | "source": [ 402 | "# Anything below 70 returns non-sensical values\n", 403 | "import time\n", 404 | "start = time.time()\n", 405 | "F,W = hottopixx(spect2,70)\n", 406 | "end = time.time()\n", 407 | "print(end-start)" 408 | ] 409 | }, 410 | { 411 | "cell_type": "code", 412 | "execution_count": 608, 413 | "metadata": { 414 | "collapsed": false 415 | }, 416 | "outputs": [ 417 | { 418 | "data": { 419 | "text/plain": [ 420 | "(63, 1029)" 421 | ] 422 | }, 423 | "execution_count": 608, 424 | "metadata": {}, 425 | "output_type": "execute_result" 426 | } 427 | ], 428 | "source": [ 429 | "W.shape" 430 | ] 431 | }, 432 | { 433 | "cell_type": "code", 434 | "execution_count": 609, 435 | "metadata": { 436 | "collapsed": false 437 | }, 438 | "outputs": [], 439 | "source": [ 440 | "# save the result to a file\n", 441 | "np.save(\"F_train\", F)\n", 442 | "np.save(\"W_train\", W)" 443 | ] 444 | }, 445 | { 446 | "cell_type": "code", 447 | "execution_count": 611, 448 | "metadata": { 449 | "collapsed": false 450 | }, 451 | "outputs": [ 452 | { 453 | "name": "stdout", 454 | "output_type": "stream", 455 | "text": [ 456 | "Frobenius: 5.14612045001\n", 457 | "INF-1: 21.0726263542\n" 458 | ] 459 | } 460 | ], 461 | "source": [ 462 | "# Frobenius norm of difference between data and factorization\n", 463 | "print('Frobenius: ' + str(np.linalg.norm(spect2-np.dot(F,W))))\n", 464 | "# Norm in paper\n", 465 | "print('INF-1: ' + str(max(np.sum(abs(spect2-np.dot(F,W)), axis = 1))))" 466 | ] 467 | }, 468 | { 469 | "cell_type": "code", 470 | "execution_count": 612, 471 | "metadata": { 472 | "collapsed": false 473 | }, 474 | "outputs": [], 475 | "source": [ 476 | "F_train = np.load(\"F_train.npy\")\n", 477 | "W_train = np.load(\"W_train.npy\")" 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": 613, 483 | "metadata": { 484 | "collapsed": false 485 | }, 486 | "outputs": [ 487 | { 488 | "data": { 489 | "text/plain": [ 490 | "[]" 491 | ] 492 | }, 493 | "execution_count": 613, 494 | "metadata": {}, 495 | "output_type": "execute_result" 496 | }, 497 | { 498 | "data": { 499 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEACAYAAACpoOGTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXu8FVX5/z8PBw6XQMikkEuicTTNvIdUoicvhVSSpaJd\n/EYX7YJZamlaX0G7WWb9iG9E36D4akoXNbEg7OKxLCMxxEuAnBQFLLyCCALn8vz+WHs5a9ZeM7Nm\n79l7z57zvF+v8zp7z6yZWWv2WuuznmfdiJkhCIIgCDb9Gh0BQRAEIZ+IQAiCIAhORCAEQRAEJyIQ\ngiAIghMRCEEQBMGJCIQgCILgJFEgiGgKEa0lovVEdGlEmDml86uJ6Mika4nocCK6h4geIKIlRDQs\nm+QIgiAIWRErEETUAmAugCkADgFwDhEdbIWZCmACM7cBOA/API9rfwTgC8x8GIBbAXw+sxQJgiAI\nmZBkQUwE0MnMG5i5C8BiANOsMKcBWAQAzLwCwAgiGpVwbRsz/7n0+fcA3ld9UgRBEIQsSRKIMQA2\nGt83lY75hBkdc+3DRKTF4kwA41LEWRAEQagDSQLhuw4HpXzuRwB8iohWAhgKYE/K6wVBEIQa0z/h\n/GaEW/fjoCyBuDBjS2EGRF3LzOsAvAMAiOhAAO90PZyIZKEoQRCECmDmtA33MpIsiJUA2ohoPBG1\nApgOYIkVZgmAcwGAiCYB2MrMW+KuJaKRpf/9AHwJpY5tF8xc2L8rr7yy4XGQtEn6JH3F+8uKWAuC\nmbuJaCaA5QBaACxg5jVEdH7p/HxmXkpEU4moE8AOADPiri3d+hwi+nTp883M/JPMUiQIgiBkQpKL\nCcy8DMAy69h86/tM32tLx+cAmJMqpoIgCEJdkZnUDaS9vb3RUagZRU4bIOlrdoqevqygLP1VWUNE\nnOf4CYIg5BEiAtehk1oQBEHoo4hACIIgCE5EIARBEAQnIhCCIAiCExEIIVPuvhuQcQWCUAxEIITM\n2LgRmDwZePzxRsdEEIQsEIEQMmPHjkbHQBCELBGBEDJDu5bExSQIxUAEQhAEQXAiAiEIgiA4EYEQ\nMkNcTIJQLEQgBEEQBCciEEJmiOUgCMVCBELIHBEKQSgGiQJBRFOIaC0RrSeiSyPCzCmdX01ERyZd\nS0QTiejvRLSKiO4lojdlkxxBEAQhK2IFgohaAMwFMAXAIVBbhR5shZkKYAIztwE4D6X9pROu/SaA\nLzPzkQD+u/RdaHLEchCEYpFkQUwE0MnMG5i5C8BiANOsMKcBWAQAzLwCwAgiGpVw7b8BDC99HgFg\nc9UpEXKDCIUgFIOkPanHANhofN8E4FiPMGMAjI659jIAdxPRtVAi9eZ00RbyiAiDIBSLJIHwLfJp\nt7ZbAOAzzHwrEZ0JYCGAU1wBZ82a9fLn9vZ22Uu2CRChEIT60tHRgY6OjszvmyQQmwGMM76Pg7IE\n4sKMLYUZEHPtRGY+ufT5lwB+FBUBUyAEQRCEcuzG8+zZszO5b1IfxEoAbUQ0nohaAUwHsMQKswTA\nuQBARJMAbGXmLQnXdhLRCaXPJwJ4pPqkCI1GLAdBKBaxFgQzdxPRTADLAbQAWMDMa4jo/NL5+cy8\nlIimElEngB0AZsRdW7r1eQD+h4gGAnip9F0oCCIUglAMiHNcmomI8xw/Iczq1cARRwDr1gEHHtjo\n2AhC34WIwMxp+4bLkJnUgiAIghMRCCFzxOgThGIgAiFkhgiDIBQLEQhBEATBiQiEkBmyYZAgFAsR\nCEEQBMGJCISQOWJBCEIxEIEQMkOEQRCKhQiEIAiC4EQEQsgM6aQWhGIhAiEIgiA4EYEQBEEQnIhA\nCJkhLiZBKBYiEIIgFA5mYMuWRsei+RGBEDJDLAeh0QwbBtx8M3DjjcCoUY2OTfMjAiFkjgiF0Che\nfBH429+AZ55pdEyKQaJAENEUIlpLROuJ6NKIMHNK51cT0ZFJ1xLRYiJaVfp7jIhWZZMcQRAEgKre\nKkcAErYcJaIWAHMBnAxgM4B7iWiJsXUoiGgqgAnM3EZExwKYB2BS3LXMfLZx/bUAtmadMKH+SCe1\n0AjOOAPYay9g4cLgWD/xjWRC0mucCKCTmTcwcxeAxQCmWWFOA7AIAJh5BYARRDTK51oiIgBnAbip\n6pQIgtAnuflmYPHi8DERiGxIeo1jAGw0vm8qHfMJM9rj2skAtjDzv3wjLOQXsRyEPMAsApEVsS4m\nAL5FvlKP3zkAbowLMGvWrJc/t7e3o729vcJHCfVChEKoN3ae62sC0dHRgY6OjszvmyQQmwGMM76P\ng7IE4sKMLYUZEHctEfUHcDqAo+IiYAqEIAiCD31NIOzG8+zZszO5b9JrXAmgjYjGE1ErgOkAllhh\nlgA4FwCIaBKArcy8xePakwGsYeYnM0iHkAPEchDyQl8TiFoRa0EwczcRzQSwHEALgAXMvIaIzi+d\nn8/MS4loKhF1AtgBYEbctcbtp0M6pwuJCIXQaGSYazYkuZjAzMsALLOOzbe+z/S91jg3wz+aQjMg\nwiA0ir7eB1Er5DUKgtD02AIhFkQ2iEAImSOWRDG46y7gmmsaHYvKEAsiG+Q1CpkhwlAsrroKuOyy\nRsfCD3Ex1QZ5jYIgFA4RiGyQ1yhkhqzFJJh0dQF//Wt9niUWRG2Q1ygIQk342c+At761Mc+WTups\nEIEQMkcsiOKye7d/2K6u2sXDRiyI2iCvUcgMEYbiM2gQ8NhjjY5FMiIQ2SCvURCEVGxtgt1bRCCy\nQV6jkBnSSS3kBemDyAYRCEEQakI9K2npg6gN8hqFzBDLQcgLIhDZIK9RyBwRCqHeiAVRG+Q1CoLg\npJn9+CIQ2SCvUcgMsRyKRTP9nmZcmZtb3PKECISQOc1UsQjFRAuE5MXqSBQIIppCRGuJaD0RXRoR\nZk7p/GoiOtLnWiK6gIjWENFDRNSkiwoLJlIYhTxAFOTFnp7GxqXZid1RjohaAMyF2j96M4B7iWiJ\nuXUoEU0FMIGZ24joWADzAEyKu5aI3gbgNACHMXMXEY2sSeqEhiBCUQyqddPkwc3T29voGDQ3SRbE\nRACdzLyBmbsALAYwzQpzGoBFAMDMKwCMIKJRCdd+EsDXS8fBzE9nkhpBEDKjmYVex10EojqSBGIM\ngI3G902lYz5hRsdc2wbgeCL6GxF1ENExaSMu5I9mrlCEYiICUR2xLiYAvkU+rTHZH8ArmXkSEb0J\nwM8BHOAKOGvWrJc/t7e3o729PeWjhHojQlEM8uAiimP7dmDoUPe5vmZBdHR0oKOjI/P7JgnEZgDj\njO/joCyBuDBjS2EGxFy7CcAtAMDM9xJRLxG9ipmftSNgCoSQb0QYhHqy117AD34QH6avdFLbjefZ\ns2dnct8kF9NKAG1ENJ6IWgFMB7DECrMEwLkAQESTAGxl5i0J1/4KwImlaw4E0OoSB0EQGke1gl8P\nC2TVqvjnSKOlOmItCGbuJqKZAJYDaAGwoDQK6fzS+fnMvJSIphJRJ4AdAGbEXVu69UIAC4noQQB7\nUBIYoRhIoRTqxTPPuI/LysLZkORiAjMvA7DMOjbf+j7T99rS8S4AH0oVUyH3SGEsFnnvgwCUQJjz\nHoRskZnUgiA0La4+BmaxILJCBELIDCmUgkk9LBCi5rB0mhURCEEQCoc0VrJBBELIHCmUQj0RC6J2\niEAImSHCIOQFsSCyQQRCEISmRayH2iICIWSGtNqEvCF5sTpEIARBaFqiRjGJMGSDCIQgCDUhD8Nc\nRSiqQwRCyAxxMRWLZvbvS17MBhEIQRCcNEvl2sxClndEIITMaJYKRSg+YkFkgwiEkDlSKIV6IUtt\n1BYRCEEQnBSh4pXGSnWIQAiZIWZ9sWiW31GGudaORIEgoilEtJaI1hPRpRFh5pTOryaiI5OuJaJZ\nRLSJiFaV/qZkkxxBEPJCvYa5xiFCUR2xAkFELQDmApgC4BAA5xDRwVaYqQAmMHMbgPMAzPO4lgFc\nx8xHlv5+m2GahAYhhbFYNLOLSfJiNiRZEBMBdDLzhtIucIsBTLPCnAZgEQAw8woAI4holMe1TZz9\nhDiKWDhfeKGY6So68ptVR5JAjAGw0fi+qXTMJ8zohGsvKLmkFhDRiFSxFgSDZ54B7rqrts8YPhxY\nsKC2z8gbzVC5ylIbtSVJIHxfc1prYB6A/QEcAeDfAL6d8nohhzSqUF5+OdDeXvvnPPFE7Z8hZIsI\nRXX0Tzi/GcA44/s4KEsgLszYUpgBUdcy81P6IBH9CMDtURGYNWvWy5/b29vRXo+aQKiKohbKoqYr\nimbpg7Dj2Rf3pO7o6EBHR0fm900SiJUA2ohoPIAnAUwHcI4VZgmAmQAWE9EkAFuZeQsRPRt1LRHt\ny8z/Ll1/OoAHoyJgCoSQb/pKYRSEvGE3nmfPnp3JfWMFgpm7iWgmgOUAWgAsYOY1RHR+6fx8Zl5K\nRFOJqBPADgAz4q4t3foaIjoCyoX1GIDzM0mN0CepV0tXBDAdsppr85NkQYCZlwFYZh2bb32f6Xtt\n6fi56aIpNBP1LpTN4gppdvJa2Uonde2QmdRCZhS9UBY9fb4003toprjmkUQLQhDSUlQLoi9XNq9/\nPTB5cqNjUY4Mc60tIhBCZjSqUIqLqfasW5d+ZFAtf5eVK/3CiVBUh7iYBMGTvl7Z5GXo6MMPA9//\nfnyYRsexKIhACJlTVBeTkA8OPRTYvVt9lsX6aosIhJAZRXcx9fXKJi8WBAD09gafpQ+idkgfhND0\niEDUh0anf+NGYPPmdNc0Os7NjlgQQmbkqYUp1I5GdVK///3Am9+c7jmSF6tDBMKDt72t763k2UyI\nBVEf8pR+HZeo3z5PcW1mRCA86OgAbrml0bHIP0Xvg+grXH89cM890efzVvmKBVE7RCA8kYzmT1Hf\nVbOn6957/dJw7rnABRcE3y+1Nhpu1HtI89xm/63yggiEIHjS7JXOxIlAZ2f66775TfU/T+k3XUxi\nQdQOEQhPJKMlIy6m/NPV5RcubuhoM5SFZohjMyACIWROUSfKFaHSMecPJOHaiKeRmM/3jUuj49zs\niEAImVH0wliE9KURiKj05mEtpriJcuaOckJ1iEB4IhnOn6JaEEUgjUBEkaeyIEtt1JZEgSCiKUS0\nlojWE9GlEWHmlM6vJqIjfa8loouJqJeI9q4uGUJfRlxM/vgKhCuteUq/GRdpINSOWIEgohYAcwFM\nAXAIgHOI6GArzFQAE5i5DcB5AOb5XEtE4wCcAuDxzFJTQ/JUOPKKdFLnn2r6IDR5GObqO1FOym11\nJFkQEwF0MvMGZu4CsBjANCvMaQAWAQAzrwAwgohGeVx7HYAvZJAGIWcUtVDmJV3/+U/l1/b0+Ie1\n05uX9APxw1ylwZAdSQIxBsBG4/um0jGfMKOjriWiaQA2MfMDFcRZyClFtyDyUkHuu6//hjk21fRB\n5KlVnhSHPMW1mUlazdX39XoXUSIaDOByKPdS6usbhWS0/NIXW4zbtlV2XRbDXPPkYgJkolwtSRKI\nzQDGGd/HQVkCcWHGlsIMiLj2dQDGA1hN6pcdC+A+IprIzE/ZEZg1a9bLn9vb29He3p4Q5dogGc2f\nor6rPKWrUkugGgsi7VLbtRRuWawvTEdHBzo6OjK/b5JArATQRkTjATwJYDqAc6wwSwDMBLCYiCYB\n2MrMW4joWde1zLwGwGv0xUT0GICjmfk5VwRMgRDyTdFdTHmi0nedZhRTtfMgzPBZ/0ZiQYSxG8+z\nZ8/O5L6xAsHM3UQ0E8ByAC0AFjDzGiI6v3R+PjMvJaKpRNQJYAeAGXHXuh6TSUpqTF/JaM1IX+uD\nAGovEFmQ9ftKM5M6T79VM5O4oxwzLwOwzDo23/o+0/daR5gDkqMpNAON9lHXmjylqx4upqzcN2JB\nNC8yk9oTyWhCnmikBZF2HaSsyo5LCIiAflYtJmU1O0QghMwp6lIbeap4msnFVIv3lmRBFN2arRci\nEDXij38E2toaHYv6Ip3U9aPWApFHt00lw1yF6kjsgxAUvoVi1y4V9s47K9ucRUiPWBD+VLMWU1pq\n2YrX6YjaMEgsiGwQCyJjTjgBOOaYRseiMRS9UOYpXZW6itIstRFFJZ3UWSMWRH0QgfDEN5Pffz/w\nz3/WNi5CmL5YQTRTJ3VWRLmY7E5q83yeRL0ZEYEQmp6+6GKqtKJvRBpq+UzZk7q2iEDUiL6YMaXV\nVj8qfcf1dDHlYRSTUB0iEJ5IhssvfdGCyMMw14svBp59Nvp81gIhi/XVHxGIjOmL/nBN0Ye55qmy\nqXYm9SmnxO8rEZdWfe6664C7764sHtUindT1QQTCk7Rm9S231C4ueScP4+SLTtq0rl2r/muB+P3v\nK99TwqS1Nfj89NPhyroeLqaoZTzE3ZkNIhA1Yo1rWUKhJugK4ic/qe1z8lTZpLEgnn8eOLi02e+v\nf63m6gCV90fYrh5mtT/FN76RHD4rdNyZ3aOYhGyQiXJCZjSq1aYFYuPG+HBFIs077u4OPt90E3Dy\nyepzFgLxzncCc+eqY9dd5w5Xiz4IM+5iQdQO0V5PfDOa+EMbR0tLbe+fp8omTVzsPKmtjyxGNAHA\nunXZ3CcNOg21WClWCBCByJg8VSL1ptGd1H1JIKoZjaSFwbQs0sAcfn5vr/vd1LIVb7qYxIKoHSIQ\nDUBn6qJm3ka5mPr3IYdpNe84Cwtiz57y+9nU0sWkxU0siNqSKBBENIWI1hLReiK6NCLMnNL51UR0\nZNK1RHR1Kez9RPQHIhrnum+eyNLF1NWl/mdl4guKWgtEngS9mrjofJeU/+I2DNq9O/geZUGY4bPG\nHD0l8yBqR6xAEFELgLkApgA4BMA5RHSwFWYqgAnM3AbgPADzPK79JjMfzsxHAPgVgCuzS1L+0a2v\nSk38vNJoF1NfEohqXEzVWhDMwUiouLjUeomN/faLHsWUp9+qmUmyICYC6GTmDczcBWAxgGlWmNMA\nLAIAZl4BYAQRjYq7lpm3G9cPBfBM1SmpMVlmuKIKhKZRhVNcTG7sFravBRH3DB8Xk899KqW7O3DT\nigVRO5KK1BgA5uDBTQCO9QgzBsDouGuJ6KsAPgRgJ4BJqWLd5GgXk/5fFMSCqB9ZWBDVdFKb1zai\nk7qrK/jd9X/T7aWfuX07hCpIKlK+P23qbiJmvgLAFUR0GYDvAJjhCjdr1qyXP7e3t6O9vT3tozLh\nL38B7rsPOPro6u9VdAui3kgndbqw1fZBAOUCUW+6upRryXQxXXGF+m+m9+ST8yXstaKjowMdHR2Z\n3zepSG0GYHYgj4OyBOLCjC2FGeBxLQDcCGBpVARMgWg0J5+sZqWm5ZJLgI99DHj969V3bTkUVSAa\nVSD70jDXagTCpw8i7v7vfnd4pYB6jWIycbmYnn66/Nl9BbvxPHv27Ezum9QHsRJAGxGNJ6JWANMB\nLLHCLAFwLgAQ0SQAW5l5S9y1RGTu1jwNwKqqU5Jjvv1t4IYbgu9FtSDExVQ/0rTan3oq/D0LC8K8\ntl4uJnuYq+1iqnUDoS8SW6SYuZuIZgJYDqAFwAJmXkNE55fOz2fmpUQ0lYg6AexAyVUUdW3p1l8n\nooMA9AD4F4BP1iJxecLM3EUVCI3Mg6g9S5cC738/sNdeyWEPPTT83bcPIq7CNQWims7uSunqAgYM\nCFsQ5mimPIl5M5NYpJh5GYBl1rH51veZvteWjp+RLpr5IK5F9alPAf/zP9HnzQwrndS1oS+5mG6/\nHfje9wK/OwDMmAFs2ADceWf8tb7DXH0FIsmCqAVmH4RLIIRskFeaAczAvHn+Zn/RLYh6k6eKu9aY\nae3XD9ixQw2gAJRo+PRT+rqY4ipcM+/qFV2jqKWLSQSitsgrTUHczFJAFbikMIB0UmfNwIH1eW4e\nhMgWiOuuA447Lt09dEMmqUFTrYupHp3UQCAM4mLKHhGIFERlurSjQopqQTS6UD7wQHmHbJY0On12\nHIiAnTvT38NnBjVz9S4m815ZIy6m+iCvNAPSCoQWhqL1QTQK/W6vugr40IcaG5daY1sQ5oxmX3xd\noWkEwkXWwpDGxZQHMS8CIhAp8HExRWFm2GpnsuaVRi2x7LLOav2cRmFbEJWk11wq2/dZUfcAki2I\nWuRz10xqsSCyR15pBqS1IPTnogmEUHtsgTCtUF8BMzfb8Qnnwmcmtb7/Sy/5xSsNvb3xLqY8iHkR\nEIFIQZQF4WMR9AULolHUqzLIQ6Vju5gqcVP69EEQxYdL42KqpJ/EBxnFVHvklWaAj4vJFb5ofRA6\nXWed1dh41Iq8CYRtQfjim0/jLIg4F5P9nnbuBF54wT9+Udj3lVFMtUcEIgPSuph8TXzBj7j3uGpV\nsd5zFp3UPn1FzOkEwr72+eeBe+5R33fuBA48MHtLQkYx1R55pSlIcjEtXx7eSMXE1cIqUsUFhGf1\n1pO493jUUcC999b+OfUii05q3/yXpg/Czt+XXRasP7ZzJ7BtW/Z9EUTAn/4UzBwXgcieQr/S3l7g\n8cdr/xxdOP73f5PDmJ8rXSb5N78BvvSlyq6tJbWcg1ANRXLl2RaEmYd8BSwLgUiyIHbsCL7v3KmO\nmWJGBCwrW4QnHbYgmA24PIh5ESi0QPzkJ8D48bV/ju/M1ErD2zz5JLBxY3K4JH70I+CnP63+PkB5\ngax1Ad2yJagQkp5V6ab2N96Yv0rHtiCqvUccvgLx/PPlm/WYYrBzp7qXGQYAHnzQLx5R2Omv9H0I\n0RR6/cutW+vzHJ8KP0sLoqsrm01aPv5xYPBg4AMfqP5ednx6emq7uuqWLcHnWlXc//xn+HseBaKS\nSnH+/OQwgL9ArF6t/jS2QOj8arvDqn2fcQKRh9+qCBTagqgXlU6Ui8vE27cDr3mN+1x3d3a7eGVV\nkOz41HqXsTQVY61b2vXEdjFV02pOSp/vMFfXfc14dXWVi4bP85OQPofaU2gLImuSOqnjKkWXrzgu\n/L//Xe7T37IFWL8+KHBZsGsX8OKLwNCh1d3Hjo/vUMpKSdNazMr1kAfByMKCcN3Ldc63k9p17ZAh\n4bAuF1Pa9xk1zNUnrFAZhdbgevkkfdfX37UL+Pvf0wuK5sILgcmTs7UggPKCWwkuF1MtydqdMGwY\n8Mgj1d+n1mRpQSTlobjzH/1o9Dlm5brU6EECWS+DYqd9wIDg+UI2eAkEEU0horVEtJ6ILo0IM6d0\nfjURHZl0LRF9i4jWlMLfQkTDq09OY/CxCJjVnhHHHusfPupYFn0Q5v69WQhpvV1MWbsXXnyxvNPU\nfi95qHjiOqnTxq8agYiDORwX3QCppYtp5Mjw7np5+K2KQGIxI6IWAHMBTAFwCIBziOhgK8xUABOY\nuQ3AeQDmeVx7B4A3MPPhAB4B8MVMUhSKV9Z3dOPbSa3nSPj0QcQJRHd39QXAFIgsaBYL4v77gTvu\ncJ+rZvG6epHGzZL2XjaVCkRvb/haLRAuF9Mf/gAcdJDffePSPmZM7RslfRGfdthEAJ3MvIGZuwAs\nBjDNCnMagEUAwMwrAIwgolFx1zLz75hZ/6QrAIytOjUNwlcg7PHnaS0ITdYupixolk7qM88E3vEO\nd7g8CEAS9mCHRrmY4rD7L7Tl4HIx/fGPlbv2zLTrdZmEbPHppB4DwBx1vwnAsR5hxgAY7XEtAHwE\nwE0ecWkoSct9JxUoO1ylApHVMFdNFvfKswVhho2a6e5znzxUQPUSiAceqPy+zOHfvx4uJr3shr5v\nHn6rIuAjEL6vuqKsSkRXANjDzDe6zs+aNevlz+3t7Whvb09x70pilJ4sLIhVq4DDDgs2aUmyIKot\nAK5ht9VQb4Ewn5vmXcR1yNtpyOPEq3paEJViWxBxLqa09zWxBUI/M4tBF81GR0cHOnw2JE+Jj0Bs\nBjDO+D4OyhKICzO2FGZA3LVE9GEAUwGcFPVwUyDyiq9A2H0PZoY/6ijg+uuBD36w/Jx5DyAbCyJJ\nIB5/HLj9dmDmTL/71dvF5DtyDCi2BRF1Lu29siSNi6maOJi/q2lB7NqVj9+qntiN59mzZ2dyX58+\niJUA2ohoPBG1ApgOYIkVZgmAcwGAiCYB2MrMW+KuJaIpAD4PYBozxxTbyvFtXd17r1/YaneUS3Ix\nmfMeksag19rF9N3vAhdcUPk9am1BmO/cNUs9irjWZTMKRDNYEFECkeVM6r5uQdSKRAuCmbuJaCaA\n5QBaACxg5jVEdH7p/HxmXkpEU4moE8AOADPiri3d+nsAWgH8jtQvfQ8zfyrj9Hmxdm111/su353U\nSW2umZ9kQdTaxZR2aeZ6C4Qpsmn6IOLG4mc5QqhWNKNAZOVisonqg+iLFkSt8JpJzczLACyzjs23\nvjudEa5rS8fb/KNZGVkU8EcfLb/f7t3AwIHBcZfLyMZVIacVCE3WFoSrMk+7NHNeXEw+v7meUGXz\n978D06ZFzyrPQ6Xj23f0l78ADz8MnHdedJhaCoT5u8S5mNLe1yTKgohzIwrpKPRMal/iCv4f/hAO\nt2cPMGhQOIyPBWG2dKME5TvfARYvTr5XPfogbIFgBu66K/p+eXQxRb3nqEl2c+YAV1+d/MxGYr7n\nOAvi858Hzj/f/15Zss8+4Uq6VqOYpA+i9hROIIiANWuSw/liWgpAUPG5KlhXptRr1phmd1ynthak\nWo9iMvFxMa1dC8QNIGuUi8kWCFcY+3zcuzP3MbDJQ6UT52KKm2WddK+sMffgiHIxRbkH9+xRExqT\niHMxCdlQOIEAgIceUv+zcDGZ1oI5GccsAHEupq99LTqcq2K2O7xd92yEBZF2UlVS+AsuqG42t4+L\nKUqIfRehy0sfxH33Ba3vqN+tkv6TWroBq3Exff/7wJFHJoczBYJIXEy1oJACoVu/vgU8riVlWxA6\nE5oCEWdBbN4cnPOZKLd9uxK4uDBZ9EEkCUTakSBpLYi5c9Us2kox32WWFoRL+JN46CG/zW+uvrqy\nWcPHHKM2d7LjFCcQPmtV1VIgXBZEnIvJ/OxbwY8zBtCbfRAvvZQPa68IFFIgXB2sK1ZUdi9TIMxW\nipnZ4wqaLqiuiXKuTPzznwNvfGOyBVFtAbB92XHn094PqK+LKSlMpQJhE3Xd4YerSY5J/Pd/x29L\nG4euZJnIxp2pAAAgAElEQVSBsWOBz31OWWA6r1cyya+3Vw2MSGMpuba6Pf308mOmJeYzium73w0+\nu+Jjlh/NyJHBZ+mkrg2FFAjbf84MTJoUvYZ9XAFpbQ1/d1kQcRW+vrfpK/eZWOcKo6/PwoKI21PY\nRVIlUu9RTFGd1K44pJlQpvPIM88At9zif10Udp9Gpe/F3F61Xz+138J11wVxrGQ0V28v8Oyz4WNJ\nS2y47vvmN5cfMxtkPi6miy4Ktr91PWPq1PId/oiAS0vrQ9sCIRZENhRSIHSrSmc0Xeg//GF3+LjM\nZGfWtC4mbUE895xfH4Sm0RZEtbNy085wTkutOqn17zpypBomCgB33hl/XdTxTZvKh8zq97JsGXDV\nVdHxiIJZvTfbhWS/h7gRZ/a9TA4/PP4a128WtbXsq16l/kcJxL//HY7zN74R/VxzNKEZFx0f28Uk\nZEMhBcK2IHSh/+lPgSX2HPAEolrGLhfTM8+UX68L8n/+47YgXnrJvfy0nlVdKwuiKC6mtFZYEi4r\n8/rr/a832b49Ok5XXQVceWX6e0YJRCV9J5VMtEsjEL/8pZqPEeViWrhQiYR97+98x/8ZpkDod2Bb\nEK99rftaIZlCCoRdOZmFfpq9UHkCvb3A8OHh74B/gdQF+amn3H0QCxe6l5/WhaRWo5iSXExpLYhq\nXExPPAE89li65/m4mFyuv6QKMa4PIikuNq5n+cyZ2bFD9VdEPcslEGYe9/XBV5KH0ghEv35qUmJU\nJzXgFuT//MfvGeZ76NdP3X/wYHVPM39v3Fh+reBHIQVCFz6dmZMKfdIM6De9Kfgc52LSmJOtdBz2\n7CmvHOJG4JjzJ1zn7rknPMvbZJ99gM5OYNs293k7zkkCceihwN13R9/LdY80LqajjwYmTEgO73pe\nWhdT0uie3bvTu5KicIXX7yWucl69OnrCno8F8eKLfvGrtQXRr586F9cHYQ9VjSJKIEwLYs8eJUjM\nbutNSE8hBUIXPrsPIil8lKtFL8Hd0+M3imny5OCzLgCmGJjPiyoUOs6uOOnK4E9/cl/77LPAt74F\njBjhPm/fN6kl+fDDyUNSt26Nvn8S27dX7tLyGcVk3tt838uXA+vXhyvyJ56IvmcWHZ+9vWpJj3vv\njQ6j+y1cz9MCYS8XYubxuMl+9r00RMCUKcnXuPJr1NIltgXhGjrt64rUZTAqPv36qfvruFxzjd99\nhXgKKRB2wUqyIHQmdWXW3t6gkjd9/3EuJnsJAEANSfzZz8LhfQTCNVZcn4sTvlWr1P/3vhc4/vjy\nkSZmpelKt50mO+4md92lnmFS76U2zFE+GpcFYb7vKVPUchTm+Q0byvOL6araudO/EzTKxRQl7DZm\nK9hMH1F557c5cGH//cvj7uIXvwCefDL4vnx5cpxcaYqqvFtakl1Mvi69NBaEkB2FFYjbbgM++Un1\nPcmCiKtwTYEw3RlxLiaXQADBpDmz4ooSCH3/OAsiLl36WbfeCvz5z8Df/hacu+eecOWTpg/Cddxc\ngkOPTa/XKCZtmblcRzrMn/4UPMs1Ks0Wy7hK66ijysUwbbyT8qN+/nPPlZ/TeWbYsPDxqHtG7b+t\niZuXccQR5cfM96e3aknqg9B5xrcPwoVLhIiUoOtn7dmj4vKpTwGjR4fDpl2dWFAUViB+/OPge7UW\nhDmXQVcmprlsV7Dm3ImoST+u60xcFoTtMourhF2tXD1q5i1vUQvTad76VuBf/3LH0SbJFaRbtmnm\nVkQ9a9Ag98gw8xr9DlwCoONgju23hcTsV9Lf4/LLunXVbcfZ0xP9u912m6rcdEXqqtSiBCIqzknu\nJvs+JoccUp5W8/3qZWjs1QZMzBZ9kgWRtg/CvL9pQfTvXy48lYwYEwosELfdFnx3tVIefDDYByLO\ngjBbp93dQcevubmPXcGNHRt8jhvJUqmLyceCcFXQ5rh7+7we828/y+e+ruvsSvDuu4GlS+Ovtdm9\nOxiBoitGu19BW3XmjPUlS8IVf5RFp+9jpymqsnWNilq3LjkdttsrSiDmzgXmzQue74pHT49qTfta\nEElELW0OqHf1xjeGj5nvsrUVGD9ejRxy0dMTFghXH0S1FoS+P3NYIOx3Zy6lL/hTSIGwM4OroB12\nWLAgWJRA7NgBvOc9YYHQE4m0Cwcor2BGjQJ+/Wv1OcmCiBKIuNEuPn0QSRW5XeCiJpPpdETd174u\nqvP4fe8D3vlOd1x8Jirq+9qd97YFAaihzE8/7RaIJBeT+Qw7fqZA62Gor399dNxdghJnQfz+98H9\nzf+av/9dDQZwCUSlQznjLAh7tVSblhY1PNlXIJJcTJWMYjLdV9rF5LIg6rVHetHwEggimkJEa4lo\nPRFdGhFmTun8aiI6MulaIjqTiB4moh4iOqr6pATY7pKoilSPF49y2ejx2GYfhCZqe9A3vUkVnKjK\n0LyPTx9EnAWRtMVpHL4Cofen0Oi0Llmi3D92HKLcZ67OQyJVqONcOkRqn24tVLZlZVsQmpYWt0ja\n79t2MQF+Haf2MNQ4S9Gec+LbB2FXqMceC3zhC6oCtPckede73L/5n/8c/6w44vp1zPNRAtHbG/zu\nAwf6uZj0ooQ2LoFgDtK8e3e8BVHrpV+KSqJAEFELgLkApgA4BMA5RHSwFWYqgAmlXeLOAzDP49oH\nAZwOwHNMR5i5c9UQRRd2RnQV+Fe8IvisC6zt89WWiKugfPObYfeHxlxhEihvgQOqwujfP96C0Gl4\n5JFy/7peP8dcN2jLlvD1SQJhF7gogbAFQBe0adOAa6+NLoj2dVE+5Msvj4/nEUeoGfALF6rvdue9\n3UltnrfnwwBBOD23xbYg4vogKp08aA8pTmrNuiwInYbnn1fiF+f3NzEXwTPZvVtZx3Fi6Mr3rjkl\nPgIxeLB7Ap8tlh//uPtePgKhh7mKBZEdPhbERACdzLyBmbsALAZgz0c+DcAiAGDmFQBGENGouGuZ\neS0zV7D4sfrxL7ggyPxPPhmuBGxfp6vF5hIIXfGecYa6p+5viKrE9RR+M/PZlYhrGeju7qCFmyQQ\npjVkxhkArrhCFbq3vAU48MDwubQWhE2UJeDrjrH3A4hKp7nUQhz6fi4LAggqK/3b9/QE79DlYlq5\nUv1fsSI8U545/Wx7+xl2nM13Eedi0sT1QXR1qd/OXkQyDczq+paWeGsmSSB0muNcTLpiHzJE/TZr\n1ypXmcZMY5xYufKrKe7agrBdTLpciAVRGT4CMQaA6eHcVDrmE2a0x7Wp6OwMWiW6oD3/fDiMLRC+\nFoQWiJtvVkNB7clfUUQNeT3mGPXftSyCXlwsquLUaTDdYD095T7vZ55RfmC73yWqQOgC7hrN4/oe\nZUHoMFECYYZ73euC4YjmObMFmISOx7e/rVrPpqVy441BPPR76+4GPvAB9TmuD8KFHrygcfUl+JDG\ngjDvHScQunHhEoi08UsjEHGz0m13l8YUiEGDVF4+5RTlKtOYz48rby4LwrQSbReTfZ1YEJURYfiH\n8M12Ndl/68orZ2HdOlUxtre3o7W1/eVzPT0q0/3ud+q7mVlMXL5P3eq5++4g8/zqV8G6SF/5itrX\nF4hvfXR1AYsWBd9dPlq7UtIFJ64PQsdJC8T06apA2621p58OCtkBBwTbrUa1xi6+2H2+EoEAyisY\nl4tp0yZ3mLj3GtX5/fWvh793d4etEFMgtPWVViB845SE61309LgnpJlhKhWItJgL3EWdt0njYho5\nMnjfesJc3CRWu6Fnlg+XBWE2ME49Ffi//4sWiJtuUg2JotLR0YGOjo7M7+sjEJsBmJ71cVCWQFyY\nsaUwAzyujeWSS2Zhr72AmTOV31jPEAZUobrhBrV5ChBkFtvXaQvGzp1Bxpk8GbjkEjWaw5xVev/9\nwRjyuIrs3nvD+wa4BMIuFNu3J7uYNHo+wy23qFnRdmev2Vn+2GOqQxeIbjHpRQDjdvd67rlAaJIs\niBkz3PeJa7H5rEeUtLaTfo6eDKkxXUz77RdulcYJsskZZ6iVSO1nAeGRM674mERZEPa+BkD494jq\npNbnogQirfj5WhAtLe7l5+ME4oQTwhPttMWsnzdkCDBxYnhhPnvOhmmBJFkQ48ersmILRJIrtSi0\nt7ej3ZixOnv27Ezu6+NiWgmgjYjGE1ErgOkA7EWzlwA4FwCIaBKArcy8xfNaIMb60AuPXX+96oiN\nWyJCn7MFwi5oixaFW0fd3WrYq52ZtNumt1e1XE8+uTx+rnH1Gn0/u/JYsCB6lI3NZZcFn7u6yguK\nPSEubo0fE1s0r75apXfPHrVYnMb1jn/1q+D7b36j/g8ZotZr0mn9r/8q7xex79nTkzwPxHyuid7F\n7PHHw8dNC0L3JZjP89mKMypfxQmM/RsffXSwJ4MtEKNGlV9vtqR1fvW1IPRSKpW4mHwsCDPP+bqY\nhgwJf+/pCdxMQNCBHSdQ5u8QNUpMv9v+/VXDZvhwtwUhVEZicWHmbgAzASwH8E8AP2PmNUR0PhGd\nXwqzFMCjRNQJYD6AT8VdCwBEdDoRbQQwCcBviGiZ6/l6SYjt21XL3xYIl580SSD23jssBo88ovza\ndsWgn80MvOENyatRmnFQaQyut2lt9d/EXbN7d7mI2ffwnRBkC8sDD6jC9elPh1eBtd/JJZcElbOZ\nrle9Cnjb28LpjxplZloQ5j0efFC5AlzP9fUh6/fhWjdrz57w7xXVurRXQzXfsY/AbN0K/OMfwQg2\ne5hrVOezRouca70vl0BMnhw/nyEKXxdTlEDEuX/sY1ogdL5jThYI81zUKgc6Pnpo66te1TctiFrh\nNQ+CmZcx80HMPIGZv146Np+Z5xthZpbOH87M/4i7tnT8VmYex8yDmXkUM5/qerbZit+5U1VCmiiB\nsCs/uxIdNiyccZYuVZnXzqzms4lUoS9/N+HvZgUZl/n320+1ftOMrnAJhL2sse3HjUKLaFtb+Pj9\n94dFxo6fuXmOmXZdKO3wriXHo/ogPvYx4P3vV5/tdxf1Lu0Woulisn35u3eHW6JRomO7OvTw5127\nwvGI+u0+/Wn137VftDm6ClDxsY/p68zK+7OfVf+3bVN5wEx3a2tluwymcTFp9t23/HzUvU3+9a/w\nXAg9kipOoMzfIWqVA1MgALXUvemGFQuiOnI/k9qsAO15CrZARE0gc41qsjOwy4KwBcK1zv6kSeHv\nSQKh99CdMEH1GaQZXbFrV7JA+KIrob33Dh9fuTIssL6T8XRBtMX51lvLC6m+55Yt4fvr9/XQQ/7j\n2LW7Rs9w13mkuzu4h7nTmI+f/q9/DX/XaersDB93/b7r1wedoToutgVhN1jsyYL6Ot+VTn0m37lI\n62J69lm1+q193kSveeRquZuuqN7eZIHYti1YRbi7W7lmTUwXk37eK18pLqYsyb1AmBPAXAJhZkRz\nNMuphj1iFsi3vjVaIOydqEyB8K3IkwRC79P7ilco4UojED4WhC+60nMVILNjN2kfCI2+j+3eGziw\nfFKXTvMXvgD8/OfBcf2+3vjGYNkJTVRrXYu2/V5eeim4n97PePFi93pAAHDwwe7jQFh0TFy/rx4k\nYF63YkVwzOVi6upyC0SUC9K2FLS1FJWXovaZ9nUx6Xe7997Jy29oC9AlEOa6Tz4upqefBs4+Oygn\n9j1tFxNQ3h9i5m+ZC5Ge3AvEOecEn22BeOmlcCY96KDgs9lJ9tvfBp/Hjo23IMxC+YtfqP+uVl8U\nZuGNE4jWVtXaTDPYwCUQvruH2cQJhLnOlC9Rvt7W1nKBiCqo5sqhrsaAC+2GsNNx3HHlld9FF0W7\n4E480X3cjIvtLrPv//zz4Q5+fd373hcst+5yBXV1hfPXzp3xrWvXCK+HHnLn0bFj1Y6ALnxdTEn7\nQbuO2flh1iw1H0ajXUxxz9fCun27Cuea/W8LxIAB4RWAzXh84hPRzxLc5F4gTGz3RWdntB/UFAg9\n0gZQhXjWrPIMPGhQuUBomKNbnjZm4T3vvPLzWiD0f3uJjDh27SovJD67h5lDYTVZCITLxWQTZ0HE\n4dtJrStRVz74/veTn6OJc0VELe9ti80994Tzifnb6g2bduwoHxbqsiCGD1fHzMaNxp6419MT3VCI\nazX7upjSdPRGCcSgQeGNjPQoprh9Gi66SP2PEgiXBTFggJrLpBdTNDvvbdehkExTCcTOncpFpNmw\nIVog7GUpNHfcoUYtRbmYXAKRxoIwC+QFFwBjrHnjelmHuO1Ao3jssSDeui/Dx4LQm/gAQQHWriBX\n4U9jQRx2mPofVcHaE9mAYJmLOJZZY9qSRKXambKu/HLAAfHXaP+4Rov1u98dfc2LL7oFwu6k1gIx\nb175PaImNbqIO9evX/pOal9cAmEvLZ7kYtK88EIwesvE3J9F57/WVuWinD1bjYoz31/aTnyhCQTC\nrHjsYa5dXdHbPyYN+/OxILRp3ttbmQUBlA871emJW2zt1lujz9kVcVoXk7nukOt++++fzoLQ94uq\nROwWcE8P8Oijyfe++eby6+JIO2QYCDZN+uhH3RPPTGF1MX9++Lt2SdpzAEyiLAgzf734ompAdHVF\nj97RLFsWLCniIs6CIALuuy/6vB4NFCX+UcNf7XPvfS8wdapbIHzQFoTOY3qwwNatbgtCc+ihgaUO\nVL5nRl8m9wJhtrR37gybxEOGqAlZLqKm/2vsiktbEGanrx4CWqkFAZR3Irsyso1dWZn9MHZF7LtB\nvcYetXT00eHvJ5zgLxB/+1uwnHRUJWJXort3V9axrmd2R7HXXunud/HFanY+oCpKV+Vx0knxrefH\nHgt/1wIRZb0C0QJhdu5v3Qq8+tV+AjFlSnzao9bKAtSmWnr1AFeD5dWvVv+j3kHUc2+6KXDxAErs\n29rC78Xc7CcJ28Wk+zI2biwXCLvsmP0klTQi+jq5Fwizw8kWCLN1YGNnvqj16DXagjAF57rr1P9q\nLAggXDD691eVZpwbIm6dHV1YdcZ/7LHyXb/iMN/ZNdcEvnHN3nurQmdX7C7Mfa51AX344WA9Kxe7\ndlU+8iqKX/5Sze42RwvZtLWpZcM1kyaFt3C1K9KNG9V6XHGd11HEWRDbtqnBCfvtp+aU7L9/sIgd\noEZTbd2qrJeovTLs1vo++0Q/77nnklcM7tdPuUNttGjNmKGsAJsogTj7bJU+G9OCIIrv9zH77/T8\nE1OoTj9d7bliu5hcoqN3S/QdNiwE5F4gTHbuVK1bXdDjfIp2JTt2rDvznHCC+q8tCL0U8YgRan0X\nIFkg9tsvmEDkckuYrbOBA1Xmj+uDiCs42uevWbdOVeq6NZyEKRD2ujWA2ocYiG8FuzZ10fc55BDg\nNa+JvnbFCiUQ5oSramlrU+/Ato5MzjorLIb2Uit25TFokKrEzGVFfIkTCM2GDWpI7F57hS22nh7V\n+T1yZLQFYTdCXNuG7rOPcrO+5z3RcdDl4e1vL3c9/vCHwZDdyy4rd/kB6Wdvm3mqX78gz8ydGw53\nyinhYcd6yLIpELfcAlx4YbyLSaPztFgQ6WkqgViwQFkURx2lCom9SqiJnVFcYnLFFcFKrIMHh/3c\nZgXCrNa7iWLYsEAE9FIRJmYlrMO5RODDH3bf32wBvuMd5WlZtUpZO3q8fxQHHBCuTAYMKJ9JrTfS\nsQXiuOPC19mYhTfO7z11qhKIU53z5iub2KRdNq98pfv8iBHqtzbjbca3qysQrNtvD8fDp7KPio8P\nAwaEtwt95BHl9tEuJh8LwmUhfOITamLbL36hFsWLejag8rrtRvr4xwMXUxRp3Xr2Evu6vL3hDcHx\nyy9Xq7KOHh0c0wLhWsMqycVk8vTT6eIrNJlAaNfEgAHJM2LtSszVyXn88UGLevz46P1xu7riBcJc\n4dPVqjIrPT2RxzX66sc/Vv/jLCNdkGfODFryL7yg0nviie6lLTQ9PeFRVf37q3TryU1AULnZFaOZ\nBpdAmOeTOpQffbR8dJemko5EHWdtlc2bF4x5P/dc4IknVBiz8rBn4H/2s6pC1fcy0xMlZkCwaVSl\n2AKhn/fKV6p4uX5PV/645prw90GD1L1bWpT7qKenfG2sqNWGfZg+vXz3xCRsS0fvv2Ie/+pXlRCY\nDZKXXlLxP+CA6BFcPn17ZnjBj6YSCI1P55bdktCZ2Szs/foF4fbfP7yUglmBmIV0v/3UcgOuRcui\n6N8fuPNO9dlnq0i7BW628nW8xoxRo2+AYDQOoFp1l1yiPtt9E3pVWu2rt0epPPpoUEHaFoTZwnR1\nWiYJiOlTvueecAvRF9va0WjRNZdb0L/r974XiHaUQGj3xd57B8JopsFcf0pz1lnq/5w5aiKcyeDB\nQaf6uHGB29He31s/xxaIY45Rx7u63NaYq5LTv5ce1WfPKO7XTy3vYqJdLlEjAeM4/vj0y4vbeUq7\nbV2un9Gjlft0+vSgDyJqyW/AXyCuuipdnPs6uRcIV2eXj0DYYZYuVf+1CwEIBGLjxvIWs5n5n3su\n+HzqqcAPflD+rKRWvx5T7yMQ5gzRq68GvvzloIK1LY9nny3vYNSFRvehmMcHDw6WXtDvSFes++8f\nLRB6BVcgqNC0b9e8BxCeMasxrRSg3IKwBUNX5vvuG7ggzE5mE/u33rkz+P1MN0iUQJgVlMuCcLW0\n9aSvY48FPvOZ8PP37Al2/uvfP/Dv63d4xx3huNsCsffe6vijj7rdIi4Xln4Hra1qQyJ7nwwXOt2V\nzsZPi7YU9PDxOIEA1HLxEyYELqaofakBPxcToCbJCv7kXiA2bFAteLOTLG7ood5QSBeYffZRo1x0\nK9LVEh47tjzzmRWInjG7alXYlNct9QED1OinM890x6l//+BZvgKh0ftU6FFFtvvG1TEbta2oFg4d\nB+13N9Me5WL69KeBP/1J9XWcfjpw7bVBB+611yoR03zuc+XxsuNtC4Ldaa9H5hx3XDCazHZRbNqk\n3HK2L3zMGCWa+jqNKSTmYAKzk1i/G/PduSpk3eoeODB+eXBm5QpcuDCovMxRRwMGKBcYEJwfNkwd\nv/tu1TixrQFXfLQg9uunOp19+k70fh3r14f7b3zmqUQR14ekGx16RWYtEJMnR6+FNXiwu5Nao/O6\nzk9RAmFuXiT40xRrHe61l6rENXECod0QOqOMHl3uAtDE7Q2gW8c//nFQudmZTFeCra3Rpuu4ceEh\nlUkCccUVwQqxO3aUF3QfH70WAtMKamsLb+Dz5S8Hfl6z30RXRq5NYCZPVn+Amkeg95m+6KLws4YN\nU5ZN3NLaI0eq/p+f/1zNNzArvcsuUyL02teq337dOnXcdJEMG6aEwO7YNysSvdOgRldeTz0VCMTh\nh4f7EbSImHEfNKjcQtTzTwYOVHMRvvIV4EtfUsfMjuXBg9VIOT1a7o1vDFvFW7aopbB/+EOgvV39\nRsOGBfn3jDOAr31NNXIuvzy4p03ckhVR/PrXKg9v2xauoM0lMaJwWcwrV5ZbrSZa4PW71QLR0qLy\nomuuy5Ahate5fv3c/XY6rw8apAawRA19X7Wquu1m+yqJFgQRTSGitUS0nogujQgzp3R+NREdmXQt\nEe1NRL8jokeI6A4iSlx4QleUv/hFkAkWLlT/9T7LRx1V3gkctz9ClF/9ySeDzV4+/OHA12+j3Qhx\nZu2//hUWGVcmN8faf+UrQYXtagX6jOXW7huzQDz0UHiG9lVXBRXNVVcFi8yZ+2i7hja68Cl4tkAM\nG6YK9IknqrkcZuv+gx9UI29GjVLv4LDD1IxhM0yUUMY1HojUKDDTulm5MrxlrO9ez9oKGDhQvbNz\nzw3OuVxWmgceCD9f92+97nVBK3jo0EDAFi1S4m4Ob3bli0rcRIMHB42G448PuxGTsPszgPJJlzba\ngrjwQpW3zKHjp5/unj+z115qGGzUyLirrw6W04ibF2Vizq0S4okVCCJqATAXwBQAhwA4h4gOtsJM\nBTCBmdsAnAdgnse1lwH4HTMfCOAPpe+x6IJ20knBsXe9S/2/9lrVornvvqCC0CMkzP4DG7Oy1cJy\n223K9eIzVFEXqLg+ET2SxFWxX3xxBwDgi19MHqKq8encvegilW7TMmhtjY7nK14RroDmzFHPee97\nVcVszxjWvPrV0SuF2huomwV81KiwS2P8+HDcbHdTS4tqpZtUumzCb38bFpH+/cNukZEj41vQL72k\n+gVOOqkDP/lJcK9x44JWtSkQSa4ene6hQ4MKdPt2NWfjlluC68386Bo9VGk/gnb3EIVXQ7Z/P5Pe\n3vjJkFH07682yTrgAJW33v72wCo/9VT3woRJQ2mPOSb9Kq0jR8anTwhIsiAmAuhk5g3M3AVgMYBp\nVpjTACwCAGZeAWAEEY1KuPbla0r/Y6bzKHQFbmaYffYJt/5Mtm0DliwJlj8w0QXZHJ2kK9NKFtHz\n6TTfd99gj2TN0KEdYFZ7XfvM2GWOHh5q0tKiKuA5c5TlkJYLLggqvilTot0GQ4aoBdFcdHR0vDzH\n5IYbgpnCJ52kdq2zOe+8YEmRuJagHoFVq1mxgwfH++AHDVL57i9/6XAu8/Ld7wad1gcdpOZ9xKEn\n7+n5J5/7XDBCyGzR6zkAGzcGrqws+Pa3A3ExRTeuAq3GVWO68z7/+aDPMApd7n0ngvpy550d2d6w\noCT1QYwBYI6x2ATgWI8wYwCMjrn2NcysF0PeAiBm7q1C+y/N1p9diDT33KNatq4Zpia23zaqIywJ\nH7fE4MGVzcqthuHDy2fI1pNzz1Ujj6ZPDzaVHzLE3Q/zwQ+qP70bWxQTJyqXTl43f7nwwuDz2rXJ\n4RcsUJ3YutK1O9Y1hxyi3KVRDZgrrgiv2eVLS0tguZx5ZmV7gdQSbY1973vZ3O+GG9SAkrlzVf/P\nV7/q75rqiyQJhO+0Ep82Bbnux8xMRInPGT7cf5KLvQ2oi7vuKg9XiTjMmKEqQMHN8uXB56iZzmn5\n6lf918bKO3H7OtvEWbdDh5Yvw5KWiROTBbrevPWt8avVpkE3AJ94QnX4z5+vBge86U3KC2D+LVlS\nWQ9u5TcAAAS6SURBVH1QOJg58g/AJAC/Nb5/EcClVpgfADjb+L4WyiKIvLYUZlTp874A1kY8n+VP\n/uRP/uQv/V9c3e77l2RBrATQRkTjATwJYDoA25BdAmAmgMVENAnAVmbeQkTPxly7BMB/Abim9N/p\nfGFmGZgmCILQIGIFgpm7iWgmgOUAWgAsYOY1RHR+6fx8Zl5KRFOJqBPADgAz4q4t3fobAH5ORB8F\nsAHAWTVImyAIglAFxLJ6lSAIguAgl0tt+EzOyztENI6I7iSih4noISL6TOl45CRBIvpiKc1riejt\njYu9H0TUQkSriOj20vcipW0EEf2SiNYQ0T+J6NiCpe+Lpbz5IBHdSEQDmzl9RLSQiLYQ0YPGsdTp\nIaKjS+9kPRH9v3qnI4qI9H2rlD9XE9EtRDTcOJdN+rLoyMjyD8od1QlgPIABAO4HcHCj41VBOkYB\nOKL0eSiAdQAOBvBNAF8oHb8UwDdKnw8ppXVAKe2dAPo1Oh0JabwIwE8BLCl9L1LaFgH4SOlzfwDD\ni5K+UhwfBTCw9P1nUH2BTZs+AJMBHAngQeNYmvRob8rfAUwsfV4KYEqj0xaTvlP07wDlts88fXm0\nIHwm5+UeZv4PM99f+vwigDVQ80OiJglOA3ATM3cx8waoHzViq5fGQ0RjAUwF8CMEw5yLkrbhACYz\n80JA9acx8zYUJH0AXgDQBWAIEfUHMARqIEnTpo+Z/wzAXlgnTXqOJaJ9AQxj5tK+kvg/eEzirQeu\n9DHz75hZzwhaAUCvWJdZ+vIoEFET75qW0kiuI6F+xKhJgqOh0qrJe7q/A+DzAMwpa0VJ2/4Aniai\nHxPRP4jof4noFShI+pj5OQDfBvAElDBsZebfoSDpM0ibHvv4ZjRHOgHgI1AWAZBh+vIoEIXqNSei\noQBuBnAhM283z7Gy8+LSm8t3QUTvAvAUM69CxCTJZk1bif4AjgLwfWY+Cmp0Xmi9sGZOHxG9DsBn\nodwPowEMJaIPmmGaOX0uPNLTtBDRFQD2MHPm0xzzKBCbAZjLkY1DWPWaBiIaACUO1zOznuuxpbRW\nFUom31Ol43a6x5aO5ZG3ADiNiB4DcBOAE4noehQjbYDKb5uY+d7S919CCcZ/CpK+YwD8lZmfZeZu\nALcAeDOKkz5Nmvy4qXR8rHU81+kkog9DuXrN+eaZpS+PAvHy5DwiaoWaYLekwXFKDRERgAUA/snM\n3zVO6UmCQHiS4BIAZxNRKxHtD6ANqkMpdzDz5cw8jpn3B3A2gD8y84dQgLQBqv8IwEYi0jtonAzg\nYQC3owDpg1rJYBIRDS7l05MB/BPFSZ8mVX4s/e4vlEasEYAPIWISbx4goilQbt5pzLzLOJVd+hrd\nOx/RY38q1KifTgBfbHR8KkzDcVD++fsBrCr9TQGwN4DfA3gEwB0ARhjXXF5K81oA72h0GjzTeQKC\nUUyFSRuAwwHcC2A1VAt7eMHS9wUo0XsQqgN3QDOnD8qSfRLAHqg+zBmVpAfA0aV30glgTqPTFZO+\njwBYD+Bxo375ftbpk4lygiAIgpM8upgEQRCEHCACIQiCIDgRgRAEQRCciEAIgiAITkQgBEEQBCci\nEIIgCIITEQhBEATBiQiEIAiC4OT/A/8nVaVkOIaFAAAAAElFTkSuQmCC\n", 500 | "text/plain": [ 501 | "" 502 | ] 503 | }, 504 | "metadata": {}, 505 | "output_type": "display_data" 506 | } 507 | ], 508 | "source": [ 509 | "# row of W\n", 510 | "%matplotlib inline\n", 511 | "import matplotlib.pyplot as plt\n", 512 | "plt.plot(range(W_train.shape[1]), W_train[1,:])" 513 | ] 514 | }, 515 | { 516 | "cell_type": "code", 517 | "execution_count": 614, 518 | "metadata": { 519 | "collapsed": false 520 | }, 521 | "outputs": [ 522 | { 523 | "data": { 524 | "text/plain": [ 525 | "(63, 1029)" 526 | ] 527 | }, 528 | "execution_count": 614, 529 | "metadata": {}, 530 | "output_type": "execute_result" 531 | } 532 | ], 533 | "source": [ 534 | "W_train.shape" 535 | ] 536 | }, 537 | { 538 | "cell_type": "code", 539 | "execution_count": 615, 540 | "metadata": { 541 | "collapsed": false 542 | }, 543 | "outputs": [ 544 | { 545 | "data": { 546 | "text/plain": [ 547 | "[]" 548 | ] 549 | }, 550 | "execution_count": 615, 551 | "metadata": {}, 552 | "output_type": "execute_result" 553 | }, 554 | { 555 | "data": { 556 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEACAYAAAC57G0KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXuUHGWd9z+/TO7hEgIkhiRu0M0uBAVFTfCG46oYUGSP\numJeWW7y6lHxXcX1gu55SdajexFRVxRRuSkH2V02LslZAkRkVvBdE6KA4RJNgITJDQhMAkwuMz3z\nvH88XXR1TXV3VXf1dHXP93NOn+murql6eqbq29/6Pr/nKXPOIYQQovMY1+oGCCGEaA4SeCGE6FAk\n8EII0aFI4IUQokORwAshRIcigRdCiA6lpsCb2bVm9pSZbaiyzr+Y2SYze9DMXpttE4UQQtRDEgd/\nHbCk0ptmdgbwp865BcDHgKsyapsQQogGqCnwzrl7gL4qq7wPuKG47lpgupnNyqZ5Qggh6iWLDH4O\n0Bt6vQ2Ym8F2hRBCNEBWnawWea35D4QQosWMz2Ab24F5oddzi8vKMDOJvhBC1IFzLmqiE5GFg18J\nnAtgZqcAe5xzT8WtODzscK49H5dddlnL2zBW29/ObVf7W/9o9/Y3Qk0Hb2Y/A94GHGVmvcBlwAQA\n59zVzrnbzOwMM9sM9AMXVNrWxo1w/PENtVcIIURCagq8c25pgnUuTrKze+6RwAshxGgxqiNZf/Wr\n0dxbtnR3d7e6CQ3Rzu1v57aD2t9q2r39jWCNZjyJd2TmXv5yx9ato7I7IYToCMwM18JO1sQcOIAE\nXgghRolRFfhTT/U5vBBCiOYz6gLfzjm8EEK0E6Mq8G98I6xbN5p7FEKIbNi9G3btanUr0jGqAn/Y\nYbBv32juUQghsuH66+Hyy1vdinSMqsB3dcHQ0GjuUQghsmFgACZMaHUr0jGqAj9+PBQKo7lHIYTI\nhsFBCXxV5OCFEO1AoQCbNpUvGxiAiRNb0556kcALIUSEtWvh/PPLl8nB12D8eAm8ECL/HDzoH2EG\nB+Xgq9LVpQxeCJF/Bgd9JBNGnaw1UEQjhGgHBgf9I7pMAl8FCbwQoh2Ic/CKaGqgMkkhRDtQKIx0\n8IpoaiAHL4RoByo5eAl8FSTwQoh2oFIGr4im2s6Kexsern8bmzf7GlUhhGgWqqKpk0Zd/G23+Ul/\nhBCiWaiKpk4aFfihIcU8QojmEjj48B1NNVVBAhqtpBkaUiWOEKK5FApe3MNmUg4+AVk4eAm8EKKZ\nBPFMOKZRJ2sCJPBCiLwTCHu4o1WdrAlQBi+EyDuVHLwEvgbK4IUQeSfOwSuiSYAiGiFE3gk0Juzg\nFdEkQAIvhMg7lRy8BL4GWUQ0yuCFEM1EVTR1IgcvhMg7qqKpEwm8ECLvqIqmThq9bZ8iGiFEswk0\nSlU0KWn0xtty8EKIZhONaJyTg0+EIhohRN6JRjSFgjenZq1rUz20ncAXChJ4IURziTr4duxgBZVJ\nCiHECII4JuzkO1LgzWyJmW00s01m9sWY948ys9vN7AEze8jMzq+2PUU0Qoi8UyjAtGklB9+OHaxQ\nQ+DNrAu4ElgCLASWmtnxkdUuBu53zr0G6Aa+aWbjK21TAi+EyDuDgzB1anlU04kOfhGw2Tm3xTk3\nCNwMnBVZZydwWPH5YcCzzrmKEqzJxoQQeScQ+LCDb0eBr+i0i8wBekOvtwGLI+v8CPilme0ADgU+\nVG2Dmi5YCJF3og6+XSOaWgLvarwP8GXgAedct5m9ElhjZic5516Irrhs2TIefxx+8hOYNKmb7u7u\n1A2WgxdCNJvBwfIMfjQjmp6eHnp6ejLZVi2B3w7MC72eh3fxYd4EfA3AOfeYmT0B/DmwPrqxZcuW\n8dvfwtlnQx3aDkjghRDNZ3AQZsxoTUTT3V1ufpcvX173tmpl8OuBBWY238wmAmcDKyPrbATeCWBm\ns/Di/nilDWokqxAi7xQKYyCicc4VzOxi4A6gC7jGOfeomX28+P7VwNeB68zsQfwXxhecc89V2qYy\neCFE3ol2srZrFU2tiAbn3GpgdWTZ1aHnu4Ezk+5QZZJCiLwTZPAdP9Apa1QmKYTIO3EOvh0jmrab\ni0YCL4RoNnLwdZJVBu+SFHAKIUQdBJ2sHT1VQTPIIqIBGB7Opj1CCBFlrExVkDlZTBcc/imEEFkT\nHeikiCYhWUQ0IIEXQjSPTpmqoC3vyRr+KYQQWRJoy+TJ7V8H35b3ZAU5eCFEcygUvJhPnKiIJjWK\naIQQeWZw0BvR6B2dFNEkQAIvhMgzgVsPO3hFNAnJqkxSGbwQohmEBV4DnVKSxMFv3gzPPBP/nhy8\nEKKZBGI+YYIGOqUmicB/4xtwyy3x7w0N+T+8BF4I0QzCnawa6JSSJGWSg4OV1xkagkmTFNEIIZpD\nuJNVDj4lScokC4XqAj9xohy8EKI5xGXwcvAJSRLRVLupR+DgJfBCiGZQKYOXwCcgSURTy8FL4IUQ\nzaJSFY0imgRkEdEogxdCNIugkzXs4BXRJCRJRKMMXgjRKoJOVk1VUAeNCnyhoIhGCNE8whm8IpqU\nJBnJqk5WIUSr0FQFDZBFRKMMXgjRLDRVQQM0IvDO+YcyeCFEswjEvKvL683QkCKaxDRSJjk0BGaa\nqkAI0TwKBR8lQ8nFK6JJSJIyyaGhygI/fnzjM1IKIUQlwnFMUCopB5+QpBFN3DpDQ/73G51TXggh\nKhEWeDn4lDQa0XR1ycELIZpHJQcvgU9AIyNZJfBCiGYTdfCKaFLQSBWNBF4I0WyCqQqgNNhJEU1C\nks4mWU3glcELIZpFMFUBlDt4CXwCkrjvWp2scvBCiGYRzeAHBxXRJEYRjRAiz8Rl8IpoEpKFwCep\nxBFCiHqIK5NURJOQrMoklcELIZpBtJNVVTQpSDqSNW6dQkERjRCiuUQ7WTu6isbMlpjZRjPbZGZf\nrLBOt5ndb2YPmVlPte0pgxeiOTz8cKtb0BlEO1kPHixNk9JuVBV4M+sCrgSWAAuBpWZ2fGSd6cD3\ngDOdc68CPlhtm1lENCqTFGIkb30rPPdcq1vR/kQz+P5+/9qste2qh1oOfhGw2Tm3xTk3CNwMnBVZ\n538B/+Gc2wbgnNtdbYMaySpEczhwoHSDClE/UQcfCHw7Ukvg5wC9odfbisvCLABmmNndZrbezP66\n2gazGOgkgRdiJIODOi+yINzJGjj4duxgBaiVKrkE25gAnAy8A5gK/I+Z/cY5tylu5VoRTTDBfqWB\nTpouWIiROOfPieAORKJ+wp2s7e7gawn8dmBe6PU8vIsP0wvsds7tB/ab2a+Ak4ARAr9s2TL6+mD3\nbujp6aa7u3vEDgNhVwYvRHKC80XGp3GiGfy+faMr8D09PfT09GSyrVoCvx5YYGbzgR3A2cDSyDq3\nAlcWO2QnAYuBK+I2tmzZMp58En7+c4jRdqD6gaqIRoh4guxd50XjxHWyjmZE091dbn6XL19e97aq\nCrxzrmBmFwN3AF3ANc65R83s48X3r3bObTSz24HfA8PAj5xzj1TaZi33LYEXIj3hm0OLxuikTtaa\nlZ3OudXA6siyqyOvLwcuT7LDWhl8kohGAi9EOYGw67xonGgn6+7d7SvwuRvJGhygumWfEMmRg8+O\nuE7Wdq2iyd1kY4pohEiPHHx2VBro1I7kbrKxQqGygEvghYhHDj474jJ4OfiEJIloJk2SwAuRBlXR\nZIccfAPUimiGhmoLvDJ4IcqRg8+OTqqiyWVEM3mybtknRBqUwWdHEBND+09V0BKBHx72Q6vjCAQ+\n7kDVfPBCxCOBzw45+AYwg3HjvMjHkSSD1y37hChHEU12tHqqgiwZdYGH6hn60JD/o8a5fN2yT4h4\n5OCzI+6erIpoUlDNgQejyOK+BJTBCxFPUEUjB9840Ygm/LPdaInAV3PgQQdHnIhL4IWIRw4+O6Kd\nrCCBT0W1iKaWwI8frwxeiCjK4LMjzsEroklBrYimUkeqMngh4pGDz45oBg9y8KmoJtDV7tqkiEaI\neOTgs0MOvkGSRjTqZBUiGXLw2SEH3yC1IppaDl4ZvBDlqIomG4aH/aOry79WFU0dNNLJqgxeiJHI\nwWdDoD9m/nXg4BXRpKBWmWSSTlYdyEKUkMBnQzieATn4uqg1klWdrEKkQ52s2RAVeGXwdZA0g6/U\nyarpgoUoJxhOL+PTGJUEXhFNCjSSVYhsGRiAKVPk4BslPIoVFNHURa1O1koZvKYLFiKewUGYOlXn\nRaMoosmAahGNMngh0jM4KAefBZU6WRXRpKDRMkll8EKUIwefDXLwGZA0g9dIViGSEQi8HHxjRAU+\nGPAkB5+CRkeySuCFKEcOPhuiAm/mxV0OPgX1drJqumAh4gmqaHReNEa0iga8uEvgU9DobJLjxvnb\n+VW6r6sQYw1FNNkQdfDgHbwimhQ02slqpvlohAgTVNHIwTdGnMDLwaek0ZGsoBxeiDAqk8yGSg5e\nAp+CRkeygnJ4IcKokzUbFNFkQK3JxmrNJgmKaIQIoww+G+I6WefPhxkzWtKchhlfe5XsabRMEhTR\nCBFmYEAOPgviHPxdd7WmLVmQOwevDF6I9CiDz4Y4gW9nlMEL0QEog88GCXwG1IpolMELkQ5l8Nkw\n5gTezJaY2UYz22RmX6yy3hvMrGBm76+1zXrv6BSIPyiiESKMHHw2xHWytjNVBd7MuoArgSXAQmCp\nmR1fYb1/Am4HrNZOs4hoJPBClNBAp2wYaw5+EbDZObfFOTcI3AycFbPep4FbgGeS7DSLTlZl8EKU\n0B2dsmGsCfwcoDf0eltx2UuY2Ry86F9VXORq7TSrMkll8EJ4FNFkw1gT+JpiDXwb+JJzzuHjmYYj\nmqSdrDqYhfCoTDIbOk3ga3UnbAfmhV7Pw7v4MK8DbjYzgKOA081s0Dm3MrqxZcuWAbB2LRw82A10\nj9hhktkkQQIvRBg5+GzIQydrT08PPT09mWyr1kdZDywws/nADuBsYGl4BefcK4LnZnYdsCpO3KEk\n8Pv2wZFHxu+wVkQT/PF12z4hSqhMMhvy4OC7u7vp7u5+6fXy5cvr3lZVgXfOFczsYuAOoAu4xjn3\nqJl9vPj+1fXsNGkn68BA+Xty8ELEo6kKsiH4ouwUal6MOOdWA6sjy2KF3Tl3QaKdqkxSiExRBp8N\neXDwWZK7kaxJZ5NUmaQQnsAsTZrkzwmXpDRCxCKBz4BG7+gEKpMUIiAQpXHj/EO3sqyfPHSyZknb\nTjamiEYIz+Bg6YYU48crpmkEOfgM0HTBQmRHWJQmTNB50QgHDvioq1PIXQafdKCTMnghPAMDJYGX\ng2+Mp5+GmTNb3YrsyF1Ek2agkzJ4IcodvK5sG2PnTpg9u9WtyI5cRzTK4IWoTTSikYOvn1274GUv\na3UrsiOXEU2lDF7zwQsxEjn4bBgchL4+OProVrckO9rWweclg7/rLrjvvla3QoxlwlU0cvD18/TT\nXtwDjekEcpfBt9st+269FX7xi1a3Qoxl5OCzYefOzopnIIcRTbvNJnnggH8I0SrCVTRy8PWza1dn\ndbBCG0c0eRL4/ftb3QoxlpGDzwY5+IxIOpK1HW7ZJwcvWo0GOmWDHHxGNHLLvmCeiLxk8BJ40Wqi\nDl4RTX3IwWdEtYgm6WySebkUlcCLVhOtosnDedGOyMFnhDJ4IbJDDj4b5OAzIovZJJXBC+GJzkWT\nh/OiHZGDz4h6RrIGc1yPK7ZYGbwQHk1V0DjOdd40BZCziGZ42P+hx40b+SUQdu+QH6cigRetptEy\nyZtugnvuyb5d7cTevb4fo5Puxwo5i2iiVTLVBD5PEY0yeNFKGnXwmm6jM/N3yFlEE75dVi2Bl4MX\nwhO9o1Pa82L/fpmUTszfIWcRTVqBVwYvxMipCtIK/L59/jGWkYPPkEriHBX48DrhqYKD9/Pg4OV+\nRKtptExSx7AcfKZUi2gqlUHmMYMvFPxDDl60kkanKpDAy8FnSqWIJk0nax4imoMH/U8JvBhtNmwo\nxSpy8I0jB58hWWXwrXbwBw7AYYf5n861ti1ibPG5z8GaNf55o2WSyuA7716sAbnO4NtB4KdN8+3S\n4BIxmrz4oq/dhsbv6CQH35mDnCDnGXz4SyCPGfyBAzB5sn8ophGjSX9/SeAbnapAAi8Hnyn1ZvDB\ne8H7rc7gJfCiVfT3w549/nmjA53GusAXCv7LcsaMVrckexTRNIAEXrSKsINXBt8YL77oo9ZxLVHD\n5pK7iKYdBX7KlLHtgMToU0ng0zr4oSG//lg+fvft8wLfieQqogkL/LhxYFaaRTKvGfyUKXLwYnRx\nLjsHHwj7WBb4/n4JfKZUi2gqiXge6+AV0YhWcPCgNz6VqmjSCHwQzYzliKa/v/NmkQzIVUQT15Fa\nTeDz4OAl8GK06e/3P4NO1mgVTZqIZv9+OOIIOXg5+AxJEtFAdYHPS0SjDF6MNoHAV8rg00Y0gcCP\n1cF6EviMyULg5eDFWKW/Hw49tHIGn9bBH3qo7/Maq4P1xrzAm9kSM9toZpvM7Isx73/EzB40s9+b\n2a/N7MRq20tSJhldTxm8EJ7+fpgzJ5tO1n37fP48derYzeHHtMCbWRdwJbAEWAgsNbPjI6s9Dpzq\nnDsR+Crww2rbrJbBV4ph8jhdcFjgFdGI0aK/H44+2gt78Ki3THL/fh8xjuWYcayXSS4CNjvntjjn\nBoGbgbPCKzjn/sc5V/QTrAXmVttgJ2bwcvBitAgc52GHeRffyB2dJPBj3MEDc4De0OttxWWV+Chw\nW9WdjvMdOkGNe0C7ZfD79yuiEaNPIEiHH+4raaJ3dJKDT0cnC/z42quQuG/dzN4OXAi8Oe79ZcuW\nvfR83Lhuhoa6y4YHpxX4PGTwhx8ugRejSyBI06eXHLwy+PoJOq3zQk9PDz09PZlsK4nAbwfmhV7P\nw7v4Moodqz8Cljjn+uI2FBb4f/gHL9DBgQnxA52qdbK22sErgxetIOzgowJfT5mkHHy+pgru7u6m\nu7v7pdfLly+ve1tJIpr1wAIzm29mE4GzgZXhFczs5cAK4Bzn3OYkO47L4dMMdFIGL8Yq1QS+njJJ\nCfwYjmiccwUzuxi4A+gCrnHOPWpmHy++fzXwf4EjgKvMDGDQObeo6o5jHHi7ZfAqkxStIKj6yMLB\n79tXEvixHNGMWYEHcM6tBlZHll0den4RcFGaHcc5+FoCr/nghfCCNHNmeSdruIomrYMPMvix6uDH\neplkU6hH4PPq4Mfy5a0YfZTBZ0snO/iWCXycA0/TyZqnDD5rB18owPnnZ7c90VnUqqJRBp8OCXwT\niBPotLNJ5iGiacZ88H19cMMNin1EPLU6WZXBp6OTBT5RBt8MsohoBgaa385qNKtMMpgGdu9ev20h\nwgTzlzsXH9Eog0+H5oNvApUimqQCP2mS/9lKl9usMsm+4iiCQOiFCBN28H19/hxp5I5Oimg618Hn\nKqJJM5ukmZ/Hui92SNXo0KwMXgIvqhEW+Gef9eeJr07WVAX1IIFvApUGOiW9ZR90rsAHwi6BF3GE\nO1l37y4fDa6pCtLhXOlv0Im0bQYP+RL4LN2PHLyoRiDwEyfCM8+U99OoTDIdBw74uDeqLZ1CSzP4\nNCNZoyWUkB+BVwYvRpNwRBOeSRJUJpmWTo5nQA6+IQKBP3gw+4hm3DgJvIgnEKVJk7y4hwW+EQc/\nFiOaThf4XGXwaQY6QWsFPjiJxo9vTifr3LkSeDGSQsE/Jk3yHauHH96Ygw9n8GPVwXdq/g4tdPBx\nZZJpBjqBF/hWiWDg3qGUwTtXqmZohL4+OPZYCbwYSeA4g+Ps8MPL31eZZDrk4JvE5MkjLwnbKaIJ\n7uYEvl1pnVM1JPCiElFBmj59ZERT7Tg8eLB0zjgngZfAN4mZM30FQJh2Eviwg4dsY5o9e2D+fAm8\nGElUkKIRTVeXvxVm9HaYATfdBJ/+tH8+MOD7esaPVwbfqbRU4J9+unyZBN4jBy8qUUvgzarHNE88\nAdu3++fBNAUwdjP4Tp4qGHIo8NU6WcdHegzyJvBZnSASeFGJWgIP1QV+2zZ46in/PIhnoP0imt5e\neMtbGt+OHHyTmDmzdKAF1NPJmheBz6oWfngYnn8e/uRPWlsCKvJJEoGvVirZ29sZAv/YY3D//b4f\noREk8E1i1qz2j2iCkwOyi2heeMFfLh91lBy8GEmcwAd3cwqo1uHf2wvPPeffD6YKhtLxWym7bzXf\n/nb5Z9q507f/+ecb264Evkl0YgafhQPq6/Ofa8oU7040J7wIE82Mo1U0UNnBO+cF/rDD/LkXzuDH\njcvvrScLBfjc52Dr1tKynTv9zx07Gtt2p9fBt7XAT5vmv9VbMS98syKaPXu8wJv5k1cuXoRJmsHH\nOfi+Pr/uK17hY5pwRAP5jWl27fJXFtu2lZZlKfBy8E0gTuDjZpOsNpI1EMFWuPhmVdH09fnPBBJ4\nMZKoIL3iFb5DPkwlB9/b60dIz5rlRTNO4PNYKhkIe1jgd+zwX2SB0NdLpwt8y0ayHnpoKQcMLpHi\nHHzgKOIEHkoCP2tW89scppkCf8QR/rkEXkSJClJ3t3+EqeTge3th3rxSgcP06e3h4OMEfudOOOEE\nOfhatMzBm43saE0b0UDrcvhmZfBBRAMSeDGSJJlxpTLJQOBnzSpFNOFt5bUWfts235EcFfjXv75x\ngVcdfBOJxjRppwuG/Ah8Vhm8IhpRjSSOs9J0BXEC3y4O/uSTRwr8614nB1+L3Al8mjs6QesmHFNE\nI1pBEkGq5OC3bSsX+HCZJOQ7g1+8uCTw+/b5c00RTW1yJfBxA52qdbJCfhx8VgKviEZUI6mD76SI\nZvv2coHfuRNe9jI45phsOlk7uUyyZZ2sMHI0a7tl8OEDI8s6eEU0ohJJHXy1iObAgfaLaF73Oj9A\na2DAi/oxx8Ds2d7BNzJNtxx8E0mTwedR4BvJ4Pfvh5UrRy5XRCOqUa+Dd84L5dy53v3mReA3bYKN\nGyu/PzzsRfzlL/ft3rnTP2bPLt3VqpFzRALfRDqtiiaNwK9ZA2edBZ//fPl8GhJ4UY16Hfwzz/jf\nmzoVjjzSH1fPP9/6DP7yy/0o1Uo884wfeTt5sv9y2ratJPDgnXySHP6Pf/TnWxQJfBNJ0sma1ww+\nfMMPSB/RrF8Pn/wk3HsvXHhh6XPu2aOIRlSmXgcfxDPgz6Mjj/TLWp3Br18Pd95ZWaSDqw6IF/gg\npqnF3XfDf/3XyC8wlUk2kSSdrGEHH50uGPLt4Hfvhkceif/99evhtNPgF7/ws+LdfrtfLgcvqlGv\ngw8LPPir5y1bWhvRHDgAjz4KH/oQ3Hhj/DpxAr9jR3oHv26d15Df/a60bGiodH/bTiVXAt/OEU1c\nBn/lld6lR3HOC/zrX+9P1o98xLsL58oFvpX3nBX5pN4yyTiBf+KJ7CKa66/3M6GmYcMGWLDAnyPX\nXRc/9W8lB3/MMX5Z0kqatWvhjW/0PwOCCpos7qOcV1oq8Ecf7V1uMEVpOwt8nIO/4w749a9HHvi9\nvf6zBAfpe97jBX7//tKsftB6B18o5Hf62LFKvQOdghr4gFmz/LaycPB9fXDRRXDrrel+LzA5b3qT\nP7/D4htud7WIJomDf+EF/2V20UXeyQd0ev4OLRb4CRN8B8pzz/nX7Sbw0fngwydHXx889JCv3+3p\nKf/d4MAOnMPxx3thv/feUv4OrRf4T34SLrusdfsX5QwPj6xdjyOpg4faGfyNN8Kll1bf3+rVfiqB\nuKqwaoTPg/PP91cBUbLI4Nevh5NOgje/Od7BdzItFXgoj2nSziYJftKy/fur30k+CQcPplu/loO/\n6y5/S7H3vc93IoW57z54wxtKr828i7/xxlI8E2wz2Ndo8+KL8K//CtdeW/ofiNawZg1ccYW/2p08\n2ZuBakQdvHPw4IPwyleWlgUCX8vBf/ObPmrs76+8v1Wr4Mtf9u1MM3X3+vWl8+Dcc+Hf/33k1W5U\n4B9/3Ff/HHWUX5bEwa9b543WggWwd29p7I0cPGBmS8xso5ltMrMvVljnX4rvP2hmr03TgLDA1+Pg\ns5g3/emnfY3tP/5j8luA1crg77wT3v1u35F6xx3lvxs4lzDveQ+sWFEu8NC66ZBXrIBTT4U5c3xH\nsGgud98dL1RDQ/5K6rbb/JVeEkGKOvhf/tL/POWU0rJKAh/O4O+/319dn3oq3HJL/L4GBnyBwEc/\nCscdB//937XbB34/mzbBq1/tX8+dC+94hzcUYbZvLwn87Nm+bHLmzNKXXBKBX7sWFi3yv/OGN5Ri\nmjEv8GbWBVwJLAEWAkvN7PjIOmcAf+qcWwB8DLgqTQMaFXhoPKb56lfhjDO8g47WpQf0RHKWag7e\nOS/wp50GJ57oHccTT5TeW7/ej8wL8/a3+/fCEQ2M/PJyDr77XXjyyXSfMdr+Wlx/PZx3nn/EXTqP\nJmnbnjdqtX/rVjjzTDjnnJF9Hv/2b958rFnjxfOf/7n2/qJlkpdfDpdcUt6ZmMTBX3cdXHABnHJK\nzwjhDbjnHu+MZ8/2nyFpTPPAA7BwYXkFyyWXwHe+U7piDAZnzZlT+lzBFAUBs2f7yKaaMbvnnh4W\nL/bPFy+WwIdZBGx2zm1xzg0CNwPR4QLvA24AcM6tBaabWeLZ2cPTFTQi8FEH/8IL8f/0oSH4xjdK\nufhjj8HPfubv+firX/lO0QsvHJlh1hL4KVP8F1V/vx9UMTRUytbf9a5STPP44z5Wis5fP3mydzBx\nDj782b7+dX+5vnhxcrcU1/5qbN0Kv/+9P2E//GGfse7dm3xfWdNMgd+8Gf7+75tbHhhuf6EAH/sY\n/NVf+WPIOfjUp+ALX/Cu9oc/LP3e8DB87Wvwd3/nxflVr/KCW4twmeTDD3sn/pGPlK9TK4M/cABu\nusln4wMDPWzc6B13lFWrfAwJ/ueqVcmuguOuYk85xbcr6Kzt6/NfAIccUlpn7txS/g7+vJs2rdSP\nF2X7dujv73nppiiLFpVy+E6vgYfaAj8H6A293lZcVmuduUkbEB7NWm02yUrTBUN5jPHii96FH3UU\nnH56+UG5d68/CH/+cy9c3/0ufOUr8JnP+IqeGTN8HLFzJ3zwg9Wz76jAH3ecv/HCW94CP/6xd++B\nYzrttJIv1OFNAAAHK0lEQVTAxx3YAeed5w/A6GcLBH7FCvjBD/yX0E9+AmefDV/6Uvm9KrPgpz/1\nf59Jk/yAmHe+0zvJoSFftxz8v5zzTuyHP/QdypVO7D/8wV/Cf+AD8JvfZNvWRli92ne83X23/7+l\nvSpKy8GD/u+6ZYv/4j/jDO+St2zx/8drr/ViHvw/b73VC9hpp6XbT9jBX3GF/wIJH6tQ28GvXAmv\neQ3Mn+/Pu3POGXkl55xf78wz/esTTvDH/IYNtdtY6Tz47GfhW9/yz8P5e0BU4MG/fvjh+P2sW+ev\nAIJzcdEi3wc2PDw2HHytycYSJtJEK0mT/h6zZvns+4EHvOsIO/iJE/08FWee6Z1W9N6TAUce6YX6\nyiu9W3nHO7xTvvlmX/u6aJE/SB96CN77Xn/Q9/b6ocu7d8M115S2NW2aP2jPPdcfgME3/x/+AL/9\nbWm93bvLT46uLrjhBn8l8Ld/668KAk47DT7xCf85Nm3y247jAx8YueyII/xJ//3ve3G8/XZ/iXrM\nMd6JXHGFj3sWLvT356xEtP3V+M1vfOYbcN55vs2XXOL/X88+WxKMQw7xf9+vfc3/v447rnxb/f3+\n737xxX5O76VL/VXbzJnJ2pK27UkZHPRCtGKFL9P71rf85wh3fmdF0P4nn/QdnatW+eP8E5/wpXv3\n3uv/dgsX+mNn0SJ/3G3dClddlb5Oe9Ik+N73/BfYunXxzvvoo30bwg5+2jT/NznzTP/z618vvXfB\nBfDWt/oru4DBQf+lf+KJ/rWZN1DnnltesRPHr38dP0XB+9/vDdrpp3tjExX4efNKHawB55zjz51D\nDvF/w3An9GOPlSIe8Mfv9Ol++7t28VJ006mYq3I9ZWanAMucc0uKry8Fhp1z/xRa5wdAj3Pu5uLr\njcDbnHNPRbaVWPSFEEKUcM7VNRyrloNfDywws/nADuBsYGlknZXAxcDNxS+EPVFxb6SBQggh6qOq\nwDvnCmZ2MXAH0AVc45x71Mw+Xnz/aufcbWZ2hpltBvqBBN1AQgghmk3ViEYIIUT70vSRrEkGSuUJ\nM5tnZneb2cNm9pCZ/Z/i8hlmtsbM/mhmd5rZ9FrbaiVm1mVm95vZquLrtmm/mU03s1vM7FEze8TM\nFrdZ+y8tHj8bzOwmM5uU1/ab2bVm9pSZbQgtq9jW4mfbVDynU9b3ZE+F9n+jeOw8aGYrzOzw0Hu5\nb3/ovc+Z2bCZzQgtS9X+pgp8koFSOWQQ+Kxz7gTgFOBTxTZ/CVjjnPsz4K7i6zzzN8AjlCqa2qn9\n3wFuc84dD5wIbKRN2l/sr/rfwMnOuVfjo80Pk9/2X4c/P8PEttXMFuL74RYWf+f7Ztbq6U7i2n8n\ncIJz7iTgj8Cl0Fbtx8zmAe8CtoaWpW5/sz9ckoFSucI5t8s590Dx+YvAo/ha/5cGdBV//mVrWlgb\nM5sLnAH8mFIJa1u0v+i23uqcuxZ8P5Bzbi9t0n7gebxJmGpm44Gp+AKFXLbfOXcPEB0HXqmtZwE/\nc84NOue2AJvx53jLiGu/c26Ncy4YE7yW0rictmh/kSuAL0SWpW5/swU+yUCp3FJ0Y6/FHySzQtVB\nTwGJR+u2gG8BnwfCA9/bpf3HAs+Y2XVm9jsz+5GZTaNN2u+cew74JvAkXtj3OOfW0CbtL1Kprcfg\nz+GAdjifLwSCUR1t0X4zOwvY5pz7feSt1O1vtsC3bQ+umR0C/AfwN865sjnunO+ZzuVnM7P3Ak87\n5+5n5AA0IN/tx1d2nQx83zl3Mr4yqyzOyHP7zeyVwGeA+fgT8hAzOye8Tp7bHyVBW3P7OczsK8CA\nc+6mKqvlqv1mNhX4MhCeqLtaiXnV9jdb4LcD4TFt8yj/BsolZjYBL+4/dc79Z3HxU2b2suL7s4Gn\nK/1+i3kT8D4zewL4GfAXZvZT2qf92/Du5b7i61vwgr+rTdr/euD/Oeeedc4VgBXAG2mf9kPlYyV6\nPs8tLssdZnY+PqYMz8LTDu1/Jd4cPFg8h+cCvzU/v1fq9jdb4F8aKGVmE/EdBClvCzC6mJkB1wCP\nOOe+HXprJXBe8fl5wH9GfzcPOOe+7Jyb55w7Ft+590vn3F/TPu3fBfSa2Z8VF70TeBhYRRu0H98h\nfIqZTSkeS+/Ed3a3S/uh8rGyEviwmU00s2OBBcC6mN9vKWa2BB9RnuWcC88olfv2O+c2OOdmOeeO\nLZ7D2/Ad9k9RT/udc019AKcDf8B3CFza7P1l0N634LPrB4D7i48lwAzgF/he+TuB6a1ua4LP8jZg\nZfF527QfOAm4D3gQ74APb7P2fwH/pbQB30k5Ia/tx1/l7QAG8P1lF1RrKz4+2Iz/Int3Dtt/IbAJ\nX30SnL/fb4P2Hwz+/pH3Hwdm1Nt+DXQSQogOpdU1oEIIIZqEBF4IIToUCbwQQnQoEnghhOhQJPBC\nCNGhSOCFEKJDkcALIUSHIoEXQogO5f8D0Imk+lVLkq0AAAAASUVORK5CYII=\n", 557 | "text/plain": [ 558 | "" 559 | ] 560 | }, 561 | "metadata": {}, 562 | "output_type": "display_data" 563 | } 564 | ], 565 | "source": [ 566 | "# row of F\n", 567 | "plt.plot(range(F_train.shape[0]), F_train[:,0])" 568 | ] 569 | }, 570 | { 571 | "cell_type": "code", 572 | "execution_count": 618, 573 | "metadata": { 574 | "collapsed": false 575 | }, 576 | "outputs": [ 577 | { 578 | "data": { 579 | "text/plain": [ 580 | "[]" 581 | ] 582 | }, 583 | "execution_count": 618, 584 | "metadata": {}, 585 | "output_type": "execute_result" 586 | }, 587 | { 588 | "data": { 589 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEACAYAAAC57G0KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXuUXVWd57+/qiSVCokhvCIkURAKF9ij4gPwNdb4GKML\nRV29RMbuERxsunsy4mMpoKuHsOy2ux3o6XHhAx/Qtt2IjdoO9gJfSGnrIIoiASFQIQTykPAKJKSe\nt+6eP3Zt7r777td53fO4v89atarq3lP37jp3n+/5nu/+7X1ICAGGYRimeQyV3QCGYRimGFjgGYZh\nGgoLPMMwTENhgWcYhmkoLPAMwzANhQWeYRimoQQFnoiuIqK9RHSnZ5vPENEkEd1BRKfk20SGYRgm\nDTEO/moAG11PEtFbAJwghBgD8CcAPp9T2xiGYZgMBAVeCPHvAPZ5NnkbgK8ubnsrgEOJaG0+zWMY\nhmHSkkcGvw7ATu33XQDW5/C6DMMwTAbyGmQl43de/4BhGKZkluTwGrsBbNB+X7/4WBdExKLPMAyT\nAiGEaaKjyMPBXw/gvwIAEZ0O4EkhxF7bhkKIWn6NjwucffYlpbcjy9cll9S3/XVuO7e//K+6tz8L\nQQdPRF8H8FoARxDRTgCXAFi6KNhXCiFuIKK3ENE2AAcBnJupRRVkYQFot8tuBcMwTDKCAi+EODti\nm035NKeatNtAxhMpwzBM3+GZrBEsLAAnnzxedjMyMT4+XnYTUlPntgPc/rKpe/uzQFkznug3IhL9\neq+8OfVU4CMfAc46q+yWMAwzaBARRImDrI1nYUF+MQzD1AkW+AjabR5kZRimfrDAR8AOnmGYOsIC\nHwELPMMwdYQFPgIWeIZh6ggLfAScwTMMU0dY4CNgB88wTB1hgY+ABZ5hmDrCAh8BCzzDMHWEBT4C\nzuAZhqkjLPARsINnGKaOsMBHwALPMEwdYYGPgAWeYZg6wgIfAWfwDMPUERb4CNjBMwxTR1jgI2CB\nZximjrDAR8ACzzBMHWGBjyAmg19YAG65pT/tYRiGiYEFPoIYB3/vvcA55/SlOQzDMFGwwAcQQn6F\nBL7V4hiHYZhqwQIfQEUzIfHmnJ5h+svcXNktqD4s8AGUaMdk8CzwDNMfdu4ETjut7FZUHxb4AEq0\nYxx8q1V8exiGAQ4cAJ56quxWVB8W+ACxAs8ZPMP0DzZUcbDAB+AMnmGqBxuqOFjgA3AGzzDVgwU+\nDhb4ABzRMEz1YEMVBwt8gCSDrNzhGKY/tFqcwcfAAh+AM3iGqR58vMXBAh8gNoPniIZh+gcfb3Gw\nwAfgiIZhqgcLfBws8AGSCLxat4ZhmGJhQxUHC3yA2AxeDfhwp2OY4uFB1jhY4AMkqYPXvzMMUxyx\nx+WgExR4ItpIRFuJaJKILrQ8fwQRfY+IfktEdxHROYW0tCSSRDRAvKsQAnjssfTtYphBhq+Y4/AK\nPBENA7gCwEYAJwM4m4hOMjbbBOB2IcSLAYwDuJyIlhTQ1lJIMtEpZjvFnXcCZ5yRvl0MM8iwwMcR\ncvCnAtgmhNghhJgHcC2AM41tfg/gWYs/PwvA40KIxqRjSergY7ZTTE0B09Pp28UwgwxHonGEnPY6\nADu133cBMFdh/hKAHxPRHgCrALwrv+aVT1EZPFcBMEx6lIPngVY/IYGPKfr7OIDfCiHGieh4AD8k\nohcJIQ6YG27evPmZn8fHxzE+Pp6gqeVQVETDVQAMk54mO/iJiQlMTEzk8lohgd8NYIP2+wZIF6/z\nSgB/BQBCiPuJ6AEAzwdwm/liusDXhaSDrOzgGaZ4mpzBm+b30ksvTf1aoQz+NgBjRHQsES0DcBaA\n641ttgJ4AwAQ0VpIcd+eukUVo6gMnm9YwDDpabLA54nXwQshWkS0CcD3AQwD+IoQ4h4iOn/x+SsB\nfArA1UR0B+QJ42NCiCcKbnffWFgAli2LW4tGbR8DT7VmmPQ0OaLJk2A5oxDiRgA3Go9dqf38GIC3\n5t+0arCwACxdyg6eYaoED7LGwTNZAygHzxk8w1QHdvBxsMAHaLeLcfBcRcMwfh54ALjuOvtznMHH\nwQIfQEU0eWfw7OAZxs9ttwH/+I/251jg42CBD5A0ool15ezgGcZPqwXMztqf44gmDhb4AO02Z/AM\nUwYLC26B50HWOFjgA8RW0aSJaLhzMoybVguYmXE/B7BJCsECHyA2g08zyMqdk2Hc+Bw8RzRxsMAH\nKLJMUgi+YQHDuPBl8Hk7+AceyOd1qgYLfIDYDD5NRJNke4YZNBYW3BFNnsdPqwWcZN7loiGwwAco\naiYrZ4gM4yfGwecxjqXep4lX0yzwAYrK4JOWVTLV4x3vALY3Zlm96uFz8HkapCabLRb4ALEZfJrF\nxpJsz1SP7dv5vrpF0q86+CaXXLLAByiyDh5oZqcaFHiyWrH0a5C1ycciC3yAIleTTLI9Uz1Y4ItF\nTQa07eM8RZkd/ACTdC2aJEsVJNmeqR4s8MWi9q3NxXMGHwcLfIAkdfBE7OAHCV5uoljUvrUNtBYh\n8E08WbPAB0iSwY+McAY/SLCDLxafg19YAIaHOYMPwQIfIElEE3Mi0LdXr8/UExb4YlHHhiuiSXK8\n+WAHP8AsLABLlshlBYTwb5ekwzXZNQwKLPDFovatLaJRV8w8yOqHBT6AuhQcGvKLd9KIhh18/eEM\nvlj65eCbPB7GAh+g3ZYCH8r7Wi3O4AcNdvDF4nPwrRawfDlHNCFY4AMoBz887M/h00Y0TXQNgwIL\nfLGEBlmTGKqY92niZ8kCH2BhQcYzMREND7IOFizwxcKDrNlhgQ+gO/g8M3iOaOoPZ/DF0q9B1iZf\nTbPAB0iSwXNEMzi027Kqik/QxcEOPjss8AGKyuCb3KkGAf78iqfVkrPDXQLPg6xhWOADJMngk1wy\nsoOvN00WhaqwsAAccog/ouGZrH5Y4APEZvBpI5omdqpBgE/QxdNqSYHniCY9LPABYjN4rqIZLJos\nClVBCXy/ZrI28VhkgQ+gIpqYDJ6raAYHFvjiURFNv2ayNvGzZIEPELtUQZrFxpYsaaZrGARY4Isn\nFNHwIGsYFvgASSKapA4+r0tMpv9wBl88/RpkZYEfYIqc6JTXJSbTf5osClWBB1mzwwIfIDaDTxPR\nsIOvL00Whargc/B5Hj9NvhpjgQ+QZLngpGWSeV1iMv2HBb54fA4+zyvgJn+WQYEnoo1EtJWIJono\nQsc240R0OxHdRUQTubeyRDiDZ2w02fVVhVAVDQ+yhlnie5KIhgFcAeANAHYD+BURXS+EuEfb5lAA\nnwXwJiHELiI6osgG95uiJjolXT+eqRZNFoWqEFMHz2WSfkIO/lQA24QQO4QQ8wCuBXCmsc1/AfAt\nIcQuABBCPJZ/M8sjSR08RzSDAwt88bRawIoVPMiahZDArwOwU/t91+JjOmMADiOim4noNiL64zwb\nWDZJMvgkkQsPstabJotCVejXIGuTZ7J6IxoAnttMP8NSAC8B8HoAKwDcQkS/EEJMZm1cFShyuWB2\n8PWFM/jicQ2yqivppUs5ogkREvjdADZov2+AdPE6OwE8JoSYBjBNRD8F8CIAPQK/efPmZ34eHx/H\n+Ph48hb3mZgMXnW4JDNTVaTTxE41CLCDLx7XIKuaBZ7XTPCqfZYTExOYmJjI5bVCAn8bgDEiOhbA\nHgBnATjb2Ob/ArhicUB2BMBpAP7O9mK6wNeFmAx+YUF2tpDL1+FB1npTNVFoIq5B1tjChyTvo38v\nG9P8XnrppalfyyvwQogWEW0C8H0AwwC+IoS4h4jOX3z+SiHEViL6HoAtANoAviSEuDt1iypGTAbf\nasltkjp4zuDrS5Nz26oQcvDDw5zBhwg5eAghbgRwo/HYlcbvlwG4LN+mVYOYDD6No+AMvt40Obet\nCq4MXhf4vDL4ppotnskaIEa8s0Q0TexUg0DVLuubiKuKpoiIpqnHIgt8gNgMPo2D58XG6ouqmmqi\nKFSFkIPPc5B1+fJmfpYs8AGSZPBpIpomdqpBgAfJi6dfg6xNPhZZ4APEZvBcRTNYNFkUqsLCgpzJ\nOjcHCG1GTppB1h07gLe/3f5cnuvaVA0W+ACxGTw7+MGiyZf1VUHFYEuWAPPz3Y8nNVSPPAJMOqZe\ncgY/wMRk8GkiGnbw9YYFvnjUlfHISHdMk8ZQzc93nyTM92nqZ8kCHyAmg9cjmthOwmWS9YZP0PH8\n7GfA3r3J/kaIzrE3MtI90JpmkLXVch+b7OAHmKLr4JvYqQYB/vziufxy4Oabk/1Nuy1NFZF010U6\n+CZfjbHAB4jpTBzRDB5NFoW8mZ9Pvp/UMQW4HXySK+YYgW/iscgCH6CItWj0xclYIOoJC3w8PnF1\noY4pQO5nl8DnlcE39WqMBT5AbAafxMGrzpvXRA2m//AVWDxpBN508EVHNCzwA0pMBp90sTE90mli\npxoEmuz68iatwCsHn8cgK2fwjJUi1qJRr8kOvr40WRTyJm1Eoxx8HoOsviqaJle0scAHKGItmryX\nO2X6T5Mv6/OmKAefdJC11eqeEau/XlNP1izwAVREk+daNOzg6w9n8PFUZZBV/a1Jk0/WLPABiopo\n2MHXmybPfsybqgyy6t91mvxZssAH0COavKpo0twBiqkWTb6sz5usDj6vQVb9uw47+AFGF++81qLR\nX7OJnWoQaLIo5M3cXDYHb4to8nTwPNFpgInJ4JNGLk2og5+fBw4cKLsV5cEZfDx5DLKaEU2aQVb9\nu/leTb0aY4EPUMRywU2og7/uOuDDHy67FeXRZFHIm6xlkr4qmiRlkvp3872aejXGAh8gJoMfxCqa\np5+WX4NKkwfm8iarg89rsTH9u/leLPADSkwGn6UOvq4CPzcnvwYVdTOKdtteW810qMMga1NP1izw\nAZJm8IMyyMoCn+9J+rvfBR58MPvrVJGsZZJ51sG7yiSbOp7CAh8gSQYf6yiaMsia9KBtErqLzOMk\nfdVVwC9+kf11qoYQcv9kdfCuiIYHWf2wwAcoIoNvwiDroDt4/SSdx2fY1P2p9k3WiU5FOnjO4AcU\nIbpXk8xrPfgmDLKmqW1uEnmfpJsq8D5h9WEuVcAzWdPBAu9BCHnLMKJi1oOvs4Ofn2+mIMWSZqDP\nR1Mjr7QCH+Pg1S39XMbLfD39u/kcZ/ADiF6LGxPRDA3Jk0KoqqIJSxWwg+eIJgb1P2Utk7QJPBBv\nkjiDZ3pQ+TsQt9hYyOnr23MGX284g48jS0QTWmwMiL9qZoFnejA7UqgOXm2nOspNNwF//df27ZtQ\nRdNEQYol7wyeI5puYtaDB/IReJ7JOqCoAVYgbj14oLvDbd8O3Hmne/u6O/gmClIseWfw7OC7iRlk\nBZIJ/NKlXEXDaMR2JL0z6tu5Dlp28PWHM/g4ihxkBeKPofl5YMUKXk2S0UiSwdtOBK6Dlh18/eEM\nPo75eTk2VcQdnYD4Y6jVAkZHe7dVpdDLlrlv6VdnWOA9xGbwrojG5+CT1vFWjaYKUiycwcfhc84+\nYu7oBCSLaEZHe9uhXmtoSH7FlFzWCRZ4D7EZvCuimZ3tdh7m9nm5vzKokyC128D0dL6vWbcM/l3v\nArZsKe71XaQV+JjFxoBkAm9rR5qrgTrBAu8hSQafNqJhB188N9wAnHdevq9Ztwz+oYeARx4p7vVd\n5OHgzYgmrYO3tUM/kdTZcLkICjwRbSSirUQ0SUQXerZ7ORG1iOid+TaxPMwMPiai0R1dKKKp8yDr\n3Fx9MsunnpJfeVK3DN51NZkXF10EPPFE7+OuaCSE745OeQ6yuo7dpuAVeCIaBnAFgI0ATgZwNhGd\n5NjubwF8DwAV0M5SKLqKRs18rWPul7Y6ogyKELc8M3ghio+85uaKFfjrrgN27bK/b9ERTexM1lBE\nM4gO/lQA24QQO4QQ8wCuBXCmZbv/AeCbAB7NuX2lkiSDTxPRAPV1Der/qkNMU4Q7zjODV6JStIMv\n8vVd+ziPiGbZMvn3ygjlOcg66AK/DsBO7fddi489AxGtgxT9zy8+VIOL9jiSrkVjbhdy8Gr7Onaq\ntGuMlEFRDj6viKYfJ8uiHbxrH+cxyErU7eLTDLK6yiRds9CbQkjgY8T67wFcJIQQkPFMoyKamAw+\naUTTBAevDthBdfB5ZvD9EvgyMv48HDzQPZvVZah8DGpEsyTw/G4AG7TfN0C6eJ2XAriWiADgCABv\nJqJ5IcT15ott3rz5mZ/Hx8cxPj6evMV9RI9oklTR6JfcvkFWc/s6wQ4+vwy+H+MZRQ+y+iKa0VHZ\n59Xy2zHopgmQr6EE3qx8yVPgq2C2JiYmMDExkctrhQT+NgBjRHQsgD0AzgJwtr6BEOJ56mciuhrA\nd23iDnQLfB3QhTjNWjQxEU1VOlVS5ubkPhlUB59nBl93By+E38EvWyb3k/o5BtPBj44CU1Od59IO\nsppt1I/xqjh40/xeeumlqV/LG9EIIVoANgH4PoC7AXxDCHEPEZ1PROenfteakLUO3tXpzRNCFTpV\nUubngZUr6yHwnMEX6+CVO3c5+KVL3Qt9udBFHJDirAQ+zzr4pk90Cjl4CCFuBHCj8diVjm3Pzald\nlaCoDN50DXV18IccUo+IpuoZfNERzcKC/CpK4NXruhx8GoHXjxFAirOajVzUTNaqOPg84ZmsBu12\nZ/JO2gxeF3h1cJnb+1zD/DwwNpb9fykSJfCD7ODzyuCLjruKvkJQ+7ZIB29GNHmuRVP3uNQHC7zB\nZZcBl18uf84jgwf8s+dsHXRqCti2rdoToFREM6gOPu8MvsiTpXrdohy87/WzOHgzolEOPqkot9vy\ny3bXJrOijR18w3n0UeDxx+XPSZYLdkU0+nfb9rZO5bvkrQIqbx10B59nBt8PgS/awdv28dxcegdv\nRjRpB1nV9rY2cEQzYMzMdDpq7HLBrkzddWCFHHzVBV79vyMj9RB4V1SWFhXjDQ3ll8EXeTVUdH/y\nnUDycvB6RJN0kNXXhqYPsrLAG8zMdOptYzP4UERjc/C+y0L1/lUV+Lk5We6W9KAtC7Uf82qr+vyI\n8svgi6xICjn4nTvtC4XFUsQgq83Bpx1k9bWhimWSecICb+By8GnWg4+JaOro4NVl97Jl9XHwQH77\nM+/JMUVHNKH+9KlPAddck/71Qw4+jRnwOfg8Bb6KE53yJFgmOWhMT3eqaJJk8C4Hv2KFP6KxdSp1\nIOpLpFYJddCqRaCqjq/KIw1557ZlD7JOT2e7IUq/HLwtookR5SQC3zQHzwJv4Ls1WNJb9s3O2i+9\nQ0sV1CmiGUQHn/dNIvqVwbs+Kz2WzPv185zo5IpoYgdZbZ8VRzQDhh7RpL1ln74WzapV9unRvsvC\nOkU0dXDweVeR5D0TeW5ORhBAMRFB6ASXtcqoqDJJ11IFPMgaDwu8ga+KJk1Es2pVuIrGVSZZ9Yim\nLg4+7xNmERm8EqAi9ufcnOxnRTv4vCMa11IFRWbwLPANZ3q6e9W6mAzeFtH4asVDDr4uEU2dBll9\nApeUIjL4Ivfn7Kz9SlKRVeD7USbZryqapg2yssAbpK2DNzucOjmMjibP4OsS0dSpTPJZz6p2Bl/k\noHWoDDMPBz88XOwgqyuiidn/7OCZZ0ibwZsO3ufKYqtoqirwuiDVxcHborK0FJHBF1l2GnLweWTw\nrv0bMgM/+AFw4429j/fLwbPADxi6m4nN4G0RjU/gQ4uNqfevagav/291cfA+gUtKERl8kWMarsF+\nRR4O3vX6IQf/k58AN93U+3hsmSQPsvphgTeYnu6OaNKuRRPr4Os80akug6xFOPg6RTQqoioyonHt\n39BEp+lpYP/+3sfznOiUpEySM/iGk9daNCEH78sQqy7wdZzolKeDzzuDL3qQNeTg84po0jj4mRng\nqad6H+/XUgUc0QwQQrjXokm6XHCSiKaOVTSD7uCLyOCL2p8+hw3k4+Bdg9ghgXc5+Ng7OvEgqx8W\neI25uc79JYFkdfCmYM/OytUWly3r7fihNajrUgdfh0FWtW9HR6ufwZfl4PMok/RFNCEHX3REw4uN\nMQBkZ9OFIMtaNLERjSuDt50YqkKdBlnn5uSJNs+ljeuWwYfKJLNGNFkGWX0O3hXR8CBrPCzwGjMz\n8kBot+UHbS4XnGQtGiWCNmGJueHH6tXVFvi6RDTqZJnnCbOIDL7oMskVK+TVqSmGrZZ8rMhB1jQZ\nvK1MkleTTA4LvIZy8MuXy06bZLlgM5PNUkUzM5PvxJy8qdMga1EOPu8MPmmZ5D/8A/Dgg/Gv74oL\n8yjJzTLIGuvg1TGpzFdSgVdVNL7bZ3JE03Cmp6XAj4z0CnyeZZIxi40961nVzeDrtJpkEQ6+Chn8\n174G/PrXcduqfWA7yeVRseUrw4zN4NUS3QrTwRNJkVcuXkWnMaLcasn3X7Kkc39W2/uwwPeJzZuB\nb36z/+87MyM70ciI/FnP4IeGZCc0OyKQbSara6mCOkQ0dRhkbWoGPzVld7429H1gc/ArVhTn4EMz\nWdX9Fw4e7H7cdPCAbOf+/d3CnySiIer9vLiKpgS2b5e3Ees3SuD1y0HVyYikyNty+DRlkr7JFTMz\n1Rb4tHfpKYOmZvCuaMOGvg/M11d9rayZrOp9zf/FdPCAvLo+cKBb+JMIPNDbDh5kLYGpqc6lWD/R\nHbwZ0QDuHD5LRONy8L4MfscO+5VEv6jTapJNzeDzcvB6X0vbp1SVzvx872vEzGRdvrx3oNXl4A8c\nSO/ggd528EzWEihL4FVn0yOaGLdQxGJjPlf1pjcBk5Pp/sc80B3nIDr4vDP4NPMKpqak2MXg64sq\noslyslZzPmwnqBgHv3Zt78nKnOgEFCPwHNGUQJkO3qyiGdL2kKsz2SIa30Sn0HLBoSqa/fvjD+4i\nqNMNP/QKkqpm8GnmFaSJaFwZvHL3aWMa1ddtV0kxVTQ2gY+NaJLMZAVY4CtBVSIaPYMH3LXwWSMa\nl4N3CXxZ+0dRRwdvE7e0VCGDTxPR2MzG7Gxn3CmtwPteP8bBH3WU3cHnNcjqE3GeyVoCVRB4W0Rj\ny+DbbZk76mVboYlOMUsV+Bx8FQR+0B18mRl8u+2e4m/DVyZpFhakwfX6QvgFXv1++OFxDl5FNEUO\nsnIG3wemp8vN4G0TnQB7B3Btk3WxMVcGPzcnO6VZVtZP6rQWTREOvsgMPuaKyFV54iJUJpk1onG9\nvoo4h4bsAq9OLqtXxw2yqoiGM/h4KinwU1OddSf6icrgbXXwgFvgbR0uySBrEgevTnxlO/i6RDR1\nyeCTRDTq2IgdhwmVSWaNaFwD2T5hVf/H6Kjs67EOPksdvK0dPJO1BMqOaGx18IA9g0/r4EOLjbky\n+CoIfJ0GWX3xRFrKXg9effZ5lUlmjWhcpaghgVfHm03gfWWSeQ6y8kzWEihb4JPUwZsdMWYtmlCn\n8k10UtFMmRFN3VaTLKJMsswMfmpK9o8kAu9z8HlV0Zj7WF2ZAMkdvK1MkiOa5FRO4Ofn5U6uSh28\nK6J529vk9lkjGpeDd61FUwUHX7fVJIteqqDfGfz0NPDsZ+dXJplXRGNz8MuWyZ+TOHgheq+egWwR\njf558SBriZQpYL7VJIFOBxBC3gn+0Uf9EY0r+/XNnlOVB1XO4Ou2mmTRE536ncFPTQFr1kgRjNm+\nrDLJmIhmdLR3kFUZK6Lu7dNW0ajFxlQ7uEyyRKan5QdbhYjGlcGrSpZ9++wRjZrolGapgtlZ2Qld\nmWgVBL5OSxUU4eCLyuBjr4imp6XYrVoVN9AaKpN05fOx+Bx8KKKxOXjbACvAEU0aogSeiDYS0VYi\nmiSiCy3Pv4eI7iCiLUT0cyJ6YdoGKXdSVhVNTB3800/L3598Mi6iCd2yT++guiDZ1gepSgbvi2hm\nZoAf/aj/7bJRpww+iYN3Zdeu1/eVSWZx8EJ0z/lI4+DN/8M2wAoUM8g68AJPRMMArgCwEcDJAM4m\nopOMzbYD+I9CiBcC+CSAL6Zt0NQUcMQR1amDt2XwSuD37QtX0YTu6GRz8MuXd2qHzb+tgoNXEY06\nOZmVRbffDnzwg+W0zaTqGbxan3x4OFkGv2JFvMD7riazRjStluyrqv1JyyTLdvChZUPqToyDPxXA\nNiHEDiHEPIBrAZypbyCEuEUIoVK0WwGsT9ugqSn5gassup/odfC+DF4XeFdEk3a5YHXJDNgd19QU\ncOih1YhoiOwH7oED5a6Vo1P1DF5fq7wsB58lolEnUMAf0djupqQ7eD2D9zn4IurgB/2WfesA6Kuz\n71p8zMV/A3BD2gZNTckPUr8HY78wIxpXBm86+KxVNKaD9wn8wYPyCqcKEQ1gd51lL4amU/UMXvUT\nIFmZZKyDV4KlHHbeE53UyQPI5uAPHOjEkS4HX/RSBU2MaCy7sYfoVaKJ6D8BeB+AV9me37x58zM/\nj4+PY3x8vGcbdfmpBH716th3z4450Sk2gzczwYUF2VFsAt9ud24eAvR2UHXJDLgdfFkRlsIsfzNF\nQzl4IXorIfpNPzL4LK5P35dJZrLGDrKqeAaQ/enxx3ufzyLw5usnneg0OiqPmeXLpWlZudLt4EdH\ne+vjY2KVUJlk1apoJiYmMDExkctrxQj8bgAbtN83QLr4LhYHVr8EYKMQYp/thXSBd1Gmg9fr4FUV\nTSiDd0U0LoE3tzc7lR7R2A66qSngyCOB3bvz+Z/TYDp4m8CryV7qfymLfmTwWR2872rIRpKIxuew\ngeyLjfleP2aikzIz6n9RAu9y8IDdUPkIlUlWzcGb5vfSSy9N/VoxEc1tAMaI6FgiWgbgLADX6xsQ\n0XMAfBvAHwkhtqVuDboFvt+VNLa1aFwZPFF4kHVkpBPrqE4YWsAsJqI58sjyIxrdddoyeP17mZhR\nWR53wspb4NM6+BiB9zlsIPtMVvP101TRAN3/iy+iAYrL4AdykFUI0QKwCcD3AdwN4BtCiHuI6Hwi\nOn9xs/8JYA2AzxPR7UT0y7QNqkIGH1qL5umn5UzCmDJJNXimOpW5vSkQTYlo9O9lok6YqtIjjwO4\nChl8GgcfWosmawYfmslq7ifTwauBVl9EA/AgaxJiIhoIIW4EcKPx2JXaz+cBOC+PBqnOW/Ygq5pw\n5Mrg169i8UoXAAAYdklEQVR3O/hWq9vZKGe2fLk70tHbEKqiOe648qtoQoOsQDUE3nTI6nPNgi2D\nTzveYGbwScokieT9eX3Y+qFO1ohGv+JMM5P10EPlz/raOiEHn+cgK89k7TPKwY+OlpvB+9aiefpp\nYMOGuDJJoLvjxzj4mAy+bIGvi4P3Ocy06K5PDZjb7vQV2z7feIYNdYysWpXdwecd0aSZyQp0X434\nyiQB//FjY5CraCon8GYVTT+JiWh0Bx8T0QDdB25MBu+LaA4elHfAUWWcZRBynUrY1WB0mZgONo9K\nGnMQMIswpMng9YgmdBINvX6ZZZJJM3i1bRoHH7PYGAt8HygrgxcivFywnsErBx+ayQp0O5skVTSu\niGblStnOMpZzAOKqaFavbq6DD12FJW1f0gw+yyBr3hl8UgevD3IndfBLl8p9neaerDGLjfFqkn2g\nrCoadZYfHo5bi+bIIztiHxPR6A7e10FDVTRq/xxySDkxjcqb1f/simiOProaAl+Ug0/qInXUcrhA\ntwiWVSaZZSZrrINXt+4zx5t0B68GWV0OHpB9nxcbi6eSAl/GIKvuJmLWolm1Sg4QPf54sojG5uBd\nAm9zVQcPSnFfsaKcUkkVz6gBRVdEc8wx1RB4U4DyzuCB5MLw2c8Cl1zSaV/ZZZJZHbzLkOjCCvSK\nq37M6YOsrjp4QGoDz2SNp5ICX0ZEozo64F8uWAn8ypVy1cvHHkvu4G1VN7526JRZRgp0xzOA3cHv\n359d4P/5n4Errkj/94pQRJGGrAK/a5f8ArJl8HkMsuZRJhkT0QC94urK4F0RDdDr4PMYZG1yFU1U\nmWQ/Ue6k31U0emfzTXRSsczKldLB+wTe5hxt+W2aiKYsgdcHWAG3g1+3Ltsg609/mk8eWkUHv2+f\n/FLty5LBJ1mqwDXImtdiY76ZrIDfwccMsgLy/867TJIdfB+pgoOPWYtGd/A2wXYNRLru4aqoegYf\ncvBCyOgoawZ/330dEcxCEQ4+VAkV4okn5BcQHoi0oZdJ6ot02fA5+Ha7c8IuapBVNwN5OHi1do0i\n75msPMhaMGUJvO4m9CoaVwbvi2hmZ+V39bdJBlnNE41+0CnxVPunjAxed5xAr4M/eFC2O2sVzeRk\nRwSzUCcHPzQU91oqohkelt99/cDn4FU8Q1R8mSQQdvBFDrJymWRFKKuKRhdW9YEroVaYAu+KaKam\nekVQn+jkK5P0Ofi5uU6ZWJUiGl00DhyQzjL2dnI2nn5aLqaWh8BXMYN3CTwQF9OoiAYID7SaGbn+\n/4dKcmOILZME/A7eHGT1ZfBJrp7a7d6qL74na4mUVUWjdzagU2duy+DVsqa2iMYl8L6IJjaDVyc/\noLoRjRL4lSvTC/y2bfKAb6qDNyMa3wnTZGGh+29CA60+h+27Woyl3w7ejGhC+16/oYqtDfpnqWYk\nlzWBsAgqKfBlRzSA7LQHD/Zm8HNzshOPjrod/PR090GrO5vQJBnbWIBCxTNAdSOa/fvlwbpqVfpB\n1slJ4NRTswu8ypiVyFQlg9+3T/aR2dneE2aoFl5fhwYID7T6HLY+a7qMMkndVB12WOfzztPB+9og\nRPdnSdS8BccqJ/BlLVWgCysgf56a6s3g9++X7plIOvgnnrC7fJcrS7pcsH7QTU3J9waqE9G4HHyW\niOa++4BTTpHvlUZ0zLbqNftlO3gV8R1+uBR62wnT10b9Kg6Ii2hcAmyLaJIup5ylTFI3VatWyeem\npvLN4H1tUPd70BeJa9pAa+UEvqzFxkyBHxmR72+KsbopASAFvt3u7XBAfESTJIPXD+6q1MGbjjMv\ngX/+86Wry1JJo7tXoBoZ/JNPyn1zxBF2gQ9l8NPT3VFiSOD1faD6nRIwc9yJKHkGHbuaJNAtrq2W\nPHbU80Rydvijj4YnOuUl8Lb3aVoOXymBn5/vfOhlZ/DKPZsC/9RTHYFXS52a26i/V6StoqlqBl/0\nIOt99wFjY92X7Wnb6vocspBF4Pftk8ZAXf2ZAhSKaLI4eKC7T+kRDZAupkk7yKr6ue6ejzqqI/Cu\niOaNbwRe8YrO7zECb35WgyTwlZropOeL/a6iMTN49bMe0QwNdQv8mjXyu03g83Dwtgxej2j27k32\nP+ZBqLbZHGRNs0765CRw4onZBd7m4PNabCxtBq8EXl2dzM11C3ZMRKMbkdAg6+ysPAmYr79ihT2W\nnJmRrxlL2kFW01ABHQfvi2jOOKP799B6/CEHb55ImibwlXLweuctO4NXndbn4JXAJ4loki42Zmbw\n/Yxovvxl4F//tfsxW0SjC9L+/VIgRkbkAZdUUB9/XB6ARx1VjIMvO6LZt0/+X2vWpMvg9RJJIFmZ\nJNDt4PUM3nwuFrO/6m33zWQ1DRXQHdG4HLyJytBdlS/6SpKqDeqzsp1IeJC1QPTOW1WB1zN4X0QT\nO8jqq6IpO6KZmAB+/vPux2IGWZVjTBPTTE7KeIaoug4+i8A/8UR3RJM0gzcdfKiKxneSczn4JPjq\n7H1Xe2kdvA3fFVTSDN6cWV53KiXwZQ4i2urgAb+DX71aClGMwKuOn6UOvt9lknv2yC+d2EFWIL3A\nn3ii/DnrIGtVHbwe0STN4JM6eN9JLq8MXu1jJZ62pZD159X/kYeDB/IVeI5oCkQX+KVLZa4Wsz52\nHtjq4AF/Bj80JA+woiIaM4Pvd5mkS+BjyiSBdAJ/330dgVcuNy1VzeCzRDTmIGuSiU7q9fOOaFT7\nzRvMF5HB20gr8ObnCLDAF4rZefvp4tNk8IA8UEMO3pzolCSiKTOD371bfumEVpPUBT7NbFZVQQPk\nk8G7rqSyUGREE5PB68J4xBHAI4+4t/ed5PKKaFwnkKQOXq+iSSLwvv3PDr5CmPliPytpbJ0d8Gfw\nQK/Aq5swx1bRqCsEdVlblaUKDhyQ771nT/fkl9Agax4RTV4Cr+9LoBoZvK2KJmkGr5ug5z0P2L7d\nvX0/yiRd+ziNg3/kkfwjmqRlkjzIWhBlO3izDh7oFXi10Jji0EN7O+PwcHxEA3QLRFUy+N27gec+\nt3PVoggJkqqiAZIvVyCEXIemyQ7ejGhiMvh77wW2bpU/mybo2GOBnTvd7286eL0vxkQ0QoSXI3bt\n47QZfJ4RTaiKxnbssoMvCHMAqZ8CH5vBA70O3jYS75pgY3MnegfVD7oyM/g9e+RNO9at687hYyKa\ntFU0jz8u94UqP62qg8+SwesRTWwG/9nPyi+g9xgZGQHWru3cIcokVCYZcvBXXQVccIH7/8nbwfMg\na75USuDLdvAxEQ1QnINXi2PF1MEXHdHs3i1vu3fMMd05fOxqkkBygb//fuD44zu/N9XBq4gmNoOf\nnJRfQO8xAvhjGl9GHhPR3HEHsGWL+//xnUCSOvjVq+X7T00ld/CcwduptMD3cz2a2EFWoFvgTzlF\nHmA6IYF3OXh1sKgZeWVGNC4HH1pNMovAb9/evS+r6uDzWqpAOfjQPW63bZNfQO8gK+AX+NAgayii\nue8++eXC3MexEY3NwRPJQeOHH07m4H25+aDPZK3UUgVVcvCxAr9pU+9r+QTe16lCB5we0SxfLl/T\ndsLIgz17ZL47PNwb0ej7yTfIunJl+KbQOqaDX71aZvhJqyoURTh4Iew3Y0+awS9f3im79Z0w5+eB\nhx7q/Jyng4+JaCYngd//vnfsSeE7gdjGa3wOHpCVNL//fadUNoYsZZI8yNpHyqyiMZ2Ray0awN7R\ndZJGNKqDmpfM5gp/+sFd9Ho9u3d3HHwoolEHzOysFD8lKEkHWU0HPzQkRf7JJ9P9DzZ3mdXBq9s4\n6uuexIrC/Lz8vNQJ8LDDpFv1RTQPPihjsnXrgB073A7+/vvt75llotPcnPzsn//8TkRkkrZM0ubg\nAZnDJ3XweVbR8CBrgdTRwduwCbzrln1Ax8GbgqTaoQ46PaIBio1p9uzpZPChiEYJhnLvSvyyRjRA\nttmsvnw4LVly23375JiN2j9r1shBRV9V0uQkcMIJ8mty0u7gjz8+3sGH1qLRBX77dmD9euAFL3AL\nfNpBVpeDP/JI6eDLWqqAI5oCKbOKxiXw5g0/gGwO3ldFYx5wqh2zs/I9+3kCVIOsw8PdDt52wOgC\nr69cmHWQFcg2m7UIB59V4A87rPP7mjW9ZXxmRKPKRok6Ap8lgzfLJE0Hr58A1bIRJ57ozuGTlkkq\nQzIz091XFKqSJq9BVnP/Dg93YrZBmMlaKYEvy8ELIbNQlW8DsrPbhBhILvC+W/YB3Q7edDX6Qadn\n8EBx+6fdli7qmGNk22IHWfX8HUgm8DMzcpLL+vXdj2cZaC3KwaetnVYlkgr1sy+iUQ5+aEiKvWmC\nADkwOTcnoyy1AJ7Ctw9sEY0+50HNKh4bA37yk97/Rwj762d18PpNsmNIMshK1GkHT3TqM2UJ/OSk\ndEXPfnbnsZGRbvcOJMvg09TBuyIaXeD1/VNUqeRjj0lxXr5c7pO9ezszbWMiGkWSpQp27ACe85ze\nAy6LwBeVwacVBVVBo1Bu3ifwysH7Ihoi6eIfeKD3PZOuRaNHNMrBj43ZHbzqy/pxElsm6crgjzpK\nfi8qotHbwRl8nymrTPKmm4DXva574GxkxO3gdRdtY8mS5IOstioa1Y5+Z/CqRFK1fc2aznonvoPW\n5uBjB1lt+TtQTQefJaIJOXhXBj821nHwNmF0xTRJ16IxI5qxMSnytgzeZkiyzGQFpIMHipvopLdj\nECKaygm8WUXTD4H/8Y+B17+++zFXRDM6Gu58SevglQO0RTT6zZD7dYWjBlgV+kBrEgefJKKx5e9A\n9Rx83hk84M7gVYnk854HHHecXJLgqad6HTxgr6SxRShJJjqplT2POkq+jvk5uK44085kBToCX5aD\nZ4EvENsga9Flku02cPPN0sHruBx8KJ5R2yUdZJ2ftx8wylUpJ6b/bVERjSqRVOilkr6bOOzf7x5k\n/bd/Az7zGfd71snB55XBhyKaBx8Ejj5atnvZss7nYBNGWyVNqyXjE729oYlOSuCnpmRUt2GDvLId\nG+t18eb+Ve2fne0Irv7eRTp430xWm4izwC9CRBuJaCsRTRLRhY5tPrP4/B1EdEraxpSRwW/ZAhx+\neO/gniuDzyrwtojmFa8AvvpVfxWNGc8AxUY0PgfvWk3S5eDbbeCjHwX+4i/cJY/bt9fDweeZwYcG\nWfWF1wAZ1QBuB28KvJm/A+G1aNRz27bJ11RCa4tpfA7eNAJAcQ5+7Vrgrrvsz5lVNKodrZb9ZD0y\nItdEagpegSeiYQBXANgI4GQAZxPRScY2bwFwghBiDMCfAPh82saUIfA33dQbzwC9Ec3ExEQuDt4W\n0fzlXwLXXAPcdps9olHrc9gEPnb/TExMxG0Iv4P3rSZpCrxaduFf/kXut3e8A/jc5+zvef/9bgc/\nORnfdp2qZPBq38dENPr+VPm7Qol9bAZv5u9AuExSOXh92eaJiQnrQKvPwR886I5GALeDV2s7JXHw\nF18M/NVf2Y+F+Xlg164JaztsJ+tzzwUuu6z4u6X1i5CDPxXANiHEDiHEPIBrAZxpbPM2AF8FACHE\nrQAOJaK1aRpThsDb8negN6LJKvCuW/YB0rVs3iw7lstxmSWSQHEC73LwU1Pyu+mKXYOsgPz9kkuA\niy4CLrxQxjRmm4WQFSAugTcP0FhMB6ucdpYyuCwCb4tozAhF35+mgx8bk+9lCicgl3beuVPWkCts\nAnzIIcA998jnfBm8KfBJHPzMDPBnfwa8853dz8U4+KEheTwkcfAvf7m8CrZFgPPzwIMPTljbYfss\nX/1q4DWvAf7mb+Lfv8qEBH4dgJ3a77sWHwttYwQekv37ZedyHQzmIKutiuaJJ4Bf/Qq48UbgW9/q\nvaWcEDJT/9M/lbGHL8Ofnwd+9jNgfLz3OdcgaxERDQCcfz7wB3/gzuBtDr6fGfyWLXI/vfSlwOmn\nd54bHpYRzLvfDXzxi71riCjBf/vbgZNOAl75SuDqq7u3efhh+b+YJwdAiuD0NHDrrcA558gT4d69\ncf+H6WCJsi84liWDt0U0PodtOvgTTrDHM4D8vy64QC4rsGkT8Jvf2CO/d75TRpIbN8oKJ9csV/3W\niYC9VNIl8F/4gly++Moru5/TlwlwOXggucAD0sFffnlvnDc/3/t5+QQeAD79aXmlaSs7rRuh3ehZ\n6r8LMn63/t3RR3fqql/84u7ODsgOp3fglSvlcqVvfav8MLZulfnY2FinE7z//TI/f+5z5d9MTsoP\n9D3vAb7xDeAjHwFOO603TwfkZdjxx8sOb7Jihf0S0yZCJsuWdZ+o1JoyZ5whs8KLLur9m+Fh4J/+\nqXfdldFR4FOfku0xHfzKlcCXvwz88pfhNt17L/DrX4e3A+R+1gV+/Xp5Uv3kJ4FPfKK7nJRIOvPj\nj5dXIOZYxurVwAc+0DnILrxQfp7f+548GR840FnvxMbhh8uD9uyzgT//cxnlnHQS8LKXyf3abssv\nNTtR/7rrLukkdZYvlyKXZvEyQFaxmP1i2TLg+uvdgqD2/ZYt3RHN4YfbI7lbbpH76Oc/l6KlOPFE\nf4nupz8NfOhDwBVXyP6/a5c85nRGR6Ux+vCHpRHS33/FCuB3v5PvfeutwHvf23lubKzznGLfvl6B\nHx2Vr/md79j/t1tvla/x0EPuk9XatfarFB8nngj84R/Kq3G9D951l+wvZjs2bZLH2qte1fta69fL\n/fjmN3dfQQHAmWcC552XrG1lQsJzuxYiOh3AZiHExsXfLwbQFkL8rbbNFwBMCCGuXfx9K4DXCiH2\nGq8Ve7JgGIZhNIQQpomOIuRjbgMwRkTHAtgD4CwAZxvbXA9gE4BrF08IT5rinqWBDMMwTDq8Ai+E\naBHRJgDfBzAM4CtCiHuI6PzF568UQtxARG8hom0ADgI4t/BWMwzDMEG8EQ3DMAxTXwqfyRozUapK\nENEGIrqZiH5HRHcR0QcWHz+MiH5IRPcR0Q+I6NDQa5UJEQ0T0e1E9N3F32vTfiI6lIi+SUT3ENHd\nRHRazdp/8WL/uZOIriGikaq2n4iuIqK9RHSn9pizrYv/2+TiMf2fy2l1B0f7/9di37mDiL5NRKu1\n5yrffu25jxBRm4gO0x5L1P5CBT5molQFmQfwISHECwCcDuC/L7b5IgA/FEKcCOCmxd+rzAUA7kan\noqlO7f8/AG4QQpwE4IUAtqIm7V8cr3o/gJcIIf4DZLT5blS3/VdDHp861rYS0cmQ43AnL/7N54io\n7OVObO3/AYAXCCFeBOA+ABcDtWo/iGgDgDcCeFB7LHH7i/7nYiZKVQohxMNCiN8u/vw0gHsga/2f\nmdC1+P3t5bQwDBGtB/AWAF9Gp4S1Fu1fdFuvEUJcBchxICHEU6hJ+wHshzQJK4hoCYAVkAUKlWy/\nEOLfAZgLSLjaeiaArwsh5oUQOwBsgzzGS8PWfiHED4UQiwtc41Z05uXUov2L/B2AjxmPJW5/0QIf\nM1Gqsiy6sVMgO8larTpoL4BUs3X7xP8G8FEAbe2xurT/OACPEtHVRPQbIvoSER2CmrRfCPEEgMsB\nPAQp7E8KIX6ImrR/EVdbj4E8hhV1OJ7fB+CGxZ9r0X4iOhPALiHEFuOpxO0vWuBrO4JLRCsBfAvA\nBUKIrkVvhRyZruT/RkRnAHhECHE7eiegAah2+yEru14C4HNCiJdAVmZ1xRlVbj8RHQ/ggwCOhTwg\nVxLRH+nbVLn9JhFtrez/QUSfADAnhLjGs1ml2k9EKwB8HMAl+sOeP/G2v2iB3w1gg/b7BnSfgSoJ\nES2FFPevCSG+s/jwXiJ69uLzRwN4pKz2BXglgLcR0QMAvg7gdUT0NdSn/bsg3cuvFn//JqTgP1yT\n9r8MwP8TQjwuhGgB+DaAV6A+7QfcfcU8ntcvPlY5iOgcyJjyPdrDdWj/8ZDm4I7FY3g9gF+TXN8r\ncfuLFvhnJkoR0TLIAYLrC37PTBARAfgKgLuFEH+vPXU9ADV5+70AvmP+bRUQQnxcCLFBCHEc5ODe\nj4UQf4z6tP9hADuJSK2E8gYAvwPwXdSg/ZADwqcT0ehiX3oD5GB3XdoPuPvK9QDeTUTLiOg4AGMA\nIhbK6C9EtBEyojxTCKHdwqT67RdC3CmEWCuEOG7xGN4FOWC/F2naL4Qo9AvAmwHcCzkgcHHR75dD\ne18NmV3/FsDti18bARwG4EeQo/I/AHBo2W2N+F9eC+D6xZ9r034ALwLwKwB3QDrg1TVr/8cgT0p3\nQg5SLq1q+yGv8vYAmIMcLzvX11bI+GAb5InsTRVs//sATEJWn6jj93M1aP+s2v/G89sBHJa2/TzR\niWEYpqGUXQPKMAzDFAQLPMMwTENhgWcYhmkoLPAMwzANhQWeYRimobDAMwzDNBQWeIZhmIbCAs8w\nDNNQ/j88WrdTzbL6FwAAAABJRU5ErkJggg==\n", 590 | "text/plain": [ 591 | "" 592 | ] 593 | }, 594 | "metadata": {}, 595 | "output_type": "display_data" 596 | } 597 | ], 598 | "source": [ 599 | "plt.plot(range(F_train.shape[0]), F_train[:,10])" 600 | ] 601 | }, 602 | { 603 | "cell_type": "code", 604 | "execution_count": null, 605 | "metadata": { 606 | "collapsed": true 607 | }, 608 | "outputs": [], 609 | "source": [] 610 | } 611 | ], 612 | "metadata": { 613 | "kernelspec": { 614 | "display_name": "Python 3", 615 | "language": "python", 616 | "name": "python3" 617 | }, 618 | "language_info": { 619 | "codemirror_mode": { 620 | "name": "ipython", 621 | "version": 3 622 | }, 623 | "file_extension": ".py", 624 | "mimetype": "text/x-python", 625 | "name": "python", 626 | "nbconvert_exporter": "python", 627 | "pygments_lexer": "ipython3", 628 | "version": "3.4.3" 629 | } 630 | }, 631 | "nbformat": 4, 632 | "nbformat_minor": 0 633 | } 634 | -------------------------------------------------------------------------------- /Mini_batch_K_Means.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "https://algorithmicthoughts.wordpress.com/2013/07/26/machine-learning-mini-batch-k-means/" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 51, 13 | "metadata": { 14 | "collapsed": true 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "from __future__ import division\n" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 52, 24 | "metadata": { 25 | "collapsed": true 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "\n", 30 | "def get_closest(x, centers):\n", 31 | " distances = []\n", 32 | " \n", 33 | " #get the distance from the point(x) to every centroid\n", 34 | " #and get the index of centroid that yields \n", 35 | " #the minimum distance \n", 36 | " \n", 37 | " for mu in centers:\n", 38 | " distances.append(np.linalg.norm(x-mu))\n", 39 | " index = np.argmin(distances)\n", 40 | " \n", 41 | " return index, min(distances)\n", 42 | " " 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 83, 48 | "metadata": { 49 | "collapsed": true 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "'''\n", 54 | "This function returns the value of the k-means objective function\n", 55 | "\n", 56 | "Input:\n", 57 | "X- data points to be clustered\n", 58 | "k- number of clusters\n", 59 | "C- Information about which cluster each data-point corresponds to\n", 60 | "centers- list of k centroids \n", 61 | "\n", 62 | "Output:\n", 63 | "sum_distance - value obtained by evaluating the k means objective function\n", 64 | "\n", 65 | "'''\n", 66 | "\n", 67 | "def k_means_objective(X, k, C, centers):\n", 68 | " \n", 69 | " sum_distance = 0\n", 70 | " for j in range(k):\n", 71 | " for x in X[C==j]:\n", 72 | " sum_distance += (np.linalg.norm(x-centers[j]))**2\n", 73 | " \n", 74 | " return sum_distance" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 53, 80 | "metadata": { 81 | "collapsed": true 82 | }, 83 | "outputs": [], 84 | "source": [ 85 | "def mykmeans_plus_plus(p, k, max_iter):\n", 86 | " centers = []\n", 87 | " centers_list = []\n", 88 | " distortion = []\n", 89 | " centers.append(random.sample(p,1))\n", 90 | " \n", 91 | " for r in range(k-1):\n", 92 | " x = random.random()\n", 93 | " num_dx = (get_closest(p[0],centers)[1])**2\n", 94 | " den_dx = 0\n", 95 | " for x_i in p:\n", 96 | " den_dx += (get_closest(x_i,centers)[1])**2\n", 97 | " index = 0\n", 98 | " while num_dx/den_dx < x:\n", 99 | " \n", 100 | " index += 1\n", 101 | " num_dx += (get_closest(p[index],centers)[1])**2\n", 102 | "\n", 103 | " centers.append(p[index])\n", 104 | " \n", 105 | " centers_list.append(centers)\n", 106 | " for t in range(max_iter): \n", 107 | " C=[0]*len(p)\n", 108 | " \n", 109 | " \n", 110 | " #assign clusters\n", 111 | " for i in range(len(p)): \n", 112 | " index = get_closest(p[i], centers)[0]\n", 113 | " C[i] = index\n", 114 | " \n", 115 | " C = np.array(C)\n", 116 | " \n", 117 | " distortion.append(k_means_objective(p, k, C, centers))\n", 118 | " \n", 119 | " centers = [p[C==j].mean(axis = 0) for j in range(k)]\n", 120 | " centers_list.append(centers)\n", 121 | " \n", 122 | " C = np.array(C)\n", 123 | " \n", 124 | " return C, centers, distortion, centers_list " 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 84, 130 | "metadata": { 131 | "collapsed": false 132 | }, 133 | "outputs": [ 134 | { 135 | "data": { 136 | "text/plain": [ 137 | "" 138 | ] 139 | }, 140 | "execution_count": 84, 141 | "metadata": {}, 142 | "output_type": "execute_result" 143 | }, 144 | { 145 | "data": { 146 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEACAYAAACqOy3+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHk1JREFUeJzt3X2Q1PV9B/D35zj3QOAOjsOSKLiokNDwdKgkmkxdJjGx\nTKfINEGd0sY4fZCSRhxLo1LrUetISicySax2Jg+aMtE4NphSsYYYVuNIkYc7DgQFCweIQeGOe0DE\nA+7TP3Z/yz78fru/3d/z7vs1w8zdPn485LPf+3w/389PVBVERBQ9dUEHQERElWECJyKKKCZwIqKI\nYgInIoooJnAioohiAiciiihHCVxEhovIFhHpEJE9IvKIW4EREVFx4rQPXEQuVtXTIlIP4DUAf6eq\nr7kSHRERWXJcQlHV0+kvYwCGAehx+ppERFSa4wQuInUi0gHgfQCbVHWP87CIiKgUN1bgQ6o6G8Bl\nAP5ARBKOoyIiopLq3XohVe0TkRcAXAMgadwuIhy2QkRUAVWVYvc77UJpEZEx6a9HALgRQLtJEKH/\n8+CDDwYeA+NknFGNkXG6/8cOpyvwTwB4SkTqkPow+A9VfdnhaxIRkQ2OEriq7gIwx6VYiIioDDyJ\nmZZIJIIOwRbG6a4oxBmFGAHGGQTHB3lKvoGIev0eRETVRkSgXm5iEhFRcJjAiYgiyrU+cKIw6t50\nCIcfT3W2TlrSinHzLg84IiL3sAZOVat70yF03r4BQ2fOAQDqhtdj5pPzmcQpElgDp5p2+PH2TPIG\ngKEz5zKrcaJqwARORBRRTOBUtSYtaUXd8AvbPHXD6zFpSWuAERG5izVwqmrcxKSoslMDZwInIgoh\nbmISEVUxJnAioohiAiciiiiexIywzTs6sXbdBgDA4oXzcd2cmQFHRER+4iZmRG3e0YnlD6/Bx4OD\nAICGWAyrVyxjEieqEtzErGJr123IJG8A+HhwMLMaJ6LawARORBRRTOARtXjhfDTEYpnvG2IxLF44\nP8CIiMhvrIFHGDcxiaoXT2ISEUUUNzGJiKoYEzgRUUQ5TuAiMlFENonImyKyW0S+5UZgRERUnOMa\nuIhMADBBVTtEZBSA7QBuVtW96ftZAyfXcUwsVTs7NXDHR+lV9RiAY+mvT4nIXgCfBLDX6WsTmcm/\n1mXvlt/xWpdUk1ytgYtIHEArgC1uvi5RNl7rkijFtWFW6fLJcwDuUtVT2fe1tbVlvk4kEkgkEm69\nLRFRVUgmk0gmk2U9x5U+cBG5CMB/A3hRVdfk3ccaOLkqv4RSN7yeJRSqOr4c5BERAfAUgG5Vvdvk\nfiZwcsRsw5KbmFTt/ErgXwDwKoBOAMaL3aeq/5O+nwmcKsbVNtUqv7pQXgMPBJFHrDYsmcCJeEUe\noooUK+GwvEN+YQKnUJu0pBW9W36XU0KZtKQ10JiK9aEH1aPOD43axNIHhdq4eZdj5pPz0XzDRDTf\nMDEU9e9ifehB9KgbHxo9rxxBzytH0Hn7BnRvOuTpe1I4cAVOoTdu3uWBJ+0w4z5B7eIKnKhMk5a0\nom74hbVPdlmn2H1e6N50CP07P/Ds9SnceEEHogqEYRMzv95uYKtldeAVeYgsVMPhoPZFz6PnlSM5\nt9WPacD0J74S+tipNF/6wIn84GZyNesUid99Dboe3WareyTMib5x1iWhioe8xQROoed2a57Vpp+d\njcAwjbINY4sl+YubmBR6YRofG6ZYwthiSf7iCpxqjtXKNbuEEpXVLFssaxsTOIWe26UCY+WaX8du\nnHVJydo2yxYUJuxCoUgI08ah37GE6b+d/MM2QqKI4zjd2mUngXMTk6iE7k2H0L7oebQvet73GSNh\n2jSl8GENnChLfrkCQGjaBonyMYETpZn1eF88ZWygg6K4aUrFMIETpZmVK84c6Q8wIuuOGSKACZyo\nqOETR+P0mfOBroDZ601WmMCJ0szKFVetuB4AQr8CZqthbWIbIVGWKCZCthpWJ/aBh9DmHZ1Yu24D\nAGDxwvm4bs7MgCOiUsKe1M3GyjbfMBGtz94cUETkBo6TDZnNOzqx/OE1+HhwEACwc88+rF6xjEk8\nxMI0fZAon+ODPCLyYxF5X0R2uRFQNVu7bkMmeQPAx4ODmdU4+cM4lLPlS89gy5eeLnk4JwoHafy+\njBuFhxsr8J8A+D6An7rwWkSesboEWdRX1Ww1rF2OV+Cq+lsAJ12IpeotXjgfDbFY5vuGWAyLF84P\nMKLakr+aNhRbVeevbiHAYM8Z34/UlzJu3uVoffZmtD57M5N3DXFlE1NE4gDWq+oMk/uqdhOzkg1J\nbmIGx2yzz1Bs06970yG88/DrOPVmNzCU+n85TJ0eYd9kpcqEZhOzra0t83UikUAikfDjbT1V6Ybk\ndXNmMmkHJL/P21CqZjxu3uWpBDl0YSHi95F6K042WZn4wyWZTCKZTJb1HK7AK7T0gVV4o2N3zm1z\nZ0/HYw/dm3MbV9zhYiStwZ4zABSx5hG2kldYW/UqjYu94+EXmhV4rWLbYPhUciy9e9OhVMIXAOm1\nSNQ7PexexJnCzY02wqcBvA5gqogcEZFvOA8r/OxsSLJtMPqMleqpXcdTybtOMGpGS2hWq2OuvzT1\nwZIW9Q8WKo/jFbiq3uZGIFFz3ZyZWL1iGcsjVSa/LlzQuTKUKrvYSd5NY5vQ31s4zbBxTCP6Tva5\nEmvXo9syvxWgThC/+xpbsXFMbXVgCcWBUhuSixfOx849+zKrcLYNhlt+Xfjk60dRFxtW8ev19/bj\n2mdmF9y+9daOil8zm9mHS+/rR4Fl15Z8LnvHqwMTuIe4So+W/ISoZ4dw/uxQzmOqaaXq1phadrME\nhwncY3baBq06VdjBEi71Yxow/YmvhCJBHVyzFX3bjuXcFsSHC2fFBIsJPGBWnSp73zmIJ9Y+B6MF\nkx0s3rPqEzc0zrokFInp4JqtOPDI/+bcNjzehE+vusH3+NjNEiwm8ICZdao89tSz2HfwELL7540O\nFiZw72TXhQd7PsKHb/dAB1MllKBLJ9llivyVNwCc6z3DpFmDmMBD6HcfHIfV4SeWVbyVXRc2jtCf\nOTKA4RMbcx5np+7bOKbRdMOycUxjwW3FWA3hCgN2swSLCTwA2Un46hnTCjpVJlzSgv5TH+Y8R0Rw\n9YxpPBjks9P7ezF05hxO9R5H5+0bMPPJVBeRnbqvk1bB7A+IwZ6PSibvoJImu1mCxQTuM7Oa9x23\nLMD2XXsBINNmmP0YEcGdi7+K7bv2mh4MYgL3RrFZ4F7WfQtW3HWFp6mHxxtxrvdjAKmkOdlG66BX\neNHl4DCB+8ys5r19196CGSpm7YdGkqfqZtbfnX+M/9OrEkyaxAQeNsVq3DwY5K9i9V2/676jpo9H\nrHl4Jq5iyZt92bWDFzV2kZ0NxvwSSkMshtUrluGlVzfjxU2vYShrZKlxX/brcBPTmXKTm9XjvUyS\nTiYFFnsuE3u08Kr0PrJKzFZJPDsJv/TqZrzw8m9NX9dsRC1VxqsRqsUSY6VJs9LnWY2XnbSklYk9\nYjhO1kdWkwfNEnj+6cy7/+lffYmx1pV76MROYit2EtHJKUW3NwaLbcjyJGV0OR4nS6Vt3tGJpQ+s\nwtIHVmHzjk7bz2ONOzhG8u155Qh6XjmCzts3mF4Hs1hiDOKK9uVeoT6IGMk9XIG7xGqDMb+0snXn\nm5gyeRK++fVbAKRW7mMbG3HiZG/O67WMHYN/XPZXrHG7qJxDJ1E4Im72G0Kxvmyz/3Ym62hjAneJ\n1eTBpQ+syimtqCr2HTiEex76LgDg7Ln0PygRQFJ1r5sSn0fbsr/2/z+iynlx6KTYh4IXpxSzLwn3\n4Vvd0PS0xOzSh1n5pdzETtHATUyPmV070wo3LMPDbGDUFfd9zvTAjBebmGZKHamv9Bqd3MQMJ25i\nhkB+aYXCxSp59b5+tOCxVhdLKLbhaNx++PH2zPtUmiALDvi4hCcpo4sJ3GNGaSV/wuBF9akfvVFC\nERH09Z/C5h2drHv7pJIukUr6yHOu8rP5KEZ+qhmx5hFltxX27/zA8n6vSh/HT/Shqys1/XBs82ic\n7BkAAMTjEzC+pcn196PysITio/z+bwD4wVM/x/6DhzOJvVj/OLnLrGd61IwWfPbXt5n2jMfvvgZd\nj26z1UduJPr+nR9kZpbks9uHblU6kVhdRR8Gdh0/0Yf2jv05h8sysdcJWmdPYRL3EEsoIWN2dZ61\n6zZw7neInHqzG92bDplu+tntTLE7/tVuZ4tZ6cSPqwN1dR0zTd4AMDSk6Oo6xgQeMCZwqlmTlrSi\n59UjF67qDgBDmkmq+bVhuy13XtWqs4Xl6kAULB7kCdjihfPREItlvufhHf+Mm3c5Rk0fb/vxZodk\nxlx/KdoXPY/2Rc+bHvQx1I9pwIh47mrVbt3azuGc4yf6sHXb29i67W0cP1H5HPJs8fgE1JmMsgVS\nJZR4fIIr70OVc1wDF5GbAKwBMAzAD1X1O3n3swZeAgdUBafc+SjZm5hjrr/UtCYOoGT9HHWCK779\nWdtzvIttnubXqs3q09mbkeVsQHITMzieD7MSkWEA3gbwJQBHAWwFcJuq7s16DBM4hZrbg6Nan725\n4DUPP95u+VinMW3d9ja6e/pzbhvX3Ihrr/kUAHsJnsxV+sHnBj82MecCeEdVu9Jv+AyABQB45YEy\ncAUeLC/6oI3XzO5GsePgmq3o+t52DH14NnOb0wFT+ZuR3IC0J/+D72TvQOg++JzWwC8FkL2seDd9\nG9lkzEp5o2M33ujYjeUPrylr4BUFp1RtOnsgVn4roVkd2zj9mZ28gVS3yu47X7KssefXqlmfdofV\nB1+YOF2B26qNtLW1Zb5OJBJIJBIO3zYa7KysyxlDS+FSaraKVftf46xLTMsixbpczvV+nLmocv7z\nxrc0oXX2FMtf9ePxCTjZO5BTQmGCD59kMolkMlnWc5wm8KMAJmZ9PxGpVXiO7AReK8wuXuzVAR2W\nYIJTbvmlcdYlFc0rAYr3jY9vabL81b5Ugidzfn/w5S9uV65cWfI5ThP4NgBTRCQO4D0AtwC4zeFr\nVgWrlbVxH5BKtlfPmIatO9/MOYlZThuhnx8UVJ5ypxFOWtJaMEDLLcUSPJmLwgefowSuqudE5JsA\nXkKqjfBH2R0olKu3fyAn2bbvfgsAMslbRHDHLQvKSr4swYRXueNrjZZC4/EtX5mMD375Dke9Bijs\nH3yOT2Kq6osAXnQhlqpidoEHgeQkW2OQlUFVsX3XXtyxaIGvsZJ3yi2xTF52bU5v+ISFUznqlSzx\nKL1HzC7wYHztJqsrAVG0WPUbu9niGGRPs11RiDFMmMA9ZDa8KjvZDqurw/mhoZz7r54xLef7UhuU\nVlcCoujwo984/z26e/oxevTFmDrlstAkySj0XYcNx8n6LDsh9/YP4OdPrMb5wbMFjxs5eiQ2Jl/P\nqZlz1Gx1KnWS0qv3AMo/lenlCtmPn0OUcJxsCGWvypc+sArnB8/i2mdmFzxu660d+M7jT3KDkiqS\nn2itlHMqkyvk8OE0wgCVqlUfPWbv+DVFm9snKY1E293Tj+6efrR37MfY5tGWkwXt8vpkIk+Ulo8r\n8ADZWUmnf40CwA3KamBWgnC739gs0Z7sSa2W9+17FwOnTmfuKydJDg4Wzjg3u80OP34OtbAhygQe\nclMnX46mxlEALmxQml2ajZuY4VesBOFHv7HxHpUmtnPnz9u6rRQ/fg61Uu5hAg9IdhK20hCLYenX\nF+Uk5PyTl8ZhIKOnnCcxw8uvqYBjm0cXbAaObR6d+brSJHn2bOFq2+y2Uvz4OdTKBEYmcJvcnDeS\nnYSHxS7C1ls7Ch4TG9FQkIg37+jEP6x+rOhhIG50knHRhYLbrnD2uiOGN+SUX4zbKDhM4Da4PW8k\n+/h765cXAQAEwNQr4mhqHGX6AZEfA0VPseFIpcoaYajnTp16GXa074PRFSySuq1cfgyJqpUJjEzg\nNvgxb0QBvH2gCwCwdeebuHPxV3OO1OfHYLioPvVXaKzEudEZXlabdKXqteXWc71KXuNbmnDllZc6\nvjRbPD7B8yFRURhE5QYm8AAsXjg/ZwJhPlXFE2ufw7SrJhf9kGgcNRL/vHwpAG5iRlmpem259Vyv\nktfxE304cOC9TCwHDryHpsaRJV/b6gPI6wM6YR9E5QYmcBvcnjdy3ZyZuHPxV/HE2ueKJvHsVb5Z\nDP+8fGnmfibt8LNKZG69tllbnpuKfZAUK/HUyoZiEJjAbfBi3sgdixZg2lWTsXbdBvT2D2D/gUNF\nL2/EmSfRZ5XISpU8St3vdstc09gm9PcWHrsfOXoUnnvuNwW310rLXhhxFopPSnWx/PjZX+asyI25\nJwDLI9Wi2KwPJ5uYVq8bj08oWUYxe10RsRzv8OJLb+TcdtVVl+Jkz0DRGSb5Cb7c+Su1irNQQsJO\nF0v2ihy4cECHV9upHsVW0qVKHuWWRAYHz5VcFbtR0jlw4D2MvHhE0cfUyoZiEJjAfWC3iyV//OzS\nB1ZxmFUV8SqRmX0wQFCy7uzGbJOhIQUk9Z7Ful5qYUMxCEzgRD7K7iwxkqXTxGb2weDmkKlSYhfV\nY2qAK+ww9MgHhQncB5V2sfBqO9XHqw0/sxVuqV5wN/rFjecEtcKu9Q1UbmL6pNKj+G4e4afg+XnR\nAjsrU7PHWHWhjG4ajZd/vTU1gVBSK++gV7xmP8/6+mGYNfPKyCdxO5uYTOBEDpT767vTBB5kuSCM\nx/3dutJQGLELhchDlfz67qRsEWS5wM5x/x3t+zNtsD0nBzCn1fvY8n+ehlo5LMQr8hBVqJIuDmPD\ncVxzI8Y1N5aVgL2+Io6T9963/92cU8Wqin373/U8LuPnWV8/zPP3CqOKV+Ai8jUAbQA+DeBaVd3h\nVlC1hnXu2lKNLXUfffSxrdu8ML6lCbNmXllwWKgapw/mc1JC2QVgIYB/dymWmuT2qFryj98jS4Mc\nkWr13kbdO7+EAfg7K7xWDws53sQUkU0A7rFagXMTs7g/vWsF9h04lHPb3NnT8dhD9wYUEZXD7427\nct6v0tisnmd2pfvsVW82EWBO69SaSKJe4SZmyG3e0Yn9Bw8X3P7WOwexeUcnV+ER4Hc5xO772d3w\nLJWUi12vcuu2twuSd11dHcaOGVUzK+CgFU3gIrIRgNnvaPer6nq7b9LW1pb5OpFIIJFI2H1qVVu7\nboPpONn+Ux9i+cNrWEqhitkZ4WqW5EdePMLR6NehoSEm7wolk0kkk8mynlM0gavqjU4CMmQncLKH\nc0/Ia2ZJ/qMz9jce4/EJpj3YtdC+54X8xe3KlStLPsetEkrROg1d6DTp6z8FhWJM42hcPWNazlF5\nIrdUuuE5YkQDPvzwI1vPG9/ShNGjL8bAwGnT+8l7TtoIFwL4HoAWAC+ISLuq/qFrkVURqwsS79yz\nD3fcsgDbd+1FX/8pHDj8bubaliKCq2dMCyJcqgJ2ujLMkvzUKamLFNvd/Jw65bKabN8LCx6l98HS\nB1bhjY7dpvdld5xYXdSBZRTyihtdNLU8DdBL7EKJmO279uZsarIOTl5zo4umGg8mRQWP0vtg8cL5\naIjFCm7neFgicoIrcB9kX5A4exMz/9g8538TUTlYAw8ZzkUhIoDzwImIIstOAmcNnIgoolgDjyiW\nWoiIJZQIyj8YxH5xourDEkqVWrtuQ86pTqNfnIhqC0soIWBVDmGZhIiKYQIPmNUVeQBYXqmH/eJE\nBLAGHjizOSlzZ08HANPbjbkpXJ0TVTfOQqli182ZyaRNVOO4iRmw/DkpRjnE6nYiIgNLKCHATUwi\nysej9EREEcU+cCKiKsYETkQUUUzgREQRxTZCh7jRSERB4SamAxwqRURe4UEej1kNlaokgXMlT0Tl\nYgIPAat5KEziVG2On+hDV9cxAEA8PoFXs3fI0SamiKwWkb0islNEfiEiNfW34dZpSY6HpVpw/EQf\n2jv2o7unH909/Wjv2I/jJ/qCDivSnHah/ArAZ1R1FoB9AO5zHlJ0GFebnzt7OubOns5VM1ERXV3H\nMDR0YT9saEgzq3GqjKMSiqpuzPp2C4A/cRZO9LgxVIrjYYmoEq51oYjIegBPq+rP8m6v2i4UN3ET\nk6qdUUIxVuF1dYLW2VNYB7fgyiwUEdkIYILJXfer6vr0Y1YAmKOqBStwEdEHH3ww830ikUAikSgd\nPRFVHW5iWksmk0gmk5nvV65c6f0wKxG5HcBfAviiqp4xuZ8rcCKiMnneBy4iNwFYDuAGs+RNRETe\ncbQCF5H9AGIAetI3bVbVv8l7TE2vwFnbJqJKcB54wHjUnogqxXngAeMBHSLyEhM4EVFEMYF7iBcm\nJiIvsQbuMW5iElEluIlJRBRR3MQkIqpiTOBERBHFBE5EFFFM4EREEcUETkQUUbwmJhGFBsfNlodt\nhEQUCrzgQy7Px8kSEVUqf7Vtdc3MWk3gdjCBE5Hv8lfbJ3sHMPLiEQFHFT3cxCQi35mttiGpsomh\nrk4Qj5tdzZEMXIEHiHNSiC6IXVSPqbOncBOzDNzEDAgv9kC1jBuWpXEWSojxYg9Uy8a3NKF19hSM\na27EuOZGJu8KsYRCRIEY39LEpO0QV+AB4cUeiMgp1sADxE1MIrLCCzoQEUUUNzGJiKpYxQlcRB4S\nkZ0i0iEiL4vIRDcDIyKi4iouoYjIaFUdSH/9twBmqepfmDyOJRQiojJ5WkIxknfaKAAnKn0tIiIq\nn6M+cBF5GMCfATgN4HOuRERERLYUTeAishGA2TSZ+1V1vaquALBCRO4F8CiAb5i9TltbW+brRCKB\nRCJRabxERFUpmUwimUyW9RxX2ghFZBKADao63eQ+1sCJiMrkaQ1cRKZkfbsAQHulr0VEROVzUgN/\nREQ+BeA8gP8DsMSdkIiIyA6exCQiCiGexCQiqmJM4EREEcUETkQUUUzgREQRxQRORBRRTOBERBHF\nBE5EFFFM4EREEcUETkQUUUzgREQRxQRORBRRTOBERBHFBE5EFFFM4EREEcUETkQUUUzgREQRxQRO\nRBRRTOBERBHFBE5EFFFM4EREEcUETkQUUUzgREQR5TiBi8g9IjIkIs1uBERERPY4SuAiMhHAjQAO\nuRNOcJLJZNAh2MI43RWFOKMQI8A4g+B0Bf5dAH/vRiBBi8pfKuN0VxTijEKMAOMMQsUJXEQWAHhX\nVTtdjIeIiGyqL3aniGwEMMHkrhUA7gPw5eyHuxgXERGVIKpa/pNEpgN4GcDp9E2XATgKYK6qfpD3\n2PLfgIiIoKpFF8YVJfCCFxE5COBqVe1x/GJERGSLW33gXGUTEfnMlRU4ERH5z9eTmGE/9CMiD4nI\nThHpEJGX033uoSMiq0VkbzrWX4hIU9Ax5RORr4nImyJyXkTmBB1PPhG5SUTeEpH9IvLtoOMxIyI/\nFpH3RWRX0LEUIyITRWRT+u97t4h8K+iYzIjIcBHZkv73vUdEHgk6JisiMkxE2kVkfbHH+ZbAI3Lo\n519UdZaqzgbwPIAHgw7Iwq8AfEZVZwHYh1RHUNjsArAQwKtBB5JPRIYB+AGAmwD8PoDbRGRasFGZ\n+glSMYbdWQB3q+pnAHwOwNIw/jxV9QyAeel/3zMBzBORLwQclpW7AOxBifK0nyvw0B/6UdWBrG9H\nATgRVCzFqOpGVR1Kf7sFqS6gUFHVt1R1X9BxWJgL4B1V7VLVswCeAbAg4JgKqOpvAZwMOo5SVPWY\nqnakvz4FYC+ATwYblTlVNTrnYgCGAQhd44WIXAZgPoAfokR7ti8JPEqHfkTkYRE5DODrAFYFHY8N\ndwDYEHQQEXMpgCNZ37+bvo0cEpE4gFakFhahIyJ1ItIB4H0Am1R1T9AxmXgUwHIAQ6UeWPQgTzmi\ncuinSJz3q+p6VV0BYIWI3IvUD/IbvgaYVirO9GNWABhU1Z/5GlyanRhDijv3HhCRUQCeA3BXeiUe\nOunfXGen941eEpGEqiYDDitDRP4IwAeq2i4iiVKPdy2Bq+qNFgFNBzAZwE4RAVK/7m8XkYJDP36w\nitPEzxDgyrZUnCJyO1K/Zn3Rl4BMlPGzDJujALI3qCcitQqnConIRQD+E8BaVX0+6HhKUdU+EXkB\nwDUAkgGHk+16AH8sIvMBDAfQKCI/VdU/N3uw5yUUVd2tqr+nqpNVdTJS/1DmBJG8SxGRKVnfLgDQ\nHlQsxYjITUj9irUgvTETdmEbs7ANwBQRiYtIDMAtAP4r4JgiS1Irsx8B2KOqa4KOx4qItIjImPTX\nI5BqqgjVv3FVvV9VJ6Zz5a0AfmOVvIFgLugQ5l9fHxGRXekaWQLAPQHHY+X7SG2ybky3Gv1b0AHl\nE5GFInIEqa6EF0TkxaBjMqjqOQDfBPASUjv9P1fVvcFGVUhEngbwOoCpInJERAIp59nweQCLkerq\naE//CWP3zCcA/Cb973sLgPWq+nLAMZVSNF/yIA8RUUTxkmpERBHFBE5EFFFM4EREEcUETkQUUUzg\nREQRxQRORBRRTOBERBHFBE5EFFH/D26G27fodMHWAAAAAElFTkSuQmCC\n", 147 | "text/plain": [ 148 | "" 149 | ] 150 | }, 151 | "metadata": {}, 152 | "output_type": "display_data" 153 | }, 154 | { 155 | "data": { 156 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEPCAYAAACp/QjLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGStJREFUeJzt3X+w3XV95/HnKwnht1JWREAqWTeIcdlCbRmrrF52NSOO\nC9KZFXQrtGVXO1hEu2tNmKlJdYqxVtTdFXd2pBbpSidr/QF2BPmRS9HWZLXhZ4yELVHjQnC26AoE\nDOS9f5zvNYf7Pffm3PR+78299/mYOXO+5/P9cT73M5nzyufz/X6+31QVkiT1WzTbFZAkHXgMB0lS\ni+EgSWoxHCRJLYaDJKnFcJAktXQeDkkWJ9mc5Ibm89okO5qyzUnO7tt2dZJtSbYmWdl13SRJgy2Z\nge+4DNgCHNl8LuDKqrqyf6MkK4DzgRXACcAtSU6uqj0zUEdJUp9Oew5JXgi8Afg0kLHivuV+5wLX\nVdXuqtoOPACc0WX9JEmDdT2s9DHgvUD///4LuDTJXUmuTnJUU348sKNvux30ehCSpBnWWTgkeSPw\nSFVt5tk9hU8By4DTgIeAj05yGO/tIUmzoMtzDq8EzknyBuAQ4DlJPltVF45tkOTTwA3Nxx8CJ/bt\n/8Km7FmSGBiStB+qatCQ/kCd9Ryq6vKqOrGqlgEXALdV1YVJjuvb7Dzgnmb5euCCJEuTLAOWA5sm\nOLavKtasWTPrdThQXraFbWFbTP6aqpm4Wgl6w0pjtfvjJL/UfH4QeAdAVW1Jsp7elU1PA5fU/vxF\nkqR/tBkJh6oaBUab5bdNst0VwBUzUSdJ0sScIT2HjYyMzHYVDhi2xV62xV62xf7LXBu5SeJokyRN\nURLqQDghLUmauwwHSVKL4SBJajEcJEkthoMkqcVwkCS1GA6SpBbDQZLUYjhIkloMB0lSi+EgSWox\nHCRJLYaDJKnFcJAktRgOkqQWw0GS1GI4SJJaDAdJUovhIElqMRwkSS2GgySppfNwSLI4yeYkNzSf\nj05yc5L7k3wtyVF9265Osi3J1iQru66bJGmwmeg5XAZsAar5vAq4uapOBm5tPpNkBXA+sAJ4PXBV\nEns2kjQLOv3xTfJC4A3Ap4E0xecA1zTL1wBvapbPBa6rqt1VtR14ADijy/pJkgbr+n/mHwPeC+zp\nKzu2qnY2yzuBY5vl44EdfdvtAE7ouH6SpAGWdHXgJG8EHqmqzUlGBm1TVZWkBq0b22RQ4dq1a3++\nPDIywsjIwMNL0oI1OjrK6Ojofu+fqsl+m/dfkiuAtwFPA4cAzwG+APwqMFJVDyc5DthQVackWQVQ\nVeua/W8E1lTVxnHHra7qLEnzVRKqKvvesqezYaWquryqTqyqZcAFwG1V9TbgeuCiZrOLgC81y9cD\nFyRZmmQZsBzYNPjYXdVakgQdDisNMPaTvg5Yn+RiYDvwZoCq2pJkPb0rm54GLpmoi7B7Nyxd2n2F\nJWmh6mxYqStJ6tFHi6OO2ve2kqSeA2ZYqUtPPDHbNZCk+c1wkCS1GA6SpJY5GQ6PPz7bNZCk+W1O\nhoM9B0nqluEgSWoxHCRJLYaDJKnFcJAktRgOkqQWw0GS1DInw8F5DpLUrTkZDvYcJKlbhoMkqcVw\nkCS1GA6SpBbDQZLUYjhIkloMB0lSy5wMB+c5SFK35mQ42HOQpG4ZDpKkFsNBktTSWTgkOSTJxiR3\nJtmS5ENN+dokO5Jsbl5n9+2zOsm2JFuTrJzo2E8/3XtJkrqxpKsDV9WTSc6qqieSLAG+nuRMoIAr\nq+rK/u2TrADOB1YAJwC3JDm5qvaMP/Zhh8GuXXDkkV3VXpIWtk6HlapqbABoKbAYeLT5nAGbnwtc\nV1W7q2o78ABwxqDjHnaYQ0uS1KVOwyHJoiR3AjuBDVV1X7Pq0iR3Jbk6yVFN2fHAjr7dd9DrQbQY\nDpLUrc6GlQCaIaHTkjwXuCnJCPAp4APNJh8EPgpcPNEhBhU+9thaPvIReP7zYWRkhJGRkemtuCTN\ncaOjo4yOju73/qka+Ps77ZL8AbCrqv6kr+wk4IaqOjXJKoCqWtesuxFYU1Ubxx2nfuVXik9+Es4Y\nOOgkSRovCVU1aEh/oC6vVnre2JBRkkOB1wGbk7ygb7PzgHua5euBC5IsTbIMWA5sGnRsh5UkqVtd\nDisdB1yTZBG9ELq2qm5N8tkkp9EbMnoQeAdAVW1Jsh7YAjwNXFITdGsMB0nqVpeXst4D/PKA8gsn\n2ecK4Ip9Hfvwww0HSerSnJwhbc9BkrplOEiSWgwHSVLLnA0Hn+kgSd2Zs+Fgz0GSumM4SJJaDAdJ\nUovhIElqmZPh4CQ4SerWnAwHew6S1C3DQZLUMmfDwXkOktSdORsO9hwkqTuGgySpxXCQJLUYDpKk\nljkZDoceCk8+CXv2zHZNJGl+mpPhsGgRHHJILyAkSdNvToYDOLQkSV2a0+HgXAdJ6saSYTZKshg4\ntn/7qvp+V5Uahj0HSerOPsMhyaXAGuAR4Jm+Vad2ValhGA6S1J1heg7vBl5SVf+368pMheEgSd0Z\n5pzD94H/N9UDJzkkycYkdybZkuRDTfnRSW5Ocn+SryU5qm+f1Um2JdmaZOVkxzccJKk7w/QcHgQ2\nJPkr4GdNWVXVlZPtVFVPJjmrqp5IsgT4epIzgXOAm6vqj5O8D1gFrEqyAjgfWAGcANyS5OSqGjib\nwWc6SFJ3hu053AIsBY4Ajmxe+1RVYz/fS4HFwKP0wuGapvwa4E3N8rnAdVW1u6q2Aw8AZ0x0bHsO\nktSdffYcqmotQJIjm88/HfbgSRYBfwe8GPhUVd2X5Niq2tlsspPeVVAAxwPf7Nt9B70exECGgyR1\nZ5irlU4FPgv8k+bzj4CLqurefe3bDAmdluS5wE1Jzhq3vpLUZIcYVLh27Vruvht+8ANYvnyEkZGR\nfVVFkhaU0dFRRkdH93v/VE322wxJ/ha4vKo2NJ9HgCuq6pVT+qLkD4BdwL8HRqrq4STHARuq6pQk\nqwCqal2z/Y3AmqraOO44VVWsXg1HHgmXXz6VWkjSwpSEqsqw2w9zzuGwsWAAqKpR4PAhKvK8sSuR\nkhwKvA7YDFwPXNRsdhHwpWb5euCCJEuTLAOWA5smrJTDSpLUmaGuVmr+138tEODfAX8/xH7HAdc0\n5x0WAddW1a1JNgPrk1wMbAfeDFBVW5KsB7YATwOX1CTdmsMOg0cfHaIWkqQpGyYcfhv4Q+ALzec7\nmrJJVdU9wC8PKP8H4LUT7HMFcMUQdbLnIEkdGuZqpX8ALp2BukyJ4SBJ3ZkwHJJ8oqouS3LDgNVV\nVed0WK99chKcJHVnsp7DZ5v3jw5YN/klTjPAnoMkdWfCcKiqbzeLp1XVx/vXJXk3cHuXFdsXn+cg\nSd0Z5lLWiwaU/eY012PK7DlIUncmO+fwFuCtwLJx5x2OBGb99t2GgyR1Z7JzDn8DPAQ8D/gTenMc\noHf77rs7rtc+GQ6S1J3Jzjl8L8kPgaeqalbPLwxiOEhSdyY951BVTwPP9D+Q50BhOEhSd4aZIf04\ncE+Sm5tl6M1zeFd31dq3sXCoggx9KylJ0jCGCYcvNK+xuQ3hAJjnsGRJ7/Wzn8HBB892bSRpfhnm\n9hl/luRg4OSmaGtV7e62WsMZm+tgOEjS9NrnPIfm+Q33A59sXtuSvKbjeg3F8w6S1I1hhpWuBFZW\n1XcBkpwM/AUD7rg60wwHSerGMDOkl4wFA0BV3c9wodI5w0GSujHMj/y3k3wa+HP2PuznW53WakiG\ngyR1Y5hw+B3gd4GxS1fvAK7qrEZTYDhIUjeGCoeq+ih9t+5Ochnwic5qNSTDQZK6Mcw5h98cUPZb\n01yP/eIDfySpG3P2rqzgMx0kqSvD3JX1GA7Au7KCw0qS1JVJ78oKfC/Ja4FdVfVMkpcALwHumakK\nTsZwkKRuDHPO4Xbg4CQnADcBbwP+rMtKDctwkKRuDBMOi6rqCeDXgauq6t8C/3yYgyc5McmGJPcl\nuTfJu5rytUl2JNncvM7u22d1km1JtiZZOdnxDQdJ6sZQM52T/Bq9yW8XN0XDhArAbuA9VXVnkiPo\nTai7md5dXa+sqivHfc8K4HxgBXACcEuSk6tqz6CDGw6S1I1hfuTfDawGvlhV9yV5MbBhmINX1cNV\ndWez/BjwHXo/+rD3BHe/c4Hrqmp3VW0HHgDOmOj4hoMkdWOf4VBVt1fVOVX14ebz/96fB/0kOQk4\nHfhmU3RpkruSXN33pLnjgR19u+1gb5i0OM9Bkrox2TyHT1TVZePmOIypqjpn2C9phpQ+D1xWVY8l\n+RTwgWb1B+nNvr54gt1bDxZau3YtAN/9Lnz/+yPAyLBVkaQFYXR0lNHR0f3eP1WDH+qW5OVV9e3m\neQ7jVVXdPtQXJAcBXwG+WlUfH7D+JOCGqjo1yarm4OuadTcCa6pqY9/2NVbnW2+FP/ojuO22YWoi\nSQtXEqpq6IcqTzbP4dvN+2iSY5rlH02xMgGuBrb0B0OS46rqoebjeeydN3E98LkkV9IbTloObJro\n+J5zkKRuTDasFGANvTuyLm7KngH+S1X94ZDHfxXwG8DdSTY3ZZcDb0lyGr0howeBdwBU1ZYk64Et\nwNPAJTVR1wbDQZK6Mtmw0u8BZwNvr6oHm7J/Cvw34Mbxl6HOlP5hpW3b4Oyz4YEHZqMmkjR3THVY\nabKrlS4E3joWDABV9ff05jtcuP9VnD72HCSpG5OFw5JB5xiaMh8TKknz2GThsHs/180Yw0GSujHZ\nOYdngIl+eg+tqlnpPfSfc6iCgw6CXbt675KkwabzUtbF01Ol7iR7ew/Pfe5s10aS5o9hb6B3wHJo\nSZKmn+EgSWoxHCRJLYaDJKnFcJAktRgOkqSWOR8Ohx8Ojz8+27WQpPllzoeDPQdJmn6GgySpxXCQ\nJLUYDpKkFsNBktRiOEiSWgwHSVLLnA8H5zlI0vSb8+Fgz0GSpp/hIElqMRwkSS2dhkOSE5NsSHJf\nknuTvKspPzrJzUnuT/K1JEf17bM6ybYkW5Os3Nd3GA6SNP267jnsBt5TVS8DXgG8M8lLgVXAzVV1\nMnBr85kkK4DzgRXA64GrkkxaR8NBkqZfp+FQVQ9X1Z3N8mPAd4ATgHOAa5rNrgHe1CyfC1xXVbur\najvwAHDGZN9hOEjS9Juxcw5JTgJOBzYCx1bVzmbVTuDYZvl4YEffbjvohcmEDAdJmn5LZuJLkhwB\n/CVwWVX9NMnP11VVJalJdm+tW7t27c+XX/GKER5/fGTa6ipJ88Ho6Cijo6P7vX+qJvtd/sdLchDw\nFeCrVfXxpmwrMFJVDyc5DthQVackWQVQVeua7W4E1lTVxr7jVX+dq2DxYti9u/cuSWpLQlVl31v2\ndH21UoCrgS1jwdC4HrioWb4I+FJf+QVJliZZBiwHNk3+HXDoobBr1/TWXZIWsq6HlV4F/AZwd5LN\nTdlqYB2wPsnFwHbgzQBVtSXJemAL8DRwSQ3RtRk773DEER38BZK0AHU+rDTdxg8rAbzoRXD77XDS\nSbNTJ0k60B1Qw0ozxSuWJGl6GQ6SpBbDQZLUMm/CwWc6SNL0mRfhcPjh9hwkaTrNi3BwWEmSppfh\nIElqMRwkSS2GgySpxXCQJLUYDpKklnkTDs5zkKTpM2/CwZ6DJE2feREOToKTpOk1L8LBnoMkTS/D\nQZLUYjhIkloMB0lSi+EgSWqZN+HgPAdJmj7zJhzsOUjS9Jk34bBrF1TNdk0kaX6YF+GweDEsXQpP\nPjnbNZGk+aHTcEjyp0l2Jrmnr2xtkh1JNjevs/vWrU6yLcnWJCun8l0OLUnS9Om65/AZ4PXjygq4\nsqpOb15fBUiyAjgfWNHsc1WSoetnOEjS9Ok0HKrqDuDRAasyoOxc4Lqq2l1V24EHgDOG/S7DQZKm\nz2ydc7g0yV1Jrk5yVFN2PLCjb5sdwAnDHtBwkKTps2QWvvNTwAea5Q8CHwUunmDbgdcfrV279ufL\nIyMjjIyMONdBkvqMjo4yOjq63/unOr7+M8lJwA1Vdepk65KsAqiqdc26G4E1VbVx3D41qM6vfS38\n/u/DyimdxpakhSEJVTVoSH+gGR9WSnJc38fzgLErma4HLkiyNMkyYDmwadjjHn44fOtb8Nhj01dX\nSVqour6U9Trgb4CXJPlBkt8GPpzk7iR3Aa8B3gNQVVuA9cAW4KvAJQO7CBN4+9vhppvgBS+AM8+E\n978fNmxw7oMk7Y/Oh5Wm20TDSmOeeAK+8Y1eMNx2G9x7L7z85fCLvwjPfz4cc0zvNbb8C78ABx+8\n97V0ae99yRLI0B0wSTqwTXVYad6Fw3g/+Qls2gQPPQSPPAI/+tGz3x99FH72M3jqqb3vTz0Fe/b0\nZl4nsGhR7338siQdSF79avjKVwavMxymyTPP9F5VvdeePc9elqQDzeLFvfOvg0w1HGbjUtY5YfHi\n3kuSFqJ5ceM9SdL0MhwkSS2GgySpxXCQJLUYDpKkFsNBktRiOEiSWgwHSVKL4SBJajEcJEkthoMk\nqcVwkCS1GA6SpBbDQZLUYjhIkloMB0lSi+EgSWoxHCRJLYaDJKml03BI8qdJdia5p6/s6CQ3J7k/\nydeSHNW3bnWSbUm2JlnZZd0kSRPruufwGeD148pWATdX1cnArc1nkqwAzgdWNPtclcSezSRGR0dn\nuwoHDNtiL9tiL9ti/3X641tVdwCPjis+B7imWb4GeFOzfC5wXVXtrqrtwAPAGV3Wb67zH/5etsVe\ntsVetsX+m43/mR9bVTub5Z3Asc3y8cCOvu12ACfMZMUkST2zOmxTVQXUZJvMVF0kSXul9/vc4Rck\nJwE3VNWpzeetwEhVPZzkOGBDVZ2SZBVAVa1rtrsRWFNVG8cdz8CQpP1QVRl22yVdVmQC1wMXAR9u\n3r/UV/65JFfSG05aDmwav/NU/jhJ0v7pNBySXAe8Bnhekh8A7wfWAeuTXAxsB94MUFVbkqwHtgBP\nA5dU190aSdJAnQ8rSZLmnjk1jyDJ65sJctuSvG+26zOTpjqhcL5KcmKSDUnuS3Jvknc15QuxLQ5J\nsjHJnUm2JPlQU77g2mJMksVJNie5ofm8INsiyfYkdzdtsakpm1JbzJlwSLIY+K/0JsitAN6S5KWz\nW6sZNfSEwnluN/CeqnoZ8Argnc2/gwXXFlX1JHBWVZ0G/AvgrCRnsgDbos9l9Iamx4ZEFmpbFL0L\nf06vqrH5YlNqizkTDvQmxD1QVdurajfwF/Qmzi0IU5xQOG9V1cNVdWez/BjwHXoXMCy4tgCoqiea\nxaXAYnr/RhZkWyR5IfAG4NPA2IUrC7ItGuMv3plSW8ylcDgB+EHfZyfJTTyhcEFoLpM+HdjIAm2L\nJIuS3Envb95QVfexQNsC+BjwXmBPX9lCbYsCbknyrST/oSmbUlvMxqWs+8sz55OoqlpIc0CSHAH8\nJXBZVf002fufpIXUFlW1BzgtyXOBm5KcNW79gmiLJG8EHqmqzUlGBm2zUNqi8aqqeijJMcDNzfyy\nnxumLeZSz+GHwIl9n0/k2bfbWIh2JnkBQDOh8JFZrs+MSHIQvWC4tqrG5sksyLYYU1U/Af4KeDkL\nsy1eCZyT5EHgOuBfJbmWhdkWVNVDzfuPgC/SG5afUlvMpXD4FrA8yUlJltK7g+v1s1yn2TY2oRCe\nPaFw3kqvi3A1sKWqPt63aiG2xfPGrjhJcijwOmAzC7AtquryqjqxqpYBFwC3VdXbWIBtkeSwJEc2\ny4cDK4F7mGJbzKl5DknOBj5O78Tb1VX1oVmu0ozpn1BIb7zw/cCXgfXAL9JMKKyqH89WHWdCczXO\nXwN3s3eocTW92fQLrS1OpXdicVHzuraqPpLkaBZYW/RL8hrgP1bVOQuxLZIso9dbgN6pg/9RVR+a\nalvMqXCQJM2MuTSsJEmaIYaDJKnFcJAktRgOkqQWw0GS1GI4SJJaDActOEkea95flOQt03zsy8d9\n/sZ0Hl+aKYaDFqKxyT3LgLdOZcck+7of2epnfVHVq6ZyfOlAYThoIVsH/MvmgSiXNXc4/UiSTUnu\nSvJ2gCQjSe5I8mXg3qbsS80dL+8du+tlknXAoc3xrm3KxnopaY59T/MQljf3HXs0yf9M8p0kfz5W\nuSTr0nuo0V1JPjKjLaMFby7dlVWabu8D/lNV/RuAJgx+XFVnJDkY+HqSrzXbng68rKq+13z+rap6\ntLmn0aYkn6+qVUneWVWn933HWC/l14FfovdQnmOA/5Xkr5t1p9F7gNVDwDeSvArYCrypqk5p6vac\nDv5+aUL2HLSQjX8YykrgwiSbgW8CRwP/rFm3qS8YAC5rnqPwt/TuELx8H991JvC56nkEuB34VXrh\nsamq/k/17mVzJ/Ai4MfAk0muTnIesGu//0ppPxgO0rP9bvNoxdOr6sVVdUtT/vjYBs3zAv418Irm\nEZ2bgUP2cdyiHUZjvYqn+sqeAQ6qqmfo3Wb588AbgRv354+R9pfhoIXsp8CRfZ9vAi4ZO+mc5OQk\nhw3Y7znAo1X1ZJJT6D3LeszuCU5a3wGc35zXOAZ4Nb07yY4PDJrvPhw4qqq+CvwevSEpacZ4zkEL\n0dj/2O8CnmmGhz4D/GfgJODvmudGPAKc12zff/viG4HfSbIF+C69oaUx/x24O8m3m+cJFEBVfTHJ\nrzXfWcB7q+qRJC+l/ZTDohdaX05yCL0Aec+0/OXSkLxltySpxWElSVKL4SBJajEcJEkthoMkqcVw\nkCS1GA6SpBbDQZLUYjhIklr+P3Q8tZ1ibWyVAAAAAElFTkSuQmCC\n", 157 | "text/plain": [ 158 | "" 159 | ] 160 | }, 161 | "metadata": {}, 162 | "output_type": "display_data" 163 | } 164 | ], 165 | "source": [ 166 | "k1 = 3\n", 167 | "C2, centers2, distortion, centers_list2 = mykmeans_plus_plus(X_scaled, k1,50)\n", 168 | "centers2_pca = pca.transform(centers2) \n", 169 | "\n", 170 | " \n", 171 | "centers_list2 = np.array(centers_list2)\n", 172 | "plot_kmeans(C2, centers2_pca, X_pca, k)\n", 173 | " \n", 174 | "plt.figure()\n", 175 | "index1= [i for i in range(50)]\n", 176 | "plt.plot(index1, distortion)\n", 177 | "plt.xlabel('Iterations')\n", 178 | "plt.ylabel('Distortion')\n" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 94, 184 | "metadata": { 185 | "collapsed": false 186 | }, 187 | "outputs": [], 188 | "source": [ 189 | "def mini_batch(X, k, b, max_iter):\n", 190 | " v = [0]*k\n", 191 | " centers = random.sample(X, 3)\n", 192 | " distortion = []\n", 193 | " \n", 194 | " for i in range(max_iter):\n", 195 | " M = random.sample(X,b)\n", 196 | " C=[0]*len(M)\n", 197 | " for i,x in enumerate(M): \n", 198 | " index = get_closest(x, centers)[0]\n", 199 | " C[i] = index\n", 200 | " \n", 201 | " \n", 202 | " for i, x in enumerate(M):\n", 203 | " index = C[i]\n", 204 | " v[index]+=1\n", 205 | " n = 1/v[index]\n", 206 | " \n", 207 | " centers[index] = (1- n)*centers[index] + n*x\n", 208 | " \n", 209 | " D=[0]*len(X)\n", 210 | " for i in range(len(X)): \n", 211 | " index = get_closest(X[i], centers)[0]\n", 212 | " D[i] = index\n", 213 | " \n", 214 | " D = np.array(D)\n", 215 | " \n", 216 | " distortion.append(k_means_objective(X, k, D, centers))\n", 217 | "\n", 218 | " \n", 219 | " return centers, distortion" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 95, 225 | "metadata": { 226 | "collapsed": true 227 | }, 228 | "outputs": [], 229 | "source": [ 230 | "def plot_kmeans(C,centers,X_pca,k):\n", 231 | " \n", 232 | " #choose random colour for the centroids\n", 233 | " marker_c = [random.random() for y in range(3)]\n", 234 | "\n", 235 | " for i in range(k):\n", 236 | " \n", 237 | " #plot data-points corresponding to a specific cluster\n", 238 | " #with a random colour and plot the centroid of that \n", 239 | " #cluster\n", 240 | " \n", 241 | " x = X_pca[C==i]\n", 242 | " colour = [random.random() for c in range(3)]\n", 243 | " plt.scatter(x[:,0],x[:,1], color=colour)\n", 244 | " plt.scatter(centers[i][0], centers[i][1], marker='s', c = marker_c, s = 40)\n", 245 | " " 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": 97, 251 | "metadata": { 252 | "collapsed": false 253 | }, 254 | "outputs": [ 255 | { 256 | "data": { 257 | "text/plain": [ 258 | "" 259 | ] 260 | }, 261 | "execution_count": 97, 262 | "metadata": {}, 263 | "output_type": "execute_result" 264 | }, 265 | { 266 | "data": { 267 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEACAYAAACqOy3+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHmRJREFUeJzt3XuQlfV5B/Dvw15ByBIrKlfxAgQvRcRuA3XqCUajhpTa\nJiSMWEOi00mxZBygRsnIYuPYhlgdEzp1JFITExKaJqGgVA14NIxbNiHrBsKGm0EEBCTKym2XXfbp\nH+fCOe95z3ve897fc76fGWb23B92zz7nt8/v+f1+oqogIqL4GRB2AERE5AwTOBFRTDGBExHFFBM4\nEVFMMYETEcUUEzgRUUy5SuAi0igim0XkTRHZLiKPeRUYERFZE7d94CIySFVPiUgtgE0AFqrqJk+i\nIyKiolyXUFT1VPrLegA1AN53+5xERFSa6wQuIgNE5E0AhwG8qqrb3YdFRESleDEC71fVawGMAvCX\nIpJwHRUREZVU69UTqWqXiLwA4HoAycz1IsLNVoiIHFBVsbrdbRfKBSIyNP31QAA3A2g3CSLy/5Ys\nWRJ6DIyTccY1Rsbp/T873I7AhwN4TkQGIPVh8H1V3eDyOYmIyAZXCVxVtwK4zqNYiIioDFyJmZZI\nJMIOwRbG6a04xBmHGAHGGQbXC3lKvoCI+v0aRESVRkSgfk5iEhFReJjAiYhiyrM+cKIoOr6rD++9\n0QsAGDatDkPG8S1PlYM1cKpYx3f1Ye+qHmhf6rLUAmNnNzCJUyywBk5V7b03erPJGwC0D9nROFEl\nYAInIoopJnCqWMOm1UFyqiVSm7qOqFKwBk4VjZOYFFd2auBM4EREEcRJTCKiCsYETkQUU0zgREQx\nxRmdGNvX2Yb2jasBAJOnz8KYic0hR0REQeIkZkzt62zD+hUPo6+3BwBQW9eA2+55hEmcqEJwErOC\ntW9cnU3eANDX25MdjRNRdWACJyKKKSbwmJo8fRZq6xqyl2vrGjB5+qwQIyKioLEGHmOcxCSqXFyJ\nSUQUU5zEJCKqYEzgREQx5TqBi8hoEXlVRH4nIttEZL4XgRERkTXXNXARuRjAxar6pogMBrAFwF+r\namf6dtbAyXPcJpYqnZ0auOt3vaoeAnAo/fUJEekEMAJAp9vnJjJjPOvy5Ns9GDsbTOJUdTytgYvI\nWACTAWz28nmJcvGsS6IUz4Ys6fLJTwB8VVVP5N7W0tKS/TqRSCCRSHj1skREFSGZTCKZTJb1GE/6\nwEWkDsA6AOtV9UnDbayBk6eMJRSpBcbObmAJhSpKIAt5REQAPAfgj6p6v8ntTODkitmEJScxqdIF\nlcBvAPA6gN8CyDzZg6r6v+nbmcDJMY62qVoF1YWyCVwQRD4pNmHJBE7EE3mIHLEq4bC8Q0HhO4si\nbdi0Opx8O7+EMmxaXagxWfWhh9Wjzg+N6sSfMkXakHG1GDsbkUpOVmWdMEo+XNhUvfgTpsgbMq6W\nycgC5wmqFycfico0bFodJCc35pZ1rG7zw/FdfTh9sN+356do44EORA5EYRLTWDrJYKtlZeCJPERF\nVMLioLeeO40Te/JH3zUDgTGfZfKuBIH0gRMFwcvkajbpd+GN/TjyWq+ticAoJ/qBIwZEKh7yF2vg\nFHmZhHtiTz9O7OnH3lU9OL6rr/QDizCb9Dtqc4dDr2NxI+h6O0UPEzhFXpS2j41SLKkWywYMvnwA\nBl8+gHXvKsSfNlUds8VBF0yryyuhxGU0yxbL6safPEWe16sxiy0OGjRiQMnadhRXhlL1YhcKxUKU\nJg6DjiVK/3cKDtsIiWKO2+lWL7YREnkgzBEwl8mTFb4LiHIYkzUAbhRFkcV3IVGa2QKfhmES6giY\nk6ZkhQmcKM2sXNF7LNz5myhup0vRwXcCkYW6oYL+Xg11BMxebyqG7wqiNLNyxfBP1gOI/giYrYbV\niW2ERDnimAjZaliZ2EYYQfs629C+cTUAYPL0WRgzsTnkiCiXWbki6kmdrYbViz/hAO3rbMP6FQ+j\nr7cHAHDorW247Z5HmMQjjOdNUpS5fheKyLMAPg3giKpe4z6kytW+cXU2eQNAX28P2jeuZgIPUGY0\n3XcqVdarHSSWo+o4jG7Zali9vHgXrgTwbQDf8+C5iHxjfgSZxn5UzVbD6uV6P3BV/SWADzyIpeJN\nnj4LtXUN2cu1dQ2YPH1WiBFVF+NoOsNqT2/joQkA0HdKQzvEoZgh42px2d0DcdndA5m8q4gnXSgi\nMhbAWrMSSiV3oTiZkOQkZnjMzpDMGHz5AFx290DT247v6sO7vziD7kMKpN/KUer0iPokKzkTmS6U\nlpaW7NeJRAKJRCKIl/WV0wnJMRObmbRDYqwVZ5SqGQ8ZV5tKkDkDkajUwt1MsjLxR0symUQymSzr\nMRyBO7Rm+ULs37El77pRE6Zg5rxv5V3HEXe0lDuJmWE2ercatQfFaVzsHY++yIzAqxXbBqPHybL0\n47v6sgk/I+6dHnHorqHSXE9iisgqAG8AGC8i74jIXPdhRZ+dCclibYMUH5mRave7meI30DhcIjNa\nPW9sTd7luH+wUHlcvwNVdbYXgcTNmInNuO2eR1geqTDGunBB54qmyi5RSN7Hd/XhyGs53TMCXHij\nvVo2e8crQ/jvwhgrNSE5efosHHprW3YUzrbBaDPWhU/8oaeghbBcQz86FF3HugqubxrahGMfHHP1\n3GYfLif3ngVuLP1Y9o5XBv7EfMRRerwUJMR+QM/k36fckWrXsS7cu2ZJwfXPzFzqMErveLVNLbtZ\nwsPvtM/stA0W61RhB0u01AwExnw2GrXvw6+dwcl38rtPwiiDcK+YcPG7HLJinSpH9u1A24srkWnB\nZAeL/4r1iWcMHDEgEonp8GtncHhD/srRuvOBUZ8O/sOF3Szh4nc5ZGadKq1rn8HRA3vyFo5w4yv/\n5daF+04peo4o9GzqtrAn+XLLFMaRNwD0n+aotxrxJx5Bx98/nJe8c7Gs4q/cunBmCX3vMUXd0Pz1\nFEHWfc034YoGdrOEiyfyhCA3CY+8YhK2vPyDvE6VoReNxtH9u/MeIyJovn1uwX1ZVvFPsdWKAGyv\nYnTahZL7AdF3Ss/1oRdx0U11uOjGejv/Lc9xEtMfXIkZQWY17ym33IkDuzsAINtmmHsfpJP3gd0d\n3E88QMXqu5mvjdebJS4nrYIFI26TX+G681NlEwC4YFp4yRvgocth4nc9YGY17wO7Owr2UDFrP8wk\neapsZv3duaQ2nAlLih6+AyLGqsbNhUHBsqrvBl33bRwuqB0k2biskjdLGtWDNXAP2ZlgNJZQMnXs\nnVs2YOevfgHVcx0GZjVuTmK6U25yK3Z/P5Okm50CrR7LxB4vdmrgTOAeKZaYiyXx3CS8c8sG7Gh7\n2fR5zbaoJWf82kLVKjE6TZpOH1dse9lh0+qY2GOGk5gBKufAYuPqzHVPPxRIjNWu3EUndhKb1UpE\nN6sUvZ4YtJqQ5UrK+HK9nSyVtq+zDWuWL8Sa5Quxr7PN9uNY4w5PJvme2NOPE3v6sXdVj+k5mFaJ\n0eo2vxjP8CxVnw8jRvIOP2Y9UmyC0VhaObDzN/iTkZdj6mfuBZAauQ8c3IRTH76f93yDPnI+brrz\nAda4PVTOopM4LBE3+wvBapdBs/87k3W8sQbuIbMJRrOj1wBgQE0qcfSfTf8CyYBUu68Ixl9/Ez45\n58GAoq4uduu9do8qKzVp6HXNPfdIuO7DCvTbf26z/zuPVosuTmJGQLEEboYTltFhtmFUsdWOfkxi\nmim1pN7pGZ2cxIwmTmJGgLG0QtFSLHmd3Hu24L7FDkuwmnDMXP/eG73Z13GaIAsW+HiEKynjiz81\nn2UOdTDuMFhYQhF0n+zCvs421r0D4qRLxEkfed5r7O1Bw4VnUDtIym4rPH2wcBfCDL8WE3VsasW6\nlc8DAK5snoLtbam/JmfMnYNJN0z1/PWoPCyhBMhYIweA1rXP4I8H9mT3/eYGVcExq3M3DheM/8og\n09rwhTfW4chrvbbqxZlEf/pgP86eNn99u/XmYqUTqQEaLpSyPwzs6tjUisfnL8KZ7sK/HusbG7Dg\nqWVM4j5iCSVizE7nad+4Gsp9vyOj+5Di+K4+024Ou50pdrd/tdvZYlY6CeJ0oHUrnzdN3gBwprsH\n61Y+zwQeMiZwqlrDptXhxB5DgtJzSdVYG7bbcudXrTpXVE4HonBxIU/IJk+fhdq6huxlLt4JzpBx\ntWgcbvkXah6zRTLnja3BW8+dxlvPnTZd6JNRMxCoPz//tezWre0szjm+q89WHOWYMXcO6hsbTG+r\nb2zAjLlzPHkdcs51DVxEbgXwJIAaACtU9V8Nt7MGXgI3qApPuX3QuZOY542tMa2JA4UHPhjr5xDg\noun29/Eu1apY6v+QOxlZzgQkJzHD43sfuIjUANgB4JMADgD4FYDZqtqZcx8mcIo0rzeOuuzugQXP\n+d4bvbYWBjmJqdSiI+NkJCcg7XP6weeFICYxmwHsVtW96Rf8EYCZADqtHkT5OAIPlx990JnnzO1G\nsePwa2fw3i970X/m3HVuN5gyTkZyAtIe4wffjvaOyH3wua2BjwTwTs7l/enryKbMXin7d2zB/h1b\nsH7Fw2VteEXhKVWbzt0Qy9hKaFbHzqz+zE3eQKpbZd9PzDfTshMHOVPsgy9K3A47bNVGWlpasl8n\nEgkkEgmXLxsPdkbW5WxDS9FitXEUULz9b+CIAaZlkaMWXS5nT6fq6mYj8VJxzJg7BzvaO/JKKJyA\njJ5kMolkMlnWY9wm8AMARudcHo3UKDxPbgKvFmaHF/u1QIclmPCUW34ZOMLZfiWAdd+4VRyTbpiK\nBU8tC62WG1dBf/AZB7dLly4t+Ri3CfzXAMaJyFgABwF8HsBsl89ZEYqNrDO3AalkO/KKSTiw8zd5\nKzHLaSMM8oOCylPO9rVA6nR54wZaXpl0w1Qm7TLF4YPPVQJX1T4RuQ/AS0i1EX43twOF8nWf7MpL\ntgd3/xYAzq3EFMGUW+4sK/myBBNdpUobRpmWwkwpZciEGnRtOxvo4cmUL+offK6n3lV1PYD1HsRS\nUcwOeACQl2yzG1llqOLA7g5c/6m7AouT/FVuieWiG+vzesOPX8OtXqk4vht8ktmFMLdckvnaS8VO\nAqJ4Kdb37WWLY5g9zXbFIcYo4W6EATLWq2VADRY+/TpOdxfWPZuGNuHYB8dsTVByEjPegjgVx9jT\nLCK45GPjMfv+eZFJklxwlI8n8kRQbrLtPtmFLzywAveuWVJwv2dmLsXb2zfnJXxuNVuZ7B7f5saj\nX56Hra2F6wvKTZJ+jpDNYrxmajMWf3e5Z68RJ9xONoJyt5Rds3yh5X1f+68nOUFJjhgTbTHlrMqM\nw8rEasPdCENUqlb94dF3A4qEwuT1SspMot3a2oatrW14fP4iXNk8pejOgnb5vTLRuPshFxyVxhF4\niGyNpEWyx7BxgjL+zEoQ5bYblmKWaLe3bcGCp5Zh1RPfwds7dkH7U++pcpLk8WPHbF1nh9n3weu+\n62qYEGUCj7gLRl6OxvOaAJyboDQ7mo2TmNFnVYII4mDhTJJ0mthOnzxl67pSrL4PXvVdV0u5hwk8\nJJkkPGhQI56ZWbhkdmBjHWrrGjD1M/fmJWRjJ0tmMVCmp5wrMaMrqF0Br2yeUjAZeGXzlOzXTpPk\nia4PbV1XShDfh2rZgZEJ3CYvW/Vyk/A3v/xxiAiab5+L6z91l+Xr7Otsw8vPfcNyMRAnOilz6ILx\nujv+/kuunnfYiItx0pCwh4242NVzkjtM4DZ4vd+Icfm7qmLzCyuxp+N1NJ7XZPoBYYyB4sdqc6RS\nZY0o1HNn338fls1bgL7e1KChtq4Os++/r+znCWKTqGrZgZF94DasWb4Q+3fkj2pGTZiCmfO+5dnz\n5RHBn6dH5KUeM6Am1a2QGYmzVzzazBJxqQUs5S5w8XNBzM+efhbr/vMHqfi/eKftUb1ZW6PfH0hR\n+NBzg33gETV5+qy8HQgLqKLtxZW4cMwEy0TcMGgIbrn76wA4iRlnpeq15dZz/dpFr2NTK3729LPZ\nWH729LO47KqJJZ+72ISi3wt0or4RlReYwG3wer+RMROb0Xz7XGx+cWW2RdBIVfNq2WYx3HL317O3\nM2lHX7FE5tVzm7Xlecnqg8RqtFstE4phYAK3wWxjKrcJ8/pP3YULx0xA+8bV6D7ZhaP798DqgCM/\nYqBgFUtkpeq1pW4Pu2Uu7NevZkzgNuUugXfCrLsk9zl//dL30fbiyoKDHYyPc1p3p+gqVfIodbvV\nCslSZZRy6sRXNk/Btv/7VfY9KiK4snlKyRF2tUwohoGTmAEwdpAUm2g0W6DDzawqh1+Ti2abQI2d\nOB4H//C25WuVE4/xvhn1jQ0Ycekl2Nu5M+964yZUcZ9QDAMnMSPC7qk5xlH+muULuZlVBfFrctFs\nhAtIybpzqZHz0I8ORdexroLXq6+rxR1XXJt9DCCob2ywHGFXw4RiGJjAiQKU21mSSeRuE5vZB4MX\nm0x1HesqutVxriFDm0I9O7KaR/dM4AFw2sXC03Yqj18TfmYj3FJ1Zy9q05nHhDXCrvYJVNbAA+J0\nKT5P26ksQR5aYGdkanUfESk6An/gb+7EkKFNoY94zb6f5zV9BPOXfSP2SZw18Ahx2sXitvuF/BX0\nn+/lvJ6dUbHTkbPZUWxRKWWc7PoQj89fVBUjcR7oQOSQ2cEJHZtaLR/j5tACJ6/nF+Nrd2xqxbJ5\nC7KxLZu3IJDYjN/PDK8Pm4gqjsCJHHKywtBNJ0rQKxqbhjaZbnVcX1db8Nqrnlie3eQKAPp6e7Hq\nieW+j4Az38+nFn29YKfEauA4gYvI5wC0APgYgD9T1d94FVS1YZ27usSlpe7YB+dO2yl2KHLGewcL\nj/8zu84Pk26YivnLvlHQ014Ni4XclFC2ArgDwOsexVKVMot89u/Ygv07tmD9ioexr7P4LwpFR9Bn\nOIZ5ZmSx1+7Y1IpHvzwPfb19BY8Jcq/wzEj8mqnNuGZqc1XUvwEPulBE5FUAC4qNwNmFYu3H37wX\nR/fvzrvOzVa1FKwoT2I6ja3Y48y2hDVbnQmk9gpftPzxqkiifrHThcIEHqJ9nW1Y9x9fK9hWNrNN\nLEsp5JTdZfKlkrLV8nqzskrDoEEYP+nq0NsLK4HrNkIReQWA2d9BD6nqWruBtLS0ZL9OJBJIJBJ2\nH1rR2jeuNt0TvOfUcaxf8TD3PSHH7Ex4mi2CGXHpJa4mSs90n2bydiiZTCKZTJb1GMsErqo3uwko\nIzeBkz3c94T8Zpbk3zt4yPbjZ8ydk7c7IQBov3Kvb4eMg9ulSws7gIy8aiO0HObTuU6T7pOpzYEa\nz2vCyCsm5S2VJ/KK02Xyw0YMR29Pj63HTbphKi752Hjs7dzhXeBUFjdthHcAeArABQBeEJF2Vb3N\ns8gqSLEDiQ+9tQ1TbrkTB3Z3oPtkF95/9+1zp8yLYOQVk0KIliqBnX5zsyQ/+/55AOyfVzn7/nlV\n2b4XFdwLJQBWhxjndpyYHerAOjj5yYsumqgsoa803AslZg7s7sirJ7IOTn7zYlFRXBYmVSLuhRKA\nydNnobaucL8Gbg9LRG5wBB6A3AOJcycxjcvmuf83EZWDNfCI4b4oRAQEtBLTRhBM4EREZbKTwFkD\nJyKKKdbAY4qlFiJiCSWGjAuD2C9OVHlYQqlQ7RtX563qzPSLE1F1YQklAoqVQ1gmISIrTOAhM5ZD\nDr21Dbfd8wgAmF4/ZmIz+8WJCAATeOisyiFm14+Z2Jy3MAjg6JyoWjGBx1QmkRNR9eIkZsiM+6Rk\nyiHFriciymAbYQRwEpOIjLiUnogoptgHTkRUwZjAiYhiigmciCim2EboEicaiSgsnMR0gZtKEZFf\neKixz4qtonSSwDmSJ6JyMYFHQLH9UJjEqdJ0bGrFupXPAwBmzJ3D0+xdcjWJKSLLRKRTRDpE5Kci\n0uRVYHHg1WpJbg9L1aBjUysen78IW1vbsLW1DY/PX4SOTa1hhxVrbrtQXgZwlapOArATwIPuQ4qP\nzKZSoyZMwagJUzhqJrKwbuXzONN9bqByprsnOxonZ1yVUFT1lZyLmwH8rbtw4seLTaW4PSwROeFl\nH/iXALzo4fNVDY7kqRrMmDsH9Y3nSo71jQ2YMXdOiBHFX8k2QhF5BcDFJjc9pKpr0/dZDOA6VS0Y\ngYuILlmyJHs5kUggkUi4iZmIYoqTmMUlk0kkk8ns5aVLl/q/mZWIfBHAvQBuUtVuk9srtg+ciMgv\nvveBi8itABYBuNEseRMRkX9cjcBFZBeAegDvp69qVdV/MNynqkfgXKBDRE5wP/CQcak9ETnF/cBD\nxgU6ROQnJnAiophiAvcRDyYmIj+xBu4zTmISkROcxCQiiilOYhIRVTAmcCKimGICJyKKKSZwIqKY\nYgInIoopnolJRJHB7WbLwzZCIoqEzJmZmWPX6hsbsOCpZVWbxH3fTpaIyCnjaLvYmZnVmsDtYAIn\nosAZR9s72jsw4tJLQo4qfjiJSUSBMxttA8IzM8vEEXiIuE8K0TlDhjZhwVPLOIlZBk5ihoSHPVA1\n44RladzMKsLWLF+I/Tu25F03asIUzJz3rZAiIgoWWwatsQuFiCJr0g1TmbRd4iRmSHjYAxG5xRJK\niDiJSUTFsAZORBRTPNCBiKiCOU7gIvLPItIhIm+KyAYRGe1lYEREZM1xCUVEhqjq8fTX/whgkqre\nY3I/llCIiMrkawklk7zTBgM46vS5iIiofK76wEXkUQB3ATgF4OOeRERERLZYJnAReQXAxSY3PaSq\na1V1MYDFIvI1AE8AmGv2PC0tLdmvE4kEEomE03iJiCpSMplEMpks6zGetBGKyBgAL6rq1Sa3sQZO\nRFQmX2vgIjIu5+JMAO1On4uIiMrnpgb+mIhMAHAWwB4AX/EmJCIisoMrMYmIIogrMYmIKhgTOBFR\nTDGBExHFFBM4EVFMMYETEcUUEzgRUUwxgRMRxRQTOBFRTDGBExHFFBM4EVFMMYETEcUUEzgRUUwx\ngRMRxRQTOBFRTDGBExHFFBM4EVFMMYETEcUUEzgRUUwxgRMRxRQTOBFRTDGBExHFFBM4EVFMuU7g\nIrJARPpF5HwvAiIiIntcJXARGQ3gZgBvexNOeJLJZNgh2MI4vRWHOOMQI8A4w+B2BP5vAP7Ji0DC\nFpcfKuP0VhzijEOMAOMMg+MELiIzAexX1d96GA8REdlUa3WjiLwC4GKTmxYDeBDALbl39zAuIiIq\nQVS1/AeJXA1gA4BT6atGATgAoFlVjxjuW/4LEBERVNVyYOwogRc8icgfAExR1fddPxkREdniVR84\nR9lERAHzZARORETBC3QlZtQX/YjIP4tIh4i8KSIb0n3ukSMiy0SkMx3rT0WkKeyYjETkcyLyOxE5\nKyLXhR2PkYjcKiK/F5FdIvJA2PGYEZFnReSwiGwNOxYrIjJaRF5N/7y3icj8sGMyIyKNIrI5/fu9\nXUQeCzumYkSkRkTaRWSt1f0CS+AxWfTzTVWdpKrXAvg5gCVhB1TEywCuUtVJAHYi1REUNVsB3AHg\n9bADMRKRGgDfAXArgCsBzBaRieFGZWolUjFGXS+A+1X1KgAfBzAvit9PVe0G8In07/efAviEiNwQ\ncljFfBXAdpQoTwc5Ao/8oh9VPZ5zcTCAo2HFYkVVX1HV/vTFzUh1AUWKqv5eVXeGHUcRzQB2q+pe\nVe0F8CMAM0OOqYCq/hLAB2HHUYqqHlLVN9NfnwDQCWBEuFGZU9VM51w9gBoAkWu8EJFRAG4HsAIl\n2rMDSeBxWvQjIo+KyD4AdwP4l7DjseFLAF4MO4iYGQngnZzL+9PXkUsiMhbAZKQGFpEjIgNE5E0A\nhwG8qqrbw47JxBMAFgHoL3VHy4U85YjLoh+LOB9S1bWquhjAYhH5GlLfyLmBBphWKs70fRYDOKOq\nPww0uDQ7MUYUZ+59ICKDAfwEwFfTI/HISf/lem163uglEUmoajLksLJEZAaAI6raLiKJUvf3LIGr\n6s1FAroawKUAOkQESP25v0VEChb9BKFYnCZ+iBBHtqXiFJEvIvVn1k2BBGSijO9l1BwAkDtBPRqp\nUTg5JCJ1AP4bwPOq+vOw4ylFVbtE5AUA1wNIhhxOrmkA/kpEbgfQCOAjIvI9Vf07szv7XkJR1W2q\nepGqXqqqlyL1i3JdGMm7FBEZl3NxJoD2sGKxIiK3IvUn1sz0xEzURW2bhV8DGCciY0WkHsDnAfxP\nyDHFlqRGZt8FsF1Vnww7nmJE5AIRGZr+eiBSTRWR+h1X1YdUdXQ6V34BwMZiyRsI50CHKP/5+piI\nbE3XyBIAFoQcTzHfRmqS9ZV0q9G/hx2QkYjcISLvINWV8IKIrA87pgxV7QNwH4CXkJrp/7GqdoYb\nVSERWQXgDQDjReQdEQmlnGfDXwCYg1RXR3v6XxS7Z4YD2Jj+/d4MYK2qbgg5plIs8yUX8hARxRSP\nVCMiiikmcCKimGICJyKKKSZwIqKYYgInIoopJnAiophiAiciiikmcCKimPp//UlSArT8h+AAAAAA\nSUVORK5CYII=\n", 268 | "text/plain": [ 269 | "" 270 | ] 271 | }, 272 | "metadata": {}, 273 | "output_type": "display_data" 274 | }, 275 | { 276 | "data": { 277 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEPCAYAAABcA4N7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu4XFV9//H3JwkhF0ISCCAhCQmUIIEAQe5yGSgg3qD0\nUblotdhfFWkVFVvFp0rQpxEvoNZKrdaA4kOQIqVQBZSWkSB3CBBJuCYhCYEg4eQCyQknyff3x9rD\nmZycmTPn5Ow5Z858Xs8zz5lZe8+eNTuwP7PW2mtvRQRmZmaVDOrrCpiZWf/moDAzs6ocFGZmVpWD\nwszMqnJQmJlZVQ4KMzOrKregkDRb0kpJ88vKjpT0oKR5kh6SdETZskskPSvpKUmn5VUvMzPrnjxb\nFFcDp3co+xbwlYiYAXw1e42kacDZwLTsPVdJcmvHzKwfyO1gHBFzgZYOxS8Bo7PnY4AXs+dnAnMi\noi0ilgDPAUfmVTczM6vdkDp/3peAeyR9hxRSx2Tl44H7y9ZbDuxV57qZmVkn6t2981PgMxExCfgc\nMLvKur62iJlZP1DvFsWREXFK9vxG4D+y5y8CE8vWm0B7t9RbJDk8zMx6ICLU0/fWu0XxnKQTs+cn\nA89kz28BzpE0VNIUYD/gwc42EBF+RHDppZf2eR36y8P7wvvC+6L6Y3vl1qKQNAc4ERgnaRnpLKdP\nAD+UtCOwIXtNRCyQdAOwANgEXBi98e3MzGy75RYUEXFuhUVHVVh/FjCr1u3/6ldw1FEwYUJPamdm\nZrWq9xhFr/nRj0Bq3qAoFAp9XYV+w/uinfdFO++L3qNG6uGR9FaP1HHHwQUXwEc+0seVMjPr5yQR\nDTSY3WvWr08PMzPLl4PCzMyqclCYmVlVDgozM6uqoYNiw4a+roWZ2cDXkEERkULCLQozs/w1ZFC0\ntqa/Dgozs/w1ZFCUAsJBYWaWPweFmZlV5aAwM7OqHBRmZlZVQwbFhg2w004OCjOzemjIoFi/HsaN\nc1CYmdWDg8LMzKpq2KDYdVcHhZlZPTRsULhFYWZWHw0bFKNHp+dtbX1bFzOzgS63oJA0W9JKSfPL\nyq6XNC97LJY0LyufLGlD2bKrqm17wwYYMSI93KowM8tXnvfMvhr4AfDzUkFEnFN6Luk7wOqy9Z+L\niBm1bHj9+q2DotS6MDOz3pdbiyIi5gItnS2TJOBDwJyebLtjUJiZWX76aozieGBlRDxfVjYl63Yq\nSjqu2pvXr4fhwx0UZmb1kGfXUzXnAteVvV4BTIyIFkmHATdLOjAi1nV848yZM7nnHnjhBdi0qcD6\n9YU6VdnMrDEUi0WKxWKvbU8R0Wsb22bj0mTg1oiYXlY2BFgOHBYRKyq87y7g4oh4tEN5RATnnQfv\nex/85Cfw1a/CSSfl9hXMzBqeJCJCPX1/X3Q9nQIsLA8JSeMkDc6e7wPsByyqtAGPUZiZ1U+ep8fO\nAe4FpkpaJun8bNHZbDuIfQLweHa67H8Cn4yI1VTgoDAzq5/cxigi4twK5ed3UnYTcFOt23ZQmJnV\nT8POzPZZT2Zm9dGwQeEWhZlZfTRkUJQu4TF8uIPCzCxvDRkUblGYmdWPg8LMzKpquKCISF1PHsw2\nM6uPhguK1lYYOhQGDXJQmJnVQ8MFRanbCRwUZmb10HBBUTrjCRwUZmb10HBB4RaFmVl9NXxQbNjQ\nt/UxMxvoGjIohg9Pz92iMDPLX0MGhbuezMzqp+GCwoPZZmb11XBB4RaFmVl9NXRQlC4KmOPdXM3M\nml5DB8XgwWmW9saNfVsnM7OBrCGDonTWE7j7ycwsbw0ZFKUWBTgozMzylltQSJotaaWk+WVl10ua\nlz0WS5pXtuwSSc9KekrSaZW2W37WEzgozMzyNiTHbV8N/AD4eakgIs4pPZf0HWB19nwacDYwDdgL\nuFPS1IjY0nGj69fD+PHtrx0UZmb5yq1FERFzgZbOlkkS8CFgTlZ0JjAnItoiYgnwHHBkZ+9115OZ\nWX311RjF8cDKiHg+ez0eWF62fDmpZbENB4WZWX3l2fVUzbnAdV2s0+nsiEcfncnmzfDMM1AoFBgx\nouCgMDMrUywWKRaLvba9ugeFpCHAWcBhZcUvAhPLXk/IyrYxadJMPv5xeNe70usf/cgtCjOzcoVC\ngUKh8Nbryy67bLu21xddT6cACyNiRVnZLcA5koZKmgLsBzzY2Zt91pOZWX3leXrsHOBeYKqkZZLO\nzxadTfsgNgARsQC4AVgA3AZcGNH5hTk8RmFmVl+5dT1FxLkVys+vUD4LmNXVdh0UZmb15ZnZZmZW\nVUMGha/1ZGZWPw0ZFG5RmJnVT8MFxYYNW7coSvekMDOzfDRcUAwdmu5DUeIWhZlZvhouKMq7nUqv\nHRRmZvlxUJiZWVUNFxTl4xPgoDAzy1vDBYVbFGZm9eWgMDOzqhwUZmZWlYPCzMyqclCYmVlVDRcU\nHc96GjYM3nwTNm/um/qYmQ10DRcUHVsUUgqPDRv6pj5mZgNdwwdFqcxBYWaWjwETFB6nMDPLh4PC\nzMyqclCYmVlVuQWFpNmSVkqa36H805IWSvqjpG9mZZMlbZA0L3tcVWm7Hc96AgeFmVmehuS47auB\nHwA/LxVIOgk4Azg4Itok7Va2/nMRMaOrjbpFYWZWX7m1KCJiLtDSofhTwDcioi1b50/d3a6Dwsys\nvuo9RrEfcIKk+yUVJR1etmxK1u1UlHRcpQ04KMzM6ivPrqdKnzc2Io6WdARwA7APsAKYGBEtkg4D\nbpZ0YESs67iB66+fyX33peeFQoFCoeCgMDMrUywWKRaLvbY9RUSvbWybjUuTgVsjYnr2+jbg8oj4\nffb6OeCoiFjV4X13ARdHxKMdyuOhh4LDy9shwEUXwT77pL9mZrY1SUSEevr+enc93QycDCBpKjA0\nIlZJGidpcFa+D6mLalFnG3DXk5lZfeXW9SRpDnAisKukZcBXgdnA7OyU2TeBj2arnwB8TVIbsAX4\nZESs7my7Dgozs/rKLSgi4twKi/6qk3VvAm6qZbuVgmJ1p7FiZmbbq6agyLqF9ihfPyKW5lWpatyi\nMDOrry6DQtKngUuBV4Dyuz5Mz6tS1VSamf3GG/Wvi5lZM6ilRfFZYP+OZyb1lcGDty0bOdItCjOz\nvNRy1tNSYG3eFdkeO+0Er7/e17UwMxuYamlRLAbukvRr0plKABERV+ZXre5xUJiZ5aeWoFiaPYZm\nDwH5zdLrAQeFmVl+ugyKiJgJIGlU9nqby2r0NQeFmVl+uhyjkDRd0jzgSeBJSY9IOij/qtXOQWFm\nlp9aBrN/DHw+IiZFxCTg4qys33BQmJnlp5agGBERd5VeREQRGJlbjXpgxAjYsAG2bOnrmpiZDTy1\nBMViSV/Jblc6RdI/UeGCfX1l0KA0Ec9zKczMel8tQfFxYHfStZh+BeyWlfUr7n4yM8tHLWc9vQZ8\nug512S4OCjOzfFQMCknfj4iLJN3ayeKIiDNyrFe3OSjMzPJRrUXx8+zvFZ0s61cT7sBBYWaWl4pB\nERGPZE8PjYjvlS+T9Fng93lWrLscFGZm+ahlMPtjnZT9dS/XY7s5KMzM8lFtjOJc4DxgSodxilFA\nv7jkeLmRIx0UZmZ5qDZGcS/wEjAO+A7pYoCQLjn+RM716ja3KMzM8lGx6ykiXgDuATZGxO8jopg9\nHo2ITV1tWNJsSSslze9Q/mlJCyX9UdI3y8ovkfSspKckndbdL+KgMDPLR9V5FBGxSdJmSWMiYnU3\nt3018APaz55C0knAGcDBEdEmabesfBpwNjAN2Au4U9LUiKj5ohwOCjOzfNRyP4o3gPmSfpc9hzSP\n4jPV3hQRcyVN7lD8KeAbEdGWrfOnrPxMYE5WvkTSc8CRwP01fQtSUKzqdyMnZmaNr5aguCl7lOZO\nbM+Ni/YDTpA0C2gFvhARDwPj2ToUlpNaFjVzi8LMLB+1XMLjGkk7AlOzoqdKLYIeft7YiDha0hHA\nDcA+lT66s8KZM2e+9bxQKFAoFAAHhZlZSbFYpFgs9tr2ugwKSQXgZ8ALWdEkSR+LiJ5MuFtOap0Q\nEQ9J2iJpHPAiMLFsvQlZ2TbKg6JcLUERAW1tMHRot+ttZtYwyn9EA1x22WXbtb1aJtxdCZwWESdE\nxAnAacB3e/h5NwMnA0iaCgyNiFeBW4BzJA2VNIXURfVgdzZcS1Dcfjt88IM9qbaZWfOqZYxiSEQ8\nXXoREc9IqqUlMgc4EdhV0jLgq8BsYHZ2yuybwEezbS6QdAOwANgEXBgR3RoHqSUoVq6Ehx/uzlbN\nzKyWoHhE0n8AvyANZH8Y6PJwGxHnVlj0VxXWnwXMqqE+naolKNauhRUr4LXXYJddevpJZmbNpZau\npwuAhcBnSPeleJJ0mmu/UmtQAMyfX309MzNrV1NQRMQVEfGX2eO7pPDoV7oTFE/0uwuQmJn1X7UE\nxV93UnZ+L9dju5UuClhtZGPtWjjoILcozMy6Y8BcPXboUBg8GDZuhGHDOl9n7Vo47jh47LH61s3M\nrJHVcvXY3WiAq8dCe/dTpaBYswbe8x74xS9gyxYYVEt7ysysyVW9emxEFIFTgHuy5y+RJsOp0vv6\nUlfjFGvXwt57w+jRsGRJ3aplZtbQavlN/XtgR0l7AXeQTm+9Js9K9VQtQbHzzjB9uscpzMxqVUtQ\nDIqI9cBfAldFxAeBg/KtVs/UGhQHH+ygMDOrVU299JKOIU20+3V33ldvO+0Eb7xReblbFGZm3VfL\nAf+zwCXAf0XEk5L2Be7Kt1o9U61FEeGgMDPriVouM/570jhF6fXzpFna/U61oNiwAXbYIT3e/nZY\ntAhaWyufIWVmZkm1eRTfj4iLOsyhKImIOCPHevVItaAotSYAdtwR9t0XFi6EGTPqVz8zs0ZUrUVR\nutf1FZ0s6+kd7nJVa1BA+4C2g8LMrLqKQRERj2R/i5J2y57/qdL6/UF3gsLjFGZmtak4mK1kpqRX\ngWeAZyS9KunS+lWve7obFL44oJlZ16qd9fQ54J3AERExNiLGAkcC75T0+brUrpvcojAz633VguKj\nwHkRsbhUEBGLSPMpPpp3xXqiWlCsWZMu3VGy997Q0lJ93oWZmVUPiiGdjUlkZbXcGa/uutOikGDs\nWFi9uj51MzNrVNWCoq2Hy/pMd4ICUgtjzZr862Vm1siqBcXBktZ19gCmd7VhSbMlrZQ0v6xspqTl\nkuZlj9Oz8smSNpSVX9WTL9PdoBgzxi0KM7OuVDs9dvB2bvtq4Ae0z8eANP/iyoi4spP1n4uI7ZrV\n0FVQTJ68dZlbFGZmXcvt4n4RMRdo6WRRbveycNeTmVnv64urwH5a0uOSfippTFn5lKzbqSjpuJ5s\nuDtnPYG7nszMalHvs5f+Dfha9vzrpMuD/A2wApgYES2SDgNulnRgRKzruIGZM2e+9bxQKFAoFN56\n7RaFmRkUi0WKxWKvbU8R+V22SdJk4NaI2Gbwu4tldwEXR8SjHcqjWn0jYMgQ2Lgx/S136KFwzTXp\nb8k//3OaRzFrVs1fycys4UgiInrc7V/XridJe5a9PAuYn5WPkzQ4e74PsB+wqPvbr3zzIp/1ZGbW\nM7l1PUmaA5wIjJO0DLgUKEg6lHT202Lgk9nqJwBfk9QGbAE+GRE9OoSXup86jke468nMrGdyC4qI\nOLeT4tkV1r0JuKk3PrezcYrS3e1Gjdq6fPRotyjMzLrSL+99vT06C4rWVhg0KN2wqNyYMW5RmJl1\npSmCYu3abbuiwF1PZma1aJqg6Dg+Ae56MjOrRVMHhbuezMy61tRBMWpUOpV28+b61M3MrBE1dVAM\nGpTWX7u2PnUzM2tETR0U4O4nM7OuNEVQdHZBwBKf+WRmVl1TBEW1FoXPfDIzq67pg8JdT2Zm1Q24\noBg5svstCgeFmVllAy4o3PVkZta7mj4o3PVkZlZdUwSFz3oyM+u5pggKdz2ZmfVc0weFu57MzKpr\n+qBw15OZWXUDLihGjoS2tq3vm+2uJzOznhtwQTF4MBx1FNx9d3q9cWP62/HudiXuejIzqy63oJA0\nW9JKSfPLymZKWi5pXvZ4d9mySyQ9K+kpSadtz2efeir87nfpebUznsBdT2ZmXcmzRXE1cHqHsgCu\njIgZ2eM2AEnTgLOBadl7rpLU47qVB0W1bidILQp3PZmZVZZbUETEXKClk0XqpOxMYE5EtEXEEuA5\n4Miefvbhh8OLL8JLL3UdFMOHw6ZN7V1UZma2tb4Yo/i0pMcl/VTSmKxsPLC8bJ3lwF49/YDBg+Gk\nk+DOO7sOCsndT2Zm1Qyp8+f9G/C17PnXgSuAv6mwbnRWOHPmzLeeFwoFCoVCp28udT994APVgwLa\nB7R33736emZmjaBYLFIsFntte4ro9HjcOxuXJgO3RsT0asskfQkgIi7Plt0OXBoRD3R4T9Ra3+ef\nh+OPh29+E+64A37xi8rrvuMd8O//nrqszMwGGklERGfd/jWpa9eTpD3LXp4FlM6IugU4R9JQSVOA\n/YAHt+ez9t0Xhg2D++6rftYTeC6FmVk1uXU9SZoDnAiMk7QMuBQoSDqU1K20GPgkQEQskHQDsADY\nBFxYc9OhilNPhZtugvPPr76e51KYmVWWW1BExLmdFM+usv4sYFZv1uHUU+HHP+56jKKrweyf/AQ+\n8pF0hpSZWbMZcDOzy518cjqrqZagqNT1tGoVfOITcPXVvV8/M7NGMKCDYpdd0gB1rWc9deb++2Gv\nveDb307zLczMms2ADgpI3UbvfW/1dap1Pd13XxrjmDQJfvnL3q+fmVl/N+CD4pBDUsuimmpdT/fe\nC8ceC5dcApdfDjmeTWxm1i8N+KCoRaWup02b4KGH4Oij4V3vSjO+f/Ob+tfPzKwvOSio3PU0fz5M\nnAhjx6ZB8S99Cb7xjfrXz8ysLzkoqNz1VOp2KvnAB+Dll+G3v61f3czM+pqDgspdT/fdB8cc0/56\nyBD44Q/TnIprr61f/czM+pKDgspdTx1bFJDGKv7v/+Cyy+Bzn/Mps2Y28DkoaO96Kj+j6eWXU9n+\n+2+7/kEHpUHuhQvTpL7nn992HZ8dZWYDhYMCGDo0Pdavby+77750ttOgCnto7Fj49a/hzDPTPbqv\nvBI2b04zub/1LZgyBd79bli3rj7fwcwsLw6KTMfup3vv3Xp8ojODB8PFF8MDD8D//E9qafzZn8GC\nBXDDDemMqZNOgpUr8627mVmeHBSZjmc+3XfftuMTley7L/zv/8L3vw9PPw3XXANHHpnucfHe98I7\n39l595SZWSOo9x3u+q3yM5/efBPmzUsH+1pJcNpp25ZddhnsuWcKi2uvTVe0NTNrJG5RZEpdT2vX\npkt1TJ0Ko0b1zrYvuADmzIGPfQy+/nXYsqV3tmtmVg8Oiszo0TBrFkyenMYYfvaz3t3+SSfBww+n\n+3i/973w+uu9u30zs7zkes/s3tade2Z31zXXwDPPwIUXwoQJuXwEAG1t8PGPp+6t669P3VNmZnna\n3ntmOyj6QGsrHHccfPjDadKemVmetjcocut6kjRb0kpJ8ztZdrGkLZJ2yV5PlrRB0rzscVVe9eoP\nhg2DG2+Eb34T5s7t69qYmVWX51lPVwM/AH5eXihpInAq8EKH9Z+LiBk51qdfmTw5jYOcc066zeqr\nr8ILL6SJfBdc0Ne1MzNrl1uLIiLmAi2dLLoS+Me8PreRvOtd8MUvwqWXwi23pHkcX/taOjXXzKy/\nqOs8CklnAssj4gltO4o7RdI8YA3wTxFxTz3r1lc+85n0KNltt9Qldf31fVcnM7NyuQ5mS5oM3BoR\n0yWNAO4CTo2ItZIWA4dHxCpJQ4GREdEi6TDgZuDAiFjXYXsDYjC7mnXr0nWi7r8/XQ7EzGx7be9g\ndj1bFPsCk4HHs9bEBOARSUdGxCvAmwAR8aik54H9gEc7bmTmzJlvPS8UChQKhbzrXVejRsGnPgXf\n/na6BIiZWXcVi0WKxWKvba9uLYpOli0G3hERr0kaB7RExGZJ+wB3AwdFxOoO7xnwLQqAP/0pXd78\nySfT5T/MzLZHfz49dg5wLzBV0jJJ53dYpfyIfwKppTEP+E/gkx1Dopnstlu6i973vtfXNTEz84S7\nfuuFF+Cww9JVbKdO7evamFkj67ctCts+e++drjx77LGpdbFwYV/XyMyalYOiH/v7v0/3sTjwQCgU\n4PTT4aqrYNmyvq6ZmTUTdz01iDfegN/8Jk3Mu+22dPe8009Pj2OPhR122Hr9TZtg/nxYuhTe9750\nNz4za06+KGAT2rQpjV3ccUd6PPMMTJoEu+ySLgGyZk26pPnEibDjjukS6tdem153R2srLFqU7gM+\ndmza/pgx6fM3bkzL99gj3W/czPovB4WxahW8+CK89lp6jBgBRx2VDu6bN8O3vgXf/S788IfwwQ9W\n39bChem+HPfcAy+9lMZKxo1Llxd57bX0d8iQFEBDh8L69fDnfw7veQ+8+90wfnx9vrOZ1c5BYTV5\n6KF0WfPhw1NX1PvfD4cfnoKkrS2Ne8yalVoon/88fOAD6cKFQ7qYkrlyJdx+O/z613DnnenU3kIB\nTj4ZzjrLrQ2z/sBBYTXbvBkeeABuvTU9FixIQbDDDmlG+IUXwmc/Czvv3PPtz58PxWLa/qJF6SKH\n5503MMZIIlKL6pVX0mPNGth999Slt8ceMKgHp4Zs2pS209KSxqH23z9dht6sNzkorN+6+2645JJ0\nIPzwh2HffWGffdpnm5f+Kd/2tq1bLgsWwC9/CY8/DieckAbsDzig/W6AEfDUU6kFc+edaf1DDoGj\nj05dbnvtlcZSdt656xZRRxGpy62lJV13a+1aeOKJNCZ0773pYL7HHikgdt45BcayZek7Dh2aDvxt\nbelgv88+6bH33ilE169Pj1Wr0me89FJ63+jRqb7Dh8OSJWn+zIknwqmnwjHHbHuigll3OSisX4tI\nZ2kVi6mFsWhROkBK6Rf45s3poDx5cppYuGRJGgv50IdS19jdd6eurTffTOMia9akg/eECXDKKelx\n0EHpYH7//amL7eWX0y//tWth113TdqdOTRdZ3H33VLbrrqkOra3psWRJGpeZOzcd7MeNS62sUaNS\nSB1zTHpMntz57WtbW1NAlFpo69en7/r88+nMsyFD0tjRiBHps/fcMwXkrrtu3RJ5/fUUSMVi6gZc\nvDgFxoknpjAZNSoFyqJF6RIvf/xjaq0deijMmJFOpR47Nq27007951a7b7wBV1wB//IvMHJkCvMJ\nE9K/yyGHwMEHp7KWlnRvltXZdRl22CHtu7e9LZ2w0ZNWmzkobADYsCEdUJ9+Ov1aP/bYrQ8IEelA\nvmVLe0uhll/ZW7ak0Hj22XRm2PPPp4PQq6+mX/WQwmfYsHTgPu44OP74dPXe/nKAXbEiBe0DD6Tg\nW7cuhdCUKSkgDzwwBdu8efDYY6mltXp1erS2poNyKfDGj08H5OnT08F5+vTujSEtXZo+v7TPhgxJ\n+zgihdUee2y731pb4brr4CtfSa3DmTPTv92LL6bHwoUp5B9/PP2AKIX42LHp/aUW2vLl6fsfcEDq\nnpsyJT323DO16BYuTC3Llpb042Pz5lSnXXZJ2xszJrX+li5NVz3YvDnVtxRAxxyT/u3LW66trem/\nzS1b0vobN6Y6rliR/u6wQ3trcNIk2G+//htkDgoz69SmTenA/vrr6e/SpWkM6YknUqgsXpwC46ij\n0i/7SZNSN9no0emg+Oab6eB6221p/s6rr6aW1saN6dHWlg6MgwaldVtb00F8//3TQX3BgnRQPfpo\nuPzy9Dnbo6UlBcJTT6WD/ZIlKWwmTIBp09JBfvfdU30GD04H99deSz8KVq9OJ1qUvuPgwelEjJUr\n0w+IP/whtV7XrEnL1q5Nnzl8eHo9aFAK1T33TIG7557t40urV7efRn7YYakuGze27/vRo9N7xo9P\nn7///ql1O3z4dv8T18xBYWY9sm5dmm/z4IPtXWRLl6aDZOn059GjU/fe+98PRxxR/Rfz6tXpIP70\n06nVd8ABaVyqkcZYVq5Mf0eP7v5JBatWwSOPpH0wbFhqxY0cmfZnqRWyZEnaP4sWpeAaPTqts9NO\nqYWz994pTMaPT8t23rk9aLYnWBwUZmYNZtOm1J22bl0av3n99dRN+sIL6fHSS+0nU7S0pNejRrVP\nml21KrWW3nijfVysNLl2zJjUdfeFL6T5TdBYNy4yMzPSwX3y5NrX37Il3adm2bI0hlIayxk5MnUB\ntrWl7q5SV9jq1b17h0y3KMzMBjhfZtzMzHLloDAzs6ocFGZmVpWDwszMqsotKCTNlrRS0vxOll0s\naYukXcrKLpH0rKSnJJ2WV73MzKx78mxRXA2c3rFQ0kTgVOCFsrJpwNnAtOw9V0lya6eKYrHY11Xo\nN7wv2nlftPO+6D25HYwjYi7Q0smiK4F/7FB2JjAnItoiYgnwHHBkXnUbCPw/QTvvi3beF+28L3pP\nXX+1SzoTWB4RT3RYNB5YXvZ6ObBX3SpmZmYV1W1mtqQRwJdJ3U5vFVd5i2fWmZn1A7nOzJY0Gbg1\nIqZLmg7cCazPFk8AXgSOAs4HiIjLs/fdDlwaEQ902J7Dw8ysBxriWk8RMR/Yo/Ra0mLgHRHxmqRb\ngOskXUnqctoPeLCTbfSTuwSYmTWPPE+PnQPcC0yVtEzS+R1Weat1EBELgBuABcBtwIW+qJOZWf/Q\nUBcFNDOz+muYuQqSTs8m4z0r6Yt9XZ96kjRR0l2SnpT0R0mfycp3kfQ7Sc9I+q2kMX1d13qRNFjS\nPEm3Zq+bcl9IGiPpRkkLJS2QdFQT74tLsv9H5ku6TtKOzbIvOpvgXO27d3eCc0MEhaTBwL+SJuNN\nA86VdEDf1qqu2oDPRcSBwNHA32Xf/0vA7yJiKvC/2etmcRGpq7LUJG7WffF94DcRcQBwMPAUTbgv\nshNn/hY4LCKmA4OBc2iefdHZBOdOv3tPJjg3RFCQJt89FxFLIqINuJ40Sa8pRMTLEfFY9vx1YCFp\n0P8M4GfZaj8D/qJvalhfkiYA7wH+g/ZTrJtuX0gaDRwfEbMBImJTRKyhCfcFsJb0g2qEpCHACGAF\nTbIvKkyAbXiuAAAEe0lEQVRwrvTduz3BuVGCYi9gWdnrpp2Ql/1ymgE8AOwREdldfllJ2VllA9x3\ngX8AtpSVNeO+mAL8SdLVkh6V9BNJI2nCfRERrwFXAEtJAbE6In5HE+6LMpW+e7cnODdKUHjEHZC0\nE/Ar4KKIWFe+LDtLbMDvJ0nvA16JiHlUmLDZLPuCdHr7YcBVEXEY8AYdulaaZV9I2hf4LDCZdCDc\nSdJHytdpln3RmRq+e9X90ihB8SIwsez1RLZOxAFP0g6kkLg2Im7OildKelu2fE/glb6qXx0dC5yR\nzcOZA5ws6Vqac18sJ10S56Hs9Y2k4Hi5CffF4cC9EbEqIjYBNwHH0Jz7oqTS/xMdj6elyc8VNUpQ\nPAzsJ2mypKGkgZhb+rhOdSNJwE+BBRHxvbJFtwAfy55/DLi543sHmoj4ckRMjIgppMHK/4uIv6I5\n98XLwDJJU7OiU4AngVtpsn1BGsQ/WtLw7P+XU0gnOzTjviip9P/ELcA5koZKmkKFCc7lGmYehaR3\nA98jnc3w04j4Rh9XqW4kHQfcDTxBexPxEtI/7g3AJGAJ8KGIWN0XdewLkk4ELo6IM7J7mzTdvpB0\nCGlQfyjwPOlyOINpzn3xj6QD4hbgUeD/AaNogn2RTXA+ERhHGo/4KvDfVPjukr4MfBzYROrKvqPq\n9hslKMzMrG80SteTmZn1EQeFmZlV5aAwM7OqHBRmZlaVg8LMzKpyUJiZWVUOCmsqkl7P/u4t6dxe\n3vaXO7z+Q29u36yvOCis2ZQmDk0BzuvOG7OrklZzyVYfFPHO7mzfrL9yUFizuhw4Prv50UWSBkn6\ntqQHJT0u6RMAkgqS5kr6b+CPWdnNkh7ObiL1t1nZ5cDwbHvXZmWl1ouybc+X9ISkD5VtuyjpP7Mb\nD/2iVDlJl2c34Xlc0rfrumfMOujqF5LZQPVF4AsR8X6ALBhWR8SRknYE7pH022zdGcCBEfFC9vr8\niGiRNBx4UNKNEfElSX8XETPKPqPUevlL4BDSjYV2Ax6SdHe27FDSDWReAv4g6Z2k6xb9RUS8Pavb\nzjl8f7OauUVhzarjJcpPAz4qaR5wP7AL8GfZsgfLQgLgIkmPAfeRrsK5XxefdRxwXSSvAL8HjiAF\nyYMRsSK7DPRjwN7AaqBV0k8lnQVs6PG3NOsFDgqzdn8fETOyx74RcWdW/kZpBUkF4M+BoyPiUGAe\nMKyL7QbbBlOptbGxrGwzsENEbCbdcexG4H3A7T35Mma9xUFhzWod6cqiJXcAF5YGrCVNlTSik/ft\nDLRERKukt5PuYV7SVmHAey5wdjYOshtwAunKv53eeCm7S92YiLgN+Dyp28qsz3iMwppN6Zf848Dm\nrAvpauBfSHdHezS7n8ErwFnZ+uWXWL4duEDSAuBpUvdTyY+BJyQ9kt0jIwAi4r8kHZN9ZgD/EBGv\nSDqAbe8sFqQA+29Jw0hh8rle+eZmPeTLjJuZWVXuejIzs6ocFGZmVpWDwszMqnJQmJlZVQ4KMzOr\nykFhZmZVOSjMzKwqB4WZmVX1/wEwUpQhqSwoPAAAAABJRU5ErkJggg==\n", 278 | "text/plain": [ 279 | "" 280 | ] 281 | }, 282 | "metadata": {}, 283 | "output_type": "display_data" 284 | } 285 | ], 286 | "source": [ 287 | "%matplotlib inline\n", 288 | "from sklearn.datasets import load_iris\n", 289 | "import matplotlib.pyplot as plt\n", 290 | "import numpy as np\n", 291 | "import random\n", 292 | "from sklearn.decomposition import PCA\n", 293 | "from sklearn import preprocessing\n", 294 | "\n", 295 | "np.random.seed(12222015)\n", 296 | "\n", 297 | "iris = load_iris() \n", 298 | "X = iris.data\n", 299 | "Y = iris.target\n", 300 | "\n", 301 | "X_scaled = preprocessing.scale(X)\n", 302 | "\n", 303 | "k = 3 \n", 304 | "\n", 305 | "centers, distortion = mini_batch(X_scaled, k, 5, 100)\n", 306 | "\n", 307 | "C=[0]*len(X_scaled)\n", 308 | "for i in range(len(X_scaled)):\n", 309 | " index = get_closest(X_scaled[i], centers)[0]\n", 310 | " C[i] = index\n", 311 | "\n", 312 | " \n", 313 | "C = np.array(C)\n", 314 | " \n", 315 | "#use PCA to reduce the dimenions to 2(for plotting)\n", 316 | "pca = PCA(n_components=2)\n", 317 | "pca.fit(X_scaled)\n", 318 | "\n", 319 | "centers_pca = pca.transform(centers) \n", 320 | "X_pca = pca.transform(X_scaled) \n", 321 | "\n", 322 | "plot_kmeans(C, centers_pca, X_pca, k)\n", 323 | "\n", 324 | "plt.figure()\n", 325 | "index1= [i for i in range(100)]\n", 326 | "plt.plot(index1, distortion)\n", 327 | "plt.xlabel('Iterations')\n", 328 | "plt.ylabel('Distortion')\n" 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "execution_count": null, 334 | "metadata": { 335 | "collapsed": true 336 | }, 337 | "outputs": [], 338 | "source": [] 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": null, 343 | "metadata": { 344 | "collapsed": true 345 | }, 346 | "outputs": [], 347 | "source": [] 348 | } 349 | ], 350 | "metadata": { 351 | "kernelspec": { 352 | "display_name": "Python 2", 353 | "language": "python", 354 | "name": "python2" 355 | }, 356 | "language_info": { 357 | "codemirror_mode": { 358 | "name": "ipython", 359 | "version": 2 360 | }, 361 | "file_extension": ".py", 362 | "mimetype": "text/x-python", 363 | "name": "python", 364 | "nbconvert_exporter": "python", 365 | "pygments_lexer": "ipython2", 366 | "version": "2.7.10" 367 | } 368 | }, 369 | "nbformat": 4, 370 | "nbformat_minor": 0 371 | } 372 | -------------------------------------------------------------------------------- /Online_Algorithm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "from __future__ import division" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": { 19 | "collapsed": false 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "%matplotlib inline\n", 24 | "import wave, struct, numpy as np, matplotlib.mlab as mlab, pylab as pl\n", 25 | "\n", 26 | "filename = \"CML_Recording_Both.wav\"\n", 27 | "w = wave.open(filename,\"rb\")\n", 28 | "\n", 29 | "#returns a named tuple (nchannels, sampwidth, framerate, \n", 30 | "# nframes, comptype, compname)\n", 31 | "waveParams = w.getparams()\n", 32 | "\n", 33 | "s = w.readframes(waveParams[3])\n", 34 | "w.close()\n", 35 | "waveArray = np.fromstring(s, np.int16)\n", 36 | "\n", 37 | "spectrum, freq, bins = mlab.specgram(waveArray, NFFT=256,Fs=waveParams[2],sides='onesided')\n" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": { 44 | "collapsed": true 45 | }, 46 | "outputs": [], 47 | "source": [ 48 | "# calculates and returns spectrogram of \n", 49 | "def get_wav(filename)\n", 50 | " filename = \"CML_Recording_Both.wav\"\n", 51 | " w = wave.open(filename,\"rb\")\n", 52 | " return w\n", 53 | "\n", 54 | "# spectrogram is the absolute value of the stft\n", 55 | "def get_sepctrogram(stft):\n", 56 | " return abs(stft)\n", 57 | "\n", 58 | "# takes in wave file as input\n", 59 | "# win_size is the length of the window in samples\n", 60 | "# overlap is the amount of overlap between windows in samples\n", 61 | "def my_stft(w, win_size, overlap):\n", 62 | " # number of samples in the wave file\n", 63 | " n = len(w)\n", 64 | " win_count= 0\n", 65 | " \n", 66 | "\n", 67 | "def my_istft(stft):\n", 68 | " " 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 3, 74 | "metadata": { 75 | "collapsed": false 76 | }, 77 | "outputs": [ 78 | { 79 | "data": { 80 | "text/plain": [ 81 | "(129, 11263)" 82 | ] 83 | }, 84 | "execution_count": 3, 85 | "metadata": {}, 86 | "output_type": "execute_result" 87 | } 88 | ], 89 | "source": [ 90 | "spectrum.shape" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 84, 96 | "metadata": { 97 | "collapsed": false 98 | }, 99 | "outputs": [], 100 | "source": [ 101 | "import numpy as np\n", 102 | "\n", 103 | "# divergence\n", 104 | "def div(v,W,h):\n", 105 | " whv = np.dot(W,h) * 1/v\n", 106 | " div = whv - np.log(whv) - 1\n", 107 | " div = np.dot( div, np.ones(div.shape) )\n", 108 | " return div\n", 109 | "\n", 110 | "# divergence gradient\n", 111 | "def div_grad(v,W,h):\n", 112 | " grad = np.dot( 1/v - 1/(np.dot(W,h)) , W)\n", 113 | " return grad\n", 114 | "\n", 115 | "# epsilon divergence\n", 116 | "def compute_obj(v,W,h,eps):\n", 117 | " \n", 118 | " whv = (np.dot(W,h) + eps)/(v + eps)\n", 119 | " if any(whv < 0):\n", 120 | " print(W)\n", 121 | " print(h)\n", 122 | " print(v)\n", 123 | " print('whv: ' + str(whv))\n", 124 | " #print('whv.shape: ' + str(whv.shape))\n", 125 | " #print('np.dot(W,h): ' + str(np.dot(W,h)))\n", 126 | " div = whv - np.log(whv) - 1\n", 127 | " #print(div)\n", 128 | " return np.sum( div )\n", 129 | "\n", 130 | "# epsilon divergence gradient\n", 131 | "def compute_grad(v,W,h,eps):\n", 132 | " #print('compute_grad start')\n", 133 | " #print('W.shape: ' + str(W.shape))\n", 134 | " #print('h.shape: ' + str(h.shape))\n", 135 | " #print('v.shape: ' + str(v.shape))\n", 136 | " #print('np.dot(W,h).shape: ' + str( (np.dot(W,h) + eps).shape ) )\n", 137 | " #print((1/(v + eps) - 1/(np.dot(W,h) + eps)).shape)\n", 138 | " grad = np.dot(W.T, (1/(v + eps) - 1/(np.dot(W,h) + eps)))\n", 139 | " #print('compute_grad end')\n", 140 | " return grad\n", 141 | "\n", 142 | "def itakura_saito(y,x):\n", 143 | " y = np.array(y)\n", 144 | " x = np.array(x)\n", 145 | " return np.sum(y/x - np.log(y/x) -1)\n", 146 | " \n", 147 | "# important! input here has to be the matrix H, not the vector h_t\n", 148 | "# this is because we need h_t but also h_(t-1) and h_(t+1)\n", 149 | "# ind_t is the index of h_t in H\n", 150 | "# lambda is the smoothness constant\n", 151 | "def compute_smooth_obj(v,W,H,ind_t,lamb,eps):\n", 152 | " \n", 153 | " # get the column h_t\n", 154 | " h = H[:,ind_t]\n", 155 | " h = h.reshape(h.shape[0],1)\n", 156 | " \n", 157 | " # compute regular objective\n", 158 | " # maybe doing this direct instead of the function call is faster:\n", 159 | " # whv = (np.dot(W,h) + eps)/(v + eps)\n", 160 | " # div = whv - np.log(whv) - 1 \n", 161 | " div = compute_obj(v,W,h,eps)\n", 162 | " \n", 163 | " # compute smoothness terms\n", 164 | " s1 = H[:,ind_t]/H[:,ind_t-1]\n", 165 | " s2 = H[:,ind_t]/H[:,ind_t+1]\n", 166 | " sm = s1 - np.log(s1) - 1\n", 167 | " sm += s2 - np.log(s2) - 1\n", 168 | " # returning properly scaled smooth objective\n", 169 | " return div + lamb * np.sum( sm )\n", 170 | " \n", 171 | "# input parameters as above\n", 172 | "def compute_smooth_grad(v,W,H,ind_t,lamb,eps):\n", 173 | " \n", 174 | " # get the column h_t\n", 175 | " h = H[:,ind_t]\n", 176 | " h = h.reshape(h.shape[0],1)\n", 177 | " \n", 178 | " # calculates gradient of regular divergence\n", 179 | " div_grad = compute_grad(v,W,h,eps)\n", 180 | "\n", 181 | " # calculating gradient of smoothness term\n", 182 | " sm_grad = 1/H[:,ind_t-1] + 1/H[:,ind_t+1] - 2/H[:,ind_t]\n", 183 | " sm_grad = sm_grad.reshape(sm_grad.shape[0],1)\n", 184 | " \n", 185 | " # returning properly scaled gradient of smooth objective \n", 186 | " return div_grad + lamb * sm_grad\n", 187 | " " 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 85, 193 | "metadata": { 194 | "collapsed": false 195 | }, 196 | "outputs": [ 197 | { 198 | "name": "stdout", 199 | "output_type": "stream", 200 | "text": [ 201 | "[[-0.91200162]\n", 202 | " [ 0.42126031]]\n", 203 | "[[-0.91199808]\n", 204 | " [ 0.42126121]]\n" 205 | ] 206 | } 207 | ], 208 | "source": [ 209 | "def grad_checker(v, W, h):\n", 210 | " eps = 1e-3\n", 211 | " (f,k) = W.shape\n", 212 | " t_grad = np.zeros(h.shape)\n", 213 | " for i in range(k):\n", 214 | " ei = np.zeros(h.shape)\n", 215 | " ei[i] = eps\n", 216 | " t_grad[i] = (compute_obj(v,W,h+ei, 1e-12) - compute_obj(v,W,h-ei,1e-12)) / (2*eps)\n", 217 | " print(t_grad)\n", 218 | " print(compute_grad(v,W,h,1e-12))\n", 219 | "grad_checker(np.random.rand(2,1), np.random.rand(2,2),np.random.rand(2, 1))" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 86, 225 | "metadata": { 226 | "collapsed": false 227 | }, 228 | "outputs": [ 229 | { 230 | "name": "stdout", 231 | "output_type": "stream", 232 | "text": [ 233 | "[ 7.53544566 9.5102638 ]\n", 234 | "[[ 7.53544693]\n", 235 | " [ 9.51026604]]\n" 236 | ] 237 | } 238 | ], 239 | "source": [ 240 | "def grad_checker(v,W,H,ind_t,lamb,eps):\n", 241 | " eps_dif = 1e-3\n", 242 | " (f,k) = W.shape\n", 243 | " t_grad = np.zeros(H[:,ind_t].shape)\n", 244 | " for i in range(k):\n", 245 | " ei = np.zeros(H.shape)\n", 246 | " ei[i,ind_t] = eps_dif\n", 247 | " t_grad[i] = (compute_smooth_obj(v,W,H+ei,ind_t,lamb,eps) - compute_smooth_obj(v,W,H-ei,ind_t,lamb,eps)) / (2*eps_dif)\n", 248 | " print(t_grad)\n", 249 | " print(compute_smooth_grad(v,W,H,ind_t,lamb,eps))\n", 250 | "grad_checker(np.random.rand(2,1), np.random.rand(2,2),np.random.rand(2, 3),1,1,1e-12)" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 98, 256 | "metadata": { 257 | "collapsed": false 258 | }, 259 | "outputs": [], 260 | "source": [ 261 | "def gradient_backtracking(v, W, h, max_iter, compute_grad, compute_obj, eps):\n", 262 | " \n", 263 | " v = v.reshape(v.shape[0],1)\n", 264 | " \n", 265 | " beta = 0.5 #backstep factor between 0.1 and 0.8\n", 266 | " opt_prec = 1-1e-4 # optimization precision\n", 267 | " n = 1e-1 #initial step size\n", 268 | " \n", 269 | " h = np.random.rand(2, 1)\n", 270 | " \n", 271 | " obj = [None]*max_iter\n", 272 | " \n", 273 | " max_backstep = 100 # maximum number of backsteps\n", 274 | " t = 0 # backstepping counter\n", 275 | " k = 0 # gradient step counter \n", 276 | " \n", 277 | " while( k < max_iter and t != max_backstep ):\n", 278 | " \n", 279 | " grad = compute_grad(v,W,h,eps)\n", 280 | " obj[k] = compute_obj(v,W,h,eps)\n", 281 | " \n", 282 | " t = 0 # reset backstepping counter\n", 283 | " n = 1/beta*n # try to increase stepsize slightly again\n", 284 | " \n", 285 | " # make sure h-n*grad is positive\n", 286 | " while(any(h - n * grad < 0) and t < max_backstep ):\n", 287 | " t += 1\n", 288 | " n = beta * n\n", 289 | " \n", 290 | " new_obj = compute_obj(v,W,(h - n*grad),eps)\n", 291 | " while( new_obj > opt_prec * compute_obj(v,W,h,eps) and t < max_backstep):\n", 292 | " t += 1\n", 293 | " n = beta * n\n", 294 | " new_obj = abs(compute_obj(v,W,(h - n*grad),eps))\n", 295 | " \n", 296 | " h = h - n * grad # update h according to gradient step\n", 297 | " k += 1 # update gradient step counter\n", 298 | " \n", 299 | " \n", 300 | " return h, obj[0:k]\n", 301 | "\n", 302 | "h, obj = gradient_backtracking(np.random.rand(10,1), np.random.rand(10,2), np.random.rand(2, 1), 100, compute_grad, compute_obj, 1e-12)" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": 99, 308 | "metadata": { 309 | "collapsed": false, 310 | "scrolled": true 311 | }, 312 | "outputs": [ 313 | { 314 | "name": "stdout", 315 | "output_type": "stream", 316 | "text": [ 317 | "[25.294090530377957, 9.523756099388935, 7.9600867384393883, 7.6123309624643438, 7.5394195057403977, 7.5046593723963975, 7.4961605757929686]\n" 318 | ] 319 | }, 320 | { 321 | "data": { 322 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEKCAYAAAAyx7/DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGGVJREFUeJzt3XuUVeWd5vHvUwUCAkJABSI3gxKJwXgb2hgNx0SjTow6\nMVHbYCRG0zPptrPSdtKm15qWdK/uTreT7qzMJTMqGCAGzWhrJLFVopQx8QYtKCpqgnKxR0CjchFE\nkN/8sXfJSVGXU6fOqX32Ps9nrbNqn3325beL4qm33ndfFBGYmVl+tGRdgJmZ9Y6D28wsZxzcZmY5\n4+A2M8sZB7eZWc44uM3McsbBbYUi6W5Jl9Z62b6StFfSB/pjX1Z88nncVi1Js4GrgQ8AW4E7gG9F\nxJYqt7cXOCIiXqxZkQ2i0mOTNBl4ERgQEXv7oTTLIbe4rSqSrga+QxLcBwEnAZOAJZIG9mXT3exz\nQB+2mzddfh/MHNzWa5IOAuYAfxIR90XEuxGxDrgQmAzMSpebI+knkuZL2irpaUkndLHNX6aTT0ra\nJunzkkqSXpb0TUmvAHMljZT0M0mbJb0uabGkw8q20ybpy+n0bEm/knRduuyLks6qctnDJf0yPY4l\nkv6npIXdfI++Ien/pfVf3uGzT0taIWmLpPWSri37uP378Gb6ffgDSVMkPSDpNUmvSvqRpBHd/iNZ\noTm4rRonA4OBfymfGRFvAXcDZ5TN/gywCBgB3AX8j842GBEfTyePiYjhEfF/0/djgPcBE4E/IvmZ\nnZu+nwjs7LDNSF/tZgDPAaOBf0zXrWbZHwOPAqNIfmnN6rDue9LAvxo4HZiafi23HZgVESOATwP/\nRdJ56Wenpl9HpN+Hx9L3fwuMA6YBE9IarEk5uK0aBwOvddEHuzH9vN1DEXFPJIMpPwI+0st97QWu\njYjdEfF2RLweEXek09uBvwNmdrP+uoiYm+5/ATBO0qG9WVbSROBE4K8iYk9E/Jrkl1BX3RkXAvMi\n4tmI2AGUt6iJiAcj4pl0ehVwS9kx7LfNiFgTEfen34PXgH/u4Zit4BzcVo3XgIMldfbzMw54tez9\nprLpHcDgLtbryqsR8U77G0kHSvo/ktZK2gI8CIyQ1FWIbmyfSEMUYFgvl30/8HpEvF227IZuah7X\n4fP15R+m3R9L0+6eN0n+khjd1cYkjZF0S9rtsgVY2N3yVnwObqvGI8Au4ILymZKGAWcB99dwXx27\nI64m6X6YkXY1zCRppdZzMO8VYJSkIWXzJvawfPnnHZf9MXAnMD4iRgL/m33/Fzvrfvk74F3gw+kx\nX4r/7zY1/+Nbr6Wn+30b+O+SzpQ0MD2N7SckLc0uB+16sAmY0sMyw0j6tbdIGkWHboh6SAdelwNz\n0mP9KHAOXfRxk3wfZkuaJunATmocBrwREe9ImgFcUratV0m6h6Z0WP4tYGs6EPuNWhyX5ZeD26oS\nEdcBfwn8N2ALycDdOuCTEbG7fTH2D7fuLhyYA8yX9Iakz3Wx/veAISTdNQ8D/9rNNnuz/56W/QLw\nUeB3wN8AtwLv0ImIuCet8wHgBZK/QMq39VXgryVtBf5ruq32dXeQDET+Oj27ZQbJL8njSb7Pi4Hb\nuzkOawLdXoAjaTBJH+Ig4ADgpxHxrbSlcyvJebtrgQsj4s36l2vWGCTdCjwbEd/OuhZrPt22uNPB\nmNMi4ljgGOA0SacA1wBLImIqSWvimrpXapYhSSem51O3SDobOJekn9qs3/XYVVI2un4A0Aq8QfJD\nOz+dPx84vy7VmTWOscBSYBvJ6Xj/OSKezLYka1Y93qskPXXrCZLBkh9ExDclvRER70s/F8mpUu+r\ne7VmZkaP935IL7I4Nr3E9l5Jp3X4PCR5oMTMrJ9UfNOeiNgi6efACcAmSWMjYqOkccDmjss7zM3M\nqhMR3V6X0G0ft6SDJY1Mp4eQ3INiBcnlvpeli11GF4M0EVHY17XXXpt5DT4+H18zHl+Rjy2isvZu\nTy3ucSTn1bakIb8wIu6XtAL4SXpntbUk92YwM7N+0G1wR3IDnOM7mf86+9/xzMzM+oGvnKxSqVTK\nuoS68vHlW5GPr8jHVqm6PbpMUtRr22ZmRSWJ6MvgpJmZNR4Ht5lZzji4zcxyxsFtZpYzDm4zs5xx\ncJuZ5YyD28wsZxzcZmY54+A2M8sZB7eZWc44uM3McsbBbWaWMw5uM7OccXCbmeWMg9vMLGcc3GZm\nOePgNjPLGQe3mVnOOLjNzHLGwW1mljMObjOznHFwm5nljIPbzCxnHNxmZjnj4DYzyxkHt5lZzji4\nzcxyxsFtZpYzDm4zs5xxcJuZ5YyD28wsZxzcZmY54+A2M8sZB7eZWc50G9ySJkhaKukZSU9L+tN0\n/hxJL0takb7O6p9yzcxMEdH1h9JYYGxErJQ0DPg34HzgQmBbRPxTN+vGzp3B4MG1LtnMrLgkERHq\nbpluW9wRsTEiVqbT24HVwGHt2++pgNtvr7BSMzOrWMV93JImA8cBj6azrpL0pKS5kkZ2ts4NN/S5\nPjMz62BAJQul3SS3AV+LiO2SfgD8dfrx3wDfBb7ccb1ly+Zw1VUwejSUSiVKpVKNyjYzK4a2tjba\n2tp6tU63fdwAkgYCPwP+NSK+18nnk4HFETG9w/z45jeDvXvhuut6VZOZWdPqcx+3JAFzgWfLQ1vS\nuLLF/hOwqrP1r7gCFiyAXbsqL9rMzLrXUx/3x4BZwGllp/6dDfyDpKckPQnMBL7e2cpHHglHHw0/\n/WltizYza2Y9dpVUvWEpIoJFi2DePFiypC67MTMrlEq6Suoe3G+/DRMmwKOPwpQpddmVmVlh9LmP\nuxYGD4ZLL4W5c+u9JzOz5lD3FjfA6tXwiU/A+vUwcGBddmdmVggN0eIGmDYNjjgCfvaz/tibmVmx\n9dvdAb/yFbj++v7am5lZcfVLVwnAzp0wfjw88QRMmlSXXZqZ5V7DdJUADBkCl1ySnBpoZmbV67cW\nN8CqVXD22bB2LQyo6C4pZmbNpaFa3ADTpyfdJffc0597NTMrln5/dJkHKc3M+qZfu0oA3noruZJy\n1So47LBOVjQza2IN11UCMHQoXHSRBynNzKrV7y1uSE4J/OxnYc0aaG2ty+7NzHKpIVvcAMcfDwcf\n7DsGmplVI5PgBrjySj+T0sysGpl0lQBs3ZpcQbl6NYwdW5cSzMxyp2G7SgAOOgguuAB++MOsKjAz\ny6fMWtwAjz2WXAb/m99AS2a/QszMGkdDt7gBZsyAYcNg6dIsqzAzy5dMg1vyIKWZWW9l2lUC8Oab\nMHly0l1yyCF1KcXMLDcavqsEYORIOO88WLAg60rMzPIh8+CG5MZTN9wAdWr8m5kVSkME98knJ2eV\nPPRQ1pWYmTW+hghuybd7NTOrVOaDk+1+9zuYMgVefBFGjapLSWZmDS8Xg5PtRo+GT38aFi7MuhIz\ns8bWMMEN+87p9iClmVnXGiq4Z86Ed96BRx7JuhIzs8bVUMHtKynNzHrWMIOT7TZvhqlTYd06GDGi\nDoWZmTWwXA1Otjv0UPjUp+Dmm7OuxMysMTVccEPSXXL99R6kNDPrTEMG9yc/mTwhZ/nyrCsxM2s8\nDRncLS1wxRUepDQz60y3g5OSJgALgEOBAK6PiO9LGgXcCkwC1gIXRsSbHdatanCy3SuvwIc+BOvX\nw/DhVW/GzCxXajE4uRv4ekQcDZwE/LGkacA1wJKImArcn76vqXHjoFSCW26p9ZbNzPKt2+COiI0R\nsTKd3g6sBg4DzgXmp4vNB86vR3G+8ZSZ2f4q7uOWNBk4DngMGBMRm9KPNgFjal4ZyWmBmzfDihX1\n2LqZWT5VFNyShgG3A1+LiG3ln6Ud2XU5ca+1Fb78ZQ9SmpmVG9DTApIGkoT2woi4M529SdLYiNgo\naRywubN158yZ8950qVSiVCr1usDLL4djjoHrroOhQ3u9uplZQ2tra6Otra1X6/R0VolI+rB/FxFf\nL5v/j+m8f5B0DTAyIq7psG6fziopd845cMEF8KUv1WRzZmYNq5KzSnoK7lOAXwJPsa875FvA48BP\ngInU6XTAcnfdBd/5Djz8cE02Z2bWsPoc3H3cec2Ce88emDQJ7r0XPvzhmmzSzKwh5fImU50ZMCDp\n6/YgpZlZTlrcAGvXwoknwoYNMGRIzTZrZtZQCtPiBpg8OQnu22/PuhIzs2zlJrjBT8cxM4McdZUA\n7N4NEyZAWxscdVRNN21m1hAK1VUCMHAgzJ4NN96YdSVmZtnJVYsb4Le/hZNPTgYpBw2q+ebNzDJV\nuBY3wBFHwPTpcOedPS9rZlZEuQtu2PdMSjOzZpS7rhKAXbuSQcpHHoEpU+qyCzOzTBSyqwSSvu1L\nL/UgpZk1p1y2uAGeey55tNmGDcnZJmZmRVDYFjck53FPnQqLF2ddiZlZ/8ptcIOfSWlmzSm3XSUA\nO3cmg5TLlyf3MjEzy7tCd5VAcpfAL3wB5s7NuhIzs/6T6xY3wNNPw5lnwrp1yX27zczyrPAtbkie\niDNpEtx9d9aVmJn1j9wHN/h2r2bWXHLfVQLw1lvJIOVTT8H48f2ySzOzumiKrhKAoUPh4oth3rys\nKzEzq79CtLgBVqyA886Dl16C1tZ+262ZWU01TYsb4LjjYMwYuO++rCsxM6uvwgQ3eJDSzJpDYbpK\nALZtg4kT4dlnYdy4ft21mVlNNFVXCcDw4fC5z8EPf5h1JWZm9VOoFjfAsmVw0UXJsylbCvVrycya\nQdO1uAFOPBEOOggeeCDrSszM6qNwwS35dq9mVmyF6yoB2LIluX/JCy/AoYdmUoKZWVWasqsEYMQI\nOP98mD8/60rMzGqvkC1ugIcfhtmz4fnnk+4TM7M8aNoWN8BHP5o8RPjBB7OuxMystgob3O2DlL6S\n0syKprBdJQCvvw4f+ACsWQOjR2daiplZRZq6qwRg1Cg45xxYuDDrSszMaqfH4JY0T9ImSavK5s2R\n9LKkFenrrPqWWb32G09l3Pg3M6uZSlrcNwEdgzmAf4qI49LXPbUvrTY+/nHYsyc5y8TMrAh6DO6I\neAh4o5OPcnGSneTbvZpZsfSlj/sqSU9KmitpZM0qqoPLLoM774Q338y6EjOzvqs2uH8AHA4cC7wC\nfLdmFdXBIYfAmWfCzTdnXYmZWd8NqGaliNjcPi3pRmBxZ8vNmTPnvelSqUSpVKpmdzVx5ZVw9dXw\n1a/6SkozaxxtbW20tbX1ap2KzuOWNBlYHBHT0/fjIuKVdPrrwH+IiEs6rJP5edzl9u6FI4+ERYtg\nxoysqzEz61wl53H32OKWtAiYCRwsaQNwLVCSdCzJ2SUvAX9Ug3rrqqUFrrgiud2rg9vM8qzQV052\ntHEjTJsG69YlD1swM2s0TX/lZEdjx8JppyXdJWZmedVUwQ2+8ZSZ5V/TBfcZZ8Crr8ITT2RdiZlZ\ndZouuFtbk0FKt7rNLK+aanCy3csvwzHHwIYNMHRo1tWYme3jwckujB8Pp5wCt96adSVmZr3XlMEN\nvvGUmeVX0wb32WcnXSWrVvW8rJlZI2na4B4wAC6/3K1uM8ufphycbLduHRx/fDJYOWRI1tWYmXlw\nskeTJiX3LbnttqwrMTOrXFMHNySDlNdfn3UVZmaVa+quEoDdu2HiRHjggeQGVGZmWXJXSQUGDoTZ\ns+HGG7OuxMysMk3f4gZYswZOOikZpBw0KOtqzKyZucVdoSlT4CMfgTvuyLoSM7OeObhTX/mKBynN\nLB/cVZLatQsmTIBf/zp5NqWZWRbcVdILgwbBF7/oQUoza3xucZd5/nmYORPWr4cDDsi6GjNrRm5x\n99IHP5i87ror60rMzLrm4O7Az6Q0s0bnrpIO3n47edDCsmVw+OFZV2NmzcZdJVUYPBhmzYK5c7Ou\nxMysc25xd+KZZ5Knwa9fn9y328ysv7jFXaWjj066SX7+86wrMTPbn4O7C34mpZk1KneVdGHHjuRK\nypUrk69mZv3BXSV9cOCBcPHFMG9e1pWYmf0+t7i7sXIlnHsuvPQStLZmXY2ZNQO3uPvo2GNh7Fi4\n996sKzEz28fB3QM/k9LMGo27SnqwbVvyTMpnnoH3vz/rasys6NxVUgPDh8PnPw833ZR1JWZmCbe4\nK7B8eRLea9ZAi3/VmVkducVdIyecACNHwi9+kXUlZmYVBLekeZI2SVpVNm+UpCWSXpB0n6SR9S0z\nW5Jv92pmjaOSFvdNwFkd5l0DLImIqcD96ftCu+QSWLIENm3KuhIza3Y9BndEPAS80WH2ucD8dHo+\ncH6N62o4I0bAZz8L8+f3vKyZWT1V28c9JiLa256bgDE1qqehtd94qiBjrmaWU32+23REhKROo2zO\nnDnvTZdKJUqlUl93l6mTTkqeBt/WBqedlnU1ZlYEbW1ttLW19Wqdik4HlDQZWBwR09P3zwGliNgo\naRywNCKO6rBOYU4HLPf978Ojj8KPf5x1JWZWRPU8HfAu4LJ0+jLgziq3kzuzZsHdd8Nrr2VdiZk1\nq0pOB1wEPAx8UNIGSV8CvgOcIekF4BPp+6YwahR85jOwYEHWlZhZs/KVk1V4/HE4/XSYOhVKpeR1\n6qnJmSdmZn1RSVeJg7tKu3YlAd7WlrweewyOOspBbmZ94+DuRw5yM6sFB3eGOgvyadP2BfkppzjI\nzWx/Du4G4iA3s0o4uBuYg9zMOuPgzhEHuZmBgzvXHORmzcnBXSAdg/zxx/cP8oMOyrZGM+s7B3eB\nvf32/kH+oQ85yM3yzsHdRBzkZsXg4G5iDnKzfHJw23sc5Gb54OC2LnUM8mXLfj/Ip09PzloZOhRa\nqr35r5n1moPbKlYe5EuXwvPPw7ZtsGMHDBuWtMZHjEi+tr96896/AMwq4+C2Pnv33STAt25NXlu2\n7JvuzfudO2H48OqDv/390KGgbn+kzfLNwW0NY88e2L69+uBvf5X/Augp6LtbZtAgaG31XwHWeBzc\nVjh79vz+XwDV/hWwa1fy1wQkAV6UV0vLvq8tLclfJ+1fy6cr+ayRly//q6uz6Z4+r3bZWuyjJw5u\nsx7s3ZsEeNFeEclr797f/9rZvO4+a9Tl25VHTPt0Z/NqsWxf1utKZyG/d6+D28ysYVTyC2HgwJ6D\ne0A9ijMzs/111aXSWx6aMTPLGQe3mVnOOLjNzHLGwW1mljMObjOznHFwm5nljIPbzCxnHNxmZjnj\n4DYzyxkHt5lZzji4zcxyxsFtZpYzDm4zs5xxcJuZ5YyD28wsZ/p0P25Ja4GtwLvA7oiYUYuizMys\na31tcQdQiojjmi2029rasi6hrnx8+Vbk4yvysVWqFl0lfXiOQ34V/YfHx5dvRT6+Ih9bpWrR4v6F\npOWSrqxFQWZm1r2+PnPyYxHxiqRDgCWSnouIh2pRmJmZda5mT3mXdC2wPSK+m773I97NzKpQt6e8\nSzoQaI2IbZKGAp8Cvl3pjs3MrDp96SoZA9yh5BnzA4CbI+K+mlRlZmZdqllXiZmZ9Y+6XDkp6SxJ\nz0n6jaS/qMc+siJpnqRNklZlXUs9SJogaamkZyQ9LelPs66pViQNlvSYpJWSnpX091nXVA+SWiWt\nkLQ461pqTdJaSU+lx/d41vXUmqSRkm6TtDr9GT2p0+Vq3eKW1Ao8D5wO/DuwDPjDiFhd0x1lRNKp\nwHZgQURMz7qeWpM0FhgbESslDQP+DTi/QP9+B0bEDkkDgF8Bfx4Rv8q6rlqS9GfACcDwiDg363pq\nSdJLwAkR8XrWtdSDpPnAgxExL/0ZHRoRWzouV48W9wzgtxGxNiJ2A7cA59VhP5lIT3d8I+s66iUi\nNkbEynR6O7AaeH+2VdVOROxIJw8AWoFCBYCk8cB/BG6kuBfHFfK4JI0ATo2IeQARsaez0Ib6BPdh\nwIay9y+n8yxnJE0GjgMey7aS2pHUImklsAlYGhHPZl1Tjf0z8A1gb9aF1EmRL/o7HHhV0k2SnpB0\nQ3r23n7qEdwe7SyAtJvkNuBracu7ECJib0QcC4wHPi6plHFJNSPpHGBzRKygoK1Skov+jgPOBv44\n7bosigHA8cD/iojjgbeAazpbsB7B/e/AhLL3E0ha3ZYTkgYCtwM/iog7s66nHtI/QX8OnJh1LTV0\nMnBu2g+8CPiEpAUZ11RTEfFK+vVV4A6SrtmieBl4OSKWpe9vIwny/dQjuJcDR0qaLOkA4CLgrjrs\nx+pAyYn5c4FnI+J7WddTS5IOljQynR4CnAGsyLaq2omIv4yICRFxOHAx8EBEfDHrumpF0oGShqfT\n7Rf9FebsrojYCGyQNDWddTrwTGfL9vVeJZ3tfI+kPwHuJRn8mVuUMxIAJC0CZgKjJW0A/ioibsq4\nrFr6GDALeEpSe6h9KyLuybCmWhkHzJfUQtJoWRgR92dcUz0VrduyGS76uwq4OW30rgG+1NlCvgDH\nzCxn/OgyM7OccXCbmeWMg9vMLGcc3GZmOePgNjPLGQe3mVnOOLjNzHLGwW1mljP/H5BNdv1LBYNw\nAAAAAElFTkSuQmCC\n", 323 | "text/plain": [ 324 | "" 325 | ] 326 | }, 327 | "metadata": {}, 328 | "output_type": "display_data" 329 | } 330 | ], 331 | "source": [ 332 | "import matplotlib.pyplot as plt\n", 333 | "\n", 334 | "index = [i for i in range(10)]\n", 335 | "plt.title(\"On training data\")\n", 336 | "plt.plot([i for i in range(len(obj))], obj)\n", 337 | "\n", 338 | "print(obj)" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": 123, 344 | "metadata": { 345 | "collapsed": false 346 | }, 347 | "outputs": [ 348 | { 349 | "name": "stdout", 350 | "output_type": "stream", 351 | "text": [ 352 | "10397\n", 353 | "1159\n", 354 | "7673\n", 355 | "1195\n", 356 | "8925\n", 357 | "8117\n", 358 | "8352\n", 359 | "9888\n", 360 | "2726\n", 361 | "1144\n", 362 | "3475\n", 363 | "3826\n", 364 | "3032\n", 365 | "1861\n", 366 | "2194\n", 367 | "5991\n", 368 | "10329\n", 369 | "9339\n", 370 | "1516\n", 371 | "9884\n", 372 | "390\n", 373 | "7362\n", 374 | "9378\n", 375 | "7485\n", 376 | "6431\n", 377 | "9810\n", 378 | "8308\n", 379 | "8573\n", 380 | "5932\n", 381 | "9897\n", 382 | "3574\n", 383 | "3142\n", 384 | "7137\n", 385 | "10108\n", 386 | "6775\n", 387 | "3772\n", 388 | "6585\n", 389 | "5474\n", 390 | "6239\n", 391 | "5812\n", 392 | "1689\n", 393 | "5567\n", 394 | "2060\n", 395 | "7920\n", 396 | "5159\n", 397 | "9849\n", 398 | "6252\n", 399 | "3475\n", 400 | "8488\n", 401 | "3944\n", 402 | "10605\n", 403 | "7137\n", 404 | "5211\n", 405 | "3008\n", 406 | "9898\n", 407 | "3394\n", 408 | "6706\n", 409 | "1929\n", 410 | "1727\n", 411 | "5845\n", 412 | "1167\n", 413 | "5818\n", 414 | "5789\n", 415 | "5076\n", 416 | "8082\n", 417 | "7449\n", 418 | "9597\n", 419 | "7358\n", 420 | "9852\n", 421 | "8601\n", 422 | "6206\n", 423 | "15\n", 424 | "1331\n", 425 | "3413\n", 426 | "3760\n", 427 | "2098\n", 428 | "1738\n", 429 | "3532\n", 430 | "7715\n", 431 | "2317\n", 432 | "10439\n", 433 | "2162\n", 434 | "7252\n", 435 | "7377\n", 436 | "5980\n", 437 | "695\n", 438 | "9375\n", 439 | "11003\n", 440 | "6948\n", 441 | "7636\n", 442 | "4654\n", 443 | "3109\n", 444 | "1814\n", 445 | "10295\n", 446 | "3446\n", 447 | "(129, 2)\n" 448 | ] 449 | } 450 | ], 451 | "source": [ 452 | "import random\n", 453 | "def online_nmf(spectrum, W, H,A, B, rho, beta, n, eps):\n", 454 | " \n", 455 | " a = np.zeros(W.shape)\n", 456 | " b = np.zeros(W.shape)\n", 457 | "\n", 458 | " t = 1\n", 459 | " W_old = W\n", 460 | " k = W.shape[1]\n", 461 | " h = np.random.rand(W.shape[1], 1)\n", 462 | " h_old = h\n", 463 | " \n", 464 | " while np.linalg.norm(W - W_old, ord = \"fro\") < n:\n", 465 | " \n", 466 | " t = t+1 \n", 467 | " \n", 468 | " ind = random.randint(0, len(spectrum.T)-1)\n", 469 | " print(ind)\n", 470 | " v = spectrum.T[ind]\n", 471 | " h_old = h\n", 472 | " h, obj = gradient_backtracking(v, W, h_old, 100, compute_grad, compute_obj, 1e-12)\n", 473 | " \n", 474 | " h = h.reshape(h.shape[0],1)\n", 475 | " v = v.reshape(v.shape[0],1)\n", 476 | " den = eps + np.dot(W, h)\n", 477 | " \n", 478 | " a += np.dot(np.dot(((eps+v)/(den**2)), h.T), np.dot(W.T,W))\n", 479 | " b += np.dot(1/den, h.T)\n", 480 | " \n", 481 | " if t % beta == 0:\n", 482 | " A = A + rho*a\n", 483 | " a = 0\n", 484 | " B = B + rho*b\n", 485 | " b = 0\n", 486 | " W_old = W\n", 487 | " W = np.sqrt(A/B)\n", 488 | " \n", 489 | " W = np.array([x/sum(x) for x in zip(*W)]).T\n", 490 | " A = np.array([x/sum(x) for x in zip(*A)]).T\n", 491 | " B = np.array([x*sum(x) for x in zip(*B)]).T\n", 492 | " \n", 493 | " if t > 95:\n", 494 | " print(W.shape)\n", 495 | " break\n", 496 | "\n", 497 | "eps = 1e-12\n", 498 | "v = spectrum.T[0]\n", 499 | "K = 2\n", 500 | "W = np.random.rand(spectrum.shape[0],K)\n", 501 | "H = np.zeros((K, spectrum.shape[1]))\n", 502 | "\n", 503 | "A = np.zeros(W.shape)\n", 504 | "B = np.zeros(W.shape)\n", 505 | "\n", 506 | "online_nmf(spectrum, W, H, A, B, 0.5, 100, 1e-3, eps)" 507 | ] 508 | }, 509 | { 510 | "cell_type": "code", 511 | "execution_count": null, 512 | "metadata": { 513 | "collapsed": true 514 | }, 515 | "outputs": [], 516 | "source": [ 517 | "eps = 1e-12\n", 518 | "random.seed(12222015)" 519 | ] 520 | }, 521 | { 522 | "cell_type": "code", 523 | "execution_count": 92, 524 | "metadata": { 525 | "collapsed": false 526 | }, 527 | "outputs": [], 528 | "source": [ 529 | "s = [sum(x) for x in zip(*W)]\n", 530 | "W = [sum(x) for x in zip(*W)]" 531 | ] 532 | }, 533 | { 534 | "cell_type": "code", 535 | "execution_count": 93, 536 | "metadata": { 537 | "collapsed": false 538 | }, 539 | "outputs": [ 540 | { 541 | "data": { 542 | "text/plain": [ 543 | "(1, 129)" 544 | ] 545 | }, 546 | "execution_count": 93, 547 | "metadata": {}, 548 | "output_type": "execute_result" 549 | } 550 | ], 551 | "source": [ 552 | "p.shape" 553 | ] 554 | }, 555 | { 556 | "cell_type": "code", 557 | "execution_count": 245, 558 | "metadata": { 559 | "collapsed": false 560 | }, 561 | "outputs": [], 562 | "source": [ 563 | "W = np.array([[1,1], [1,2], [2,2]])\n", 564 | "X = [1,2]\n", 565 | "\n" 566 | ] 567 | }, 568 | { 569 | "cell_type": "code", 570 | "execution_count": 246, 571 | "metadata": { 572 | "collapsed": false 573 | }, 574 | "outputs": [ 575 | { 576 | "data": { 577 | "text/plain": [ 578 | "array([[1, 1],\n", 579 | " [1, 2],\n", 580 | " [2, 2]])" 581 | ] 582 | }, 583 | "execution_count": 246, 584 | "metadata": {}, 585 | "output_type": "execute_result" 586 | } 587 | ], 588 | "source": [ 589 | "W" 590 | ] 591 | }, 592 | { 593 | "cell_type": "code", 594 | "execution_count": 237, 595 | "metadata": { 596 | "collapsed": false 597 | }, 598 | "outputs": [ 599 | { 600 | "name": "stdout", 601 | "output_type": "stream", 602 | "text": [ 603 | "[ 0.25 0.25 0.5 ]\n", 604 | "[0 0 0]\n", 605 | "[ 0.2 0.4 0.4]\n", 606 | "[0 0 0]\n" 607 | ] 608 | } 609 | ], 610 | "source": [ 611 | "X = W.T\n", 612 | "for i in range(2):\n", 613 | " \n", 614 | " col_sum = X[i].sum()\n", 615 | " print(X[i]/col_sum)\n", 616 | " X[i] = (X[i]/col_sum)\n", 617 | " print(X[i])\n" 618 | ] 619 | }, 620 | { 621 | "cell_type": "code", 622 | "execution_count": 251, 623 | "metadata": { 624 | "collapsed": false 625 | }, 626 | "outputs": [], 627 | "source": [ 628 | "z = np.array([x/sum(x) for x in zip(*W)])" 629 | ] 630 | }, 631 | { 632 | "cell_type": "code", 633 | "execution_count": 252, 634 | "metadata": { 635 | "collapsed": false 636 | }, 637 | "outputs": [ 638 | { 639 | "data": { 640 | "text/plain": [ 641 | "array([[ 0.25, 0.2 ],\n", 642 | " [ 0.25, 0.4 ],\n", 643 | " [ 0.5 , 0.4 ]])" 644 | ] 645 | }, 646 | "execution_count": 252, 647 | "metadata": {}, 648 | "output_type": "execute_result" 649 | } 650 | ], 651 | "source": [ 652 | "z.T" 653 | ] 654 | }, 655 | { 656 | "cell_type": "code", 657 | "execution_count": null, 658 | "metadata": { 659 | "collapsed": true 660 | }, 661 | "outputs": [], 662 | "source": [ 663 | "import numpy as np\n", 664 | "\n", 665 | "# divergence\n", 666 | "def div(v,W,h):\n", 667 | " whv = np.dot(W,h) * 1/v\n", 668 | " div = whv - np.log10(whv) - 1\n", 669 | " div = np.dot( div, np.ones )\n", 670 | " return div\n", 671 | "\n", 672 | "# divergence gradient\n", 673 | "def div_grad(v,W,h):\n", 674 | " grad = np.dot( 1/v - 1/(np.dot(W,h)) , W)\n", 675 | " return grad\n", 676 | "\n", 677 | "# epsilon divergence\n", 678 | "def eps_div(v,W,h,eps):\n", 679 | " whv = (np.dot(W,h) + eps) * 1/(v + eps)\n", 680 | " div = whv - np.log10(whv) - 1\n", 681 | " div = np.dot( div, np.ones )\n", 682 | " return div\n", 683 | "\n", 684 | "# epsilon divergence gradient\n", 685 | "def eps_div_grad(v,W,h,eps):\n", 686 | " grad = np.dot( 1/(v + eps) - 1/(np.dot(W,h) + eps), W)\n", 687 | " return grad" 688 | ] 689 | } 690 | ], 691 | "metadata": { 692 | "kernelspec": { 693 | "display_name": "Python 3", 694 | "language": "python", 695 | "name": "python3" 696 | }, 697 | "language_info": { 698 | "codemirror_mode": { 699 | "name": "ipython", 700 | "version": 3 701 | }, 702 | "file_extension": ".py", 703 | "mimetype": "text/x-python", 704 | "name": "python", 705 | "nbconvert_exporter": "python", 706 | "pygments_lexer": "ipython3", 707 | "version": "3.4.3" 708 | } 709 | }, 710 | "nbformat": 4, 711 | "nbformat_minor": 0 712 | } 713 | -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-1.wav -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-10.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-10.wav -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-2.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-2.wav -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-3.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-3.wav -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-4.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-4.wav -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-5.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-5.wav -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-6.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-6.wav -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-7.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-7.wav -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-8.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-8.wav -------------------------------------------------------------------------------- /Output Files/CML_Both_Separation_1/sm-9.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/CML_Both_Separation_1/sm-9.wav -------------------------------------------------------------------------------- /Output Files/Mary_Separation/is-1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mary_Separation/is-1.wav -------------------------------------------------------------------------------- /Output Files/Mary_Separation/is-2.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mary_Separation/is-2.wav -------------------------------------------------------------------------------- /Output Files/Mary_Separation/is-3.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mary_Separation/is-3.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-1.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-10.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-10.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-2.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-2.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-3.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-3.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-4.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-4.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-5.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-5.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-6.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-6.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-7.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-7.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-8.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-8.wav -------------------------------------------------------------------------------- /Output Files/Mozart_Separation/sm-9.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/Output Files/Mozart_Separation/sm-9.wav -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Source Separation for Audio Applications 2 | 3 | Check out our [final report](https://github.com/aartibagul/Source-Separation/blob/master/Final_Report.pdf) or [presentation](https://github.com/aartibagul/Source-Separation/blob/master/presentation.pdf). 4 | 5 | -------------------------------------------------------------------------------- /Source_Separation.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import numpy as np 3 | import random 4 | import wave, struct, numpy as np, matplotlib.mlab as mlab, pylab as pl 5 | import math 6 | import matplotlib.pyplot as plt 7 | import gc 8 | from scipy.io import wavfile 9 | import sys 10 | import cProfile 11 | 12 | # Our implementations 13 | from mini_batch import mini_batch 14 | import istft 15 | import nmf 16 | import stft 17 | 18 | 19 | # Filename of input soundfile 20 | #"CML_Recording_Both.wav" 21 | #"data.wav" 22 | filename = "Mary.wav" 23 | w = wave.open(filename,"rb") 24 | 25 | # returns bit array of mono .wav file, and framerate 26 | def get_wave(filename): 27 | w = wave.open(filename,"rb") 28 | waveParams = w.getparams() 29 | s = w.readframes(waveParams[3]) 30 | w.close() 31 | waveArray = np.fromstring(s, np.int16) 32 | return waveArray, waveParams[2] 33 | 34 | # returns wave array of one track of stereo .wav file 35 | # activate line 2 to convert from stereo file 36 | def get_wave_stereo(filename): 37 | rate, data = wavfile.read(filename) 38 | #data = data[:,0] 39 | return data, rate 40 | 41 | # power spectrogram is the absolute value SQUARED of the stft 42 | def get_spectrogram(stft): 43 | return np.square(np.absolute(stft)) 44 | 45 | # define input parameters ofr stft 46 | win_size = 256 47 | overlap = 128 48 | 49 | # get wave array 50 | waveArr, framerate = get_wave_stereo(filename) 51 | 52 | # normalize wave_arr (int16) 53 | waveArr = waveArr.astype(np.float64) 54 | #waveArr = waveArr/32768 55 | 56 | # compute stft, padded input signal 57 | stft, wave_pad = stft.my_stft(waveArr, win_size, overlap) 58 | 59 | # get power spectrogram of stft 60 | spectrum = get_spectrogram(stft) 61 | 62 | eps = 1e-12 63 | (F,N) = spectrum.shape 64 | K = 10 65 | W = abs(np.random.randn(F,K)) + np.ones((F,K)) 66 | H = abs(np.random.randn(K,N)) + np.ones((K, N)) 67 | #abs(np.random.randn(K, spectrum.shape[1]) + np.ones((K, spectrum.shape[1]))) 68 | 69 | A = np.zeros(W.shape) 70 | B = np.zeros(W.shape) 71 | 72 | r = 1 73 | beta = 1000 74 | rho = r**(beta/spectrum.shape[1]) 75 | 76 | # for profiling: 77 | #import cProfile 78 | #cProfile.run('nmf.online_nmf(spectrum, W, H, A, B, rho, beta, 1e-3, eps)') 79 | 80 | W, H, cost = nmf.online_nmf(spectrum, W, H, A, B, rho, beta, 1e-5, eps) 81 | 82 | ''' 83 | Initialization with mini batch 84 | 85 | 86 | centers = mini_batch(spectrum.T, K, 100, 100) 87 | centers = np.array(centers) 88 | W2 = centers.T 89 | 90 | H2 = abs(np.random.randn(K,N)) + np.ones((K, N)) 91 | A2 = np.zeros(W.shape) 92 | B2 = np.zeros(W.shape) 93 | 94 | W2, H2, cost2 = online_nmf(spectrum, W2, H2, A2, B2, rho, beta, 1e-4, eps) 95 | 96 | 97 | fig = plt.figure(1) 98 | plt.plot([i for i in range(len(cost))], cost, label = "Without k-means") 99 | plt.plot([i for i in range(len(cost2))], cost2, label = "With k-means") 100 | plt.legend(loc="upper right") 101 | plt.xlabel('iteration') 102 | plt.ylabel('IS divergence') 103 | fig.savefig("objective_function.png") 104 | 105 | ''' 106 | fig = plt.figure(1) 107 | plt.plot([i for i in range(len(cost))], cost, label = "Without k-means") 108 | plt.legend(loc="upper right") 109 | plt.xlabel('iteration') 110 | plt.ylabel('IS divergence') 111 | fig.savefig("objective_function.png") 112 | 113 | 114 | # according to Fevotte's Matlab code (Wiener Filtering + ISTFT) 115 | V = np.dot(W,H) 116 | 117 | 118 | print('spec norm: ', np.linalg.norm(spectrum)) 119 | print('V norm: ', np.linalg.norm(V)) 120 | print(np.linalg.norm(V-spectrum)) 121 | print('V dtype ', V.dtype) 122 | print('spec dtype ', spectrum.dtype) 123 | 124 | 125 | Tpad = win_size + (N-1)*(win_size - overlap); 126 | C = np.zeros((K,Tpad)) 127 | 128 | for i in range(K): 129 | 130 | ct = np.zeros(V.shape) 131 | 132 | ct = (np.dot(W[:,i].reshape(F,1),H[i,:].reshape(1,N))/V) * stft 133 | 134 | ct[np.isnan(ct)] = 0 135 | 136 | s1 = np.real( istft.my_istft(ct, win_size, overlap)) 137 | 138 | # scale it back to int16 139 | #s = s * 32768 140 | #print("Before:", max(s1)) 141 | s1 = s1.astype(np.int16) 142 | #print("After:", max(s1)) 143 | #for j in range(len(s1)): 144 | # s1[j] += random.randint(-10000,10000) 145 | C[i,:] = s1 146 | #s1 = s1.tolist() 147 | #print(s1.shape) 148 | noise_output = wave.open('out-{}.wav'.format(i), 'w') 149 | w_copy = list(w.getparams()) 150 | w_copy[0] = 1 151 | noise_output.setparams(tuple(w_copy)) 152 | value_str = s1.tostring() 153 | 154 | noise_output.writeframes(value_str) 155 | 156 | noise_output.close() 157 | 158 | #wavfile.write('out-{}.wav'.format(i), framerate, s1) 159 | 160 | -------------------------------------------------------------------------------- /divergence_benchmark.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import cProfile 4 | 5 | 6 | def grad(x,y): 7 | gra = 1/x - 1/y 8 | return np.sum(grad) 9 | 10 | def grad2(x,y): 11 | gra = 1/y * (1 - x/y) 12 | return np.sum(grad) 13 | 14 | 15 | x = np.random.rand(128) + 1 16 | y = np.random.rand(128) + 1 17 | start = time.time() 18 | for i in range(100000): 19 | gra = 1/x - 1/y 20 | end = time.time() 21 | 22 | start2 = time.time() 23 | for i in range(100000): 24 | gra = 1/y * (1 - x/y) 25 | end2 = time.time() 26 | 27 | print(end - start) 28 | print(end2 - start2) -------------------------------------------------------------------------------- /istft.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import numpy as np 3 | import random 4 | import wave, struct, numpy as np, matplotlib.mlab as mlab, pylab as pl 5 | import math 6 | import matplotlib.pyplot as plt 7 | import gc 8 | from scipy.io import wavfile 9 | import nmf 10 | from stft import sinebell 11 | 12 | def my_istft(stft, win_size, overlap): 13 | # power sinbell analysis window 14 | win = sinebell(win_size, overlap) 15 | (num_coeff, num_frames) = stft.shape 16 | 17 | # recover full stft by conjugate symmetry of the fourier expansion of real signals 18 | stft_full = np.zeros((win_size, num_frames), dtype = np.complex128) 19 | stft_full[:num_coeff,:] = stft 20 | 21 | # reasoning: stft[ num_coeff - 1] is the both the negative and positive Nyquist frequency if win_size is even 22 | # thus, we take the conjugate only of stft[ num_coeff - 2:0:-1 ] 23 | if win_size%2 == 0: 24 | stft_full[num_coeff:, :] = np.conj(stft[num_coeff-2:0:-1, :]) 25 | else: 26 | stft_full[num_coeff:,:] = np.conj(stft[num_coeff-1:0:-1, :]) 27 | 28 | # take inverse fft of recovered stft 29 | istft = np.fft.ifft(stft_full, axis = 0) 30 | # reconstruct padded signal by taking overlap into account 31 | x_pad = overlap_add(istft, win_size, overlap) 32 | return x_pad 33 | 34 | def overlap_add(signal, win_size, overlap): 35 | 36 | win = sinebell(win_size, overlap) 37 | (temp, num_frames) = signal.shape 38 | 39 | if temp != win_size: 40 | print("Dimensions of ISTFT are wrong!") 41 | 42 | pad_len = overlap + num_frames * (win_size - overlap) 43 | x_pad = np.zeros(pad_len, dtype = np.complex128) 44 | 45 | # index of beginning of each frame in x_pad 46 | frame_ind = np.array([i for i in range(num_frames)]) * (win_size - overlap) 47 | 48 | # do we really need the window again here? 49 | x_pad[frame_ind[0]:frame_ind[0] + win_size] = signal[:,0] * win 50 | for i in range(1,num_frames): 51 | x_pad[frame_ind[i]:frame_ind[i] + win_size] = x_pad[frame_ind[i]:frame_ind[i] + win_size] + signal[:,i] * win 52 | return x_pad 53 | -------------------------------------------------------------------------------- /mini_batch.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import numpy as np 3 | import random 4 | 5 | 6 | def get_closest(x, centers): 7 | distances = [] 8 | 9 | #get the distance from the point(x) to every centroid 10 | #and get the index of centroid that yields 11 | #the minimum distance 12 | 13 | for mu in centers: 14 | distances.append(np.linalg.norm(x-mu)) 15 | index = np.argmin(distances) 16 | 17 | return index, min(distances) 18 | 19 | def k_means_objective(X, k, C, centers): 20 | 21 | sum_distance = 0 22 | for j in range(k): 23 | for x in X[C==j]: 24 | sum_distance += (np.linalg.norm(x-centers[j]))**2 25 | 26 | return sum_distance 27 | 28 | def mini_batch(X, k, b, max_iter): 29 | v = [0]*k 30 | centers = [0]*k 31 | 32 | centers[0] = X[random.randint(0, len(X)-1)] 33 | 34 | distortion = [] 35 | 36 | for r in range(k-1): 37 | x = random.random() 38 | num_dx = (get_closest(X[0],centers)[1])**2 39 | den_dx = 0 40 | for x_i in X: 41 | den_dx += (get_closest(x_i,centers)[1])**2 42 | index = 0 43 | while num_dx/den_dx < x: 44 | 45 | index += 1 46 | num_dx += (get_closest(X[index],centers)[1])**2 47 | 48 | centers[r+1] = X[index] 49 | centers = np.array(centers) 50 | 51 | 52 | for i in range(max_iter): 53 | M = np.array(random.sample(list(X),b)) 54 | C=[0]*len(M) 55 | for i,x in enumerate(M): 56 | index = get_closest(x, centers)[0] 57 | C[i] = index 58 | 59 | 60 | for i, x in enumerate(M): 61 | index = C[i] 62 | v[index]+=1 63 | n = 1/v[index] 64 | 65 | centers[index] = (1- n)*centers[index] + n*x 66 | 67 | return centers 68 | -------------------------------------------------------------------------------- /nmf.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import numpy as np 3 | import random 4 | import wave, struct, numpy as np, matplotlib.mlab as mlab, pylab as pl 5 | import math 6 | import matplotlib.pyplot as plt 7 | import gc 8 | from scipy.io import wavfile 9 | 10 | # Objective functions section 11 | 12 | def compute_obj(v,W,h,eps): 13 | vhw = (v + eps) / (np.dot(W,h) + eps) 14 | div = vhw - np.log(vhw) - 1 15 | return np.sum( div ) 16 | 17 | def compute_grad(v,W,h,eps): 18 | (F,K) = W.shape 19 | wh = np.dot(W,h) 20 | grad = np.dot(W.T, (1 / ( wh + eps) - (v + eps) / (np.square(wh) + eps)) ) 21 | return grad 22 | 23 | # important! 24 | # Not only do we need h = h_t but also h_m = h_(t-1) and h_p = h_(t+1) 25 | # lambda is the smoothness constant 26 | def compute_smooth_obj(v,W,h,h_m,h_p,lamb,eps): 27 | 28 | h = h.reshape(h.shape[0],1) 29 | h_m = h_m.reshape(h_m.shape[0],1) 30 | h_p = h_p.reshape(h_p.shape[0],1) 31 | 32 | # compute regular objective 33 | # maybe doing this direct instead of the function call is faster: 34 | # whv = (np.dot(W,h) + eps)/(v + eps) 35 | # div = whv - np.log(whv) - 1 36 | div = compute_obj(v,W,h,eps) 37 | 38 | # compute smoothness terms with epsilon 39 | s1 = (h + eps) / (h_m + eps) 40 | s2 = (h + eps) / (h_p + eps) 41 | sm = s1 - np.log(s1) - 1 42 | sm += s2 - np.log(s2) - 1 43 | # returning properly scaled smooth objective 44 | return div + lamb * np.sum( sm ) 45 | 46 | # input parameters as above 47 | def compute_smooth_grad(v,W,h,h_m,h_p,lamb,eps): 48 | 49 | # the famous reshape trio 50 | h = h.reshape(h.shape[0],1) 51 | h_m = h_m.reshape(h_m.shape[0],1) 52 | h_p = h_p.reshape(h_p.shape[0],1) 53 | 54 | # calculates gradient of regular divergence 55 | div_grad = compute_grad(v,W,h,eps) 56 | 57 | # calculating gradient of smoothness term 58 | sm_grad = 1 / (h_m + eps) + 1 / (h_p + eps) - 2 / (h + eps) 59 | sm_grad = sm_grad.reshape(sm_grad.shape[0],1) 60 | 61 | # returning properly scaled gradient of smooth objective 62 | return div_grad + lamb * sm_grad 63 | 64 | # gradient descent function 65 | def gradient_backtracking(v, W, h, max_iter, compute_grad, compute_obj, eps): 66 | 67 | v = v.reshape(v.shape[0],1) 68 | h = h.reshape(h.shape[0],1) 69 | 70 | beta = .1 #backstep factor between 0.1 and 0.8 71 | opt_prec = 1-1e-6 # optimization precision 72 | eta = 1e-1 #initial step size 73 | 74 | #obj = [None]*max_iter 75 | 76 | max_backstep = 30 # maximum number of backsteps 77 | t = 0 # backstepping counter 78 | k = 0 # gradient step counter 79 | 80 | old_obj = compute_obj(v,W,h,eps) 81 | 82 | while( k < max_iter and t != max_backstep ): 83 | 84 | grad = compute_grad(v,W,h,eps) 85 | #obj[k] = compute_obj(v,W,h,eps) 86 | 87 | t = 0 # reset backstepping counter 88 | eta = 1/beta*eta # try to increase stepsize slightly again 89 | 90 | # make sure h-n*grad is positive 91 | while(any(h - eta * grad < 0) and t < max_backstep ): 92 | t += 1 93 | eta = beta * eta 94 | 95 | new_obj = compute_obj(v,W,(h - eta*grad),eps) 96 | 97 | while( new_obj > opt_prec * old_obj and t < max_backstep): 98 | t += 1 99 | eta = beta * eta 100 | new_obj = abs(compute_obj(v,W,(h - eta*grad),eps)) 101 | 102 | h = h - eta * grad # update h according to gradient step 103 | k += 1 # update gradient step counter 104 | 105 | old_obj = new_obj 106 | 107 | h = h.reshape(h.shape[0],) 108 | return h 109 | 110 | def online_nmf(spectrum, W, H,A, B, rho, beta, eta, eps): 111 | 112 | a = np.zeros(W.shape) 113 | b = np.zeros(W.shape) 114 | 115 | t = 1 116 | W_old = W + 1.5*eta 117 | k = W.shape[1] 118 | h = np.random.rand(W.shape[1],) 119 | n = spectrum.shape[1] 120 | cost = [] 121 | cost.append(compute_obj(spectrum,W,H,eps)) 122 | 123 | while np.linalg.norm(W - W_old, ord = "fro") > eta: 124 | 125 | t = t+1 126 | 127 | ind = random.randint(0, n-1) 128 | v = spectrum[:,ind] 129 | 130 | h = gradient_backtracking(v, W, H[:,ind], 100, compute_grad, compute_obj, eps) 131 | 132 | H[:, ind] = h 133 | 134 | h = h.reshape(h.shape[0],1) 135 | v = v.reshape(v.shape[0],1) 136 | den = eps + np.dot(W, h) 137 | 138 | a += np.dot(((eps+v)/(den)**2), h.T) * np.square(W) 139 | 140 | b += np.dot(1/den, h.T) 141 | 142 | if t % beta == 0: 143 | A = A + rho*a 144 | a = 0 145 | B = B + rho*b 146 | b = 0 147 | W_old = W 148 | W = np.sqrt(A/B) 149 | 150 | for i in range(k): 151 | s = np.sum(W[:,i]) 152 | W[:,i] = W[:,i]/s 153 | A[:,i] = A[:,i]/s 154 | B[:,i] = B[:,i]*s 155 | #print(i) 156 | 157 | #print(np.linalg.norm(compute_obj(spectrum,W_old,H,eps))- compute_obj(spectrum,W,H,eps)) 158 | gc.disable() 159 | cost.append(compute_obj(spectrum,W,H,eps)) 160 | gc.enable() 161 | 162 | #cost.append(compute_obj(spectrum,W,H,eps)) 163 | if t > 100*n: 164 | print(" W shape" , W.shape) 165 | break 166 | 167 | #print("W", np.linalg.norm(W[:,1])) 168 | #print("H", np.linalg.norm(H[1])) 169 | #print(compute_obj(spectrum,W,H.T,eps)) 170 | 171 | print("t" , t) 172 | print(cost[-1]) 173 | return W, H, cost -------------------------------------------------------------------------------- /objective_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/objective_function.png -------------------------------------------------------------------------------- /presentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aartibagul/Source-Separation/10d1502c6965b2614231ccaa766bbecd7bb71fbc/presentation.pdf -------------------------------------------------------------------------------- /stft.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import numpy as np 3 | import random 4 | import wave, struct, numpy as np, matplotlib.mlab as mlab, pylab as pl 5 | import math 6 | import matplotlib.pyplot as plt 7 | import gc 8 | from scipy.io import wavfile 9 | 10 | # takes in wave file as input 11 | # win_size is the length of the window in samples 12 | # overlap is the amount of overlap between windows in samples 13 | def my_stft(wave, win_size, overlap): 14 | # power sinbell analysis window 15 | win = sinebell(win_size, overlap) 16 | # make the frames 17 | frames, wave_pad = make_frames(wave, win, overlap) 18 | # fft, for each column 19 | stft = np.fft.fft(frames, axis = 0) 20 | # keep the spectrum associated with the positive frequencies (potentially times to the upper frequencies) 21 | if len(win)%2 == 0: 22 | stft = stft[:int(len(win)/2)+1] 23 | else: 24 | stft = stft[:int((len(win)+1)/2)] 25 | return stft, wave_pad 26 | 27 | # create sinebell window of length win_size, with overlap, DONE 28 | def sinebell(win_size, overlap): 29 | win = np.zeros(win_size) 30 | win[0:overlap] = np.sin( math.pi * (np.array([i for i in range(overlap)])) / (2*(overlap-1)) ) 31 | win[overlap : win_size - overlap] = 1 32 | win[win_size - overlap:] = np.sin( math.pi * (win_size - np.array([i for i in range(win_size-overlap, win_size)]) - 1 ) / (2*(overlap - 1)) ) 33 | return win 34 | 35 | # x is the input signal, win is the analysis window, overlap 36 | # returns frame matrix and padded input signal x_pad 37 | def make_frames(x, win, overlap): 38 | win_len = win.shape[0] 39 | win_size = 256 40 | x_len = len(x) 41 | 42 | # number of frames 43 | num_frames = int(np.ceil((x_len + overlap)/(win_len - overlap))) 44 | 45 | # initializing zero padded signal 46 | pad_len = int(overlap + num_frames * (win_len - overlap)) 47 | x_pad = np.zeros(pad_len) 48 | x_pad[overlap: overlap + x_len] = x 49 | 50 | # index of beginning of each frame in x_pad 51 | frame_ind = np.array([i for i in range(num_frames)]) * (win_len - overlap) 52 | 53 | # initialize frames matrix 54 | frames = np.zeros((win_size, num_frames)) 55 | for i in range(num_frames): 56 | frames[:,i] = (x_pad[frame_ind[i] : frame_ind[i] + win_size] * win) 57 | 58 | return frames, x_pad 59 | 60 | --------------------------------------------------------------------------------