├── LICENSE ├── README.md ├── matlab ├── matlab_1.png ├── matlab_2.png ├── matlab_3.png └── tensor_gmm.m └── python ├── python_1.png ├── python_2.png ├── tensor_gmm.ipynb └── tensor_gmm.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Stephan Rabanser 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tensor-gmm 2 | Implementation of learning a Gaussian mixture model using tensor decomposition. 3 | 4 | ## Matlab sample plots 5 | 6 | Shared parameters: 7 | * n = 10000 8 | * s = 2 9 | * dist = 15 10 | * spher = 1 11 | 12 | #### Plot 1 13 | 14 | * d = 3 15 | * k = 2 16 | 17 | ![matlab_1](./matlab/matlab_1.png "matlab_1") 18 | 19 | #### Plot 2 20 | 21 | * d = 20 22 | * k = 12 23 | 24 | ![matlab_2](./matlab/matlab_2.png "matlab_2") 25 | 26 | #### Plot 3 27 | 28 | * d = 50 29 | * k = 40 30 | 31 | ![matlab_3](./matlab/matlab_3.png "matlab_3") 32 | 33 | ## Python sample plots 34 | 35 | Shared parameters: 36 | * n = 10000 37 | * s = 2 38 | * dist = 15 39 | * spher = 1 40 | 41 | #### Plot 1 42 | 43 | * d = 3 44 | * k = 2 45 | 46 | ![python_1](./python/python_1.png "python_1") 47 | 48 | #### Plot 2 49 | 50 | * d = 20 51 | * k = 12 52 | 53 | ![python_2](./python/python_2.png "python_2") 54 | -------------------------------------------------------------------------------- /matlab/matlab_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/steverab/tensor-gmm/020f0642ec76ddd78487104bdce8b89af4b34e08/matlab/matlab_1.png -------------------------------------------------------------------------------- /matlab/matlab_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/steverab/tensor-gmm/020f0642ec76ddd78487104bdce8b89af4b34e08/matlab/matlab_2.png -------------------------------------------------------------------------------- /matlab/matlab_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/steverab/tensor-gmm/020f0642ec76ddd78487104bdce8b89af4b34e08/matlab/matlab_3.png -------------------------------------------------------------------------------- /matlab/tensor_gmm.m: -------------------------------------------------------------------------------- 1 | % ========================================================================= 2 | % Tensor Factorization: Learning Mixtures of Gaussians 3 | % ------------------ 4 | % Paper: Tensor Decompositions and their Applications in Machine Learning 5 | % Seminar: Efficient Inference and Large Scale Machine Learning 6 | % Author: Stephan Rabanser (rabanser@cs.tum.edu) 7 | % ========================================================================= 8 | 9 | % ----------------------- 10 | % Configuration and setup 11 | % ----------------------- 12 | 13 | close all; clear; clc; 14 | 15 | d = 3; % # of dimensions 16 | k = 2; % # of clusters 17 | n = 10000; % # of samples per cluster 18 | tot = k * n; % # of total samples 19 | s = 2; % isotropic variance 20 | dist = 40; % distance between gaussians 21 | spher = 1; % decide shape of the covariance matrix 22 | spher_range = 2; % range of the values of the covariance matrix 23 | csv_import = 0; % decide whether data should be imported 24 | 25 | assert(k <= d, 'k cannot be larger than d') 26 | 27 | rand('seed',11); % initilize random generator for reproducibility 28 | 29 | % -------------------- 30 | % Generate sample data 31 | % -------------------- 32 | 33 | if csv_import == true 34 | fprintf('Reading sample data ...\n'); 35 | tic 36 | 37 | A = csvread('A.csv'); 38 | X = csvread('X.csv'); 39 | 40 | for i = 1:k 41 | mean = A(:,i)'; 42 | scatter(X((i-1)*n+1:i*n,1),X((i-1)*n+1:i*n,2),10,'.') 43 | hold on 44 | scatter(mean(1),mean(2),100,'x', 'black') 45 | hold on 46 | end 47 | else 48 | fprintf('Generating sample data ...\n'); 49 | tic 50 | 51 | A = -dist+(dist+dist)*rand(d, k); % mixture component means 52 | X = []; % GMM data 53 | 54 | figure 55 | for i = 1:k 56 | mean = A(:,i)'; 57 | if spher == true 58 | covariance = s * eye(d); 59 | else 60 | a = -spher_range+(spher_range+spher_range)*rand(d,d); 61 | covariance = a'* a; 62 | end 63 | mvn = mvnrnd(mean, covariance, n); 64 | scatter(mvn(:,1),mvn(:,2),10,'.') 65 | hold on 66 | scatter(mean(1),mean(2),100,'x', 'black') 67 | hold on 68 | X = [X ; mvn]; 69 | end 70 | end 71 | 72 | toc 73 | fprintf('Data present.\n'); 74 | 75 | % --------------------------- 76 | % Computate first two moments 77 | % --------------------------- 78 | 79 | fprintf('Compute data mean ...\n'); 80 | tic 81 | 82 | mu = zeros(d,1); 83 | for i = 1:tot 84 | mu = mu + X(i, :)'; 85 | end 86 | mu = mu / tot; 87 | 88 | toc 89 | fprintf('Mean computed.\n'); 90 | 91 | fprintf('Compute data covariance ...\n'); 92 | tic 93 | 94 | Sigma = zeros(d,d); 95 | for i = 1:tot 96 | Sigma = Sigma + X(i, :)' * X(i, :); 97 | end 98 | Sigma = Sigma / tot; 99 | 100 | toc 101 | fprintf('Covariance computed.\n'); 102 | 103 | % --------------------------------------- 104 | % Compute SVD from data covariance matrix 105 | % --------------------------------------- 106 | 107 | fprintf('Performing whitening ...\n'); 108 | tic 109 | 110 | [U,S,~] = svd(Sigma); 111 | s_est = S(end,end); % Estimate sigma^2 112 | fprintf('Estimating variance %d as %d. \n', s, s_est); 113 | W = U(:,1:k) * sqrt(pinv(S(1:k,1:k)-diag(ones(k,1).*s_est))); % Obtain whitening matrix 114 | X_whit = X * W; % Whiten the data 115 | 116 | toc 117 | fprintf('Whitening performed.\n'); 118 | 119 | % ------------------------------------ 120 | % Compute whitenend third order moment 121 | % ------------------------------------ 122 | 123 | % T = zeros(k,k,k); 124 | % for t = 1 : tot 125 | % for i = 1 : k 126 | % for j = 1 : k 127 | % for l = 1 : k 128 | % T(i,j,l) = T(i,j,l) + X_whit(t,i)*X_whit(t,j)*X_whit(t,l); 129 | % end 130 | % end 131 | % end 132 | % end 133 | % T = T / tot; 134 | 135 | % ------------------------------------------------------------------------ 136 | % Apply tensor power method 137 | % ------------------------------------------------------------------------ 138 | % Inspired by: https://bitbucket.org/kazizzad/tensor-power-method/overview 139 | % ------------------------------------------------------------------------ 140 | 141 | fprintf('Performing tensor power method ...\n'); 142 | tic 143 | 144 | TOL = 1e-8; % Convergence delta 145 | maxiter = 100; % Maximum number of power step iterations 146 | V_est = zeros(k,k); % Estimated eigenvectors for tensor V 147 | lambda = zeros(k,1); % Estimated eigenvalues for tensor V 148 | 149 | for i = 1:k 150 | % Generate initial random vector 151 | v_old = rand(k,1); 152 | v_old = v_old./norm(v_old); 153 | for iter = 1 : maxiter 154 | % Perform multilinear transformation 155 | v_new = (X_whit'* ((X_whit* v_old) .* (X_whit* v_old)))/tot; 156 | v_new = v_new - s_est * (W' * mu * dot((W*v_old),(W*v_old))); 157 | v_new = v_new - s_est * (2 * W' * W * v_old * ((W'*mu)' * (v_old))); 158 | % Defaltion 159 | if i > 1 160 | for j = 1:i-1 161 | v_new = v_new - (V_est(:,j) * (v_old'*V_est(:,j))^2)* lambda(j); 162 | end 163 | end 164 | % Compute new eigenvalue and eigenvector 165 | l = norm(v_new); 166 | v_new = v_new./norm(v_new); 167 | % Check for convergence 168 | if norm(v_old - v_new) < TOL 169 | fprintf('Eigenvector %d converged at iteration %d. \n', i, iter); 170 | V_est(:,i) = v_new; 171 | lambda(i,1) = l; 172 | break; 173 | end 174 | v_old = v_new; 175 | end 176 | end 177 | 178 | toc 179 | fprintf('Tensor power method performed.\n'); 180 | 181 | % ------------------------------ 182 | % Apply backwards transformation 183 | % ------------------------------ 184 | 185 | fprintf('Perform backwards transformation ...\n'); 186 | tic 187 | 188 | A_est = pinv(W') * V_est * diag(lambda); 189 | 190 | toc 191 | fprintf('Backwards transformation performed.\n'); 192 | 193 | fprintf('Plotting results ...\n'); 194 | 195 | for i = 1:k 196 | mean_est = A_est(:,i)'; 197 | % Plot mean 198 | scatter(mean_est(1),mean_est(2),1000,'+', 'red') 199 | hold on 200 | % Plot variance 201 | viscircles([mean_est(1) mean_est(2)], sqrt(s_est)); 202 | hold on 203 | end 204 | -------------------------------------------------------------------------------- /python/python_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/steverab/tensor-gmm/020f0642ec76ddd78487104bdce8b89af4b34e08/python/python_1.png -------------------------------------------------------------------------------- /python/python_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/steverab/tensor-gmm/020f0642ec76ddd78487104bdce8b89af4b34e08/python/python_2.png -------------------------------------------------------------------------------- /python/tensor_gmm.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Estimating spherical Gaussian Mixture Models (GMMs) using Tensor Decompositions" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": { 14 | "collapsed": true 15 | }, 16 | "outputs": [], 17 | "source": [ 18 | "import numpy as np\n", 19 | "import matplotlib.pyplot as plt\n", 20 | "from matplotlib.patches import Ellipse" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "d = 10\n", 32 | "k = 6\n", 33 | "n = 1000\n", 34 | "tot = k * n\n", 35 | "s = 2\n", 36 | "dist = 20\n", 37 | "spher = True\n", 38 | "cov_range = 2" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "### Generate data" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 3, 51 | "metadata": { 52 | "collapsed": true 53 | }, 54 | "outputs": [], 55 | "source": [ 56 | "def generate_data():\n", 57 | " A = -dist+(dist+dist)*np.random.rand(d, k)\n", 58 | " X = np.zeros((tot, d))\n", 59 | "\n", 60 | " plt.axis('equal')\n", 61 | "\n", 62 | " for i in range(k):\n", 63 | " mean = np.transpose(A[:, i])\n", 64 | " if spher:\n", 65 | " covariance = s * np.identity(d)\n", 66 | " else:\n", 67 | " a = -cov_range + (cov_range + cov_range) * np.random.rand(d, d)\n", 68 | " covariance = np.matmul(np.transpose(a), a)\n", 69 | " mvn = np.random.multivariate_normal(mean, covariance, n)\n", 70 | " #plt.plot(mvn[:, 0], mvn[:, 1], '.')\n", 71 | " #plt.plot(mean[0], mean[1], 'x')\n", 72 | " X[i*n:(i+1)*n, :] = mvn\n", 73 | " \n", 74 | " return (X, A)" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "### Calculate first order data moment (mean)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 4, 87 | "metadata": { 88 | "collapsed": true 89 | }, 90 | "outputs": [], 91 | "source": [ 92 | "def calculate_first_moment(X):\n", 93 | " mu = np.zeros((d, 1))\n", 94 | " for t in range(tot):\n", 95 | " for i in range(d):\n", 96 | " mu[i] += + X[t, i]\n", 97 | " mu /= tot\n", 98 | " return mu" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": {}, 104 | "source": [ 105 | "### Calculate second order data moment (covariance)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 5, 111 | "metadata": { 112 | "collapsed": true 113 | }, 114 | "outputs": [], 115 | "source": [ 116 | "def calculate_second_moment(X):\n", 117 | " Sigma = np.zeros((d, d))\n", 118 | " for t in range(tot):\n", 119 | " for i in range(d):\n", 120 | " for j in range(d):\n", 121 | " Sigma[i, j] += np.dot(X[t, i],X[t, j])\n", 122 | " Sigma /= tot\n", 123 | " return Sigma" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "### Extract information from second order data moment" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": 6, 136 | "metadata": { 137 | "collapsed": true 138 | }, 139 | "outputs": [], 140 | "source": [ 141 | "def extract_information_from_second_moment(Sigma, X):\n", 142 | " U, S, _ = np.linalg.svd(Sigma)\n", 143 | " s_est = S[-1]\n", 144 | " W, X_whit = perform_whitening(X, U, S)\n", 145 | " return (s_est, W, X_whit)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "metadata": {}, 151 | "source": [ 152 | "### Perform whitening" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": 7, 158 | "metadata": { 159 | "collapsed": true 160 | }, 161 | "outputs": [], 162 | "source": [ 163 | "def perform_whitening(X, U, S):\n", 164 | " W = np.matmul(U[:, 0:k], np.sqrt(np.linalg.pinv(np.diag(S[0:k]))))\n", 165 | " X_whit = np.matmul(X, W)\n", 166 | " return (W, X_whit)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "metadata": {}, 172 | "source": [ 173 | "### Perform tensor power method" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 12, 179 | "metadata": { 180 | "collapsed": true 181 | }, 182 | "outputs": [], 183 | "source": [ 184 | "def perform_tensor_power_method(X_whit, W, s_est, mu):\n", 185 | " TOL = 1e-8\n", 186 | " maxiter = 100\n", 187 | " V_est = np.zeros((k, k))\n", 188 | " lamb = np.zeros((k, 1))\n", 189 | "\n", 190 | " for i in range(k):\n", 191 | " v_old = np.random.rand(k, 1)\n", 192 | " v_old = np.divide(v_old, np.linalg.norm(v_old))\n", 193 | " for iter in range(maxiter):\n", 194 | " v_new = (np.matmul(np.transpose(X_whit), (np.matmul(X_whit, v_old) * np.matmul(X_whit, v_old)))) / tot\n", 195 | " #v_new = v_new - s_est * (W' * mu * dot((W*v_old),(W*v_old)));\n", 196 | " #v_new = v_new - s_est * (2 * W' * W * v_old * ((W'*mu)' * (v_old)));\n", 197 | " v_new -= s_est * (np.matmul(np.matmul(W.T, mu), np.dot(np.matmul(W, v_old).T,np.matmul(W, v_old))))\n", 198 | " v_new -= s_est * (2 * np.matmul(W.T, np.matmul(W, np.matmul(v_old, np.matmul(np.matmul(W.T, mu).T, v_old)))))\n", 199 | " if i > 0:\n", 200 | " for j in range(i):\n", 201 | " v_new -= np.reshape(V_est[:, j] * np.power(np.matmul(np.transpose(v_old), V_est[:, j]), 2) * lamb[j], (k, 1))\n", 202 | " l = np.linalg.norm(v_new)\n", 203 | " v_new = np.divide(v_new, np.linalg.norm(v_new))\n", 204 | " if np.linalg.norm(v_old - v_new) < TOL:\n", 205 | " V_est[:, i] = np.reshape(v_new, k)\n", 206 | " lamb[i] = l\n", 207 | " break\n", 208 | " v_old = v_new\n", 209 | " \n", 210 | " return (V_est, lamb)" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "### Perform backwards transformation" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": 13, 223 | "metadata": { 224 | "collapsed": true 225 | }, 226 | "outputs": [], 227 | "source": [ 228 | "def perform_backwards_transformation(V_est, lamb):\n", 229 | " return np.matmul(np.matmul(np.linalg.pinv(np.transpose(W)), V_est), np.diag(np.reshape(lamb.T, k)))" 230 | ] 231 | }, 232 | { 233 | "cell_type": "markdown", 234 | "metadata": {}, 235 | "source": [ 236 | "### Plot result" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": 14, 242 | "metadata": { 243 | "collapsed": true 244 | }, 245 | "outputs": [], 246 | "source": [ 247 | "def plot_results(X, A, A_est, s_est):\n", 248 | " plt.axis('equal')\n", 249 | " \n", 250 | " ax = plt.subplot(aspect='equal')\n", 251 | " \n", 252 | " plt.plot(X[:,0], X[:,1], '.', zorder=-3)\n", 253 | " \n", 254 | " for i in range(k):\n", 255 | " mean = A[:, i].T\n", 256 | " mean_est = A_est[:, i].T\n", 257 | " \n", 258 | " plt.plot(mean[0], mean[1], 'x', color='y', zorder=-2)\n", 259 | " plt.plot(mean_est[0], mean_est[1], '+', color='r', zorder=-1)\n", 260 | " \n", 261 | " ell = Ellipse(xy=(mean_est[0], mean_est[1]),\n", 262 | " width=s_est, height=s_est,\n", 263 | " angle=0, color='red')\n", 264 | " ell.set_facecolor('none')\n", 265 | " ax.add_artist(ell)\n", 266 | "\n", 267 | " plt.show()" 268 | ] 269 | }, 270 | { 271 | "cell_type": "markdown", 272 | "metadata": {}, 273 | "source": [ 274 | "# Execution" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": 15, 280 | "metadata": { 281 | "scrolled": false 282 | }, 283 | "outputs": [ 284 | { 285 | "data": { 286 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAANkAAAD8CAYAAAD63wHzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXl8lNW9/9/nmSwYCBDCFpYkRELAACJBQEUrFav4Q1Co\nVbSt1Vr0ivdK29uCG7W0tXq7aHuLF9B666/KooJCqQugCGIJQiJCAkFCICGyJwEigWRmnnP/mIWZ\nyUxmfTJLzvv14pVZnnmew8x85nvO93wXIaVEoVAYhxbtASgUiY4SmUJhMEpkCoXBKJEpFAajRKZQ\nGIwSmUJhMEpkCoXBKJEpFAajRKZQGExStAfgSs+ePWVubm60h6FQBERJSckpKWUvf8fFlMhyc3PZ\nsWNHtIehUASEEKI6kOPUdFGhMBglMoXCYJTIFAqDUSJTKAxGiUyhMBglMoXCYJTIgJLqBhZurKSk\nuiHaQ1EkIDG1TxYNSqobuOflYlosOilJGq8/MJ6inIxoD0uRQHR4S1ZcVUeLRUeXYLboFFfVeT1O\nWTtFqHR4SzY+L5OUJA2zRSc5SWN8XmarY5S1U4RDhxdZUU4Grz8wnuKqOsbnZXoVz6rSWprNOpKL\n1k6JTBEoHV5kYBOaL9GUVDewYnsNjsJ5QhMcOX2ekuoGr68pqW5oU7CKjocSmQeeIllZWotFv/i8\n1SpZ9lkNK0trmT+lkIamFuexwU4rlSA7BkpkLngTifA4RgJSQrNZZ/7qMnQpSTJpfLtoAKcam1tN\nK8HmXMlISwlLkIr4RYnMBW+exsJ+3RDCJixXJGDRbQ+2WHSWbqtxe96kCTLSUpxC0iUIIDVZc64B\nPa+lRJaYdHgXvisOT6NJQHKSRuN5M0+8s7uVwALh+oLeNDS1OIUENmFeMOss2nSg1bW8eTUViYGy\nZB7MGD0ACXRNTWLx5ipC7RSwfu9xTje1kGTSsFgvCg1g/Z7jTCzo7derqUgMwhaZEGIg8P+BPth+\nrJdIKf8khOgBrABygUPAd6SUMbWT63A8ZKSl8PG+E2zYcxwdSNJAIkIWGNiml58dakAT0K9bJ2pP\nX3B7fsX2Gr5V2NdpwRZurFRiS1BEuF1dhBBZQJaUslQIkQ6UALcBPwDqpZTPCiHmARlSyrltnWvM\nmDGyvcoPLN1Ww/zVZVh1GZaYQsWkCaTdaYKUWHSpHCBxhhCiREo5xt9xYa/JpJRHpZSl9tuNwF6g\nPzANeNV+2KvYhBcTlFQ3MH91GZYoCQxA16XT6WG2Sr9hXYr4JaKODyFELnAFsA3oI6U8an/qGLbp\npLfXzBJC7BBC7Dh58mQkh+OT4qo6rHr0+rJpdmeHw+mRbBLKAZLARMzxIYToAqwE5kgpzwpxcYdJ\nSimFEF6/1VLKJcASsE0XIzWethifl0lqsubc02pvbhjWh4e+canT6QEoB0gCExGRCSGSsQnsdSnl\nKvvDx4UQWVLKo/Z124lIXCsSOOIVV5bW8lZJLVarjrWd1CaAiQW9W4lKiStxiYR3UQB/BfZKKf/o\n8tQa4F7gWfvf1eFeK9L0734J91+dy9aqOr6oPdNu133qnd3oEkwmQW5mZ/J6dnbuqylrlnhEwrs4\nAfgE2A04ovwex7YuewPIBqqxufDr2zpXe3kXS6obmPmSLRIjlnCNCFFCi30C9S6GbcmklFugVYif\ngxvCPb8RrCqtDVtg3c430v/sSc6ldKK6exYIX29B4KhUmsSkQ0Z8nGhsDvm13097hW+u3M/ofVV8\n1bUX3c83cqFLMhsn57Ogd5vbgH4RKA9jItIhRRaqzck/Wc2cN9Zx7Dtm7pv+c0rOj2NY9508xTNM\n+mMFpwuW8edrZoY8pgn5PZkzaYiyYglGhxRZqPzXe3/muQn38UVBFg+Pepbsmlt4aNMqHh3xNKfu\nHMg///Yo6/PHsbd3XtDnNmmCycOzlMASkA4Zhd8zPTXo1xQeq6TXuXreHDGJivqRbKy5hWmDl1Pw\nWgsV9SM51TmD10ZN5rufvxvSmCy6ZMHaclWoJwHpkCKbMXpA0FPGy4/tZ0vuFeiaiaE9djEx+11W\nV94FwNAeuwDYlFfEyKP7Qx5Xi1mFVSUiHXK6WJSTwcgB3YLaG7MKjWTdwtAeu/jT7qcpmNvCVJYD\n8P7cxwFYMWISVi3w3y0BbhEnmiaU0yMB6RCWzFvNxDuvzA7qHMXZI/hGVQn5aXt4dMTT5M5dS+7c\ntQDc/Nwz/NuiH3C2Uxe2Zl8e8DklNqEJIEkTLJg2XK3JEpCEF5mjlsYf1u3jnpeLnUIr6JvOgO6d\nAj5PdUY/dvXNZ/AyKxX1I92eq6gfyZfbxzGj7CNev2JyUOOT2CzYgmnDuXtccMJXxAcJP130VSH4\nnpeLaTYHtyE9d/KjLF82j/5nT/DylbdR0SuXF8fN4Psl/+CRrW/wyxt+RG03r8kGbaLrkoamlqBf\np4gPEl5k3ioEO4QXbEDZyS4ZTP/u77m3dC3/++bTZDadQQpYn38Vs6Y/yc5+BQGdR2CLW7TYo5LV\nBnRiE3bsYiQxKnbRs76hYwpptlu4UN+BVEsLLaYkpAhu1j02N4O5k4exsrQWAUwfPUCtxeKQQGMX\nO4TIvFFS3cCq0lo+3neCrzzqbxhJkgYrHrxaiSoBaLcA4Xhl37FGVmw/7KydaDQpSbYCqDN8WC1V\nTThx6ZAic63xAbY10sgB3ag6+TWNzdaIX08AT99a6NN7qKoJJzYJ78L3RnFVHbrLNNmkCebfWsiw\nrK6GXE8Cf996yGd/s0B7pCnikw4pMofHURPum8CD+6Qbds29xxpb7dV5jkcV00lMOuR0sSgng/lT\nCnmv7CiTh2c5p3EzRg9g2baakLyN3S5J4usLVqxtOJJ81b0PpEeaIn7pkCIrqW5gwdpyWiw62w/V\nU9A33dmj7MHr8li0uSroc545b/F7jNaGpWqrR5oivumQ08W21kDzbhnGbaP6RfyaApg5Nls5NTog\nHVJk/tZA+X3SQ86e9oUQ0K/7JUpgHZAOOV30twZyFD9tsegIaBUV0rdrKie/bnFWITZpYPUSBnnb\nqH6s3XUUXdrq3CuHRsekw0Z8+MN1c3h9+TG3ddozt4+goG86q0prkdgcJgArS2upPN5Is0Xnziuz\nuXtcttpkTmBUWFWEWbqtppU3UtGxUWFVEebucdlKXIqQ6JCOj0DxllGtUASLsmQ+UPGEikihLJkP\nVDyhIlIokflAxRMqIoWaLvpAxRMqIoUSWRuoeEJFJFDTRYXCYJTIFAqDUSJTKAxGiUyhMBglMoXC\nYJTIFAqDUSJLYFTsZWyg9skSFBV7GTtExJIJIV4RQpwQQpS5PNZDCLFeCLHf/ld9wmESjGVSsZex\nQ6Smi38DbvZ4bB7woZQyH/jQfl/RBm2JyFefNV+o2MvYISLTRSnlZiFErsfD04Dr7bdfBT4G5kbi\neomIv+mdN8vk+rxnmQMVexk7GLkm6yOlPGq/fQwIvjteB8KfiDz7rGWkpbBwY6XTQrkKdP6UQhqa\nWhifl8nsiYOj9V9S2GkXx4eUUgohvBYTEULMAmYBZGd33PR+b80KXXG1TBlpKc7irEmaYFhWV5rN\ntqaGF8w6T60uQ9orZCmHR/Qx0oV/XAiRBWD/e8LbQVLKJVLKMVLKMb169TJwOLGNQ0Q/+VaBT2EU\n5WQwe+JgGppanFavxSr5ovaMW8k6qy7RpU1wizcdaL//hMIrRopsDXCv/fa9wGoDr5UQOEQEtHKA\nuDpFxudlkqQFVn513Z7jLN1W49czqfbUjCMi00UhxDJsTo6eQoha4BfAs8AbQogfAtXAdyJxrUTH\n4QBpNusIAbOuzePGwr7MXLIVs1WSbBIsm3UV1xf0Zt2e4wGd85UtVdSePu/TqaL21IwlUt7FmT6e\nuiES5+9IFFfVOddXUsKizVWU1jTQYm/i3mKVLNp0gI8qvM6+vVJT3+R8vTenij+niyI8VMRHDODq\nfs9IS2nVuunzw6fd7n9xuMFZIjwQHAIDMJlsThXXa/pzuijCQ4ksypRUNzDzpWLnF/wbQ1o7f8xW\nd0HVnzOHfL20ZI3Fmw7w8b4TWPSLHki1p2YcSmRRZvGmA7RYbN0qWiw6Xxz273gIp5n86fMWt7Wc\nY3o4e+JgJS6DUFH4UaSkuoEP97o7LyzW9utNILgYcqW8i8ahLFkUKa6qc1t/CWB4/25s3n/K8Gtr\nAu4am+3sSKO8i8ahLJmB+LMOGWkpaEI4Gw5K4NNK4wXmuJjjuq7exWazzsrS2vYZQwdBWTKD8Lf3\n5Ohb7eklbK/Zoo6tHdSbJbVcP6QXmrjY7PCtklpmjB6grFmEUJbMINrK5yqpbuCFDV9e3A+L0hgl\nNmfLuj3Hsbh0CrVaVf5ZJFGWzCB87T25WriwxCUlY77aw1XVuzBJnS+yhrBp0Gh0zRTWuDVU/lmk\nUSIzCF/5XK4WThCaFRtU/xX/vea/SGs5zwdDrsasmXj006X8cv0ifjLlJ+wYUBjyuHt3TeU/bhii\npooRRInMQLzV0ndYuBaLjhACpAxqHda7sY5Vb/6Yv0+8kefzf4gUthn/u7ddzq01H7L4ld/wvTt/\nzZ4+eSGN+djZZp56ZzflR84w3e55VJvU4aF6RkeBpdtqmL+6LKRN5cc2vkKf5KNo88t5cec8KupH\n8uuSP9L1Zzt4cec8xmw4zHWHPmfW9CfDHmdKkgZSukWGKKFdJNCe0crxEQUamlrQQ/lxk5I7dm/g\n+cL7eHHnPB4e9Sy3D36N7274yCm4lcNvYFzNbjLPnfZ/Pj+YLTpmq1TFeMJETRejgGPK6PAuBkon\nSzOdW85TndEP6vuxseYWpg1eDkBF/UgAzqd04kjXXvQ+V09d5+5hjVOIi+tG5QwJHSWyKHFdfi+K\nq+o4e8ES8Guak1LQhUZG0xl+uvevfPe5j5zPHXpuCgB/uvpOep1r4EynLmGP0TGbTdIE86cUqqli\niCiRtTMl1Q3MXLLVLf0kUKTQ+OfQCcyu/Dtdf7aDm298hor6kRx6bgprPujKizvnkffpGU6ldSen\n4RgtpmROdQ5fGFJKGppawj5PR0WJrJ0prqprlboSDC+NvZ233vgp8wfdR0X6SOfjS3b8J/PWvcq1\nm7+kulsWcz5dytATB/kk9wp+O/F+vurWO6TrqX2z8FEia2fG52WSbBIhWTKAfb1yeWTy4zz/v3/g\n+ty9vD/kKtYMncD/PPcXss6d5LUbb+AXV8wBIL35HD8rf4l/rHiEW+/8S1BC+9Zlfbi+oLeztJya\nKoaO8i62M0U5GTw9dTgB1sHxyqa8Iq6ftYQvsoYwvXwjl9YfwSSt3Pez/yTjp58xtMcu5mx5nf5Z\nB+j+n5+xdszVPL7xr0FdI69nZ+4el63yzCKAsmRRIBLrm7OduvDKldN45cppvPnaz/nTNTP5RF7F\nyZ0ZPDzqWabOPUve/H/y4s55fFV4KVs230+vrxs42SUwwSz5pIobC/sqgUUAZcmiQEZaCmEkN7di\n2MmDbM2xrc8q6keyseYWADbW3EJF/UgaUzuzt/cgBtfVBHxOXcILG75USZwRQFmyKNDQ1BJy3KI3\nLJqJTuYWfrj9HeZ8usz5+PM/Ws7zLOeFa2bSydKM2RTcx71l/ym2H6pXkR5hoixZFBifl0lqskYY\nyzI3Ng0qYkrFZt6fOoI1H3Tl5ueeAeDm555hzQdd+eKafgw8fZzdffODOq8jFUZFeoSHElkUcETo\n3z0umyTNFlWhAb26pIR0vleLpvDgtpUUWUqd4VVgmzouLvkZc95/g+WX30RzUvDn16VteqsIHSWy\nKFGUk8H00QMQQiCxZSqf/Do0h0hp/2H8z/g7eOS3Gxm/7hA9zzWwcNy3mbR/G79cuJRT5/rywoS7\nQzq3RmQcNR0ZtSaLIqtKa8PamHbl1aJb2d13MPeWrOXHW14nWbcw/nAZr11xC2su+wbWIJM5k00C\nXZdqIzoCKJFFkZONzRE9X2n/YZT2Hxb2efpnXMKf77pC5ZFFCDVdjCI901Pd7g/u1ZnBvcMP7A2X\nJCGUwCKIElkUmTF6ACkmW0m4JA3G5WVy/zWDSDJFyu8YGtX1Tfz+g8B6Uyv8o0QWRYpyMlg26ypm\njstG0zSWfVbDgrXlPHDNoLDCriKBRCVqRgolsihTlJNB/+6XYLFeLB9XfvQs0a4KoaLvI4dyfMQA\nzuI6ZltxncKsrmw9UBdWY4lgMWmCH00YRPolyWSkpVB25EzENss7OsqSxQBFORnMn1KIpgl0Kfnb\n1kNMGZnVbtcXwA1De3NjYV9mTxxMQd90VpXWsuyzGrUuiwBKZDGCo7iOY8pYd67FaUkEGGpVJLbe\n0g5BtVX9WBE8SmQxgmPKaBK2tdDk4VmkJtvumzSBaIe5m0NQnmNR67LwUGuyGMFbxeGCvukUV9WR\nkZbCgrXlmC06JpOtFqLZKiNaQ9/V0eGr+rEiNFRx0zjBtccz2HK9tuw/5Vdomr2sW1vRW7eN6kd+\nn3QlqCAJtLipsmRxgmfJ7zmThrD9UD1mi44Obi7/LqkmzjVbkdgEdtfYbE40NrN+z3HP06IB+X3S\nmT1xsNH/hQ6L4SITQtwM/AkwAS9LKZ81+pqJhKsFcxWZ65QuIy2Fp9eUOaeQXzdbAZsVS07SnDXt\nP9xzHJcOSQggJVmtuYzGUJEJIUzAQuBGoBbYLoRYI6XcY+R1EwV/jQRdrVtB33S3KaQGXDO4J3Mm\n2Tq0LNxY6Ta1NGmCO68cqJr9tQNGexfHApVSyiopZQuwHJhm8DUThmBc6UU5GcyZNMTpkUxJ1pwC\ng4vZ2Bq2isC/mjacZ24foQTWDhg9XewPHHa5XwuMM/iaCYOvRoK+aMsrqDyG0SPqjg8hxCxgFkB2\ndnaURxNbhCIMbz3RAnlOYRxGi+wrYKDL/QH2x5xIKZcAS8Dmwjd4PHGHEkb8Y/SabDuQL4QYJIRI\nAe4C1hh8TYUipjDUkkkpLUKIR4APsLnwX5FSlht5TYUi1jB8TSalfBd41+jrKBSxigoQVigMRolM\noTAYJTKFwmCUyBQKg1EiUygMRolMoTAYJTKFwmCUyBQKg1EiUygMRolMoTAYJTKFwmCUyBQxSUl1\nAws3ViZE9eKoJ20qFJ74q20SbyhLpog5Qi0THqvWT1kyRcwRbG0TiG3rp0SmiDlCqW3izfrFisjU\ndFERkxTlZDA+L5PiqrqApn8ZaSloQjgLumakpficOrb3tFJZMkVM4pj+NZt1TJrgAXuDQk/LVlLd\nwKrSWlZsr8Gig0nAD67KZcHacq9Tx6Xbapi/ugyrLklN1pg/pZCGphZDy+QpkSlikuKqOprNOhKw\n6JJFm6vQBG6iKaluYOaSrbS4dNOwSnhn51dep44l1Q3MX13m7GB6wazz5Du7kdJWUXnBtOHcPS7y\nZQmVyBQxyfi8TEyacGvpq0toNuusKq0FYO5bX7gJzMGxs82Arda/q+OkuKoO3aOLkeP0Fl0yf3UZ\nBX3TI27R1JpMEZMU5WTwwIRBaB7NDyWwYnsN31n8LypPnmvzHBK4ubCvc13n8Fr66qdo1SUr7QKO\nJEpkiphk6bYaXt5yEG+96S06WPXWj3vjnZ1H+P0H+7jn5WIAXn9gPBPye3o9VgJvldRG3CGiRKaI\nOp7evqXbanjynd1uU8VwkECLfZpZXFVHYVZXn8darZHvka3WZIqo4rmJPH9KIU+tLvNqwcJBB5Z/\nVuO1M6kmIMmkYbUGvvkdDEpkiqhRUt3ACxu+dHoRzRadFdtrsEZaYXZ8tfQtyslg3uRhhnW8USJT\nRAVXC+ZoWpicpJGS1P4rmCF90g09vxKZIiq4hkFp4mJX0Ofe29vuYznXbGHmkq2YrZJkk2DZrKsi\nas2U40MRFRzudJN9g3nOpCEAUYmgL66qo8Xeb7vFGnk3vrJkiqjgLQh44cbKiDs8AqGpxep239c+\nWqgokSmihmeDQ0dfa4cjxB+Z505z5651TDi0kyTdQnmfS3lt1C0c6DnQ/4tdOHvB4rydkqQxffSA\noF7vDzVdVMQMRTkZzJ9SSE5mmt9jJx7Yzvq/PszA08dYNG4Gf7j2ezSmpLF82WM88q/lIV0/NzON\nZT+KfB5ah7VkJdUNqkl5jFFS3eCMnm+L/JPV/O7dF7jv27/gi34FAMzZ8jp/vO57/H30FJYte4wj\nXXuxavgNQV1/1MDuzo1o5fgIgLZyhhzu4z+ss4XbxELOkcLmgLhg1v2uy+7fsZpXxkyj3zW7Gdpj\nFwBzPl0GQGb2YdbcW8i/Fb8FMrgF3tpdR9v8ToRKQorMn4j81ZAIRISKyFJS3cCmfScCOvb/7fuU\nt4bfwMEzQ3h41LNOoQ3tsYuHRz3Luj7XkWppYcipar/n0rA5OkwCdCmDrisSCAk5XfSXiu6vhkQs\np7InIo4ftQvmAKJ+paRLcxOnOnfn7jXvM3XuWabyOADvz7X9rbqmjLq07nRpPu/3dI4rSmw5ZUKX\nEQ+tSkiRBVKIZfroAQj7X08BhVLIRRE4nuthR4JmQAhBbbfeXHbiIC9MuIcXJtzD7YNf4/kfLefH\nL93F25XfpZP5AveWrOVI114Bj0mXIHTJXWOzvX4nwiEhRdZWIRbPgFRXd63rh+94fUZaiiGL4Y6K\nt6pS4/MyESLwJdTyy2/i/h2r+cmUnzK0xy4mZr8LwMTsd9lbP5KiD2v5vF8Bx7p6T2nxhVXCicZm\n5V0MFM89GAe+poIl1Q3cufhfWHRI0mDFg1czPi8zZsuMxSuu73+zWWfxpgNcPrA7Ewb3ZPP+UwGd\n47UrbuHtv/+UX5U8T/c5n7Fw52NUXbOb9z8fzq/O/Yr8TwX33PHbkMa3fs9xnn13L/NuGRbS672R\nsCLzxfi8TJI04Uxbz0hLAWDRpgM4PMcW3ZbaPjYv0+0LsbK0VoksTFzffwms23Oc9XuOB7T57OBs\npy7cNfO3vPrRPPrOhLOXbqbZlMKff7eYTloXlsy+knJxachjXPJJFTcW9o3YZx2WyIQQdwBPA8OA\nsVLKHS7PPQb8ELAC/yGl/CCca4WD6zQQLi52rRKe/kc5ALtrT7u9pvLkOWoazqMJ23zdkTU7I8Lz\n9Y5GUU4Gd4wZyNJtF3O7QomkOtmlB7dMXcLgUzVMOLQTk27lFzc+xNbsESACD4wSXq4vJay0J3hG\nYh81XEtWBkwHFrs+KIS4DLgLKAT6ARuEEEOklNbWpwiftjaWva3BrC6JRS0Wnafe2e0118hs0enf\nvRO1py8A7lmzaiM7dKaPHsDK0lrnLCEcKntmU9kz9ApT3kSWbBK8VVKLxRqZZUJYIpNS7gUQrX85\npgHLpZTNwEEhRCUwFtgazvW84W0hDTidFu+VHXV+mC0Wnc37TrR6U30l80lwCgxA0wQZaSlqnRYm\nro6pxvNmXvqkyu0z8PbFN4qBPdI43NCElDZx3THGFve47LOaiG3hGLUm6w8Uu9yvtT/WCiHELGAW\nQHZ28L9Ino6MlaW1vFVS6zU0R5fuogkWXZeUHTnjDGC9YNZZtOkAowZ2V1YtSByOqcffbj2LaM9A\n/Or6JgCSNMHTU211F0uqG1hZWhuxLRy/IhNCbAD6ennqCSnl6rCuDkgplwBLAMaMGRP0++vY03J8\n8T+vbvAb+xYqUsKpxma3L8H6Pcf5cO9xZdVCJNJpJaFi1SUNTS1AaLX428KvyKSUk0I471eAa77B\nAPtjEacoJ4MfXJXLos1VSAl7jzUacRkAhCY4bf8gXFGRIRcJNvB6+ugBvFlisxpRSCVzIoHG82bn\nfV9bQKFg1HRxDbBUCPFHbI6PfOCzSF/E8YFuqAgs5i1crLrks0Ot4xg9K9V2VEJpX1SUk8GyH13c\n+H9lS1WroqXttUZ7ecvBiLruHYTrwr8d+G+gF/BPIcROKeVNUspyIcQbwB7AAsyOtGfR9QONRjat\nAwHceFkfHvzGpR3eioUa8+mwGiXVDRysa/J6TM/0FE41tp5FRBJdSkNmI2FF4Usp35ZSDpBSpkop\n+0gpb3J57jdSykullAVSyvfCH6o7rh9oNJHAxwFGjyc6nu2LgrXsxVV1SC+xVRIiLjDPtaDAlhVt\nxGwkblNdHJEDsUCLVUa86my84Ui4tOoSTQjmTykM2iK092cqXP5OyO9pmOMqLsOqHGux6wt6Bx2S\nA5BktTB536d8Z9d6Bp45ztepabxbcA0rRn6Lus7dQxqTIzyrI+JZpNRq3+rwdpyrU8TzvrdoEKOQ\n2ErRCQkpybZqWao/mR3XtViSJtA0EVTF2c7NTbyycgGarvO3olsp75NHz6bT3LFrA+//7yP8cMZ8\ndmUN8fl6TUCPLq3XBw1evI4dAc8ipXAxBG14v26UHTmDAAr7dXNrzDd/SqHzfpJm2wSePnqAMxqk\nrWI6Y3MzvDqgwG6d7BH9Jg1MwtZ+SROCKSOzWLvrKFbdFjfp6Es2f0ohAAs3VqoKwuC+FmuxStJT\nTTQ2B+5TeeaDhRzqnsW8yf+OFLbZ8m17PmbuLY9y4/5iXlr1a254YBFfp3ov5qLL1usDTZDwnkVH\nR0sJbvGbvtbGLRadJ97e7RSKq4ew2azzypYqp5BarJKl22pYWVrL6w+M5/UHxju7Z1p1e+aySaDb\nEyrnTh7Gk2/vdtuuEeBcC7p2z3SM0SGe712VywsbvuTTylO2mFQpKT9yxmdnzkgQNyJzTC1si+uL\nzduCEVi/syf4xsESnn52GgXNZVTUjwRs9SHenzqCpG/Vsr38Mm4v/4i/j54S8HlnXZuX0J7FkuoG\nZr5U7Nzkf2vHYZ6eOpzyI2c40diMpgl0L7Fpso3bnm56RwTNo8tKyeySyp1XZrNg2gjeKzvK5OFZ\nFPRNdwvyviIng33HG9HtFutX00ZQfuQMElo18nPcdnyHJg/PYvuhemdEhwRDM+HjQmSe+y+5mZ39\nNoDzxjcP7GDD4HFUNBfy8KhneXHnPKfQHPffKRzIvSX/CFhkGnBjobeAmMShuKoOs0sUjdkqecre\ndxlo1agvHGpPX6D29AW+qN1Nkt16bT9U70zudITNWazu00zAaY1W2S1iW8m6ntZuVQTDqDyJC5F5\n7r/k9epZ7OaoAAAM+klEQVQSksg6mS9wplMXKupHcvZ3Y3h/w+PO56beZKsV8cbwG7jE0hzwOXVs\nuWgvfX8MkJil5sbnZZKcpLmFq7mugx317CO9nWKxW0dHTOoqj7WaVZf0636Js/pwW9bI8zvU0NTC\n7ImDnc9HMozKk7gQmWfNjesLevNhxYmgW+wcyBzILfs+BeDJop9Qcmdvpg1ezvUTIXfuWgBmbVvp\nXKsFykcVJ1i6rYayI2cimiIRKziiMlaV1rL/eKNXp4MR+5UOA2nSBALcnCueUTbe6rK4/uD5q9sS\nyTAqT+JCZJ4Bm6tKa0PqYbV50Gh+/cGLjKkt5+uRViZmv8vqyru4nuUM7bGLmmP5fO/zd3lk6s+D\nOq9ub+rt8FpB/MQyBmp5HV/CaX/Z0m5jc/VWpqcmuQnZM8rG8zsCtArxMtJatUVciAzcf2lC7bph\n1Uz84saHWLL6V+wr1Pnl509Q0XA5RZNO8NOsZ+i9uBef5lzOF2248L1h0gS6lD5/ZWOVUGINe3ft\nBLTeAzMSi1VSfvSs00OpAZcP7N5qrK7fEW/Tx9kTB0flRy9uRObKjNEDeGPHYeecPRg25I9jadZ4\npj7/BX/WF7OnzyB6njvNiId0tkzsyuOXzQ4qfX1wr87cPyGPBWvLMVt0TC6L8Vi3YsHEGjos3sSC\n3mysOI7FXzaRlFxxZB+3l2+k17kG6tK68XbhREr6Dwvq/QWbsAqzurp5BP39gMVSWb+4FFlRTgYr\nZl3FqtJaTjQ281GQ67Pfd5nD7++TFH211xbxkZLGltzLuZDcKeixjM3L5O5x2W4u5lgXl4NAv4ie\nFm/BtBGUHTnD0m01Xo+/pOUC/73mOfLrDrP88pvYmj2C7DPH+N27L1DbrQ8P3/aYz31IX6Rfktxq\nutfWVDfSOWHhILwFZEaLMWPGyB07dvg/0I7r3tmz7+11a4HTXqz8t6vjRlSuuL53Dle2r//Hwo2V\n/GHdPueaqEdaMl0vSeaQj4j5/3n7Gc4np/LzyY9iMdl+x+dseZ0/X30Xz3ywkF7nGvjht38R8FhN\nmuCNB68CaHPN1d6fgxCiREo5xt9xcWnJoPWv68j+3XyG2hjFMLv1gvgqfBrsWsyzjF59k5n6JrPX\nY4ecPMQVRyq49qG/OgUGtg3/FybcwxM3zWbT4h8x/FglZX0Hez1Hq3P27sL68mO8vOUgVl2Smmwr\niBQvpdTjNgrfLbzKrNNsUMkBX5g0OHDy67hsSuGv4Qa4d7Upysng+oLeAZ176t7NrBr+TcymZK/P\nWzUTb46YxLQ9Hwc83r3HGlm0uQqL3XvbYtadqSmmENNq2pO4tWSO9USLWUcHdtW2n8crNzONqwf3\nZHkEKxq1J/7WYt6iIz6qOB7QuXs0nWVPnzzANkV0tDQCOPScLYpm3eBxnO3UJeTxa5pwBhPHwprL\nH3ErMsfC1jXYs72Ydd2lFPRN560dhzFbJSZNxPQvqSf+nAKelu69sqNYA5woHEvP5NK6wwDs+14q\nN099hor6kRx6bgq5c9cytMcufvzhUo6fDa1WYpImWDBtuNv+WKwTd9NFz2nMnElDSEnSIho/1xa3\njerH3ePsXxCHKzpIl3QsUJST4XPfyGHpHFOxycOzSDYF9n9cVTiRaXs20aW5yWv/sH8f+lvG/usg\nb40Ivj6TJmDi0N4U9E0P+rXRJK4sma8Fu2sHlrIjZ9i870RY9RV9IYD8PrYPuLiqDovVnqRoja/p\noj98WTpflZZdqe3el7VDr2XR27/h4dse40Xm8fCoZ3n/1kL+I/+39PpZFusG5XCwh9cynG2iS9iw\n5zib9p2Im71IiDNL5mvB7vhVLuibzqrSWo6cibzAwHusXDwsvEPB09I1NLUEPCX/5aRZ7O+ZzebF\nD/D9pZsQSwYxQpTzje82sy/tUp64aXbI43LNP4sXh1NcWbJgOmRGAkclqp7pqV4bBrbVSDDRyEhL\nCbiHmFUz8ctJD7Jo3AweOLSCseylNPlyUhcf4LUj12KpD/5rpwnbP6tuE5okfhxOcSUyfwv2jLSU\niDpAUpM1r6Xelm6rcQYEO/ZsEpmS6gae/kd50O9tRs4R+kz7lEd2Pk1F/UiGHtnVKo8vUATwzaF9\nOH72AmVHzqDrxI3DKa5EBm2nJDQ0tYRcCPO6/J6cN1s5cPIcfdJTGZ2T4dVClVQ3MH91GRb7N67F\nHB+/puGwqtR7bwF/DOr2pZugKupH8uLOeQzq9mXQIpPQumhSnDic4k5kbTE+L5PUZM0ZqJud2ZnK\nE1/7fZ0APq085VzUn25q4WDdOa8WqriqDt1lzqTFya9pOJxoDDyJ1ZX3Dn671WMV9SODFhh4z1eL\nF4dT3IrMW3Cor5wiRzatw8oJbOIQSOcaw9Vr1lZnTecmuEVHE+57NolK7/TUaA+hFbbiOvHhcIpL\nkbUVe+c5nXR1779XdpQt+0/ZphxScufYbPp1v4QvDp9m3R73iAYJvLnjcKvOmrEU3d1eTB89gOXb\nD4eUKBsOGpDXu0ur2YgGzs8wHohLkQWTB+UquoK+6W45SY41V0l1Ax9/edJWLMbFg2a2SlbZrZm3\nQpwdhaKcDO68cqDP1BYjGTuoB7UNTTSbdYTA+VlIaavxoaaLBhFqQp4vK+TaWWTn4dOsd7FqktAy\niBONGaMH8Mb2Gv/JmhFE0wQzRg9ghkuMItiWALGQjBkocSmycKZsvqyQa2eRTftOYLZKkk3C+QHH\nS1qFUdisWTavt5M10wQ+YxTjbboelyID46oLFeVksGzWVa0+xFhJZY8m00cPYMX2w87tCyOZdW2e\nM0bU21Q9HsTlIK4zo9uTRKynGApLt9Xw5Du7Dc96uDa/J3Mm2QoaxepUPeEzo9ubePv1NApHPZPF\nmw5QfuQMR89cMERwW/afYvuh+rjKgPaFEpmiTXzVAlny/THO2h9G4IhNdGRAx/NUXYlM4ROHV9Wx\nma8J3KZs4/My0YRwi4CJFI7alfGUAe0LJTKFTxxeVYeEPKdsRTkZLJg2vFX15HDRBMwcm+0WOxqP\n4nKgRKbwiWcdFW+9oF1rTjaeN7P4k6qgAjE8A7pNmuBX04ZfzD5PAJTIFD7xzDr3VZ/RYdWeeHu3\nV4F967I+XF/Q263dEthE2/2SZGd5OQHceeXAhBIYKJEp/BCMV9WbAevkkpPnsHgZaSls3HeC9XuO\nuwksNVljRgLm5oVVfkAI8TshRIUQYpcQ4m0hRHeX5x4TQlQKIfYJIW4Kf6iKWGfG6AGkmGxtjpI0\nuGdcdqvg7dkTB3P3uGwumN07pOZkpsXUHlgkCdeSrQcek1JahBDPAY8Bc4UQlwF3AYVAP2CDEGKI\nlDLw3rOKuMNXtIw3Jg/P4pP9p5z3Z13XOgM9UQhLZFLKdS53iwFHlt40YLmUshk4KISoBMYCW8O5\nniL2CXR66Vh3OXpCJ9o6zJVIrsnuB1bYb/fHJjoHtfbHWiGEmAXMAsjOTtw3WtGau8dlJ7S4HPgV\nmRBiA+Ct8/gTUsrV9mOeACzA68EOQEq5BFgCttjFYF+vUMQ6fkUmpWyz1KsQ4gfAFOAGeTHa+Ctg\noMthA+yPKRQdjnC9izcDPwemSildm1WtAe4SQqQKIQYB+cBn4VxLoYhXwl2T/QVIBdYLW3muYinl\nQ1LKciHEG8AebNPI2cqzqOiohOtd9NnFTUr5G+A34ZxfoUgE4qoWvkIRj8RUZrQQ4iRQHeLLewKn\n/B4Vm8Tz2CG+xx/O2HOklL38HRRTIgsHIcSOQFLBY5F4HjvE9/jbY+xquqhQGIwSmUJhMIkksiXR\nHkAYxPPYIb7Hb/jYE2ZNplDEKolkyRSKmCTuRRbPiaNCiDuEEOVCCF0IMcbjuZgeO9jC6uzjqxRC\nzIv2ePwhhHhFCHFCCFHm8lgPIcR6IcR++9+IJ7XFvciwJY4Ol1KOBL7EljiKR+LozcCLQghT1Ebp\nnTJgOrDZ9cF4GLt9PAuBycBlwEz7uGOZv2F7P12ZB3wopcwHPrTfjyhxLzIp5ToppcV+txhbxD+4\nJI5KKQ8CjsTRmEFKuVdK6a06aMyPHdt4KqWUVVLKFmA5tnHHLFLKzUC9x8PTgFftt18Fbov0deNe\nZB7cD7xnv90fOOzynM/E0RgkHsYeD2MMhD5SyqP228eAPpG+QFxUqzI6cdRIAhm7IjaQUkohRMTd\n7XEhsnhOHPU3dh/ExNj9EA9jDITjQogsKeVRIUQWcCLSF4j76WKCJo7Gw9i3A/lCiEFCiBRsjpo1\nUR5TKKwB7rXfvheI/OxCShnX/7A5BQ4DO+3/Frk89wRwANgHTI72WL2M/XZsa5lm4DjwQbyM3T7G\nW7B5dA9gm/5GfUx+xrsMOAqY7e/7D4FMbF7F/cAGoEekr6siPhQKg4n76aJCEesokSkUBqNEplAY\njBKZQmEwSmQKhcEokSkUBqNEplAYjBKZQmEw/wfV6e3fZpdFnwAAAABJRU5ErkJggg==\n", 287 | "text/plain": [ 288 | "" 289 | ] 290 | }, 291 | "metadata": {}, 292 | "output_type": "display_data" 293 | } 294 | ], 295 | "source": [ 296 | "X, A = generate_data()\n", 297 | "\n", 298 | "mu = calculate_first_moment(X)\n", 299 | "Sigma = calculate_second_moment(X)\n", 300 | "\n", 301 | "s_est, W, X_whit = extract_information_from_second_moment(Sigma, X)\n", 302 | "\n", 303 | "V_est, lamb = perform_tensor_power_method(X_whit, W, s_est, mu)\n", 304 | "\n", 305 | "A_est = perform_backwards_transformation(V_est, lamb)\n", 306 | "\n", 307 | "plot_results(X, A, A_est, s_est)" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": null, 313 | "metadata": { 314 | "collapsed": true 315 | }, 316 | "outputs": [], 317 | "source": [] 318 | } 319 | ], 320 | "metadata": { 321 | "kernelspec": { 322 | "display_name": "Python 3", 323 | "language": "python", 324 | "name": "python3" 325 | }, 326 | "language_info": { 327 | "codemirror_mode": { 328 | "name": "ipython", 329 | "version": 3 330 | }, 331 | "file_extension": ".py", 332 | "mimetype": "text/x-python", 333 | "name": "python", 334 | "nbconvert_exporter": "python", 335 | "pygments_lexer": "ipython3", 336 | "version": "3.6.1" 337 | } 338 | }, 339 | "nbformat": 4, 340 | "nbformat_minor": 2 341 | } 342 | -------------------------------------------------------------------------------- /python/tensor_gmm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from matplotlib.patches import Ellipse 4 | 5 | d = 10 6 | k = 8 7 | n = 1000 8 | tot = k * n 9 | s = 2 10 | dist = 20 11 | spher = True 12 | cov_range = 2 13 | 14 | def generate_data(): 15 | A = -dist+(dist+dist)*np.random.rand(d, k) 16 | X = np.zeros((tot, d)) 17 | 18 | plt.axis('equal') 19 | 20 | for i in range(k): 21 | mean = np.transpose(A[:, i]) 22 | if spher: 23 | covariance = s * np.identity(d) 24 | else: 25 | a = -cov_range + (cov_range + cov_range) * np.random.rand(d, d) 26 | covariance = np.matmul(np.transpose(a), a) 27 | mvn = np.random.multivariate_normal(mean, covariance, n) 28 | X[i*n:(i+1)*n, :] = mvn 29 | 30 | return (X, A) 31 | 32 | def calculate_first_moment(X): 33 | mu = np.zeros((d, 1)) 34 | for t in range(tot): 35 | for i in range(d): 36 | mu[i] += + X[t, i] 37 | mu /= tot 38 | return mu 39 | 40 | def calculate_second_moment(X): 41 | Sigma = np.zeros((d, d)) 42 | for t in range(tot): 43 | for i in range(d): 44 | for j in range(d): 45 | Sigma[i, j] += np.dot(X[t, i],X[t, j]) 46 | Sigma /= tot 47 | return Sigma 48 | 49 | def extract_information_from_second_moment(Sigma, X): 50 | U, S, _ = np.linalg.svd(Sigma) 51 | s_est = S[-1] 52 | W, X_whit = perform_whitening(X, U, S, s_est) 53 | return (s_est, W, X_whit) 54 | 55 | def perform_whitening(X, U, S, s_est): 56 | W = np.matmul(U[:, 0:k], np.sqrt(np.linalg.pinv(np.diag(S[0:k]) - s_est * np.eye(k)))) 57 | X_whit = np.matmul(X, W) 58 | return (W, X_whit) 59 | 60 | def perform_tensor_power_method(X_whit, W, s_est, mu): 61 | TOL = 1e-8 62 | maxiter = 100 63 | V_est = np.zeros((k, k)) 64 | lamb = np.zeros((k, 1)) 65 | 66 | for i in range(k): 67 | v_old = np.random.rand(k, 1) 68 | v_old = np.divide(v_old, np.linalg.norm(v_old)) 69 | for iter in range(maxiter): 70 | v_new = (np.matmul(np.transpose(X_whit), (np.matmul(X_whit, v_old) * np.matmul(X_whit, v_old)))) / tot 71 | #v_new = v_new - s_est * (W' * mu * dot((W*v_old),(W*v_old))); 72 | #v_new = v_new - s_est * (2 * W' * W * v_old * ((W'*mu)' * (v_old))); 73 | v_new -= s_est * (np.matmul(np.matmul(W.T, mu), np.dot(np.matmul(W, v_old).T,np.matmul(W, v_old)))) 74 | v_new -= s_est * (2 * np.matmul(W.T, np.matmul(W, np.matmul(v_old, np.matmul(np.matmul(W.T, mu).T, v_old))))) 75 | if i > 0: 76 | for j in range(i): 77 | v_new -= np.reshape(V_est[:, j] * np.power(np.matmul(np.transpose(v_old), V_est[:, j]), 2) * lamb[j], (k, 1)) 78 | l = np.linalg.norm(v_new) 79 | v_new = np.divide(v_new, np.linalg.norm(v_new)) 80 | if np.linalg.norm(v_old - v_new) < TOL: 81 | V_est[:, i] = np.reshape(v_new, k) 82 | lamb[i] = l 83 | break 84 | v_old = v_new 85 | 86 | return (V_est, lamb) 87 | 88 | def perform_backwards_transformation(V_est, lamb): 89 | return np.matmul(np.matmul(np.linalg.pinv(np.transpose(W)), V_est), np.diag(np.reshape(lamb.T, k))) 90 | 91 | def plot_results(X, A, A_est, s_est): 92 | plt.axis('equal') 93 | 94 | ax = plt.subplot(aspect='equal') 95 | 96 | plt.plot(X[:,0], X[:,1], '.', zorder=-3) 97 | 98 | for i in range(k): 99 | mean = A[:, i].T 100 | mean_est = A_est[:, i].T 101 | 102 | plt.plot(mean[0], mean[1], 'x', color='y', zorder=-2) 103 | plt.plot(mean_est[0], mean_est[1], '+', color='r', zorder=-1) 104 | 105 | ell = Ellipse(xy=(mean_est[0], mean_est[1]), 106 | width=s_est, height=s_est, 107 | angle=0, color='red') 108 | ell.set_facecolor('none') 109 | ax.add_artist(ell) 110 | 111 | plt.show() 112 | 113 | if __name__ == "__main__": 114 | X, A = generate_data() 115 | 116 | mu = calculate_first_moment(X) 117 | Sigma = calculate_second_moment(X) 118 | 119 | s_est, W, X_whit = extract_information_from_second_moment(Sigma, X) 120 | 121 | V_est, lamb = perform_tensor_power_method(X_whit, W, s_est, mu) 122 | 123 | A_est = perform_backwards_transformation(V_est, lamb) 124 | 125 | plot_results(X, A, A_est, s_est) 126 | --------------------------------------------------------------------------------