├── GPFA.m ├── README.md ├── lib ├── invPerSymm.m ├── invToeplitz │ ├── invToeplitz.m │ ├── invToeplitzFast.m │ ├── invToeplitzFastZohar.c │ ├── invToeplitzFastZohar.mexa64 │ ├── invToeplitzFastZohar.mexglx │ ├── invToeplitzFastZohar.mexmaci │ ├── invToeplitzFastZohar.mexmaci64 │ ├── invToeplitzFastZohar.mexw32 │ └── invToeplitzFastZohar.mexw64 ├── logdet.m └── minimize.m ├── startup.m └── testGPFA.m /GPFA.m: -------------------------------------------------------------------------------- 1 | classdef GPFA 2 | % Gaussian Process Factor Analysis with partially observed factors. 3 | % 4 | % This is a generalized version of the method described in Yu et al. 5 | % 2009, J. Neurophys. The implementation is based on and heavily 6 | % influenced by Byron Yu and John Cunningham's code. 7 | % 8 | % Alexander S. Ecker 9 | % 2013-01-17 10 | 11 | properties 12 | params % parameters for fitting 13 | S % basis functions for PSTH 14 | gamma % GP timescales 15 | tau % GP timescales as SD (unit: bins) 16 | C % factor loadings 17 | D % stimulus weights 18 | R % independent noise variances 19 | T % # time points per trial 20 | M % # basis functions for PSTH 21 | p % # unobserved factors 22 | q % # neurons 23 | logLike % log-likelihood curve during fitting 24 | means % method of estimating means (zero|hist|reg) 25 | end 26 | 27 | properties (Access = private) 28 | runtime % runtime of fitting process 29 | end 30 | 31 | methods 32 | 33 | function self = GPFA(varargin) 34 | % GPFA constructor 35 | % gpfa = GPFA('param1', value1, 'param2', value2, ...) 36 | % constructs a GPFA object with the following optional 37 | % parameters: 38 | % 39 | % SigmaN: Innovation noise variance for the latent 40 | % Gaussian Process (default: 0.001) 41 | % Tolerance: Stopping criterion used for fitting (default: 42 | % 0.0001) 43 | % Verbose: Verbose output? (default: false) 44 | 45 | % initialize from struct? 46 | if nargin && isstruct(varargin{1}) 47 | s = varargin{1}; 48 | for f = fieldnames(s)' 49 | self.(f{1}) = s.(f{1}); 50 | end 51 | return 52 | end 53 | 54 | % parse optional parameters 55 | p = inputParser; %#ok<*PROP> 56 | p.KeepUnmatched = true; 57 | p.addOptional('SigmaN', 1e-3); 58 | p.addOptional('Tolerance', 1e-4); 59 | p.addOptional('Verbose', false); 60 | p.parse(varargin{:}); 61 | self.params = p.Results; 62 | end 63 | 64 | 65 | function self = fit(self, Y, p, S, C, D, R, gamma) 66 | % Fit the model 67 | % self = fit(self, Y, p) fits the model to data Y using p 68 | % latent factors. Y is assumed to residuals (i.e. the mean 69 | % for each bin across trials has been subtracted). Y is a 3d 70 | % array of size #neurons x #bins x #trials. 71 | % 72 | % self = fit(self, Y, p, S) additionally uses S as basis 73 | % functions for predicting the PSTHs. 74 | % 75 | % See GPFA for optional parameters to use for fitting. 76 | 77 | self.runtime = now(); 78 | 79 | % make sure there are no non-spiking cells 80 | ok = var(Y(1 : end, :), [], 2) > 1e-10; 81 | assert(all(ok), 'Non-spiking cell found! Please exclude beforehand.') 82 | 83 | % determine dimensionality of the problem 84 | [q, T, N] = size(Y); 85 | if nargin < 4 || isempty(S) 86 | M = 0; 87 | S = []; 88 | means = 'zero'; 89 | elseif ischar(S) && strcmp(S, 'hist') 90 | M = T; 91 | S = []; 92 | means = 'hist'; 93 | else 94 | [M, Tc] = size(S); 95 | means = 'reg'; 96 | assert(T == Tc, 'The number of columns in S and Y must be the same!') 97 | end 98 | assert(q > p, 'Number of latent factors must be smaller than number of neurons.') 99 | 100 | self.q = q; 101 | self.T = T; 102 | self.M = M; 103 | self.p = p; 104 | self.means = means; 105 | 106 | if nargin < 5 107 | Yn = reshape(Y, q, T * N); 108 | 109 | switch means 110 | case 'zero' 111 | D = []; 112 | Y0 = Yn; 113 | case 'hist' 114 | D = mean(Y, 3); 115 | Y0 = Yn - repmat(D, 1, N); 116 | case 'reg' 117 | % initialize stimulus weights using linear regression 118 | Sn = repmat(S, 1, N); 119 | D = Yn / Sn; 120 | Y0 = Yn - D * Sn; 121 | end 122 | 123 | % initialize factor loadings using PCA 124 | Q = cov(Y0'); 125 | [C, Lambda] = eigs(Q, p); 126 | 127 | % initialize private noise as residual variance not accounted 128 | % for by PCA and stimulus 129 | R = diag(diag(Q - C * Lambda * C')); 130 | 131 | % initialize gammas 132 | gamma = log(0.01) * ones(p, 1); 133 | end 134 | 135 | self = self.collect(C, R, gamma, S, D); 136 | 137 | % run EM 138 | self = self.EM(Y); 139 | self = self.reorderFactors(); 140 | self.runtime = (now() - self.runtime) * 24 * 3600 * 1000; % ms 141 | end 142 | 143 | 144 | function [EX, VarX, logLike] = estX(self, Y) 145 | % Estimate latent factors (and log-likelihood). 146 | % [EX, VarX, logLike] = self.estX(Y) returns the expected 147 | % value (EX) of the latent state X, its variance (VarX), and 148 | % the log-likelihood (logLike) of the data Y. 149 | 150 | T = self.T; q = self.q; p = self.p; C = self.C; R = self.R; 151 | N = size(Y, 3); 152 | 153 | % catch independent case 154 | if p == 0 155 | EX = zeros(0, T, N); 156 | VarX = []; 157 | if nargout == 3 158 | Y = reshape(Y, q, T * N); 159 | val = N * T * (sum(log(diag(R))) + q * log(2 * pi)); 160 | normY = bsxfun(@rdivide, Y, sqrt(diag(R))); 161 | logLike = -0.5 * (val + normY(:)' * normY(:)); 162 | end 163 | return 164 | end 165 | 166 | % compute GP covariance and its inverse 167 | Kb = zeros(T * p, T * p); 168 | Kbi = zeros(T * p, T * p); 169 | logdetKb = 0; 170 | for i = 1 : p 171 | K = toeplitz(self.covFun(0 : T - 1, self.gamma(i))); 172 | ndx = i : p : T * p; 173 | Kb(ndx, ndx) = K; 174 | [Kbi(ndx, ndx), logdetK] = invToeplitz(K); 175 | logdetKb = logdetKb + logdetK; 176 | end 177 | 178 | % Perform E step 179 | RiC = bsxfun(@rdivide, C, diag(R)); 180 | CRiC = C' * RiC; 181 | [VarX, logdetM] = invPerSymm(Kbi + kron(eye(T), CRiC), p); 182 | RbiCb = kron(eye(T), RiC); % [TODO] can kron be optimized? 183 | Rbi = kron(eye(T), diag(1 ./ diag(R))); 184 | Cb = kron(eye(T), C); 185 | KbCb = Kb * Cb'; % [TODO] optimize: K is block-diagonal 186 | % KbCb = kron(K, C'); % if all Ks/taus are equal 187 | CKCRi = Rbi - RbiCb * VarX * RbiCb'; 188 | Y0 = self.subtractMean(Y); 189 | Y0 = reshape(Y0, q * T, N); 190 | EX = KbCb * CKCRi * Y0; 191 | EX = reshape(EX, [p T N]); 192 | 193 | % calculate log-likelihood 194 | if nargout > 2 195 | Y0 = reshape(Y0, q, T * N); 196 | val = -T * sum(log(diag(R))) - logdetKb - logdetM - ... 197 | q * T * log(2 * pi); 198 | normY0 = bsxfun(@rdivide, Y0, sqrt(diag(R))); 199 | CRiY0 = reshape(RiC' * Y0, p * T, N); 200 | logLike = 0.5 * (N * val - normY0(:)' * normY0(:) + ... 201 | sum(sum(CRiY0 .* (VarX * CRiY0)))); 202 | end 203 | end 204 | 205 | 206 | function Y0 = subtractMean(self, Y) 207 | % Subtract mean. 208 | 209 | switch self.means 210 | case 'zero' 211 | Y0 = Y; 212 | case 'hist' 213 | Y0 = bsxfun(@minus, Y, self.D); 214 | case 'reg' 215 | Y0 = bsxfun(@minus, Y, self.D * self.S); 216 | end 217 | end 218 | 219 | 220 | function Y = addMean(self, Y0) 221 | % Add mean. 222 | 223 | switch self.means 224 | case 'zero' 225 | Y = Y0; 226 | case 'hist' 227 | Y = bsxfun(@plus, Y0, self.D); 228 | case 'reg' 229 | Y = bsxfun(@plus, Y0, self.D * self.S); 230 | end 231 | end 232 | 233 | 234 | function [Yres, X] = resid(self, Y) 235 | % Compute residuals after accounting for internal factors. 236 | 237 | [Ypred, X] = predict(self, Y); 238 | Yres = Y - Ypred; 239 | end 240 | 241 | 242 | function [R, X] = residCov(self, Y, byTrial) 243 | % Residual covariance. 244 | % R = model.residCov(Y) returns the residual covariance using 245 | % data Y after accounting for latent factors. 246 | % 247 | % R = model.residCov(Y, true) returns the residual covariance 248 | % for spike counts summed over the entire trial. 249 | % 250 | % Note: this residuals covariance is computed using the 251 | % update rule of the EM algorithm. It is not the same 252 | % as computing the covariance of the residuals as in 253 | % cov(model.resid(Y)). 254 | 255 | if nargin < 3 || ~byTrial 256 | [R, X] = self.residCovByBin(Y); 257 | else 258 | [R, X] = self.residCovByTrial(Y); 259 | end 260 | end 261 | 262 | 263 | function [Ypred, X] = predict(self, Y) 264 | % Prediction of activity based on inference of latent factors. 265 | 266 | Ypred = zeros(size(Y)); 267 | N = size(Y, 3); 268 | X = self.estX(Y); 269 | for i = 1 : N 270 | Ypred(:, :, i) = self.C * X(:, :, i); 271 | end 272 | Ypred = self.addMean(Ypred); 273 | end 274 | 275 | 276 | function ve = varExpl(self, Y, byTrial) 277 | % Variance explained by model. 278 | % ve = model.varExpl(Y) computes the fraction of variance 279 | % explained by the model. 280 | % 281 | % ve = model.varExpl(Y, true) uses spike counts summed over 282 | % the entire trial to compute the fraction of variance 283 | % explained. 284 | 285 | if nargin < 3 || ~byTrial 286 | ve = self.varExplByBin(Y); 287 | else 288 | ve = self.varExplByTrial(Y); 289 | end 290 | end 291 | 292 | 293 | function k = covFun(self, t, gamma) 294 | % Gaussian process covariance function 295 | 296 | sn = self.params.SigmaN; 297 | sf = 1 - sn; 298 | k = sf * exp(-0.5 * exp(gamma) * t .^ 2) + sn * (t == 0); 299 | end 300 | 301 | 302 | function [self, X] = ortho(self, X) 303 | % Orthogonalize factor loadings 304 | % 305 | % Caution: After applying this transformation, the model 306 | % cannot be used for inference on new data. 307 | 308 | [self.C, S, V] = svd(self.C, 'econ'); 309 | if nargout > 1 310 | if size(X, 1) == self.q % Y passed -> estimte X 311 | X = self.estX(X); 312 | end 313 | N = size(Y, 3); 314 | X = reshape(X, self.p, self.T * N); 315 | X = S * V' * X; 316 | X = reshape(X, [self.p self.T N]); 317 | end 318 | end 319 | 320 | 321 | function [self, X] = normLoadings(self, X) 322 | % Normalize factor loadings 323 | % 324 | % Caution: Covariance function is not adjusted properly. 325 | % After applying this transformation inference of latent 326 | % factors won't be correct any more. This can be fixed but 327 | % hasn't been done yet. 328 | 329 | n = sqrt(sum(self.C .^ 2, 1)); 330 | self.C = bsxfun(@rdivide, self.C, n); 331 | if nargout > 1 332 | if size(X, 1) == self.q % Y passed -> estimte X 333 | X = self.estX(X); 334 | end 335 | for i = 1 : self.p 336 | X(i, :) = X(i, :) * n(i); 337 | end 338 | end 339 | end 340 | 341 | 342 | function [self, X] = normFactors(self, X) 343 | % Normalize factors to unit variance 344 | % 345 | % Caution: Covariance function is not adjusted properly. 346 | % After applying this transformation inference of latent 347 | % factors won't be correct any more. This can be fixed but 348 | % hasn't been done yet. 349 | 350 | if size(X, 1) == self.q % Y passed -> estimte X 351 | X = self.estX(X); 352 | end 353 | for i = 1 : self.p 354 | sd = std(X(i, :)); 355 | self.C(:, i) = self.C(:, i) * sd; 356 | if nargout > 1 357 | X(i, :) = X(i, :) / sd; 358 | end 359 | end 360 | end 361 | 362 | 363 | function s = struct(self) 364 | % Convert to struct. 365 | 366 | state = warning('off', 'MATLAB:structOnObject'); 367 | s = builtin('struct', self); 368 | warning(state) 369 | end 370 | 371 | end 372 | 373 | 374 | methods (Access = protected) 375 | 376 | function self = collect(self, C, R, gamma, S, D) 377 | self.C = C; 378 | self.R = R; 379 | self.gamma = gamma; 380 | if nargin > 4 381 | self.S = S; 382 | self.D = D; 383 | end 384 | end 385 | 386 | function [E, dEdgamma] = Egamma(self, gamma, EXX) 387 | % EXX is the average (over N) of the second moments of X 388 | 389 | sigmaf = 1 - self.params.SigmaN; 390 | t = 0 : self.T - 1; 391 | [Ki, logdetK] = invToeplitz(self.covFun(t, gamma)); 392 | ttsq = bsxfun(@minus, t, t') .^ 2; 393 | dKdgamma = -0.5 * sigmaf * exp(gamma) * ttsq .* exp(-0.5 * exp(gamma) * ttsq); 394 | dEdK = 0.5 * (Ki - Ki * EXX * Ki); 395 | dEdgamma = dEdK(:)' * dKdgamma(:); 396 | E = 0.5 * (logdetK + EXX(:)' * Ki(:)); 397 | end 398 | 399 | end 400 | 401 | 402 | methods (Access = private) 403 | 404 | function self = EM(self, Y) 405 | % Run EM. 406 | % self = self.EM(Y) runs the EM iteration until convergence. 407 | 408 | S = self.S; p = self.p; q = self.q; T = self.T; M = self.M; 409 | N = size(Y, 3); 410 | Sn = repmat(S, [1 1 N]); 411 | 412 | % catch independent case 413 | if p == 0 414 | [~, ~, self.logLike] = self.estX(Y); 415 | return 416 | end 417 | 418 | iter = 0; 419 | logLikeBase = NaN; 420 | while iter <= 2 || (self.logLike(end) - self.logLike(end - 1)) / (self.logLike(end - 1) - logLikeBase) > self.params.Tolerance 421 | 422 | iter = iter + 1; 423 | 424 | % E step 425 | [EX, VarX, self.logLike(end + 1)] = estX(self, Y); 426 | 427 | % Perform M step 428 | T1 = zeros(q, p + M); 429 | T2 = zeros(p + M); 430 | for t = 1 : T 431 | x = permute(EX(:, t, :), [1 3 2]); 432 | y = permute(Y(:, t, :), [1 3 2]); 433 | T1(:, 1 : p) = T1(:, 1 : p) + y * x'; 434 | tt = (1 : p) + p * (t - 1); 435 | T2(1 : p, 1 : p) = T2(1 : p, 1 : p) + N * VarX(tt, tt) + x * x'; 436 | switch self.means 437 | case 'hist' 438 | T1(:, p + t) = sum(y, 2); 439 | sx = sum(x, 2); 440 | T2(1 : p, p + t) = sx; 441 | T2(p + t, 1 : p) = sx'; 442 | T2(p + t, p + t) = N; 443 | case 'reg' 444 | s = permute(Sn(:, t, :), [1 3 2]); 445 | sx = x * s'; 446 | T1(:, p + (1 : M)) = T1(:, p + (1 : M)) + y * s'; 447 | T2(1 : p, p + (1 : M)) = T2(1 : p, p + (1 : M)) + sx; 448 | T2(p + (1 : M), 1 : p) = T2(p + (1 : M), 1 : p) + sx'; 449 | T2(p + (1 : M), p + (1 : M)) = T2(p + (1 : M), p + (1 : M)) + s * s'; 450 | end 451 | end 452 | CD = T1 / T2; 453 | self.C = CD(:, 1 : p); 454 | self.D = CD(:, p + (1 : M)); 455 | 456 | Y0 = self.subtractMean(Y); 457 | Y0 = reshape(Y0, q, T * N); 458 | self.R = diag(mean(Y0 .^ 2, 2) - ... 459 | sum(bsxfun(@times, Y0 * reshape(EX, p, T * N)', self.C), 2) / (T * N)); 460 | 461 | % optimize gamma 462 | self.gamma = zeros(p, 1); 463 | for i = 1 : p 464 | ndx = i : p : T * p; 465 | EXi = permute(EX(i, :, :), [2 3 1]); 466 | EXX = VarX(ndx, ndx) + (EXi * EXi') / N; 467 | fun = @(gamma) self.Egamma(gamma, EXX); 468 | self.gamma(i) = minimize(self.gamma(i), fun, -25); 469 | end 470 | self.tau = exp(-self.gamma / 2); 471 | 472 | if iter == 2 473 | logLikeBase = self.logLike(end); 474 | end 475 | 476 | if self.params.Verbose 477 | subplot(211) 478 | plot(self.logLike(2 : end), '.-k') 479 | subplot(212), hold all 480 | plot(self.C(:, 1), 'k') 481 | drawnow 482 | end 483 | end 484 | end 485 | 486 | 487 | function self = reorderFactors(self) 488 | % Re-order factors according to covariance explained. 489 | 490 | C = self.C; p = self.p; 491 | v = zeros(p, 1); 492 | for i = 1 : p 493 | v(i) = mean(mean(C(:, i) * C(:, i)')); 494 | end 495 | [~, order] = sort(v, 'descend'); 496 | C = C(:, order); 497 | C = bsxfun(@times, C, sign(median(C, 1))); % flip sign? 498 | self.C = C; 499 | end 500 | 501 | 502 | function [R, X] = residCovByBin(self, Y) 503 | % Residual covariance for spike counts per bin. 504 | 505 | T = self.T; N = size(Y, 3); p = self.p; q = self.q; C = self.C; 506 | [X, VarX] = self.estX(Y); 507 | Y0 = self.subtractMean(Y); 508 | Y0 = reshape(Y0, q, T * N); 509 | EXX = 0; 510 | for t = 1 : T 511 | x = permute(X(:, t, :), [1 3 2]); 512 | tt = (1 : p) + p * (t - 1); 513 | EXX = EXX + N * VarX(tt, tt) + x * x'; 514 | end 515 | X = reshape(X, p, T * N); 516 | Y0XC = (Y0 * X') * C'; 517 | R = (Y0 * Y0' - Y0XC - Y0XC' + C * EXX * C') / (T * N); 518 | end 519 | 520 | 521 | function [R, X] = residCovByTrial(self, Y) 522 | % Residual covariance for spike counts over entire trial. 523 | 524 | T = self.T; N = size(Y, 3); p = self.p; C = self.C; 525 | [X, VarX] = self.estX(Y); 526 | X = reshape(X, [p * T, N]); 527 | Y0 = self.subtractMean(Y); 528 | Z = permute(sum(Y0, 2), [1 3 2]); 529 | Ct = repmat(C, 1, T); 530 | CXZ = Ct * X * Z'; 531 | EXX = N * VarX + X * X'; 532 | R = (Z * Z' - CXZ - CXZ' + Ct * EXX * Ct') / N; 533 | end 534 | 535 | 536 | function ve = varExplByBin(self, Y) 537 | % Compute variance explained for spike counts per bin.. 538 | 539 | Y0 = self.subtractMean(Y); 540 | V = mean(Y0(:, :) .^ 2, 2); 541 | R = self.residCovByBin(Y); 542 | ve = 1 - diag(R) ./ V; 543 | end 544 | 545 | 546 | function ve = varExplByTrial(self, Y) 547 | % Compute variance explained for spike counts of entire trial. 548 | 549 | Y0 = self.subtractMean(Y); 550 | V = mean(sum(Y0, 2) .^ 2, 3); 551 | R = self.residCovByTrial(Y); 552 | ve = 1 - diag(R) ./ V; 553 | end 554 | 555 | end 556 | 557 | 558 | methods (Static) 559 | 560 | function [model, Y, X, S] = toyExample(N) 561 | % Create toy example for testing 562 | % [model, Y, X, S] = toyExample(N) creates a simple toy 563 | % example with 16 neurons and two latent factors, sampling N 564 | % trials of activity. 565 | 566 | if ~nargin 567 | N = 100; 568 | end 569 | T = 20; 570 | p = 2; 571 | q = 16; 572 | gamma = log(1 ./ [4; 1] .^ 2); 573 | 574 | model = GPFA(); 575 | 576 | K = toeplitz(model.covFun(0 : T - 1, gamma(1))); 577 | X1 = chol(K)' * randn(T, N); 578 | K = toeplitz(model.covFun(0 : T - 1, gamma(2))); 579 | X2 = chol(K)' * randn(T, N); 580 | X = [X1(:), X2(:)]'; 581 | 582 | phi = (0 : q - 1) / q * 2 * pi; 583 | C = [cos(phi); sin(phi)]' / sqrt(q / 2); 584 | D = rand(q, T); 585 | S = eye(T); 586 | Sn = repmat(S, 1, N); 587 | R = 0.02 * eye(q); 588 | Y = chol(R)' * randn(q, T * N) + C * X + D * Sn; 589 | Y = reshape(Y, [q T N]); 590 | 591 | model = model.collect(C, R, gamma, S, D); 592 | model.T = T; 593 | model.p = p; 594 | model.q = q; 595 | end 596 | 597 | end 598 | 599 | end 600 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | GPFA - Gaussian Process Factor Analysis 2 | ======================================= 3 | 4 | This is an implementation of Gaussian Process Factor Analysis, 5 | an analysis tool for simultaneous dimensionality reduction and 6 | smoothing. The model was developed by Byron Yu and John 7 | Cunningham. My implementation builds heavily on their original 8 | code, which can be found here: 9 | 10 | http://users.ece.cmu.edu/~byronyu/software.shtml 11 | 12 | Details about the model can be found in the following paper: 13 | 14 | Yu, B.M. et al., 2009. Gaussian-Process Factor Analysis for 15 | Low-Dimensional Single-Trial Analysis of Neural Population 16 | Activity. Journal of Neurophysiology, 102(1), pp.614-635. 17 | 18 | I used this model in the following paper: 19 | 20 | A.S. Ecker, P. Berens, R.J. Cotton, M. Subramaniyan, G.H. Denfield, 21 | C.R. Cadwell, S.M. Smirnakis, M. Bethge, A.S. Tolias (2014): State 22 | dependence of noise correlations in macaque primary visual cortex. 23 | Neuron. 24 | 25 | For more information go to: 26 | http://toliaslab.org/publications/ecker-et-al-2014/ 27 | 28 | 29 | What's the difference? 30 | ====================== 31 | 32 | I used the model in a slightly different context, whichrequired 33 | some additions but also allowed me to simplify some aspects: 34 | 35 | * Fixed trial length. In contrast to the original implementation, 36 | my code does not allow for variable trial length (fixed T). 37 | 38 | * Known regressors. I added the option to include a number of 39 | known regressors. This can be useful if you want to include an 40 | external signal as a predictor into the model. Examples of such 41 | signals could be the local field potential or a stimulus with 42 | known temporal dynamics. 43 | 44 | * Estimation of residual covariance. I added the option to 45 | estimate the residual covariance that remains after a certain 46 | number of latent factors have been accounted for. This can be 47 | useful if you want to estimate the covariance in the presence 48 | of some confounder, which is modeled by the latent factor. 49 | -------------------------------------------------------------------------------- /lib/invPerSymm.m: -------------------------------------------------------------------------------- 1 | function [invM, logdet_M] = invPerSymm(M, q) 2 | % Invert block persymmetric matrix. 3 | % [invM, logdet_M] = invPerSymm(M, q) 4 | % 5 | % @ 2009 Byron Yu byronyu@stanford.edu 6 | % John Cunningham jcunnin@stanford.edu 7 | % 8 | % Modified by AE 2012-10-12 9 | 10 | % [TODO] 11 | % check if we're actually using all knowledge about the structure of M 12 | 13 | T = size(M, 1) / q; 14 | mkr = q * ceil(T / 2); 15 | invA11 = inv(M(1 : mkr, 1 : mkr)); 16 | invA11 = (invA11 + invA11') / 2; 17 | A12 = M(1 : mkr, mkr + 1 : end); 18 | term = invA11 * A12; 19 | F22 = M(mkr + 1 : end, mkr + 1 : end) - A12' * term; 20 | res12 = -term / F22; 21 | res11 = invA11 - res12 * term'; 22 | res11 = (res11 + res11') / 2; 23 | upperHalf = [res11 res12]; 24 | 25 | % Fill in bottom half of invM by picking elements from res11 and res12 26 | idxHalf = bsxfun(@plus, (1 : q)', (floor(T / 2) - 1 : -1 : 0) * q); 27 | idxFull = bsxfun(@plus, (1 : q)', (T - 1 : -1 : 0) * q); 28 | invM = [upperHalf; upperHalf(idxHalf(:), idxFull(:))]; 29 | 30 | if nargout == 2 31 | logdet_M = -logdet(invA11) + logdet(F22); 32 | end 33 | -------------------------------------------------------------------------------- /lib/invToeplitz/invToeplitz.m: -------------------------------------------------------------------------------- 1 | 2 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 3 | % John P Cunningham 4 | % 2009 5 | % 6 | % invToeplitz() 7 | % 8 | % Invert a symmetric, real, positive definite Toeplitz matrix 9 | % using either inv() or the Trench algorithm, which 10 | % uses Zohar 1969. This is slightly different than 11 | % Algorithm 4.7.3 of Golub and Van Loan 12 | % "Matrix Computations," the 1996 version, because the Zohar 13 | % version produces the log determinant essentially for free, which 14 | % is often useful in cases where one actually needs the full matrix inverse. 15 | % 16 | % If Zohar or the MATLAB implementation of the Trench algorithm is called, 17 | % the inversion is done in O(n^2) time, which is considerably better 18 | % than the O(n^3) that inv() offers. This follows the Trench 19 | % algorithm implementation of Zohar 1969. See that paper for 20 | % all explanation, as the C code is just an implementation of 21 | % the algorithm of p599 of that paper. See Algorithm 4.7.3 of Golub and Van Loan 22 | % for an explanation of the MATLAB version of this algorithm, which is slower 23 | % due to for loops (slower than inv(), up to about n=300). 24 | % Thus, if the C can be used, always use it. 25 | % 26 | % This function also computes the log determinant, which is 27 | % calculated essentially for free as part of the calculation of 28 | % the inverse when Zohar is used. This is often useful in applications when one really 29 | % needs to represent the inverse of a Toeplitz matrix. If Zohar is not used, the computation 30 | % is reasonably costly. 31 | % 32 | % Usage: [Ti,ld] = invToeplitz(T, [runMode]); 33 | % 34 | % Inputs: 35 | % T the positive definite symmetric Toeplitz matrix to be inverted 36 | % runMode OPTIONAL: to force this inversion to happen using a particular method. This 37 | % will NOT roll over to a method (by design), so it will fail if something is amuck. 38 | % 39 | % Outputs: 40 | % Ti the inverted matrix, also symmetric (and persymmetric), NOT TOEPLITZ 41 | % 42 | % NOTE: We bother with this method because we 43 | % need this particular matrix inversion to be 44 | % as fast as possible. Thus, no error checking 45 | % is done here as that would add needless computation. 46 | % Instead, the onus is on the caller to make sure 47 | % the matrix is toeplitz and symmetric and real. 48 | % 49 | % NOTE: Algorithm 4.7.3 in the Golub book has a number of typos 50 | % Do not use Alg 4.7.3, use the equations on p197-199. 51 | % 52 | % Run time tests (testInvToeplitz) suggest that Zohar is the fastest method across 53 | % the board. (2-3 orders of magnitude at sizes from 200-1000). Trench is 13/4n^2 flops, 54 | % whereas inv() is probably 2/3n^3. However, the overhead for the MEX calls, etc., 55 | % may make the crossover point very MATLAB dependent. 56 | % 57 | % NOTE: The interested numerical linear algebra geek can dig into all this stuff by 58 | % going into the directory 'tests' and running that testInvToeplitz to find the crossover 59 | % point him/herself. Some tinkering required. This will run seven inversion methods: inv(), invToeplitzFastGolub (MEX), 60 | % invToeplitzFastZohar (MEX), invPerSymm (2x2 exploitation of persymmetry), invToeplitz 61 | % (Golub MATLAB non-vectorized...almost always the worst), and invToeplitz (Golub MATLAB vectorized). 62 | % Use figure 1 from the results. This version of invToeplitz has been pared down to 63 | % be more user friendly, so it only includes Zohar, inv(), and vectorized MATLAB Trench/Golub. 64 | % 65 | % MEX NOTE: This function will call a MEX routine. 66 | % A try block is included to default back to the best non-MEX solver (either inv() 67 | % or a vectorized Trench/Golub version in native MATLAB), so hopefully the user will 68 | % not experience failures. We have also included the compiled .mex in a number of 69 | % different architectures (hopefully all). However, the user should really compile 70 | % this code on his/her own machine to ensure proper use of MEX. That can be done 71 | % from the MATLAB prompt with "mex invToeplitzFast.c". This C code does nothing 72 | % fancy, so if you can run MEX at all, this should work. See also 'help mex' and 73 | % 'mexext('all')'. 74 | % 75 | % MEX NOTE 2: The try-catch block is to hopefully fail MEX gracefully. If the mex 76 | % code is not compiled or in the wrong architecture, this will default to the next 77 | % best MATLAB implementation, which, depending on n, is inv() or vectorized Trench. 78 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 79 | 80 | 81 | 82 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 83 | function [Ti,ld] = invToeplitz( T , runMode ) 84 | 85 | % dimension of the matrix 86 | n = size(T,1); 87 | % if runMode is specified, just run with that (for testing, etc.) 88 | % otherwise, do automated determination of runMode 89 | if nargin<2 || isempty(runMode) 90 | % then do the automated determination of runMode, based on runtime tests 91 | % always try Zohar first. 92 | tryMode = 0; 93 | % if MEX fails, use inv() or vectorized Trench in MATLAB 94 | % The n=150 is a rough estimate based on tests on 64 and 32 bit linux systems 95 | if n < 150 96 | % inv() is the best failure option 97 | catchMode = -1; 98 | else 99 | % n is >=150, so vectorized Trench is the best failure option 100 | catchMode = 1; 101 | end 102 | else 103 | % if specified, force it 104 | tryMode = runMode; 105 | catchMode = runMode; 106 | end 107 | 108 | % Invert T with the specified method 109 | try 110 | [Ti,ld] = invToeplitzMode( T , tryMode); 111 | catch 112 | [Ti,ld] = invToeplitzMode( T , catchMode); 113 | end 114 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 115 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 116 | 117 | 118 | 119 | 120 | 121 | 122 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 123 | % SUBFUNCTION FOR TRY-CATCH BLOCK 124 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 125 | function [Ti,ld] = invToeplitzMode( T , runMode ) 126 | 127 | % dimension of the matrix 128 | n = size(T,1); 129 | % Invert T to produce Ti, the inverse, by the specified method 130 | switch (runMode) 131 | 132 | case -1 133 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 134 | % just call MATLAB inv() 135 | Ti = inv(T); 136 | % and calculate the log determinant 137 | ld = logdet(T); 138 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 139 | 140 | 141 | case 0 142 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 143 | % here is the fast C-MEX implementation of this inversion, using Zohar 144 | % invToeplitzFastZohar.c should be mex-compiled with a command from the MATLAB 145 | % prompt of "mex invToeplitzFastZohar.c" This should just work. 146 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 147 | % call the MEX wrapper 148 | [Ti,ld] = invToeplitzFast(T); 149 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 150 | 151 | 152 | case 1 153 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 154 | % a faster vectorized version of Trench that wastes no computations 155 | % This is faster than 2 but considerably less fast than 0. 156 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 157 | % set up the Trench parameters 158 | [r,gam,v] = trenchSetup(T); 159 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 160 | % fill out the borders of Ti 161 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 162 | % the first element 163 | Ti(1,1) = gam; 164 | % the first row 165 | Ti(1,2:n) = v(n-1:-1:1)'; 166 | % use symmetry and persymmetry to fill out border of matrix 167 | % the first column 168 | Ti(2:n,1) = Ti(1,2:n)'; 169 | % last column - persymmetric with 1st row 170 | Ti(2:n,n) = Ti(1,n-1:-1:1)'; 171 | % last row - persymmetric with 1st column 172 | Ti(n,2:n-1) = Ti(n-1:-1:2,1)'; 173 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 174 | % fill out the interior of Ti 175 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 176 | for i = (n-1) : -1 : (floor((n-1)/2) + 1); 177 | % 4.7.5 178 | Ti(i,i:-1:(n-i+1)) = Ti(n-(i:-1:(n-i+1)),n-i) + (1/gam)*((v(i)*v(i:-1:(n-i+1)))' - v(n-i)*v(n-(i:-1:(n-i+1)))'); 179 | % symmetry 180 | Ti((n-i+1):(i-1),i) = Ti(i,(n-i+1):(i-1)); 181 | % persymmetry 182 | Ti((n-i+1),(i-1):-1:(n-i+1)) = Ti((n-i+2):(i),i)'; % note 1 fewer element assigned 183 | % symmetry 184 | Ti((n-i+1-1:i-1),(n-i+1)) = Ti((n-i+1),(n-i+1-1:i-1)); % note 1 fewer elements again 185 | end 186 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 187 | % renormalize 188 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 189 | Ti = Ti/T(1,1); 190 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 191 | % calculate the log determinant 192 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 193 | ld = logdet(T); 194 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 195 | 196 | end 197 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 198 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 199 | 200 | 201 | 202 | 203 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 204 | % SUBFUNCTION FOR TRENCH SETUP 205 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 206 | function [r,gam,v] = trenchSetup(T); 207 | 208 | % dimension of the matrix 209 | n = size(T,1); 210 | % Initial setup for Trench algorithm (from Golub book) 211 | % normalize the matrix to r0 = 1 w.l.o.g. 212 | r = [T(1,2:n)]/T(1,1); 213 | % nicely, MATLAB has an implementation of Algorithm 4.7.1 to solve Tn-1*y = (-r1,...,rn-1)'; 214 | % it does not quite seem to work as MATLAB says it does, but the following reconciles it with 4.7.1. 215 | % y = T(1:n-1,1:n-1)\(-T(1,2:end)'); % this considerably slows the implementation. 216 | y = levinson([1 r])'; % this is a reasonable piece of the computation in the fastest algorithm (2n^2 of 13/4n^2 flops) 217 | y = y(2:end); 218 | % calculate the quantity gamma 219 | % gam = 1/(1+r(1:n-1)*y(n-1:-1:1)); (This is a typo from the book) 220 | % this calculates the right answer 221 | gam = 1/(1+r(1:n-1)*y(1:n-1)); 222 | % now calculate v 223 | v(1:n-1) = gam*y(n-1:-1:1); % corresponds to v = gam*E*y, top of p198 224 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 225 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -------------------------------------------------------------------------------- /lib/invToeplitz/invToeplitzFast.m: -------------------------------------------------------------------------------- 1 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 2 | % John P Cunningham 3 | % 2009 4 | % 5 | % invToeplitzFast() 6 | % 7 | % This function is simply a wrapper for the C-MEX 8 | % function invToeplitzFastZohar.mexa64 (or .mexglx, etc), 9 | % which is just a compiled version of invToeplitzFastZohar.c, 10 | % which should also be in this folder. Please see 11 | % that code for details. 12 | % 13 | % This algorithm inverts a positive definite (symmetric) 14 | % Toeplitz matrix in O(n^2) time, which is considerably better 15 | % than the O(n^3) that inv() offers. This follows the Trench 16 | % algorithm implementation of Zohar 1969. See that paper for 17 | % all explanation, as the C code is just an implementation of 18 | % the algorithm of p599 of that paper. 19 | % 20 | % This function also computes the log determinant, which is 21 | % calculated essentially for free as part of the calculation of 22 | % the inverse. This is often useful in applications when one really 23 | % needs to represent the inverse of a Toeplitz matrix. 24 | % 25 | % This function should be called from within invToeplitz.m, 26 | % which adds a try block so that it can default to a native 27 | % MATLAB inversion (either inv() or a vectorized version of 28 | % the Trench algorithm, depending on the size of the matrix) 29 | % should the MEX interface not work. This will happen, for example, 30 | % if you move to a new architecture and do not compile for .mexmaci 31 | % or similar (see mexext('all') and help mex for some info on this). 32 | % 33 | % Inputs: 34 | % T the positive definite (symmetric) Toeplitz matrix, which 35 | % does NOT need to be scaled to be 1 on the main diagonal. 36 | % 37 | % Outputs: 38 | % Ti the inverse of T 39 | % ld the log determinant of T, NOT Ti. 40 | % 41 | % 42 | % NOTE: This code is used to speed up the Toeplitz inversion as much 43 | % as possible. Accordingly, no error checking is done. The onus is 44 | % on the caller (which should be invToeplitz.m) to pass the correct args. 45 | % 46 | % NOTE: cf. invTopelitzFastGolub.c, which uses the algorithm in Golub and 47 | % Van Loan. This newer version was written because the Zohar version 48 | % also computes the log determinant for free, which is essential in the 49 | % application for which this algorithm was coded. 50 | % 51 | % NOTE: Whenever possible, do not actually invert a matrix. This code is 52 | % written just in case you really need to do so. Otherwise, for example 53 | % if you just want to solve inv(T)*x for some vector x, you are better off 54 | % using a fast inversion method, like PCG with fast matrix multiplication, 55 | % which could be something like an FFT method for the Toeplitz matrix. To 56 | % learn about this, see Cunningham, Sahani, Shenoy (2008), ICML, "Fast Gaussian 57 | % process methods for point process intensity estimation." 58 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 59 | 60 | function [Ti,ld] = invToeplitzFast(T) 61 | 62 | [Ti,ld] = invToeplitzFastZohar(T); 63 | 64 | -------------------------------------------------------------------------------- /lib/invToeplitz/invToeplitzFastZohar.c: -------------------------------------------------------------------------------- 1 | /*================================================================= 2 | * 3 | * invToeplitzFastZohar.c completes the inversion of a symmetric 4 | * positive definite Toeplitz matrix. This 5 | * function is a subroutine of the invToeplitz.m, which follows the 6 | * algorithm of Zohar 1969. This is the algorithm of W.F. Trench. 7 | * 8 | * The calling syntax is: 9 | * 10 | * [ Ti , logdetT ] = invToeplitzFastZohar( T ) 11 | * 12 | * T is the positive definite symmetric Toeplitz matrix of dimension n+1 13 | * (following the convention of p599 in the Zohar paper) 14 | * Ti is the inverse of T, same as inv(T) in MATLAB. 15 | * logdetT is the log determinant of T, NOT of Ti. 16 | * 17 | * NOTE: This code is used to speed up the Toeplitz inversion as much 18 | * as possible. Accordingly, no error checking is done. The onus is 19 | * on the caller (which should be invToeplitz.m) to pass the correct args. 20 | * 21 | * NOTE: cf. invTopelitzFastGolub.c, which uses the algorithm in Golub and 22 | * Van Loan. This newer version was written because the Zohar version 23 | * also computes the log determinant for free, which is essential in the 24 | * application for which this algorithm was coded. 25 | * 26 | * John P Cunningham 27 | * 2009 28 | * 29 | *=================================================================*/ 30 | 31 | #include 32 | #include "mex.h" 33 | 34 | /* Input Arguments */ 35 | 36 | #define T_IN prhs[0] 37 | 38 | /* Output Arguments */ 39 | 40 | #define Ti_OUT plhs[0] 41 | #define LD_OUT plhs[1] 42 | 43 | void mexFunction( int nlhs, mxArray *plhs[], 44 | int nrhs, const mxArray*prhs[] ) 45 | { 46 | double *T, *Ti; 47 | double *logdet; 48 | double *r, *lambda, *ghat, *ghatold, *gamma, *g; 49 | unsigned int n; 50 | int i,j; 51 | 52 | 53 | /* Check for proper number of arguments*/ 54 | if (nrhs != 1) 55 | { 56 | mexErrMsgTxt("1 input arg required."); 57 | } 58 | else if (nlhs > 2) 59 | { 60 | mexErrMsgTxt("2 output args required."); 61 | } 62 | 63 | 64 | /* Check the dimensions of the arguments */ 65 | /* This expects T to be an n+1 by n+1 matrix */ 66 | n = mxGetN(T_IN)-1; 67 | 68 | 69 | /* Create a matrix for the return argument */ 70 | Ti_OUT = mxCreateDoubleMatrix(n+1, n+1, mxREAL); 71 | LD_OUT = mxCreateDoubleScalar(0); 72 | /* LD_OUT is a pointer to a 1x1 mxArray double initialized to 0 */ 73 | 74 | /* Assign pointers to the various parameters */ 75 | Ti = mxGetPr(Ti_OUT); 76 | logdet = mxGetPr(LD_OUT); 77 | T = mxGetPr(T_IN); 78 | 79 | 80 | r = (double*) mxMalloc(n*sizeof(double)); 81 | if (r == NULL) 82 | { 83 | mexErrMsgTxt("Inadequate space on heap for r."); 84 | } 85 | lambda = (double*) mxMalloc(n*sizeof(double)); 86 | if (lambda == NULL) 87 | { 88 | mexErrMsgTxt("Inadequate space on heap for lambda."); 89 | } 90 | ghat = (double*) mxMalloc(n*sizeof(double)); 91 | if (ghat == NULL) 92 | { 93 | mexErrMsgTxt("Inadequate space on heap for ghat."); 94 | } 95 | ghatold = (double*) mxMalloc(n*sizeof(double)); 96 | if (ghatold == NULL) 97 | { 98 | mexErrMsgTxt("Inadequate space on heap for ghatold."); 99 | } 100 | gamma = (double*) mxMalloc(n*sizeof(double)); 101 | if (gamma == NULL) 102 | { 103 | mexErrMsgTxt("Inadequate space on heap for gamma."); 104 | } 105 | g = (double*) mxMalloc(n*sizeof(double)); 106 | if (g == NULL) 107 | { 108 | mexErrMsgTxt("Inadequate space on heap for g."); 109 | } 110 | 111 | /* define r, the normalized row from T(1,2) to T(1,n+1) */ 112 | for (i = 0; i < n ; i++) 113 | { 114 | r[i] = T[ 0*(n+1) + (i+1) ]/T[ 0*(n+1) + 0]; 115 | } 116 | 117 | 118 | /* Initialize the algorithm */ 119 | lambda[0] = 1 - pow(r[0],2); 120 | ghat[0] = -r[0]; 121 | 122 | /* Recursion to build g and lambda */ 123 | for (i=0 ; i < n-1 ; i++) 124 | { 125 | /* calculate gamma */ 126 | gamma[i] = -r[i+1]; 127 | /* ghat, g_i, etc are i+1 elts long */ 128 | for (j=0 ; j < i+1 ; j++) 129 | { 130 | gamma[i] -= r[j]*ghat[j]; 131 | } 132 | /* calculate ghat */ 133 | 134 | /* first store ghatold..i+1 elts long */ 135 | for (j=0 ; j < i+1 ; j++) 136 | { 137 | ghatold[j] = ghat[j]; 138 | } 139 | 140 | ghat[0] = gamma[i]/lambda[i]; 141 | for (j=1 ; j < i+2 ; j++) 142 | { 143 | ghat[j] = ghatold[j-1] + gamma[i]/lambda[i]*ghatold[i+1-j]; 144 | } 145 | /* calculate lambda */ 146 | lambda[i+1] = lambda[i] - pow(gamma[i],2)/lambda[i]; 147 | } 148 | /* assign g for convenience */ 149 | for (i=0 ; i < n; i++) 150 | { 151 | g[ i ] = ghat[n-1-i]; 152 | } 153 | 154 | /* There are n lambdas, n g values, and n-1 gammas */ 155 | /* Note that n is not the dimension of the matrix, but one less. */ 156 | /* This corresponds with Zohar notation. */ 157 | 158 | 159 | /* Evaluate the matrix Ti (B_{n+1} in Zohar) */ 160 | /* NOTE ON MEX MATRIX INDEXING */ 161 | /* indexing is always (colidx * colLen + rowidx) */ 162 | 163 | /* Assign first upper left element */ 164 | Ti[ 0*(n+1) + 0 ] = 1/lambda[n-1]; 165 | /* Assign the first row and column*/ 166 | for (i = 1; i < n+1; i++) 167 | { 168 | /* first row */ 169 | Ti[ 0*(n+1) + i] = g[ i-1 ]/lambda[ n-1 ]; 170 | /* first column */ 171 | Ti[ i*(n+1) + 0] = g[ i-1 ]/lambda[ n-1 ]; 172 | } 173 | for (i = 1; i < n ; i++) 174 | { 175 | /* last row */ 176 | Ti[ n*(n+1) + i ] = g[ n-i-1 ]/lambda[ n-1 ]; 177 | /* last column */ 178 | Ti[ i*(n+1) + n ] = g[ n-i-1 ]/lambda[ n-1 ]; 179 | 180 | } 181 | Ti[ n*(n+1) + n ] = Ti[ 0*(n+1) + 0]; 182 | 183 | 184 | /* Fill in the interior of Ti_OUT */ 185 | for (i = 0; i < n/2 ; i++) 186 | { 187 | for (j = i; j < n-i-1; j++) 188 | { 189 | /* calculate the value using p599 of Zohar */ 190 | Ti[ (j+1)*(n+1) + (i+1) ] = (Ti[ j*(n+1) + i ] + (1/lambda[n-1])*(g[i]*g[j] - g[n-1-i]*g[n-1-j])); 191 | /* use symmetry */ 192 | Ti[ (i+1)*(n+1) + (j+1) ] = Ti[ ((j+1)*(n+1)) + (i+1) ]; 193 | /* use persymmetry */ 194 | /* recall there are n+1 elements, so 0<->n, 1<->n-1, and so on */ 195 | Ti[ (n-(i+1))*(n+1) + (n-(j+1)) ] = Ti[ ((j+1)*(n+1)) + (i+1) ]; 196 | Ti[ (n-(j+1))*(n+1) + (n-(i+1)) ] = Ti[ ((j+1)*(n+1)) + (i+1) ]; 197 | } 198 | } 199 | 200 | 201 | /* normalize the entire matrix by T(1,1) so it is the properly scaled inverse */ 202 | for (i = 0; i < n+1 ; i++) 203 | { 204 | for (j = 0; j < n+1 ; j++) 205 | { 206 | Ti[ j*(n+1) + i ] = Ti[ j*(n+1) + i ]/T[ 0*(n+1) + 0]; 207 | } 208 | } 209 | 210 | /* Calculate the log determinant for free (essentially) */ 211 | logdet[0] = 0; 212 | for (i=0 ; i < n; i++) 213 | { 214 | logdet[0] += log(lambda[i]); 215 | } 216 | /* renormalize based on T(1,1) */ 217 | logdet[0] += (n+1)*log(T[ 0*(n+1) + 0 ]); 218 | 219 | 220 | /* free allocated arrays */ 221 | mxFree(g); 222 | mxFree(gamma); 223 | mxFree(ghatold); 224 | mxFree(ghat); 225 | mxFree(lambda); 226 | mxFree(r); 227 | 228 | return; 229 | } 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | -------------------------------------------------------------------------------- /lib/invToeplitz/invToeplitzFastZohar.mexa64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aecker/gpfa/cf8d7fb7dd74d816e806ccfc57ee77d54723b048/lib/invToeplitz/invToeplitzFastZohar.mexa64 -------------------------------------------------------------------------------- /lib/invToeplitz/invToeplitzFastZohar.mexglx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aecker/gpfa/cf8d7fb7dd74d816e806ccfc57ee77d54723b048/lib/invToeplitz/invToeplitzFastZohar.mexglx -------------------------------------------------------------------------------- /lib/invToeplitz/invToeplitzFastZohar.mexmaci: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aecker/gpfa/cf8d7fb7dd74d816e806ccfc57ee77d54723b048/lib/invToeplitz/invToeplitzFastZohar.mexmaci -------------------------------------------------------------------------------- /lib/invToeplitz/invToeplitzFastZohar.mexmaci64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aecker/gpfa/cf8d7fb7dd74d816e806ccfc57ee77d54723b048/lib/invToeplitz/invToeplitzFastZohar.mexmaci64 -------------------------------------------------------------------------------- /lib/invToeplitz/invToeplitzFastZohar.mexw32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aecker/gpfa/cf8d7fb7dd74d816e806ccfc57ee77d54723b048/lib/invToeplitz/invToeplitzFastZohar.mexw32 -------------------------------------------------------------------------------- /lib/invToeplitz/invToeplitzFastZohar.mexw64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aecker/gpfa/cf8d7fb7dd74d816e806ccfc57ee77d54723b048/lib/invToeplitz/invToeplitzFastZohar.mexw64 -------------------------------------------------------------------------------- /lib/logdet.m: -------------------------------------------------------------------------------- 1 | function y = logdet(A) 2 | % log(det(A)) where A is positive-definite. 3 | % This is faster and more stable than using log(det(A)). 4 | 5 | U = chol(A); 6 | y = 2 * sum(log(diag(U))); 7 | -------------------------------------------------------------------------------- /lib/minimize.m: -------------------------------------------------------------------------------- 1 | function [X, fX, i] = minimize(X, f, length, varargin) 2 | 3 | % Minimize a differentiable multivariate function. 4 | % 5 | % Usage: [X, fX, i] = minimize(X, f, length, P1, P2, P3, ... ) 6 | % 7 | % where the starting point is given by "X" (D by 1), and the function named in 8 | % the string "f", must return a function value and a vector of partial 9 | % derivatives of f wrt X, the "length" gives the length of the run: if it is 10 | % positive, it gives the maximum number of line searches, if negative its 11 | % absolute gives the maximum allowed number of function evaluations. You can 12 | % (optionally) give "length" a second component, which will indicate the 13 | % reduction in function value to be expected in the first line-search (defaults 14 | % to 1.0). The parameters P1, P2, P3, ... are passed on to the function f. 15 | % 16 | % The function returns when either its length is up, or if no further progress 17 | % can be made (ie, we are at a (local) minimum, or so close that due to 18 | % numerical problems, we cannot get any closer). NOTE: If the function 19 | % terminates within a few iterations, it could be an indication that the 20 | % function values and derivatives are not consistent (ie, there may be a bug in 21 | % the implementation of your "f" function). The function returns the found 22 | % solution "X", a vector of function values "fX" indicating the progress made 23 | % and "i" the number of iterations (line searches or function evaluations, 24 | % depending on the sign of "length") used. 25 | % 26 | % The Polack-Ribiere flavour of conjugate gradients is used to compute search 27 | % directions, and a line search using quadratic and cubic polynomial 28 | % approximations and the Wolfe-Powell stopping criteria is used together with 29 | % the slope ratio method for guessing initial step sizes. Additionally a bunch 30 | % of checks are made to make sure that exploration is taking place and that 31 | % extrapolation will not be unboundedly large. 32 | % 33 | % See also: checkgrad 34 | % 35 | % Copyright (C) 2001 - 2006 by Carl Edward Rasmussen (2006-09-08). 36 | % 37 | % Permission is granted for anyone to copy, use, or modify these 38 | % programs and accompanying documents for purposes of research or 39 | % education, provided this copyright notice is retained, and note is 40 | % made of any changes that have been made. 41 | % 42 | % These programs and documents are distributed without any warranty, 43 | % express or implied. As the programs were written for research 44 | % purposes only, they have not been tested to the degree that would be 45 | % advisable in any important application. All use of these programs is 46 | % entirely at the user's own risk. 47 | 48 | INT = 0.1; % don't reevaluate within 0.1 of the limit of the current bracket 49 | EXT = 3.0; % extrapolate maximum 3 times the current step-size 50 | MAX = 20; % max 20 function evaluations per line search 51 | RATIO = 10; % maximum allowed slope ratio 52 | SIG = 0.1; RHO = SIG/2; % SIG and RHO are the constants controlling the Wolfe- 53 | % Powell conditions. SIG is the maximum allowed absolute ratio between 54 | % previous and new slopes (derivatives in the search direction), thus setting 55 | % SIG to low (positive) values forces higher precision in the line-searches. 56 | % RHO is the minimum allowed fraction of the expected (from the slope at the 57 | % initial point in the linesearch). Constants must satisfy 0 < RHO < SIG < 1. 58 | % Tuning of SIG (depending on the nature of the function to be optimized) may 59 | % speed up the minimization; it is probably not worth playing much with RHO. 60 | 61 | % The code falls naturally into 3 parts, after the initial line search is 62 | % started in the direction of steepest descent. 1) we first enter a while loop 63 | % which uses point 1 (p1) and (p2) to compute an extrapolation (p3), until we 64 | % have extrapolated far enough (Wolfe-Powell conditions). 2) if necessary, we 65 | % enter the second loop which takes p2, p3 and p4 chooses the subinterval 66 | % containing a (local) minimum, and interpolates it, unil an acceptable point 67 | % is found (Wolfe-Powell conditions). Note, that points are always maintained 68 | % in order p0 <= p1 <= p2 < p3 < p4. 3) compute a new search direction using 69 | % conjugate gradients (Polack-Ribiere flavour), or revert to steepest if there 70 | % was a problem in the previous line-search. Return the best value so far, if 71 | % two consecutive line-searches fail, or whenever we run out of function 72 | % evaluations or line-searches. During extrapolation, the "f" function may fail 73 | % either with an error or returning Nan or Inf, and minimize should handle this 74 | % gracefully. 75 | 76 | if max(size(length)) == 2, red=length(2); length=length(1); else red=1; end 77 | if length>0, S='Linesearch'; else S='Function evaluation'; end 78 | 79 | i = 0; % zero the run length counter 80 | ls_failed = 0; % no previous line search has failed 81 | [f0 df0] = feval(f, X, varargin{:}); % get function value and gradient 82 | fX = f0; 83 | i = i + (length<0); % count epochs?! 84 | s = -df0; d0 = -s'*s; % initial search direction (steepest) and slope 85 | x3 = red/(1-d0); % initial step is red/(|s|+1) 86 | 87 | while i < abs(length) % while not finished 88 | i = i + (length>0); % count iterations?! 89 | 90 | X0 = X; F0 = f0; dF0 = df0; % make a copy of current values 91 | if length>0, M = MAX; else M = min(MAX, -length-i); end 92 | 93 | while 1 % keep extrapolating as long as necessary 94 | x2 = 0; f2 = f0; d2 = d0; f3 = f0; df3 = df0; 95 | success = 0; 96 | while ~success && M > 0 97 | try 98 | M = M - 1; i = i + (length<0); % count epochs?! 99 | [f3 df3] = feval(f, X+x3*s, varargin{:}); 100 | if isnan(f3) || isinf(f3) || any(isnan(df3)+isinf(df3)), error(''), end 101 | success = 1; 102 | catch % catch any error which occured in f 103 | x3 = (x2+x3)/2; % bisect and try again 104 | end 105 | end 106 | if f3 < F0, X0 = X+x3*s; F0 = f3; dF0 = df3; end % keep best values 107 | d3 = df3'*s; % new slope 108 | if d3 > SIG*d0 || f3 > f0+x3*RHO*d0 || M == 0 % are we done extrapolating? 109 | break 110 | end 111 | x1 = x2; f1 = f2; d1 = d2; % move point 2 to point 1 112 | x2 = x3; f2 = f3; d2 = d3; % move point 3 to point 2 113 | A = 6*(f1-f2)+3*(d2+d1)*(x2-x1); % make cubic extrapolation 114 | B = 3*(f2-f1)-(2*d1+d2)*(x2-x1); 115 | x3 = x1-d1*(x2-x1)^2/(B+sqrt(B*B-A*d1*(x2-x1))); % num. error possible, ok! 116 | if ~isreal(x3) || isnan(x3) || isinf(x3) || x3 < 0 % num prob | wrong sign? 117 | x3 = x2*EXT; % extrapolate maximum amount 118 | elseif x3 > x2*EXT % new point beyond extrapolation limit? 119 | x3 = x2*EXT; % extrapolate maximum amount 120 | elseif x3 < x2+INT*(x2-x1) % new point too close to previous point? 121 | x3 = x2+INT*(x2-x1); 122 | end 123 | end % end extrapolation 124 | 125 | while (abs(d3) > -SIG*d0 || f3 > f0+x3*RHO*d0) && M > 0 % keep interpolating 126 | if d3 > 0 || f3 > f0+x3*RHO*d0 % choose subinterval 127 | x4 = x3; f4 = f3; d4 = d3; % move point 3 to point 4 128 | else 129 | x2 = x3; f2 = f3; d2 = d3; % move point 3 to point 2 130 | end 131 | if f4 > f0 132 | x3 = x2-(0.5*d2*(x4-x2)^2)/(f4-f2-d2*(x4-x2)); % quadratic interpolation 133 | else 134 | A = 6*(f2-f4)/(x4-x2)+3*(d4+d2); % cubic interpolation 135 | B = 3*(f4-f2)-(2*d2+d4)*(x4-x2); 136 | x3 = x2+(sqrt(B*B-A*d2*(x4-x2)^2)-B)/A; % num. error possible, ok! 137 | end 138 | if isnan(x3) || isinf(x3) 139 | x3 = (x2+x4)/2; % if we had a numerical problem then bisect 140 | end 141 | x3 = max(min(x3, x4-INT*(x4-x2)),x2+INT*(x4-x2)); % don't accept too close 142 | [f3 df3] = feval(f, X+x3*s, varargin{:}); 143 | if f3 < F0, X0 = X+x3*s; F0 = f3; dF0 = df3; end % keep best values 144 | M = M - 1; i = i + (length<0); % count epochs?! 145 | d3 = df3'*s; % new slope 146 | end % end interpolation 147 | 148 | if abs(d3) < -SIG*d0 && f3 < f0+x3*RHO*d0 % if line search succeeded 149 | X = X+x3*s; f0 = f3; fX = [fX' f0]'; % update variables 150 | % fprintf('%s %6i; Value %4.6e\r', S, i, f0); 151 | s = (df3'*df3-df0'*df3)/(df0'*df0)*s - df3; % Polack-Ribiere CG direction 152 | df0 = df3; % swap derivatives 153 | d3 = d0; d0 = df0'*s; 154 | if d0 > 0 % new slope must be negative 155 | s = -df0; d0 = -s'*s; % otherwise use steepest direction 156 | end 157 | x3 = x3 * min(RATIO, d3/(d0-realmin)); % slope ratio but max RATIO 158 | ls_failed = 0; % this line search did not fail 159 | else 160 | X = X0; f0 = F0; df0 = dF0; % restore best point so far 161 | if ls_failed || i > abs(length) % line search failed twice in a row 162 | break; % or we ran out of time, so we give up 163 | end 164 | s = -df0; d0 = -s'*s; % try steepest 165 | x3 = 1/(1-d0); 166 | ls_failed = 1; % this line search failed 167 | end 168 | end 169 | %fprintf('\n'); 170 | -------------------------------------------------------------------------------- /startup.m: -------------------------------------------------------------------------------- 1 | function startup() 2 | % Add folders that are needed to MATLAB path 3 | % AE 2012-10-29 4 | 5 | folder = fileparts(mfilename('fullpath')); 6 | addpath(folder) 7 | addpath(fullfile(folder, 'lib')); 8 | addpath(fullfile(folder, 'lib/invToeplitz')); 9 | -------------------------------------------------------------------------------- /testGPFA.m: -------------------------------------------------------------------------------- 1 | % Test case for three latent factors plus known stimulus terms 2 | % AE 2012-10-19 3 | 4 | % create toy example 5 | rng(1) 6 | % [grd, Y, X, S] = GPFA.toyExample(); 7 | % [grd, Y, X, S] = GPFA.toyExampleOri('gauss'); 8 | [grd, Y, X, S] = GPFA.toyExampleOri('poisson'); 9 | [grd, X] = grd.normFactors(X); 10 | 11 | % fit model 12 | model = GPFA('Tolerance', 1e-6); 13 | model = model.fit(Y, grd.p, 'hist'); 14 | % model = model.fit(Y, grd.p, S); 15 | [model, Xest] = model.normFactors(Y); 16 | 17 | 18 | %% diagnostic plots 19 | cc = model.C' * grd.C; 20 | [~, ndx] = max(abs(cc)); 21 | N = size(Y, 3); 22 | for i = 1 : model.p 23 | subplot(3, model.p, i) 24 | cla, hold on 25 | plot(X(i, :), 'k') 26 | plot(sign(cc(ndx(i), i)) * Xest(ndx(i), :), 'r') 27 | axis tight 28 | plot(repmat((1 : N - 1) * model.T, 2, 1), ylim, 'k') 29 | ylabel(sprintf('Factor %d', i)) 30 | xlabel('Time') 31 | xlim([0 100]) 32 | end 33 | 34 | for i = 1 : model.p 35 | subplot(3, model.p, model.p + i) 36 | cla, hold on 37 | plot(grd.C(:, i), 'k') 38 | plot(sign(cc(ndx(i), i)) * model.C(:, ndx(i)), 'r') 39 | ylabel(sprintf('Loading %d', i)) 40 | xlabel('Neuron #') 41 | axis tight 42 | end 43 | 44 | legend({'Ground truth', 'Model fit'}) 45 | 46 | subplot(3, model.p, 2 * model.p + 1) 47 | cla, hold on 48 | plot(grd.D(:), model.D(:), '.k') 49 | axis tight 50 | xlabel('Stim term: ground truth') 51 | ylabel('Stim term: model fit') 52 | shg 53 | --------------------------------------------------------------------------------