├── .gitattributes ├── .gitignore ├── GPtutorial.m ├── GPtutorialFcn.m ├── calcGP.m ├── hypSample.m ├── k_GP.m └── stdRegion.m /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear on external disk 35 | .Spotlight-V100 36 | .Trashes 37 | 38 | # Directories potentially created on remote AFP share 39 | .AppleDB 40 | .AppleDesktop 41 | Network Trash Folder 42 | Temporary Items 43 | .apdisk 44 | -------------------------------------------------------------------------------- /GPtutorial.m: -------------------------------------------------------------------------------- 1 | % This is Matlab code to produce Figure 6 of 'Gaussian Processes for Timeseries Modelling' 2 | % a.k.a. Figures 1 and 2 at http://arxiv.org/abs/1505.02965 3 | % Mark Ebden, Aug 2008. Comments to mebden@gmail.com 4 | % 5 | % Dependencies: calcGP.m k_GP.m, GPtutorialFcn.m, hypSample.m 6 | 7 | %%%%%%%%%%%%%%%%%%%%%%%%% 1. Initialization %%%%%%%%%%%%%%%%%%%%%%%%% 8 | 9 | % a) Parameters 10 | sigma_n = 0.3; % Noise variance 11 | f = 0; % f = 1/T. Set to zero to turn off periodicity 12 | % b) Inputs 13 | X = [-1.5 -1 -.75 -.4 -.25 0]'; n = size(X,1); 14 | y = .55*[-3 -2 -.6 .4 1 1.6]'; % My example 15 | % c) Vector indicating which variables to fit (l, sigma_n, sigma_f, and f) 16 | vars = [1 0 1 0]'; % 0 means "known/given" 17 | % d) Vector of constants for objective function 18 | % (Log transform on the parameter space prevents negative values) 19 | consts = [log([1 sigma_n 1 f])'; vars; n; X; y]; 20 | 21 | %%%%%%%%%%%%%%%%%%% 2. Fit the ML parameters %%%%%%%%%%%%%%%%%%%%%%%%% 22 | 23 | % a) Basic initializations 24 | options = optimset('GradObj','on'); 25 | Nstarts = 5; 26 | % b) Choose several varied starting points 27 | l_samp = hypSample ([0.1 10], Nstarts); % for l 28 | sf_samp = hypSample ([0.3 3], Nstarts); % for sigma_f 29 | inits = log([l_samp sf_samp]); 30 | % c) Find candidates 31 | paramVec = []; 32 | for randomStart = 1:Nstarts 33 | [params, fval] = fminunc(@(params) GPtutorialFcn(params,consts), inits(randomStart,:), options); 34 | paramVec = [paramVec; fval inits(randomStart,:) params]; 35 | end 36 | paramVec(:,2:end) = exp(paramVec(:,2:end)); 37 | paramVec(:,1) = paramVec(:,1)/max(abs(paramVec(:,1))); 38 | % d) Select best candidate 39 | paramVec = sortrows(paramVec) 40 | params = paramVec(1,size(inits,2)+2:end); 41 | l = params(1); sigma_f = params(2); 42 | 43 | %%%%%%%%%%%%%%%%%%%%% 3. Plot the regression %%%%%%%%%%%%%%%%%%%%%%%%% 44 | 45 | % a) Plot Figure 2 in GPtutorial.pdf 46 | xstar = (1:1e3)/1e3 * 2.1 - 1.8; 47 | close all 48 | [xstar, bestEstimate, bounds, K, kstar] = calcGP (@k_GP, X, y, [l sigma_n sigma_f f], 0, xstar, 1); 49 | axis tight, v = axis; axis([-1.7 v(2:4)]) 50 | v = axis; line ([v(1) v(1)], [v(3) v(4)], 'Color', 'k') 51 | xlabel ('x'), ylabel('y') 52 | % b) Calculate y* 53 | xindex = 952; % determined by inspection 54 | newX = xstar(xindex); newBounds = bounds(xindex,:); 55 | newEst = bestEstimate(xindex); newSigma = (newBounds(1)-newEst)/1.96; 56 | disp([newEst newSigma.^2]) % Answers reported in Step 2 of page 3 of GPtutorial.pdf 57 | % c) Update Figure 2 with y* 58 | errorbar (newX,newEst,newSigma,newSigma,'Color','green') 59 | plot (newX,newEst,'b.','MarkerSize',15) 60 | errorbar (X,y,sigma_n*ones(size(y)),sigma_n*ones(size(y)),'Color','red', 'Linestyle','none') 61 | plot (xstar,bestEstimate,'k'), plot (X,y,'k.', 'MarkerSize',15) 62 | % d) Plot Figure 1 63 | figure, hold on 64 | errorbar (X,y,sigma_n*ones(size(y)),sigma_n*ones(size(y)),'Color','red','Linestyle','none') % plot error bars 65 | errorbar (newX,newEst,newSigma,newSigma,'Color','green') 66 | text(newX-.01,newEst-newSigma-0.4,'?') 67 | plot (X,y,'k.', 'MarkerSize',15), plot (newX,newEst,'b.', 'MarkerSize',15) 68 | xlabel ('x'), ylabel('y'), axis(v) 69 | -------------------------------------------------------------------------------- /GPtutorialFcn.m: -------------------------------------------------------------------------------- 1 | % The objective function for optimization in GPtutorial.m 2 | % Mark Ebden, 2008 3 | 4 | function [fcn, grd] = GPtutorialFcn (params,v) 5 | 6 | % a) Initializations 7 | eparams = exp(params); evs = exp(v); 8 | whichVars = v(5:8); % indicators 9 | % Variables: 10 | varCount = 1; 11 | if whichVars(1) == 1, 12 | l1 = eparams(1); 13 | varCount = varCount + 1; 14 | end 15 | if whichVars(2) == 1, 16 | sigma_n = eparams(varCount); 17 | varCount = varCount + 1; 18 | end 19 | if whichVars(3) == 1, 20 | sigma_f = eparams(varCount); 21 | varCount = varCount + 1; 22 | end 23 | if whichVars(4) == 1, 24 | f = eparams(varCount); 25 | varCount = varCount + 1; 26 | end 27 | % Constants - whatever's left 28 | if exist('l1') == 0, 29 | l1 = evs(1); 30 | end 31 | if exist('sigma_n') == 0, 32 | sigma_n = evs(2); 33 | end 34 | if exist('sigma_f') == 0, 35 | sigma_f = evs(3); 36 | end 37 | if exist('f') == 0, 38 | f = evs(4); 39 | end 40 | n = v(9); 41 | X = v(10:n+9); y = v(n+10:end); y = y - mean(y); 42 | 43 | % b) Variance and its derivative 44 | K = zeros(n); dKdl = zeros(n); dKds = zeros(n); dKdf = zeros(n); dKdw = zeros(n); 45 | for i = 1:n, 46 | for j = 1:n, 47 | [K(i,j), dKdl(i,j), dKds(i,j), dKdf(i,j), dKdw(i,j)] = k_GP (X(i),X(j),[l1 sigma_n sigma_f f]); 48 | end 49 | end 50 | 51 | % c) Calculations 52 | L = chol (K,'lower'); 53 | alpha = L'\(L\y); 54 | invK = inv(K); 55 | alpha2 = invK*y; % alpha from page 114, not page 19, of Rasmussen and Williams (2006) 56 | 57 | % d) Log marginal likelihood and its gradient 58 | logpyX = -y'*alpha/2 - sum(log(diag(L))) - n*log(2*pi)/2; 59 | dlogp_dl = l1*trace((alpha2*alpha2' - invK)*dKdl)/2; 60 | dlogp_ds = sigma_n*trace((alpha2*alpha2' - invK)*dKds)/2; 61 | dlogp_df = sigma_f*trace((alpha2*alpha2' - invK)*dKdf)/2; 62 | dlogp_dw = f*trace((alpha2*alpha2' - invK)*dKdw)/2; 63 | % NB: Since l = exp(p1), dlogp/dp1 = dlogp/dl dl/dp1 = dlogp/dl exp(p1) = dlogp/dl l, where p1 = params(1) 64 | 65 | % e) Format output 66 | fcn = -logpyX; grd = []; 67 | if whichVars(1) == 1, 68 | grd = [grd -dlogp_dl]; 69 | end 70 | if whichVars(2) == 1, 71 | grd = [grd -dlogp_ds]; 72 | end 73 | if whichVars(3) == 1, 74 | grd = [grd -dlogp_df]; 75 | end 76 | if whichVars(4) == 1, 77 | grd = [grd -dlogp_dw]; 78 | end 79 | -------------------------------------------------------------------------------- /calcGP.m: -------------------------------------------------------------------------------- 1 | % Calculates, for xstar x-coordinates, the bestEstimate and the 95% 2 | % confidence interval (bounds) for a Gaussian process regression. 3 | % Mark Ebden, July 2008 4 | % 5 | % Inputs: 6 | % k - a handle to the covariance function 7 | % X,y - the training data x- and y-coordinates 8 | % theta - the GP parameters to be passed to the covariance function 9 | % (e.g. length, scale, periodicity, etc. 10 | % sigma_n - the noise (optional -- default 0) 11 | % NOTE: sigma_n should be left as zero if k *already* includes the noise term, as I usually do. 12 | % xstar - specify the x-coordinates to solve for (optional -- default is 13 | % to let the algorithm choose them) 14 | % graphing - produce a plot of results (optional) 15 | % Default is to plot the results, unless the function outputs were asked for 16 | % 17 | % Outputs: 18 | % xstar - x-coordinates (default 100 are chosen) 19 | % K - covariance matrix among training data 20 | % V - variance at each xstar 21 | 22 | function [xstar, bestEstimate, bounds, K, V] = calcGP (k, X, y, theta, sigma_n, xstar, graphing) 23 | 24 | if nargin < 7, 25 | if nargout == 0, 26 | graphing = 1; 27 | else 28 | graphing = 0; 29 | end 30 | end 31 | if nargin < 6 || ~any(xstar), 32 | r2 =(max(X)-min(X)); 33 | N = 1e3; % How many points to use 34 | z = .5; % How much to extend the time series on either side (e.g. .5 is 50% extension on left and on right) 35 | xstar = (0:N)/N * (z*2+1)*r2 + min(X)-r2*z; 36 | end 37 | if nargin < 5, 38 | sigma_n = 0; 39 | end 40 | if nargin < 4, 41 | theta = [1 0 1 0]; 42 | end 43 | 44 | % a) Initializations 45 | meany = mean(y); y = y - meany; 46 | n = size(X,1); lx = length(xstar); 47 | K = zeros(n); kstar = zeros(n,1); 48 | for i = 1:n, 49 | for j = 1:n, 50 | K(i,j) = k (X(i),X(j),theta); 51 | end 52 | end 53 | fstarbar = zeros(lx,1); V = zeros(lx,1); 54 | 55 | % b) One-off calculations 56 | diags = max(1e3*eps, sigma_n^2); % beef up the diagonal if sigma_n = 0 57 | L = chol (K + diags*eye(n),'lower'); 58 | alpha = L'\(L\y); 59 | logpyX = -y'*alpha/2 - sum(log(diag(L))) - n*log(2*pi)/2; % Log marginal likelihood 60 | 61 | % c) xstar loop 62 | for q = 1:lx, 63 | for i = 1:n, 64 | kstar(i) = k (X(i),xstar(q),theta); 65 | end 66 | % Mean of prediction 67 | fstarbar(q) = kstar' * alpha; 68 | % Variance of prediction 69 | v = L\kstar; 70 | ystar_noise = sigma_n^2; % recall f* has no noise itself (Algorithm 2.1 on p 19 of H152) 71 | V(q) = k (xstar(q),xstar(q),theta) - v'*v + ystar_noise; 72 | end 73 | bounds = [fstarbar+1.96*sqrt(V) fstarbar-1.96*sqrt(V)]+meany; 74 | bestEstimate = fstarbar+meany; 75 | 76 | % d) Output 77 | if graphing == 1, 78 | figure, hold on 79 | stdRegion (xstar,bounds) 80 | plot (xstar,bestEstimate,'k') 81 | plot (X,y+meany,'b+') 82 | end 83 | -------------------------------------------------------------------------------- /hypSample.m: -------------------------------------------------------------------------------- 1 | % Sample a hyperbola N times for a parameter bounded by the two components of 'bounds'. 2 | % Example usage: if p(x) = k/x from 0.1 to 4, and p(x) = 0 otherwise: 3 | % x = hypSample ([.1 4], 1e5); hist(x,100) 4 | % Bounds can be calculated as per Section 5.5.1 in 'Data Analysis: A Bayesian Tutorial', 5 | % 2nd edition, D.S. Sivia and J. Skilling, 2006. 6 | % 7 | % Mark Ebden, July 2008 8 | 9 | function x = hypSample (bounds, N) 10 | 11 | xmin = bounds(1); xmax = bounds(2); 12 | F = rand(N,1); 13 | x = xmin.^(1-F) .* xmax.^F; 14 | -------------------------------------------------------------------------------- /k_GP.m: -------------------------------------------------------------------------------- 1 | % For one-dimensional inputs, computes the covariance function and its partial derivatives wrt: 2 | % the horizontal lengthscale parameter, l 3 | % the noise parameter, sigman 4 | % the vertical lengthscale parameter, sigmaf 5 | % the frequency, f 6 | % 7 | % N.B. If your data set contains multiple identical x values, rewrite this code to accept the 8 | % indices of x rather than the values themselves; otherwise, the last if-statement below leads 9 | % eventually to the creation of a singular matrix. 10 | % 11 | % Mark Ebden, 2008 12 | 13 | function [covar, d_l, d_sigman, d_sigmaf, d_f] = k_GP (x1, x2, theta) 14 | 15 | % a) Initializations 16 | if exist('theta')==0, 17 | theta = []; 18 | end 19 | if length(theta) < 4, 20 | f = 0; 21 | else 22 | f = theta(4); 23 | end 24 | if length(theta) < 3, 25 | sigmaf = 1; 26 | else 27 | sigmaf = theta(3); 28 | end 29 | if length(theta) < 2, 30 | sigman = 0; 31 | else 32 | sigman = theta(2); 33 | end 34 | if length(theta) < 1, 35 | l = 1; 36 | else 37 | l = theta(1); 38 | end 39 | 40 | % b) Covariance and gradients 41 | covar = sigmaf^2 * exp(-(x1-x2)^2/(2*l^2)); 42 | d_l = covar * (l^-3) * (x1-x2)^2; % Differentiate (2.16) from Rasmussen and Williams (2006) 43 | d_sigmaf = 2*sigmaf * exp(-(x1-x2)^2/(2*l^2)); 44 | if f > 0, 45 | covar = covar + exp(-2*sin(pi*f*(x1-x2))^2); 46 | d_f = exp(-2*sin(pi*f*(x1-x2))^2) * (-4*sin(pi*f*(x1-x2))) * cos(pi*f*(x1-x2)) * f*pi*(x1-x2); 47 | else 48 | d_f = 0; 49 | end 50 | if x1==x2, 51 | covar = covar + sigman^2; 52 | d_sigman = 2*sigman; 53 | else 54 | d_sigman = 0; 55 | end 56 | -------------------------------------------------------------------------------- /stdRegion.m: -------------------------------------------------------------------------------- 1 | % Fill a graph with standard deviations; t is nx1 and Range is nx2 2 | % sC is the colour to use, in RGB vector format 3 | % reaxisVar = 0 to leave the axes to be overdrawn, or 1 to redraw them 4 | % Mark Ebden, 2008 5 | 6 | function theRange = stdRegion (t, theRange, sC, reaxisVar) 7 | if exist('sC') == 0, 8 | sC = [1 .8 .8]; 9 | end 10 | if exist('reaxisVar') == 0, 11 | reaxisVar = 0; 12 | end 13 | t = t(:); 14 | fill ([t; flipud(t)], [theRange(:,1); flipud(theRange(:,2))], sC, 'EdgeColor', sC); 15 | 16 | if reaxisVar == 1, 17 | v = axis; 18 | line (v([1 1]), v([3 4]), 'Color', 'k') 19 | line (v([1 2]), v([3 3]), 'Color', 'k') 20 | end --------------------------------------------------------------------------------