├── gpml ├── .octaverc ├── doc │ ├── f1.gif │ ├── f2.gif │ ├── f3.gif │ ├── f4.gif │ ├── f5.gif │ ├── f6.gif │ ├── f7.gif │ ├── f8.gif │ ├── f9.png │ ├── manual.pdf │ ├── checkmark.png │ ├── style.css │ ├── changelog │ ├── gpml_randn.m │ └── demoGrid.m ├── mean │ ├── meanShifted.m │ ├── meanShifted4d.m │ ├── meanShifted7d.m │ ├── meanShiftedBar_11d.m │ ├── meanZero.m │ ├── meanShiftedBar.m │ ├── meanShiftedBar_4d.m │ ├── meanOne.m │ ├── meanConst.m │ ├── meanGP.m │ ├── meanLinear.m │ ├── meanNN.m │ ├── meanPref.m │ ├── meanScale.m │ ├── meanDiscrete.m │ ├── meanPow.m │ ├── meanMask.m │ ├── meanPoly.m │ └── meanSum.m ├── util │ ├── logsumexp2.m │ ├── glm_invlink_expexp.m │ ├── glm_invlink_exp.m │ ├── lbfgsb │ │ ├── matlabexception.cpp │ │ ├── matlabexception.h │ │ ├── matlabscalar.cpp │ │ ├── matlabstring.cpp │ │ ├── timer.f │ │ ├── matlabstring.h │ │ ├── matlabscalar.h │ │ └── matlabmatrix.cpp │ ├── unwrap.m │ ├── elsympol.m │ ├── glm_invlink_logistic.m │ ├── glm_invlink_logit.m │ ├── rewrap.m │ ├── solve_chol.m │ ├── lik_epquad.m │ ├── cov_deriv_sq_dist.m │ └── sq_dist.m ├── prior │ ├── priorDelta.m │ ├── priorClamped.m │ ├── priorDeltaMulti.m │ ├── priorClampedMulti.m │ ├── priorLaplace.m │ ├── priorGauss.m │ ├── priorWeibull.m │ ├── priorLogNormal.m │ ├── priorT.m │ ├── priorInvGauss.m │ ├── priorTransform.m │ ├── priorMix.m │ ├── priorGamma.m │ ├── priorSmoothBox1.m │ ├── priorGaussMulti.m │ ├── priorSmoothBox2.m │ └── priorLaplaceMulti.m ├── startup.m ├── cov │ ├── covLIN.m │ ├── covConst.m │ ├── covPref.m │ ├── covLINone.m │ ├── covEye.m │ ├── covLINiso.m │ ├── covCos.m │ ├── covPoly.m │ ├── covSEisoU.m │ ├── covLINard.m │ ├── covSEiso.m │ ├── covPeriodic.m │ ├── covNoise.m │ ├── covFITC.m │ ├── covRQiso.m │ └── covSum.m ├── lik │ └── likExp.m └── Copyright ├── bayesopt ├── gpml │ ├── .octaverc │ ├── doc │ │ ├── f1.gif │ │ ├── f2.gif │ │ ├── f3.gif │ │ ├── f4.gif │ │ ├── f5.gif │ │ ├── f6.gif │ │ ├── f7.gif │ │ ├── f8.gif │ │ ├── f9.png │ │ ├── manual.pdf │ │ ├── checkmark.png │ │ ├── style.css │ │ ├── changelog │ │ ├── gpml_randn.m │ │ └── demoGrid.m │ ├── mean │ │ ├── meanZero.m │ │ ├── meanOne.m │ │ ├── meanConst.m │ │ ├── meanGP.m │ │ ├── meanLinear.m │ │ ├── meanNN.m │ │ ├── meanPref.m │ │ ├── meanScale.m │ │ ├── meanDiscrete.m │ │ ├── meanPow.m │ │ ├── meanMask.m │ │ ├── meanPoly.m │ │ └── meanSum.m │ ├── util │ │ ├── logsumexp2.m │ │ ├── glm_invlink_expexp.m │ │ ├── glm_invlink_exp.m │ │ ├── lbfgsb │ │ │ ├── matlabexception.cpp │ │ │ ├── matlabexception.h │ │ │ ├── matlabscalar.cpp │ │ │ ├── matlabstring.cpp │ │ │ ├── timer.f │ │ │ ├── matlabstring.h │ │ │ ├── matlabscalar.h │ │ │ └── matlabmatrix.cpp │ │ ├── unwrap.m │ │ ├── elsympol.m │ │ ├── glm_invlink_logistic.m │ │ ├── glm_invlink_logit.m │ │ ├── rewrap.m │ │ ├── solve_chol.m │ │ ├── lik_epquad.m │ │ ├── cov_deriv_sq_dist.m │ │ └── sq_dist.m │ ├── prior │ │ ├── priorClamped.m │ │ ├── priorDelta.m │ │ ├── priorClampedMulti.m │ │ ├── priorDeltaMulti.m │ │ ├── priorLaplace.m │ │ ├── priorGauss.m │ │ ├── priorWeibull.m │ │ ├── priorLogNormal.m │ │ ├── priorT.m │ │ ├── priorInvGauss.m │ │ ├── priorTransform.m │ │ ├── priorMix.m │ │ ├── priorGamma.m │ │ ├── priorSmoothBox1.m │ │ ├── priorGaussMulti.m │ │ ├── priorSmoothBox2.m │ │ └── priorLaplaceMulti.m │ ├── startup.m │ ├── cov │ │ ├── covLIN.m │ │ ├── covConst.m │ │ ├── covPref.m │ │ ├── covLINone.m │ │ ├── covEye.m │ │ ├── covLINiso.m │ │ ├── covCos.m │ │ ├── covPoly.m │ │ ├── covSEisoU.m │ │ ├── covLINard.m │ │ ├── covSEiso.m │ │ ├── covPeriodic.m │ │ ├── covNoise.m │ │ ├── covFITC.m │ │ └── covRQiso.m │ ├── lik │ │ └── likExp.m │ └── Copyright ├── .DS_Store ├── readme.pdf ├── extra-files │ └── .DS_Store ├── .hg_archival.txt ├── README.md ├── demo │ ├── defaultopt.m │ └── samplef.m └── pleasecite.bib ├── .DS_Store ├── admmbo ├── .DS_Store └── EI_Z.m ├── experiments └── .DS_Store ├── initial-data ├── InitialData_toy1.mat ├── initialData_toy2.mat └── initialData_toy3.mat ├── results └── toy1_results_ADMMBO.mat ├── functions ├── sinXplusY.m ├── Sum_Linear.m ├── Circular.m ├── Sum_Linear4.m ├── Sinosidual.m ├── sinXsinY.m ├── C_4d.m ├── acc_eval.m └── time_eval.m ├── README.md ├── Direct ├── gp.m └── circlecon.m └── FMINSEARCHBND ├── demo ├── fminsearchbnd_demo.m └── fminsearchcon_demo.m ├── test └── test_main.m └── doc └── Understanding_fminsearchcon.rtf /gpml/.octaverc: -------------------------------------------------------------------------------- 1 | startup 2 | -------------------------------------------------------------------------------- /bayesopt/gpml/.octaverc: -------------------------------------------------------------------------------- 1 | startup 2 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/.DS_Store -------------------------------------------------------------------------------- /gpml/doc/f1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/f1.gif -------------------------------------------------------------------------------- /gpml/doc/f2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/f2.gif -------------------------------------------------------------------------------- /gpml/doc/f3.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/f3.gif -------------------------------------------------------------------------------- /gpml/doc/f4.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/f4.gif -------------------------------------------------------------------------------- /gpml/doc/f5.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/f5.gif -------------------------------------------------------------------------------- /gpml/doc/f6.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/f6.gif -------------------------------------------------------------------------------- /gpml/doc/f7.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/f7.gif -------------------------------------------------------------------------------- /gpml/doc/f8.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/f8.gif -------------------------------------------------------------------------------- /gpml/doc/f9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/f9.png -------------------------------------------------------------------------------- /admmbo/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/admmbo/.DS_Store -------------------------------------------------------------------------------- /bayesopt/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/.DS_Store -------------------------------------------------------------------------------- /bayesopt/readme.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/readme.pdf -------------------------------------------------------------------------------- /gpml/doc/manual.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/manual.pdf -------------------------------------------------------------------------------- /experiments/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/experiments/.DS_Store -------------------------------------------------------------------------------- /gpml/doc/checkmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/gpml/doc/checkmark.png -------------------------------------------------------------------------------- /bayesopt/gpml/doc/f1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/f1.gif -------------------------------------------------------------------------------- /bayesopt/gpml/doc/f2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/f2.gif -------------------------------------------------------------------------------- /bayesopt/gpml/doc/f3.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/f3.gif -------------------------------------------------------------------------------- /bayesopt/gpml/doc/f4.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/f4.gif -------------------------------------------------------------------------------- /bayesopt/gpml/doc/f5.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/f5.gif -------------------------------------------------------------------------------- /bayesopt/gpml/doc/f6.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/f6.gif -------------------------------------------------------------------------------- /bayesopt/gpml/doc/f7.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/f7.gif -------------------------------------------------------------------------------- /bayesopt/gpml/doc/f8.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/f8.gif -------------------------------------------------------------------------------- /bayesopt/gpml/doc/f9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/f9.png -------------------------------------------------------------------------------- /bayesopt/gpml/doc/manual.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/manual.pdf -------------------------------------------------------------------------------- /gpml/doc/style.css: -------------------------------------------------------------------------------- 1 | body {font-family: sans-serif; font-size: 16px} 2 | table {font-size: inherit;} 3 | 4 | -------------------------------------------------------------------------------- /bayesopt/extra-files/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/extra-files/.DS_Store -------------------------------------------------------------------------------- /bayesopt/gpml/doc/checkmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/bayesopt/gpml/doc/checkmark.png -------------------------------------------------------------------------------- /bayesopt/gpml/doc/style.css: -------------------------------------------------------------------------------- 1 | body {font-family: sans-serif; font-size: 16px} 2 | table {font-size: inherit;} 3 | 4 | -------------------------------------------------------------------------------- /initial-data/InitialData_toy1.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/initial-data/InitialData_toy1.mat -------------------------------------------------------------------------------- /initial-data/initialData_toy2.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/initial-data/initialData_toy2.mat -------------------------------------------------------------------------------- /initial-data/initialData_toy3.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/initial-data/initialData_toy3.mat -------------------------------------------------------------------------------- /results/toy1_results_ADMMBO.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SetarehAr/ADMMBO/HEAD/results/toy1_results_ADMMBO.mat -------------------------------------------------------------------------------- /bayesopt/.hg_archival.txt: -------------------------------------------------------------------------------- 1 | repo: a57e9beab589f282db14827f4c6520b0fbc7c1b5 2 | node: 3a237f1032bb63a51bd35d2855634ba614b51d62 3 | branch: default 4 | latesttag: null 5 | latesttagdistance: 14 6 | changessincelatesttag: 18 7 | -------------------------------------------------------------------------------- /functions/sinXplusY.m: -------------------------------------------------------------------------------- 1 | function value = sinXplusY(x,y,params) 2 | %---------------------------------------------------------- 3 | % Weighted Linear sum 4 | 5 | %---------------------------------------------------------% 6 | value=sin(x)+y; -------------------------------------------------------------------------------- /functions/Sum_Linear.m: -------------------------------------------------------------------------------- 1 | function value = Sum_Linear(x,y,params) 2 | %---------------------------------------------------------- 3 | % Weighted Linear sum 4 | 5 | %-----value----------------------------------------------------% 6 | value=x+y; -------------------------------------------------------------------------------- /bayesopt/README.md: -------------------------------------------------------------------------------- 1 | # README # 2 | 3 | BayesOpt.m is a lightweight MATLAB implementation of Bayesian Optimization for Hyperparamter optimization with or without constraints. 4 | Please see the documentation in the repository for details. 5 | 6 | -------------------------------------------------------------------------------- /functions/Circular.m: -------------------------------------------------------------------------------- 1 | function [ value ] = Circular(x,y,params) 2 | %UNTITLED15 Summary of this function goes here 3 | % Detailed explanation goes here 4 | value= (x-params(1)).^2+(y-params(2)).^2-params(3); 5 | %value=x-.5; 6 | end 7 | 8 | -------------------------------------------------------------------------------- /functions/Sum_Linear4.m: -------------------------------------------------------------------------------- 1 | function value = Sum_Linear4(x,y,z,u,params) 2 | %---------------------------------------------------------- 3 | % Weighted Linear sum 4 | 5 | %---------------------------------------------------------% 6 | value=x+y+z+u; -------------------------------------------------------------------------------- /functions/Sinosidual.m: -------------------------------------------------------------------------------- 1 | function [ value ] = Sinosidual( x,y,params ) 2 | %UNTITLED16 Summary of this function goes here 3 | % Detailed explanation goes here 4 | 5 | value=(params(1)*sin(params(2)*pi*(x.^2-params(3).*y))-x-params(4).*y+params(5)); 6 | end 7 | 8 | -------------------------------------------------------------------------------- /functions/sinXsinY.m: -------------------------------------------------------------------------------- 1 | function [ value ] = sinXsinY(x,y,params) 2 | %UNTITLED15 Summary of this function goes here 3 | % Detailed explanation goes here 4 | %value= (x-params(1)).^2+(y-params(2)).^2-params(3); 5 | value=sin(x).*sin(y)+params; 6 | %value=x-.5; 7 | end 8 | 9 | -------------------------------------------------------------------------------- /gpml/doc/changelog: -------------------------------------------------------------------------------- 1 | GAUSSIAN PROCESS REGRESSION AND CLASSIFICATION Toolbox version 3.5 2 | for GNU Octave 3.2.x and Matlab 7.x 3 | 4 | Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2014-12-08. 5 | 6 | Changes and small incremental bug fixes applied since the last major release. 7 | -------------------------------------------------------------------------------- /bayesopt/gpml/doc/changelog: -------------------------------------------------------------------------------- 1 | GAUSSIAN PROCESS REGRESSION AND CLASSIFICATION Toolbox version 3.5 2 | for GNU Octave 3.2.x and Matlab 7.x 3 | 4 | Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2014-12-08. 5 | 6 | Changes and small incremental bug fixes applied since the last major release. 7 | -------------------------------------------------------------------------------- /gpml/mean/meanShifted.m: -------------------------------------------------------------------------------- 1 | function [ A] = meanShifted(hyp,x,i) 2 | %UNTITLED2 Summary of this function goes here 3 | if nargin < 2, A= ['5'];return;end 4 | y_00=hyp(1:2); 5 | z_00=hyp(3:4); 6 | rho=hyp(5); 7 | num=size(x,1); 8 | z=repmat(z_00,num,1); 9 | y=repmat(y_00,num,1); 10 | 11 | mat=x-z+(y/rho); 12 | A=(rho/2)*(sum(abs(mat).^2,2)); 13 | 14 | end 15 | 16 | -------------------------------------------------------------------------------- /gpml/mean/meanShifted4d.m: -------------------------------------------------------------------------------- 1 | function [ A] = meanShifted4d(hyp,x,i) 2 | %UNTITLED2 Summary of this function goes here 3 | if nargin < 2, A= ['5'];return;end 4 | y_00=hyp(1:4); 5 | z_00=hyp(5:8); 6 | rho=hyp(9); 7 | num=size(x,1); 8 | z=repmat(z_00,num,1); 9 | y=repmat(y_00,num,1); 10 | 11 | mat=x-z+(y/rho); 12 | A=(rho/2)*(sum(abs(mat).^2,2)); 13 | 14 | end 15 | 16 | -------------------------------------------------------------------------------- /gpml/mean/meanShifted7d.m: -------------------------------------------------------------------------------- 1 | function [ A] = meanShifted7d(hyp,x,i) 2 | %UNTITLED2 Summary of this function goes here 3 | if nargin < 2, A= ['15'];return;end 4 | y_00=hyp(1:7); 5 | z_00=hyp(8:14); 6 | rho=hyp(15); 7 | num=size(x,1); 8 | z=repmat(z_00,num,1); 9 | y=repmat(y_00,num,1); 10 | 11 | mat=x-z+(y/rho); 12 | A=(rho/2)*(sum(abs(mat).^2,2)); 13 | 14 | end 15 | 16 | -------------------------------------------------------------------------------- /gpml/mean/meanShiftedBar_11d.m: -------------------------------------------------------------------------------- 1 | function [ A] = meanShiftedBar_11d(hyp,x,i) 2 | %UNTITLED2 Summary of this function goes here 3 | if nargin < 2, A= ['23'];return;end 4 | y_00=hyp(1:11); 5 | z_00=hyp(12:22); 6 | 7 | rho=hyp(23); 8 | num=size(x,1); 9 | z=repmat(z_00,num,1); 10 | y=repmat(y_00,num,1); 11 | 12 | mat=x-z+(y/rho); 13 | A=(rho/2)*(sum(abs(mat).^2,2)); 14 | 15 | end 16 | 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ADMMBO 2 | 3 | This package is an implementation of "ADMMBO, An ADMM Framework for Bayesian Optimization with Unknown Constraints'', to appear in the Journal of Machine Learning Research (JMLR), special issue on Bayesian Optimization. 4 | 5 | This package uses the Gaussian process library gpml and Bayesian optimization library bayesopt. The current implementation is in MATLAB and a Python version will be added in future. 6 | -------------------------------------------------------------------------------- /gpml/mean/meanZero.m: -------------------------------------------------------------------------------- 1 | function A = meanZero(hyp, x, i) 2 | 3 | % Zero mean function. The mean function does not have any parameters. 4 | % 5 | % m(x) = 0 6 | % 7 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-01-10. 8 | % 9 | % See also MEANFUNCTIONS.M. 10 | 11 | if nargin<2, A = '0'; return; end % report number of hyperparameters 12 | A = zeros(size(x,1),1); % derivative and mean 13 | -------------------------------------------------------------------------------- /bayesopt/demo/defaultopt.m: -------------------------------------------------------------------------------- 1 | function opt = bo_default_opt() 2 | opt.grid_size = 20000; % Size of grid to select candidate hyperparameters from. 3 | opt.max_iters = 100; % Maximum number of function evaluations. 4 | opt.meanfunc = {@meanConst}; % Constant mean function. 5 | opt.covfunc = {@covSEard}; % Squared Exponential kernel with ARD. 6 | opt.hyp = -1; % Set hyperparameters using MLE. 7 | opt.save_trace = false; 8 | opt.trace_file = 'trace.mat'; 9 | -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanZero.m: -------------------------------------------------------------------------------- 1 | function A = meanZero(hyp, x, i) 2 | 3 | % Zero mean function. The mean function does not have any parameters. 4 | % 5 | % m(x) = 0 6 | % 7 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-01-10. 8 | % 9 | % See also MEANFUNCTIONS.M. 10 | 11 | if nargin<2, A = '0'; return; end % report number of hyperparameters 12 | A = zeros(size(x,1),1); % derivative and mean 13 | -------------------------------------------------------------------------------- /gpml/mean/meanShiftedBar.m: -------------------------------------------------------------------------------- 1 | function [ A] = meanShiftedBar(hyp,x,i) 2 | %UNTITLED2 Summary of this function goes here 3 | %tmp = num2str(length(hyp.mean)); 4 | if nargin < 2, A= ['5'];return;end 5 | 6 | y_00=hyp(1:size(x,2)); 7 | z_00=hyp((size(x,2)+1):(2*size(x,2))); 8 | rho=hyp(end); 9 | num=size(x,1); 10 | z=repmat(z_00,num,1); 11 | y=repmat(y_00,num,1); 12 | 13 | mat=x-z+(y/rho); 14 | A=(rho/2)*(sum(abs(mat).^2,2)); 15 | 16 | end 17 | 18 | -------------------------------------------------------------------------------- /gpml/mean/meanShiftedBar_4d.m: -------------------------------------------------------------------------------- 1 | function [ A] = meanShiftedBar_4d(hyp,x,i) 2 | %UNTITLED2 Summary of this function goes here 3 | %tmp = num2str(length(hyp.mean)); 4 | if nargin < 2, A= ['9'];return;end 5 | 6 | y_00=hyp(1:size(x,2)); 7 | z_00=hyp((size(x,2)+1):(2*size(x,2))); 8 | rho=hyp(end); 9 | num=size(x,1); 10 | z=repmat(z_00,num,1); 11 | y=repmat(y_00,num,1); 12 | 13 | mat=x-z+(y/rho); 14 | A=(rho/2)*(sum(abs(mat).^2,2)); 15 | 16 | end 17 | 18 | -------------------------------------------------------------------------------- /gpml/util/logsumexp2.m: -------------------------------------------------------------------------------- 1 | % Compute y = log( sum(exp(x),2) ), the softmax in a numerically safe way by 2 | % subtracting the row maximum to avoid cancelation after taking the exp 3 | % the sum is done along the rows. 4 | % 5 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 6 | 7 | function [y,x] = logsumexp2(logx) 8 | N = size(logx,2); max_logx = max(logx,[],2); 9 | % we have all values in the log domain, and want to calculate a sum 10 | x = exp(logx-max_logx*ones(1,N)); 11 | y = log(sum(x,2)) + max_logx; -------------------------------------------------------------------------------- /bayesopt/gpml/util/logsumexp2.m: -------------------------------------------------------------------------------- 1 | % Compute y = log( sum(exp(x),2) ), the softmax in a numerically safe way by 2 | % subtracting the row maximum to avoid cancelation after taking the exp 3 | % the sum is done along the rows. 4 | % 5 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 6 | 7 | function [y,x] = logsumexp2(logx) 8 | N = size(logx,2); max_logx = max(logx,[],2); 9 | % we have all values in the log domain, and want to calculate a sum 10 | x = exp(logx-max_logx*ones(1,N)); 11 | y = log(sum(x,2)) + max_logx; -------------------------------------------------------------------------------- /gpml/util/glm_invlink_expexp.m: -------------------------------------------------------------------------------- 1 | % Compute the log intensity for the inverse link function g(f) = exp(-exp(-f)). 2 | % 3 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 4 | % and likInvGauss. 5 | % 6 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 7 | 8 | function [lg,dlg,d2lg,d3lg] = glm_invlink_expexp(f) 9 | lg = -exp(-f); 10 | if nargout>1 11 | dlg = -lg; 12 | if nargout>2 13 | d2lg = lg; 14 | if nargout>2 15 | d3lg = -lg; 16 | end 17 | end 18 | end -------------------------------------------------------------------------------- /bayesopt/gpml/util/glm_invlink_expexp.m: -------------------------------------------------------------------------------- 1 | % Compute the log intensity for the inverse link function g(f) = exp(-exp(-f)). 2 | % 3 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 4 | % and likInvGauss. 5 | % 6 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 7 | 8 | function [lg,dlg,d2lg,d3lg] = glm_invlink_expexp(f) 9 | lg = -exp(-f); 10 | if nargout>1 11 | dlg = -lg; 12 | if nargout>2 13 | d2lg = lg; 14 | if nargout>2 15 | d3lg = -lg; 16 | end 17 | end 18 | end -------------------------------------------------------------------------------- /gpml/util/glm_invlink_exp.m: -------------------------------------------------------------------------------- 1 | % Compute the log intensity for the inverse link function g(f) = exp(f). 2 | % 3 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 4 | % and likInvGauss. 5 | % 6 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 7 | 8 | function [lg,dlg,d2lg,d3lg] = glm_invlink_exp(f) 9 | lg = f; 10 | if nargout>1 11 | dlg = ones(size(f)); 12 | if nargout>2 13 | d2lg = zeros(size(f)); 14 | if nargout>2 15 | d3lg = zeros(size(f)); 16 | end 17 | end 18 | end -------------------------------------------------------------------------------- /bayesopt/gpml/util/glm_invlink_exp.m: -------------------------------------------------------------------------------- 1 | % Compute the log intensity for the inverse link function g(f) = exp(f). 2 | % 3 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 4 | % and likInvGauss. 5 | % 6 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 7 | 8 | function [lg,dlg,d2lg,d3lg] = glm_invlink_exp(f) 9 | lg = f; 10 | if nargout>1 11 | dlg = ones(size(f)); 12 | if nargout>2 13 | d2lg = zeros(size(f)); 14 | if nargout>2 15 | d3lg = zeros(size(f)); 16 | end 17 | end 18 | end -------------------------------------------------------------------------------- /gpml/mean/meanOne.m: -------------------------------------------------------------------------------- 1 | function A = meanOne(hyp, x, i) 2 | 3 | % One mean function. The mean function does not have any parameters. 4 | % 5 | % m(x) = 1 6 | % 7 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-08-04. 8 | % 9 | % See also MEANFUNCTIONS.M. 10 | 11 | if nargin<2, A = '0'; return; end % report number of hyperparameters 12 | if nargin==2 13 | A = ones(size(x,1),1); % evaluate mean 14 | else 15 | A = zeros(size(x,1),1); % derivative 16 | end -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanOne.m: -------------------------------------------------------------------------------- 1 | function A = meanOne(hyp, x, i) 2 | 3 | % One mean function. The mean function does not have any parameters. 4 | % 5 | % m(x) = 1 6 | % 7 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-08-04. 8 | % 9 | % See also MEANFUNCTIONS.M. 10 | 11 | if nargin<2, A = '0'; return; end % report number of hyperparameters 12 | if nargin==2 13 | A = ones(size(x,1),1); % evaluate mean 14 | else 15 | A = zeros(size(x,1),1); % derivative 16 | end -------------------------------------------------------------------------------- /bayesopt/pleasecite.bib: -------------------------------------------------------------------------------- 1 | @inproceedings{icml2014c2_gardner14, 2 | Publisher = {JMLR Workshop and Conference Proceedings}, 3 | Title = {Bayesian Optimization with Inequality Constraints}, 4 | Url = {http://jmlr.org/proceedings/papers/v32/gardner14.pdf}, 5 | Author = {Jacob Gardner and Matt Kusner and Kilian Q. Weinberger and John Cunningham and Zhixiang Xu}, 6 | Editor = {Tony Jebara and Eric P. Xing}, 7 | Year = {2014}, 8 | Booktitle = {Proceedings of the 31st International Conference on Machine Learning (ICML-14)}, 9 | Pages = {937-945} 10 | } -------------------------------------------------------------------------------- /gpml/prior/priorDelta.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorDelta(x) 2 | 3 | % Dummy hyperparameter prior distribution to fix the value of a hyperparameter. 4 | % The function is not intended to be evaluated but exists merely to make 5 | % the user aware of the possibility to use it. The function is equivalent 6 | % to priorClamped. 7 | % 8 | % For more help on design of priors, try "help priorDistributions". 9 | % 10 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-12-06. 11 | % 12 | % See also PRIORDISTRIBUTIONS.M. 13 | 14 | error('The function is not intended to be called directly.') -------------------------------------------------------------------------------- /gpml/prior/priorClamped.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorClamped(x) 2 | 3 | % Dummy hyperparameter prior distribution to fix the value of a hyperparameter. 4 | % The function is not intended to be evaluated but exists merely to make 5 | % the user aware of the possibility to use it. The function is equivalent 6 | % to priorDelta. 7 | % 8 | % For more help on design of priors, try "help priorDistributions". 9 | % 10 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-12-06. 11 | % 12 | % See also PRIORDISTRIBUTIONS.M. 13 | 14 | error('The function is not intended to be called directly.') -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorClamped.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorClamped(x) 2 | 3 | % Dummy hyperparameter prior distribution to fix the value of a hyperparameter. 4 | % The function is not intended to be evaluated but exists merely to make 5 | % the user aware of the possibility to use it. The function is equivalent 6 | % to priorDelta. 7 | % 8 | % For more help on design of priors, try "help priorDistributions". 9 | % 10 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-12-06. 11 | % 12 | % See also PRIORDISTRIBUTIONS.M. 13 | 14 | error('The function is not intended to be called directly.') -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorDelta.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorDelta(x) 2 | 3 | % Dummy hyperparameter prior distribution to fix the value of a hyperparameter. 4 | % The function is not intended to be evaluated but exists merely to make 5 | % the user aware of the possibility to use it. The function is equivalent 6 | % to priorClamped. 7 | % 8 | % For more help on design of priors, try "help priorDistributions". 9 | % 10 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-12-06. 11 | % 12 | % See also PRIORDISTRIBUTIONS.M. 13 | 14 | error('The function is not intended to be called directly.') -------------------------------------------------------------------------------- /gpml/prior/priorDeltaMulti.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorDeltaMulti(x) 2 | 3 | % Dummy hyperparameter prior distribution to fix the value of a hyperparameter. 4 | % The function is not intended to be evaluated but exists merely to make 5 | % the user aware of the possibility to use it. The function is equivalent 6 | % to priorClampedMulti. 7 | % 8 | % For more help on design of priors, try "help priorDistributions". 9 | % 10 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-12-06. 11 | % 12 | % See also PRIORDISTRIBUTIONS.M. 13 | 14 | error('The function is not intended to be called directly.') -------------------------------------------------------------------------------- /gpml/prior/priorClampedMulti.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorClampedMulti(x) 2 | 3 | % Dummy hyperparameter prior distribution to fix the value of a hyperparameter. 4 | % The function is not intended to be evaluated but exists merely to make 5 | % the user aware of the possibility to use it. The function is equivalent 6 | % to priorDeltaMulti. 7 | % 8 | % For more help on design of priors, try "help priorDistributions". 9 | % 10 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-12-06. 11 | % 12 | % See also PRIORDISTRIBUTIONS.M. 13 | 14 | error('The function is not intended to be called directly.') -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorClampedMulti.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorClampedMulti(x) 2 | 3 | % Dummy hyperparameter prior distribution to fix the value of a hyperparameter. 4 | % The function is not intended to be evaluated but exists merely to make 5 | % the user aware of the possibility to use it. The function is equivalent 6 | % to priorDeltaMulti. 7 | % 8 | % For more help on design of priors, try "help priorDistributions". 9 | % 10 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-12-06. 11 | % 12 | % See also PRIORDISTRIBUTIONS.M. 13 | 14 | error('The function is not intended to be called directly.') -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorDeltaMulti.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorDeltaMulti(x) 2 | 3 | % Dummy hyperparameter prior distribution to fix the value of a hyperparameter. 4 | % The function is not intended to be evaluated but exists merely to make 5 | % the user aware of the possibility to use it. The function is equivalent 6 | % to priorClampedMulti. 7 | % 8 | % For more help on design of priors, try "help priorDistributions". 9 | % 10 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-12-06. 11 | % 12 | % See also PRIORDISTRIBUTIONS.M. 13 | 14 | error('The function is not intended to be called directly.') -------------------------------------------------------------------------------- /gpml/util/lbfgsb/matlabexception.cpp: -------------------------------------------------------------------------------- 1 | #include "matlabexception.h" 2 | 3 | // Function definitions for class MatlabException. 4 | // ----------------------------------------------------------------- 5 | MatlabException::MatlabException (const char* message) throw() 6 | : exception() { 7 | this->message = message; 8 | } 9 | 10 | MatlabException::MatlabException (const MatlabException& source) throw() 11 | : exception() { 12 | message = source.message; 13 | } 14 | 15 | MatlabException& MatlabException::operator= (const MatlabException& source) 16 | { 17 | message = source.message; 18 | return *this; 19 | } 20 | -------------------------------------------------------------------------------- /bayesopt/gpml/util/lbfgsb/matlabexception.cpp: -------------------------------------------------------------------------------- 1 | #include "matlabexception.h" 2 | 3 | // Function definitions for class MatlabException. 4 | // ----------------------------------------------------------------- 5 | MatlabException::MatlabException (const char* message) throw() 6 | : exception() { 7 | this->message = message; 8 | } 9 | 10 | MatlabException::MatlabException (const MatlabException& source) throw() 11 | : exception() { 12 | message = source.message; 13 | } 14 | 15 | MatlabException& MatlabException::operator= (const MatlabException& source) 16 | { 17 | message = source.message; 18 | return *this; 19 | } 20 | -------------------------------------------------------------------------------- /gpml/util/unwrap.m: -------------------------------------------------------------------------------- 1 | % Extract the numerical values from "s" into the column vector "v". The 2 | % variable "s" can be of any type, including struct and cell array. 3 | % Non-numerical elements are ignored. See also the reverse rewrap.m. 4 | 5 | function v = unwrap(s) 6 | 7 | v = []; 8 | if isnumeric(s) 9 | v = s(:); % numeric values are recast to column vector 10 | elseif isstruct(s) 11 | v = unwrap(struct2cell(orderfields(s))); % alphabetize, conv to cell, recurse 12 | elseif iscell(s) 13 | for i = 1:numel(s) % cell array elements are handled sequentially 14 | v = [v; unwrap(s{i})]; 15 | end 16 | end % other types are ignored 17 | -------------------------------------------------------------------------------- /bayesopt/gpml/util/unwrap.m: -------------------------------------------------------------------------------- 1 | % Extract the numerical values from "s" into the column vector "v". The 2 | % variable "s" can be of any type, including struct and cell array. 3 | % Non-numerical elements are ignored. See also the reverse rewrap.m. 4 | 5 | function v = unwrap(s) 6 | 7 | v = []; 8 | if isnumeric(s) 9 | v = s(:); % numeric values are recast to column vector 10 | elseif isstruct(s) 11 | v = unwrap(struct2cell(orderfields(s))); % alphabetize, conv to cell, recurse 12 | elseif iscell(s) 13 | for i = 1:numel(s) % cell array elements are handled sequentially 14 | v = [v; unwrap(s{i})]; 15 | end 16 | end % other types are ignored 17 | -------------------------------------------------------------------------------- /gpml/mean/meanConst.m: -------------------------------------------------------------------------------- 1 | function A = meanConst(hyp, x, i) 2 | 3 | % Constant mean function. The mean function is parameterized as: 4 | % 5 | % m(x) = c 6 | % 7 | % The hyperparameter is: 8 | % 9 | % hyp = [ c ] 10 | % 11 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-08-04. 12 | % 13 | % See also MEANFUNCTIONS.M. 14 | 15 | if nargin<2, A = '1'; return; end % report number of hyperparameters 16 | if numel(hyp)~=1, error('Exactly one hyperparameter needed.'), end 17 | c = hyp; 18 | if nargin==2 19 | A = c*ones(size(x,1),1); % evaluate mean 20 | else 21 | if i==1 22 | A = ones(size(x,1),1); % derivative 23 | else 24 | A = zeros(size(x,1),1); 25 | end 26 | end -------------------------------------------------------------------------------- /gpml/util/elsympol.m: -------------------------------------------------------------------------------- 1 | % Evaluate the order R elementary symmetric polynomial Newton's identity aka 2 | % the Newton–Girard formulae: http://en.wikipedia.org/wiki/Newton's_identities 3 | % 4 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-01-10. 5 | 6 | function E = elsympol(Z,R) 7 | % evaluate 'power sums' of the individual terms in Z 8 | sz = size(Z); P = zeros([sz(1:2),R]); for r=1:R, P(:,:,r) = sum(Z.^r,3); end 9 | E = zeros([sz(1:2),R+1]); % E(:,:,r+1) yields polynomial r 10 | E(:,:,1) = ones(sz(1:2)); if R==0, return, end % init recursion 11 | E(:,:,2) = P(:,:,1); if R==1, return, end % init recursion 12 | for r=2:R 13 | for i=1:r 14 | E(:,:,r+1) = E(:,:,r+1) + P(:,:,i).*E(:,:,r+1-i)*(-1)^(i-1)/r; 15 | end 16 | end 17 | 18 | -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanConst.m: -------------------------------------------------------------------------------- 1 | function A = meanConst(hyp, x, i) 2 | 3 | % Constant mean function. The mean function is parameterized as: 4 | % 5 | % m(x) = c 6 | % 7 | % The hyperparameter is: 8 | % 9 | % hyp = [ c ] 10 | % 11 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-08-04. 12 | % 13 | % See also MEANFUNCTIONS.M. 14 | 15 | if nargin<2, A = '1'; return; end % report number of hyperparameters 16 | if numel(hyp)~=1, error('Exactly one hyperparameter needed.'), end 17 | c = hyp; 18 | if nargin==2 19 | A = c*ones(size(x,1),1); % evaluate mean 20 | else 21 | if i==1 22 | A = ones(size(x,1),1); % derivative 23 | else 24 | A = zeros(size(x,1),1); 25 | end 26 | end -------------------------------------------------------------------------------- /bayesopt/gpml/util/elsympol.m: -------------------------------------------------------------------------------- 1 | % Evaluate the order R elementary symmetric polynomial Newton's identity aka 2 | % the Newton–Girard formulae: http://en.wikipedia.org/wiki/Newton's_identities 3 | % 4 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-01-10. 5 | 6 | function E = elsympol(Z,R) 7 | % evaluate 'power sums' of the individual terms in Z 8 | sz = size(Z); P = zeros([sz(1:2),R]); for r=1:R, P(:,:,r) = sum(Z.^r,3); end 9 | E = zeros([sz(1:2),R+1]); % E(:,:,r+1) yields polynomial r 10 | E(:,:,1) = ones(sz(1:2)); if R==0, return, end % init recursion 11 | E(:,:,2) = P(:,:,1); if R==1, return, end % init recursion 12 | for r=2:R 13 | for i=1:r 14 | E(:,:,r+1) = E(:,:,r+1) + P(:,:,i).*E(:,:,r+1-i)*(-1)^(i-1)/r; 15 | end 16 | end 17 | 18 | -------------------------------------------------------------------------------- /gpml/util/glm_invlink_logistic.m: -------------------------------------------------------------------------------- 1 | % Compute the log intensity for the inverse link function g(f) = log(1+exp(f))). 2 | % 3 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 4 | % and likInvGauss. 5 | % 6 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 7 | 8 | function [lg,dlg,d2lg,d3lg] = glm_invlink_logistic(f) 9 | l1pef = max(0,f) + log(1+exp(-abs(f))); % safely compute log(1+exp(f)) 10 | lg = log(l1pef); id = f<-15; lg(id) = f(id); % fix log(log(1+exp(f))) limits 11 | if nargout>1 12 | sm = 1./(1+exp(-f)); 13 | dlg = sm./l1pef; dlg(f<-15) = 1; 14 | if nargout>2 15 | sp = 1./(1+exp(f)); 16 | d2lg = dlg.*(sp-dlg); 17 | if nargout>2 18 | d3lg = d2lg.*(sp-2*dlg) - dlg.*sp.*sm; 19 | end 20 | end 21 | end -------------------------------------------------------------------------------- /bayesopt/gpml/util/glm_invlink_logistic.m: -------------------------------------------------------------------------------- 1 | % Compute the log intensity for the inverse link function g(f) = log(1+exp(f))). 2 | % 3 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 4 | % and likInvGauss. 5 | % 6 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 7 | 8 | function [lg,dlg,d2lg,d3lg] = glm_invlink_logistic(f) 9 | l1pef = max(0,f) + log(1+exp(-abs(f))); % safely compute log(1+exp(f)) 10 | lg = log(l1pef); id = f<-15; lg(id) = f(id); % fix log(log(1+exp(f))) limits 11 | if nargout>1 12 | sm = 1./(1+exp(-f)); 13 | dlg = sm./l1pef; dlg(f<-15) = 1; 14 | if nargout>2 15 | sp = 1./(1+exp(f)); 16 | d2lg = dlg.*(sp-dlg); 17 | if nargout>2 18 | d3lg = d2lg.*(sp-2*dlg) - dlg.*sp.*sm; 19 | end 20 | end 21 | end -------------------------------------------------------------------------------- /gpml/mean/meanGP.m: -------------------------------------------------------------------------------- 1 | function A = meanGP(hyp,inf,mean,cov,lik,x,y, hypz,z,i) 2 | 3 | % Mean function being the predictive mean of a GP model: 4 | % 5 | % m(z) = posterior mean of another GP at location z as given by 6 | % m(z) = gp(hyp,inf,mean,cov,lik,x,y, z) 7 | % 8 | % The hyperparameters are: 9 | % 10 | % hypz = [ ] 11 | % 12 | % Copyright (c) by Hannes Nickisch, 2014-11-01. 13 | % 14 | % See also MEANFUNCTIONS.M and MEANGPEXACT.M. 15 | 16 | if nargin<7, error('GP must be specified.'), end % check for dimension 17 | if nargin<9, A = '0'; return, end % report number of hyperparameters 18 | 19 | if nargin==9 20 | A = gp(hyp,inf,mean,cov,lik,x,y, z); % evaluate posterior mean 21 | else 22 | A = zeros(size(x,1),1); % derivative 23 | end 24 | -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanGP.m: -------------------------------------------------------------------------------- 1 | function A = meanGP(hyp,inf,mean,cov,lik,x,y, hypz,z,i) 2 | 3 | % Mean function being the predictive mean of a GP model: 4 | % 5 | % m(z) = posterior mean of another GP at location z as given by 6 | % m(z) = gp(hyp,inf,mean,cov,lik,x,y, z) 7 | % 8 | % The hyperparameters are: 9 | % 10 | % hypz = [ ] 11 | % 12 | % Copyright (c) by Hannes Nickisch, 2014-11-01. 13 | % 14 | % See also MEANFUNCTIONS.M and MEANGPEXACT.M. 15 | 16 | if nargin<7, error('GP must be specified.'), end % check for dimension 17 | if nargin<9, A = '0'; return, end % report number of hyperparameters 18 | 19 | if nargin==9 20 | A = gp(hyp,inf,mean,cov,lik,x,y, z); % evaluate posterior mean 21 | else 22 | A = zeros(size(x,1),1); % derivative 23 | end 24 | -------------------------------------------------------------------------------- /Direct/gp.m: -------------------------------------------------------------------------------- 1 | function value = gp(x) 2 | %---------------------------------------------------------- 3 | % Goldstein-Price Test Function for Nonlinear Optimization 4 | % 5 | % Taken from "Towards Global Optimisation 2",edited by L.C.W. Dixon and G.P. 6 | % Szego, North-Holland Publishing Company, 1978. ISBN 0 444 85171 2 7 | % 8 | % -2 <= x1 <= 2 9 | % -2 <= x2 <= 2 10 | % fmin = 3 11 | % xmin = 0 12 | % -1 13 | %---------------------------------------------------------- 14 | 15 | %---------------------------------------------------------% 16 | % http://www4.ncsu.edu/~definkel/research/index.html 17 | %---------------------------------------------------------% 18 | x1 = x(1); x2 = x(2); 19 | value =(1+(x1+x2+1).^2.*(19-14.*x1+3.*x1.^2-14.*x2+6.*x1.*x2+3.*x2.^2))... 20 | .*(30+(2.*x1-3.*x2).^2.*(18-32.*x1+12.*x1.^2+48.*x2-36.*x1.*x2+27.*x2 -------------------------------------------------------------------------------- /admmbo/EI_Z.m: -------------------------------------------------------------------------------- 1 | function [ EI] = EI_Z(mean_grid,variance_grid,grid_data,h_plus,opt,const_num) 2 | %% The goal of this function is to evaluate EI(.) for the grid data 3 | 4 | % preprocessing 5 | Qz=@(Z) h_plus-(opt.ADMM.rho/(2*opt.ADMM.M))*norm(opt.f.x-Z+(opt.f.y{const_num}/opt.ADMM.rho))^2; 6 | Qz_val=zeros(opt.c{const_num}.grid_size,1); 7 | 8 | for j=1:opt.c{const_num}.grid_size 9 | Qz_val(j,1)=Qz(grid_data(j,:)); 10 | end 11 | 12 | % finding the EI() regions 13 | ind_2=find(Qz_val>=0 & Qz_val<1); 14 | ind_3=find(Qz_val>=1); 15 | 16 | % Evaluating EI(z) based on regions 17 | cdf_at_0=normcdf(0,mean_grid,variance_grid); 18 | EI=zeros(length(Qz_val),1); 19 | EI(ind_2)=Qz_val(ind_2).*cdf_at_0(ind_2); 20 | EI(ind_3)=Qz_val(ind_3)-(1-cdf_at_0(ind_3)); 21 | 22 | end 23 | 24 | -------------------------------------------------------------------------------- /gpml/mean/meanLinear.m: -------------------------------------------------------------------------------- 1 | function A = meanLinear(hyp, x, i) 2 | 3 | % Linear mean function. The mean function is parameterized as: 4 | % 5 | % m(x) = sum_i a_i * x_i; 6 | % 7 | % The hyperparameter is: 8 | % 9 | % hyp = [ a_1 10 | % a_2 11 | % .. 12 | % a_D ] 13 | % 14 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-01-10. 15 | % 16 | % See also MEANFUNCTIONS.M. 17 | 18 | if nargin<2, A = 'D'; return; end % report number of hyperparameters 19 | [n,D] = size(x); 20 | if any(size(hyp)~=[D,1]), error('Exactly D hyperparameters needed.'), end 21 | a = hyp(:); 22 | if nargin==2 23 | A = x*a; % evaluate mean 24 | else 25 | if i<=D 26 | A = x(:,i); % derivative 27 | else 28 | A = zeros(n,1); 29 | end 30 | end -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanLinear.m: -------------------------------------------------------------------------------- 1 | function A = meanLinear(hyp, x, i) 2 | 3 | % Linear mean function. The mean function is parameterized as: 4 | % 5 | % m(x) = sum_i a_i * x_i; 6 | % 7 | % The hyperparameter is: 8 | % 9 | % hyp = [ a_1 10 | % a_2 11 | % .. 12 | % a_D ] 13 | % 14 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-01-10. 15 | % 16 | % See also MEANFUNCTIONS.M. 17 | 18 | if nargin<2, A = 'D'; return; end % report number of hyperparameters 19 | [n,D] = size(x); 20 | if any(size(hyp)~=[D,1]), error('Exactly D hyperparameters needed.'), end 21 | a = hyp(:); 22 | if nargin==2 23 | A = x*a; % evaluate mean 24 | else 25 | if i<=D 26 | A = x(:,i); % derivative 27 | else 28 | A = zeros(n,1); 29 | end 30 | end -------------------------------------------------------------------------------- /gpml/mean/meanNN.m: -------------------------------------------------------------------------------- 1 | function A = meanNN(c,m, hyp, x, i) 2 | 3 | % Nearest neighbor mean function. The mean function is parameterized as: 4 | % 5 | % m(z) = m_j, j = arg min_i d(ci,x) where d is the Euclidean distance and ci is 6 | % the ith cluster center. 7 | % 8 | % The hyperparameters are: 9 | % 10 | % hyp = [ ] 11 | % 12 | % Copyright (c) by Hannes Nickisch, 2014-12-04. 13 | % 14 | % See also MEANFUNCTIONS.M. 15 | 16 | if nargin<4, A = '0'; return; end % report number of hyperparameters 17 | if numel(hyp)~=0, error('No hyperparameters needed for this model.'), end 18 | if nargin==4 % evaluate mean 19 | [junk,j] = min(sq_dist(c',x')); 20 | A = m(j); A = A(:); 21 | else 22 | A = zeros(size(x,1),1); % derivative 23 | end -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanNN.m: -------------------------------------------------------------------------------- 1 | function A = meanNN(c,m, hyp, x, i) 2 | 3 | % Nearest neighbor mean function. The mean function is parameterized as: 4 | % 5 | % m(z) = m_j, j = arg min_i d(ci,x) where d is the Euclidean distance and ci is 6 | % the ith cluster center. 7 | % 8 | % The hyperparameters are: 9 | % 10 | % hyp = [ ] 11 | % 12 | % Copyright (c) by Hannes Nickisch, 2014-12-04. 13 | % 14 | % See also MEANFUNCTIONS.M. 15 | 16 | if nargin<4, A = '0'; return; end % report number of hyperparameters 17 | if numel(hyp)~=0, error('No hyperparameters needed for this model.'), end 18 | if nargin==4 % evaluate mean 19 | [junk,j] = min(sq_dist(c',x')); 20 | A = m(j); A = A(:); 21 | else 22 | A = zeros(size(x,1),1); % derivative 23 | end -------------------------------------------------------------------------------- /gpml/mean/meanPref.m: -------------------------------------------------------------------------------- 1 | function A = meanPref(mean, hyp, x, varargin) 2 | 3 | % meanPref - mean function for preference learning. 4 | % 5 | % m(x) = m_0(x1)-m_0(x2), where x1=x(:,1:D), x2=x(:,D+1:2*D), D = size(x,2)/2. 6 | % 7 | % The hyperparameters are: 8 | % 9 | % hyp = [ hyp_m0 ] 10 | % 11 | % This function doesn't actually compute very much on its own, it merely does 12 | % some bookkeeping, and calls another mean function to do the actual work. 13 | % 14 | % See Collaborative Gaussian Processes for Preference Learning, NIPS 2014. 15 | % 16 | % Copyright (c) by Hannes Nickisch and Roman Garnett, 2014-11-01. 17 | % 18 | % See also MEANFUNCTIONS.M and COVPREF.M. 19 | 20 | if nargin<3, A = strrep(feval(mean{:}),'D','D/2'); return; end % no of params 21 | 22 | A = feval(mean{:}, hyp, x(:,1 :end/2), varargin{:}) ... 23 | - feval(mean{:}, hyp, x(:,1+end/2:end ), varargin{:}); 24 | -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanPref.m: -------------------------------------------------------------------------------- 1 | function A = meanPref(mean, hyp, x, varargin) 2 | 3 | % meanPref - mean function for preference learning. 4 | % 5 | % m(x) = m_0(x1)-m_0(x2), where x1=x(:,1:D), x2=x(:,D+1:2*D), D = size(x,2)/2. 6 | % 7 | % The hyperparameters are: 8 | % 9 | % hyp = [ hyp_m0 ] 10 | % 11 | % This function doesn't actually compute very much on its own, it merely does 12 | % some bookkeeping, and calls another mean function to do the actual work. 13 | % 14 | % See Collaborative Gaussian Processes for Preference Learning, NIPS 2014. 15 | % 16 | % Copyright (c) by Hannes Nickisch and Roman Garnett, 2014-11-01. 17 | % 18 | % See also MEANFUNCTIONS.M and COVPREF.M. 19 | 20 | if nargin<3, A = strrep(feval(mean{:}),'D','D/2'); return; end % no of params 21 | 22 | A = feval(mean{:}, hyp, x(:,1 :end/2), varargin{:}) ... 23 | - feval(mean{:}, hyp, x(:,1+end/2:end ), varargin{:}); 24 | -------------------------------------------------------------------------------- /gpml/startup.m: -------------------------------------------------------------------------------- 1 | % startup script to make Octave/Matlab aware of the GPML package 2 | % 3 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch 2014-11-30. 4 | 5 | disp(['executing gpml startup script...']); 6 | 7 | OCT = exist('OCTAVE_VERSION') ~= 0; % check if we run Matlab or Octave 8 | 9 | me = mfilename; % what is my filename 10 | mydir = which(me); mydir = mydir(1:end-2-numel(me)); % where am I located 11 | if OCT && numel(mydir)==2 12 | if strcmp(mydir,'./'), mydir = [pwd,mydir(2:end)]; end 13 | end % OCTAVE 3.0.x relative, MATLAB and newer have absolute path 14 | 15 | addpath(mydir(1:end-1)) 16 | addpath([mydir,'cov']) 17 | addpath([mydir,'doc']) 18 | addpath([mydir,'inf']) 19 | addpath([mydir,'lik']) 20 | addpath([mydir,'mean']) 21 | addpath([mydir,'prior']) 22 | addpath([mydir,'util']) 23 | 24 | clear me mydir -------------------------------------------------------------------------------- /bayesopt/gpml/startup.m: -------------------------------------------------------------------------------- 1 | % startup script to make Octave/Matlab aware of the GPML package 2 | % 3 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch 2014-11-30. 4 | 5 | disp(['executing gpml startup script...']); 6 | 7 | OCT = exist('OCTAVE_VERSION') ~= 0; % check if we run Matlab or Octave 8 | 9 | me = mfilename; % what is my filename 10 | mydir = which(me); mydir = mydir(1:end-2-numel(me)); % where am I located 11 | if OCT && numel(mydir)==2 12 | if strcmp(mydir,'./'), mydir = [pwd,mydir(2:end)]; end 13 | end % OCTAVE 3.0.x relative, MATLAB and newer have absolute path 14 | 15 | addpath(mydir(1:end-1)) 16 | addpath([mydir,'cov']) 17 | addpath([mydir,'doc']) 18 | addpath([mydir,'inf']) 19 | addpath([mydir,'lik']) 20 | addpath([mydir,'mean']) 21 | addpath([mydir,'prior']) 22 | addpath([mydir,'util']) 23 | 24 | clear me mydir -------------------------------------------------------------------------------- /gpml/util/glm_invlink_logit.m: -------------------------------------------------------------------------------- 1 | % Compute the log intensity for the inverse link function g(f) = 1/(1+exp(-f)). 2 | % 3 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 4 | % and likInvGauss. 5 | % 6 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 7 | 8 | function varargout = glm_invlink_logit(f) 9 | varargout = cell(nargout, 1); % allocate the right number of output arguments 10 | [varargout{:}] = glm_invlink_logistic(f); 11 | if nargout>0 12 | elg = exp(varargout{1}); 13 | varargout{1} = f - elg; 14 | if nargout>1 15 | dlg = varargout{2}; 16 | varargout{2} = 1 - elg.*dlg; 17 | if nargout>2 18 | d2lg = varargout{3}; 19 | varargout{3} = -elg.*(dlg.^2+d2lg); 20 | if nargout>3 21 | d3lg = varargout{4}; 22 | varargout{4} = -elg.*(dlg.^3+3*d2lg.*dlg+d3lg); 23 | end 24 | end 25 | end 26 | end -------------------------------------------------------------------------------- /bayesopt/gpml/util/glm_invlink_logit.m: -------------------------------------------------------------------------------- 1 | % Compute the log intensity for the inverse link function g(f) = 1/(1+exp(-f)). 2 | % 3 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 4 | % and likInvGauss. 5 | % 6 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 7 | 8 | function varargout = glm_invlink_logit(f) 9 | varargout = cell(nargout, 1); % allocate the right number of output arguments 10 | [varargout{:}] = glm_invlink_logistic(f); 11 | if nargout>0 12 | elg = exp(varargout{1}); 13 | varargout{1} = f - elg; 14 | if nargout>1 15 | dlg = varargout{2}; 16 | varargout{2} = 1 - elg.*dlg; 17 | if nargout>2 18 | d2lg = varargout{3}; 19 | varargout{3} = -elg.*(dlg.^2+d2lg); 20 | if nargout>3 21 | d3lg = varargout{4}; 22 | varargout{4} = -elg.*(dlg.^3+3*d2lg.*dlg+d3lg); 23 | end 24 | end 25 | end 26 | end -------------------------------------------------------------------------------- /bayesopt/demo/samplef.m: -------------------------------------------------------------------------------- 1 | % function [Lval,Cval] = samplef(X,b) 2 | % Sample function for bayesopt.m 3 | % To use bayesopt.m you need an opt struct (see demo.m or the readme) and a function handle to a function like this. 4 | % The function should return two arguments, the objective function value and the constraint function value. 5 | % 6 | % The function handle should ultimately have only one argument, a vector of parameters X. 7 | % The function itself can have additional parameters that are passed in as constants. For example: 8 | % b = 1; 9 | % F = @(X)samplef(X,b); 10 | % 11 | % This lets you, for example, pass in datasets when tuning ML algorithm parameters. 12 | function [Lval,Cval] = samplef2(x,y,b) 13 | L = @(x,y) cos(2.*x).*cos(y) + sin(b.*x); 14 | C = @(x,y) -(-cos(x).*cos(y)+sin(x).*sin(y)); 15 | Lval = L(x,y) + 1e-4*randn(1,size(x,2)); 16 | Cval = C(x,y) + 1e-4*randn(1,size(y,2)); 17 | end 18 | -------------------------------------------------------------------------------- /Direct/circlecon.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------% 2 | % DIRECT Sample Call circlecon.m % 3 | % Purpose: Return the value of the function % 4 | % c(x) = x^2 + y^2 - 1 % 5 | % % 6 | % You will need the accompanying programs % 7 | % Direct.m and gp.m to run this code % 8 | % successfully. % 9 | % % 10 | % These codes can be found at % 11 | % http://www4.ncsu.edu/~definkel/research/index.html % 12 | %------------------------------------------------------% 13 | function retval = circlecon(x) 14 | 15 | %retval = (x(1)-2)^2 + (x(2)-2)^2 - 4; 16 | %retval = (-0.5*sin(2*pi*(x(1)^2-2*x(2)))-x(1)-2*x(2)+2); 17 | retval=sin(x(1)).*sin(x(2))+.95; -------------------------------------------------------------------------------- /gpml/prior/priorLaplace.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorLaplace(mu,s2,x) 2 | 3 | % Univariate Laplacian hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-abs(x-mu)/b)/(2*b), where b = sqrt(s2/2), 8 | % 9 | % mu(1x1) is the mean parameter, s2(1x1) is the variance parameter and 10 | % x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-09. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<3 % return a sample 19 | u = rand-1/2; lp = mu - sqrt(s2/2)*sign(u)*log(1-2*abs(u)); return 20 | end 21 | 22 | b = sqrt(s2/2); 23 | lp = -abs(x-mu)/b - log(2*b); 24 | dlp = -sign(x-mu)/b; -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorLaplace.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorLaplace(mu,s2,x) 2 | 3 | % Univariate Laplacian hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-abs(x-mu)/b)/(2*b), where b = sqrt(s2/2), 8 | % 9 | % mu(1x1) is the mean parameter, s2(1x1) is the variance parameter and 10 | % x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-09. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<3 % return a sample 19 | u = rand-1/2; lp = mu - sqrt(s2/2)*sign(u)*log(1-2*abs(u)); return 20 | end 21 | 22 | b = sqrt(s2/2); 23 | lp = -abs(x-mu)/b - log(2*b); 24 | dlp = -sign(x-mu)/b; -------------------------------------------------------------------------------- /gpml/prior/priorGauss.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorGauss(mu,s2,x) 2 | 3 | % Univariate Gaussian hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-(x-mu)^2/(2*s2)) / sqrt(2*pi*s2), where 8 | % 9 | % mu(1x1) is the mean parameter, s2(1x1) is the variance parameter and 10 | % x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-09. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<2, error('mu and s2 parameters need to be provided'), end 19 | if ~(isscalar(mu)&&isscalar(s2)) 20 | error('mu and s2 parameters need to be scalars') 21 | end 22 | if nargin<3, lp = sqrt(s2)*randn+mu; return, end % return a sample 23 | lp = -(x-mu).^2/(2*s2) - log(2*pi*s2)/2; 24 | dlp = -(x-mu)/s2; -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorGauss.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorGauss(mu,s2,x) 2 | 3 | % Univariate Gaussian hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-(x-mu)^2/(2*s2)) / sqrt(2*pi*s2), where 8 | % 9 | % mu(1x1) is the mean parameter, s2(1x1) is the variance parameter and 10 | % x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-09. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<2, error('mu and s2 parameters need to be provided'), end 19 | if ~(isscalar(mu)&&isscalar(s2)) 20 | error('mu and s2 parameters need to be scalars') 21 | end 22 | if nargin<3, lp = sqrt(s2)*randn+mu; return, end % return a sample 23 | lp = -(x-mu).^2/(2*s2) - log(2*pi*s2)/2; 24 | dlp = -(x-mu)/s2; -------------------------------------------------------------------------------- /gpml/util/lbfgsb/matlabexception.h: -------------------------------------------------------------------------------- 1 | #ifndef INCLUDE_MATLABEXCEPTION 2 | #define INCLUDE_MATLABEXCEPTION 3 | 4 | #include 5 | 6 | // Class MatlabException 7 | // ----------------------------------------------------------------- 8 | // This class just makes it easier for me to throw exceptions. Its 9 | // functionality really has nothing to do with MATLAB. 10 | class MatlabException : public std::exception { 11 | public: 12 | MatlabException (const char* message) throw(); 13 | ~MatlabException() throw() { }; 14 | 15 | // The copy constructor makes a shallow copy. 16 | MatlabException (const MatlabException& source) throw(); 17 | 18 | // The copy assignment operator makes a shallow copy as well. 19 | MatlabException& operator= (const MatlabException& source); 20 | 21 | // Return the message string. 22 | virtual const char* what () const throw() { return message; }; 23 | 24 | private: 25 | const char* message; // The error message. 26 | }; 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /bayesopt/gpml/util/lbfgsb/matlabexception.h: -------------------------------------------------------------------------------- 1 | #ifndef INCLUDE_MATLABEXCEPTION 2 | #define INCLUDE_MATLABEXCEPTION 3 | 4 | #include 5 | 6 | // Class MatlabException 7 | // ----------------------------------------------------------------- 8 | // This class just makes it easier for me to throw exceptions. Its 9 | // functionality really has nothing to do with MATLAB. 10 | class MatlabException : public std::exception { 11 | public: 12 | MatlabException (const char* message) throw(); 13 | ~MatlabException() throw() { }; 14 | 15 | // The copy constructor makes a shallow copy. 16 | MatlabException (const MatlabException& source) throw(); 17 | 18 | // The copy assignment operator makes a shallow copy as well. 19 | MatlabException& operator= (const MatlabException& source); 20 | 21 | // Return the message string. 22 | virtual const char* what () const throw() { return message; }; 23 | 24 | private: 25 | const char* message; // The error message. 26 | }; 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /functions/C_4d.m: -------------------------------------------------------------------------------- 1 | function [ value ] = C_4d( x,y,z,u,params ) 2 | %UNTITLED16 Summary of this function goes here 3 | % Detailed explanation goes here 4 | %value=3+20*exp(-0.2*sqrt(.25*((3*x-1)^2+(3*y-1)^2+(3*z-1)^2+(3*u-1)^2)))... 5 | % +exp(.25*((cos(2*pi*(3*x-1)))+(cos(2*pi*(3*y-1)))+(cos(2*pi*(3*z-1)))+(cos(2*pi*(3*u-1)))))-20-exp(1); 6 | C=[1;2;3;3.2]; 7 | a=[10 .05 3 17;3 10 3.5 8;17 17 1.7 .05;3.5 .1 10 10]; 8 | p=[.131 .232 .234 .404;.169 .413 .145 .882;.556 .830 .352 .873;.012 .373 .288 .574]; 9 | 10 | for i=1:4 11 | c=C(i); 12 | if i==1 13 | t=x; 14 | elseif i==2 15 | t=y; 16 | elseif i==3 17 | t=z; 18 | else 19 | t=u; 20 | end 21 | 22 | for j=1:4 23 | temp(j)=a(j,i)*(t-p(j,i))^2; 24 | end 25 | valC(i)=c*exp(-sum(temp)); 26 | 27 | clear temp c 28 | end 29 | value=-1*((1/0.8387)*(-1.1+sum(valC))); 30 | % value=(params(1)*sin(params(2)*pi*(x.^2-params(3).*y))-x-params(4).*y+params(5)); 31 | end 32 | 33 | -------------------------------------------------------------------------------- /gpml/prior/priorWeibull.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorWeibull(lam,k,x) 2 | 3 | % Univariate Weibull hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = (k/lam) * (x/lam)^(k-1) * exp(-(x/lam)^k) 8 | % 9 | % where lam(1x1) is the scale parameter, k(1x1) is the scale parameter 10 | % and x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<2, error('lam and k parameters need to be provided'), end 19 | if ~(isscalar(lam)&&isscalar(k)) 20 | error('lam and k parameters need to be scalars'), end 21 | if nargin<3,lp = lam*(-log(rand))^(1/k); return, end % return a sample 22 | 23 | lp = log(k/lam) + (k-1)*log(x/lam) - (x/lam).^k; 24 | dlp = (k-1)./x - (k/lam)*(x/lam).^(k-1); 25 | lp(x<0) = -inf; dlp(x<0) = 0; -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorWeibull.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorWeibull(lam,k,x) 2 | 3 | % Univariate Weibull hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = (k/lam) * (x/lam)^(k-1) * exp(-(x/lam)^k) 8 | % 9 | % where lam(1x1) is the scale parameter, k(1x1) is the scale parameter 10 | % and x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<2, error('lam and k parameters need to be provided'), end 19 | if ~(isscalar(lam)&&isscalar(k)) 20 | error('lam and k parameters need to be scalars'), end 21 | if nargin<3,lp = lam*(-log(rand))^(1/k); return, end % return a sample 22 | 23 | lp = log(k/lam) + (k-1)*log(x/lam) - (x/lam).^k; 24 | dlp = (k-1)./x - (k/lam)*(x/lam).^(k-1); 25 | lp(x<0) = -inf; dlp(x<0) = 0; -------------------------------------------------------------------------------- /gpml/mean/meanScale.m: -------------------------------------------------------------------------------- 1 | function A = meanScale(mean, hyp, x, i) 2 | 3 | % meanScale - compose a mean function as a scaled version of another one. 4 | % 5 | % m(x) = a * m_0(x) 6 | % 7 | % The hyperparameters are: 8 | % 9 | % hyp = [ a; 10 | % hyp_m0 ] 11 | % 12 | % This function doesn't actually compute very much on its own, it merely does 13 | % some bookkeeping, and calls other mean functions to do the actual work. 14 | % 15 | % Copyright (c) by Carl Edward Rasmussen & Hannes Nickisch 2014-11-01. 16 | % 17 | % See also MEANFUNCTIONS.M. 18 | 19 | if nargin<3 % report number of parameters 20 | A = [feval(mean{:}),'+1']; return 21 | end 22 | 23 | [n,D] = size(x); 24 | a = hyp(1); 25 | if nargin==3 % compute mean vector 26 | A = a*feval(mean{:},hyp(2:end),x); 27 | else % compute derivative vector 28 | if i==1 29 | A = feval(mean{:},hyp(2:end),x); 30 | else 31 | A = a*feval(mean{:},hyp(2:end),x,i-1); 32 | end 33 | end -------------------------------------------------------------------------------- /gpml/mean/meanDiscrete.m: -------------------------------------------------------------------------------- 1 | function A = meanDiscrete(s, hyp, x, i) 2 | 3 | % Mean function for discrete inputs x. Given a function defined on the 4 | % integers 1,2,3,..,s, the mean function is parametrized as: 5 | % 6 | % m(x) = mu_x, 7 | % 8 | % where mu is a fixed vector of length s. 9 | % 10 | % This implementation assumes that the inputs x are given as integers 11 | % between 1 and s, which simply index the provided vector. 12 | % 13 | % The hyperparameters are: 14 | % 15 | % hyp = [ mu_1 16 | % mu_2 17 | % .. 18 | % mu_s ] 19 | % 20 | % Copyright (c) by Roman Garnett, 2014-08-14. 21 | % 22 | % See also COVDISCRETE.M, MEANFUNCTIONS.M. 23 | 24 | if nargin==0, error('s must be specified.'), end % check for dimension 25 | if nargin<=2, A = num2str(s); return; end % report number of hyperparameters 26 | mu = hyp(:); 27 | if nargin==3 28 | A = mu(x(:)); % evaluate mean 29 | else 30 | A = zeros(numel(x),1); % derivative 31 | A(x==i) = 1; 32 | end -------------------------------------------------------------------------------- /gpml/prior/priorLogNormal.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp]=priorLogNormal(mu,s2,x) 2 | 3 | % Univariate Log-normal hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-(log(x)-mu)^2/(2*s2)) / (sqrt(2*pi*s2)*x) 8 | % 9 | % where mu(1x1) is the log scale parameter, s2(1x1) is the shape parameter 10 | % and x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<2, error('mu and s2 parameters need to be provided'), end 19 | if ~(isscalar(mu)&&isscalar(s2)) 20 | error('mu and s2 parameters need to be scalars'), end 21 | if nargin<3, lp = exp(sqrt(s2)*randn+mu); return, end % return a sample 22 | 23 | lx = log(x); 24 | lp = -(lx-mu).^2/(2*s2) - log(2*pi*s2)/2 - lx; 25 | dlp = -(lx-mu+s2)./(x*s2); 26 | lp(x<0) = -inf; dlp(x<0) = 0; -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanScale.m: -------------------------------------------------------------------------------- 1 | function A = meanScale(mean, hyp, x, i) 2 | 3 | % meanScale - compose a mean function as a scaled version of another one. 4 | % 5 | % m(x) = a * m_0(x) 6 | % 7 | % The hyperparameters are: 8 | % 9 | % hyp = [ a; 10 | % hyp_m0 ] 11 | % 12 | % This function doesn't actually compute very much on its own, it merely does 13 | % some bookkeeping, and calls other mean functions to do the actual work. 14 | % 15 | % Copyright (c) by Carl Edward Rasmussen & Hannes Nickisch 2014-11-01. 16 | % 17 | % See also MEANFUNCTIONS.M. 18 | 19 | if nargin<3 % report number of parameters 20 | A = [feval(mean{:}),'+1']; return 21 | end 22 | 23 | [n,D] = size(x); 24 | a = hyp(1); 25 | if nargin==3 % compute mean vector 26 | A = a*feval(mean{:},hyp(2:end),x); 27 | else % compute derivative vector 28 | if i==1 29 | A = feval(mean{:},hyp(2:end),x); 30 | else 31 | A = a*feval(mean{:},hyp(2:end),x,i-1); 32 | end 33 | end -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanDiscrete.m: -------------------------------------------------------------------------------- 1 | function A = meanDiscrete(s, hyp, x, i) 2 | 3 | % Mean function for discrete inputs x. Given a function defined on the 4 | % integers 1,2,3,..,s, the mean function is parametrized as: 5 | % 6 | % m(x) = mu_x, 7 | % 8 | % where mu is a fixed vector of length s. 9 | % 10 | % This implementation assumes that the inputs x are given as integers 11 | % between 1 and s, which simply index the provided vector. 12 | % 13 | % The hyperparameters are: 14 | % 15 | % hyp = [ mu_1 16 | % mu_2 17 | % .. 18 | % mu_s ] 19 | % 20 | % Copyright (c) by Roman Garnett, 2014-08-14. 21 | % 22 | % See also COVDISCRETE.M, MEANFUNCTIONS.M. 23 | 24 | if nargin==0, error('s must be specified.'), end % check for dimension 25 | if nargin<=2, A = num2str(s); return; end % report number of hyperparameters 26 | mu = hyp(:); 27 | if nargin==3 28 | A = mu(x(:)); % evaluate mean 29 | else 30 | A = zeros(numel(x),1); % derivative 31 | A(x==i) = 1; 32 | end -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorLogNormal.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp]=priorLogNormal(mu,s2,x) 2 | 3 | % Univariate Log-normal hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-(log(x)-mu)^2/(2*s2)) / (sqrt(2*pi*s2)*x) 8 | % 9 | % where mu(1x1) is the log scale parameter, s2(1x1) is the shape parameter 10 | % and x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<2, error('mu and s2 parameters need to be provided'), end 19 | if ~(isscalar(mu)&&isscalar(s2)) 20 | error('mu and s2 parameters need to be scalars'), end 21 | if nargin<3, lp = exp(sqrt(s2)*randn+mu); return, end % return a sample 22 | 23 | lx = log(x); 24 | lp = -(lx-mu).^2/(2*s2) - log(2*pi*s2)/2 - lx; 25 | dlp = -(lx-mu+s2)./(x*s2); 26 | lp(x<0) = -inf; dlp(x<0) = 0; -------------------------------------------------------------------------------- /gpml/mean/meanPow.m: -------------------------------------------------------------------------------- 1 | function A = meanPow(d, mean, hyp, x, i) 2 | 3 | % meanPow - compose a mean function as the power of another mean function m0. 4 | % 5 | % The degree d has to be a strictly positive integer. 6 | % 7 | % m(x) = m_0(x) ^ d 8 | % 9 | % The hyperparameter is: 10 | % 11 | % hyp = [ ] 12 | % 13 | % This function doesn't actually compute very much on its own, it merely does 14 | % some bookkeeping, and calls other mean function to do the actual work. 15 | % 16 | % Copyright (c) by Carl Edward Rasmussen & Hannes Nickisch 2013-11-02. 17 | % 18 | % See also MEANFUNCTIONS.M. 19 | 20 | d = max(abs(floor(d)),1); % positive integer degree only 21 | if nargin<4 % report number of parameters 22 | A = feval(mean{:}); return 23 | end 24 | 25 | [n,D] = size(x); 26 | if nargin==4 % compute mean vector 27 | A = feval(mean{:},hyp,x).^d; 28 | else % compute derivative vector 29 | A = ( d*feval(mean{:},hyp,x).^(d-1) ).* feval(mean{:},hyp,x,i); 30 | end -------------------------------------------------------------------------------- /gpml/util/rewrap.m: -------------------------------------------------------------------------------- 1 | % Map the numerical elements in the vector "v" onto the variables "s" which can 2 | % be of any type. The number of numerical elements must match; on exit "v" 3 | % should be empty. Non-numerical entries are just copied. See also unwrap.m. 4 | 5 | function [s v] = rewrap(s, v) 6 | 7 | if isnumeric(s) 8 | if numel(v) < numel(s) 9 | error('The vector for conversion contains too few elements') 10 | end 11 | s = reshape(v(1:numel(s)), size(s)); % numeric values are reshaped 12 | v = v(numel(s)+1:end); % remaining arguments passed on 13 | elseif isstruct(s) 14 | [s p] = orderfields(s); p(p) = 1:numel(p); % alphabetize, store ordering 15 | [t v] = rewrap(struct2cell(s), v); % convert to cell, recurse 16 | s = orderfields(cell2struct(t,fieldnames(s),1),p); % conv to struct, reorder 17 | elseif iscell(s) 18 | for i = 1:numel(s) % cell array elements are handled sequentially 19 | [s{i} v] = rewrap(s{i}, v); 20 | end 21 | end % other types are not processed 22 | -------------------------------------------------------------------------------- /gpml/util/solve_chol.m: -------------------------------------------------------------------------------- 1 | % solve_chol - solve linear equations from the Cholesky factorization. 2 | % Solve A*X = B for X, where A is square, symmetric, positive definite. The 3 | % input to the function is R the Cholesky decomposition of A and the matrix B. 4 | % Example: X = solve_chol(chol(A),B); 5 | % 6 | % NOTE: The program code is written in the C language for efficiency and is 7 | % contained in the file solve_chol.c, and should be compiled using matlabs mex 8 | % facility. However, this file also contains a (less efficient) matlab 9 | % implementation, supplied only as a help to people unfamiliar with mex. If 10 | % the C code has been properly compiled and is available, it automatically 11 | % takes precendence over the matlab code in this file. 12 | % 13 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch 2010-09-18. 14 | 15 | function X = solve_chol(L, B) 16 | 17 | if nargin ~= 2 || nargout > 1 18 | error('Wrong number of arguments.'); 19 | end 20 | 21 | if size(L,1) ~= size(L,2) || size(L,1) ~= size(B,1) 22 | error('Wrong sizes of matrix arguments.'); 23 | end 24 | 25 | X = L\(L'\B); -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanPow.m: -------------------------------------------------------------------------------- 1 | function A = meanPow(d, mean, hyp, x, i) 2 | 3 | % meanPow - compose a mean function as the power of another mean function m0. 4 | % 5 | % The degree d has to be a strictly positive integer. 6 | % 7 | % m(x) = m_0(x) ^ d 8 | % 9 | % The hyperparameter is: 10 | % 11 | % hyp = [ ] 12 | % 13 | % This function doesn't actually compute very much on its own, it merely does 14 | % some bookkeeping, and calls other mean function to do the actual work. 15 | % 16 | % Copyright (c) by Carl Edward Rasmussen & Hannes Nickisch 2013-11-02. 17 | % 18 | % See also MEANFUNCTIONS.M. 19 | 20 | d = max(abs(floor(d)),1); % positive integer degree only 21 | if nargin<4 % report number of parameters 22 | A = feval(mean{:}); return 23 | end 24 | 25 | [n,D] = size(x); 26 | if nargin==4 % compute mean vector 27 | A = feval(mean{:},hyp,x).^d; 28 | else % compute derivative vector 29 | A = ( d*feval(mean{:},hyp,x).^(d-1) ).* feval(mean{:},hyp,x,i); 30 | end -------------------------------------------------------------------------------- /bayesopt/gpml/util/rewrap.m: -------------------------------------------------------------------------------- 1 | % Map the numerical elements in the vector "v" onto the variables "s" which can 2 | % be of any type. The number of numerical elements must match; on exit "v" 3 | % should be empty. Non-numerical entries are just copied. See also unwrap.m. 4 | 5 | function [s v] = rewrap(s, v) 6 | 7 | if isnumeric(s) 8 | if numel(v) < numel(s) 9 | error('The vector for conversion contains too few elements') 10 | end 11 | s = reshape(v(1:numel(s)), size(s)); % numeric values are reshaped 12 | v = v(numel(s)+1:end); % remaining arguments passed on 13 | elseif isstruct(s) 14 | [s p] = orderfields(s); p(p) = 1:numel(p); % alphabetize, store ordering 15 | [t v] = rewrap(struct2cell(s), v); % convert to cell, recurse 16 | s = orderfields(cell2struct(t,fieldnames(s),1),p); % conv to struct, reorder 17 | elseif iscell(s) 18 | for i = 1:numel(s) % cell array elements are handled sequentially 19 | [s{i} v] = rewrap(s{i}, v); 20 | end 21 | end % other types are not processed 22 | -------------------------------------------------------------------------------- /bayesopt/gpml/util/solve_chol.m: -------------------------------------------------------------------------------- 1 | % solve_chol - solve linear equations from the Cholesky factorization. 2 | % Solve A*X = B for X, where A is square, symmetric, positive definite. The 3 | % input to the function is R the Cholesky decomposition of A and the matrix B. 4 | % Example: X = solve_chol(chol(A),B); 5 | % 6 | % NOTE: The program code is written in the C language for efficiency and is 7 | % contained in the file solve_chol.c, and should be compiled using matlabs mex 8 | % facility. However, this file also contains a (less efficient) matlab 9 | % implementation, supplied only as a help to people unfamiliar with mex. If 10 | % the C code has been properly compiled and is available, it automatically 11 | % takes precendence over the matlab code in this file. 12 | % 13 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch 2010-09-18. 14 | 15 | function X = solve_chol(L, B) 16 | 17 | if nargin ~= 2 || nargout > 1 18 | error('Wrong number of arguments.'); 19 | end 20 | 21 | if size(L,1) ~= size(L,2) || size(L,1) ~= size(B,1) 22 | error('Wrong sizes of matrix arguments.'); 23 | end 24 | 25 | X = L\(L'\B); -------------------------------------------------------------------------------- /gpml/util/lbfgsb/matlabscalar.cpp: -------------------------------------------------------------------------------- 1 | #include "matlabscalar.h" 2 | #include "matlabexception.h" 3 | 4 | // Function definitions. 5 | // ----------------------------------------------------------------- 6 | double& getMatlabScalar (const mxArray* ptr) { 7 | if (!mxIsDouble(ptr)) 8 | throw MatlabException("Matlab array must be of type double"); 9 | if (mxGetNumberOfElements(ptr) != 1) 10 | throw MatlabException("The Matlab array must be a scalar"); 11 | return *mxGetPr(ptr); 12 | } 13 | 14 | double& createMatlabScalar (mxArray*& ptr) { 15 | ptr = mxCreateDoubleScalar(0); 16 | return *mxGetPr(ptr); 17 | } 18 | 19 | // Function definitions for class MatlabScalar. 20 | // --------------------------------------------------------------- 21 | MatlabScalar::MatlabScalar (const mxArray* ptr) 22 | : x(getMatlabScalar(ptr)) { } 23 | 24 | MatlabScalar::MatlabScalar (mxArray*& ptr, double value) 25 | : x(createMatlabScalar(ptr)) { 26 | x = value; 27 | } 28 | 29 | MatlabScalar::MatlabScalar (MatlabScalar& source) 30 | : x(source.x) { } 31 | 32 | MatlabScalar& MatlabScalar::operator= (double value) { 33 | x = value; 34 | return *this; 35 | } 36 | -------------------------------------------------------------------------------- /bayesopt/gpml/util/lbfgsb/matlabscalar.cpp: -------------------------------------------------------------------------------- 1 | #include "matlabscalar.h" 2 | #include "matlabexception.h" 3 | 4 | // Function definitions. 5 | // ----------------------------------------------------------------- 6 | double& getMatlabScalar (const mxArray* ptr) { 7 | if (!mxIsDouble(ptr)) 8 | throw MatlabException("Matlab array must be of type double"); 9 | if (mxGetNumberOfElements(ptr) != 1) 10 | throw MatlabException("The Matlab array must be a scalar"); 11 | return *mxGetPr(ptr); 12 | } 13 | 14 | double& createMatlabScalar (mxArray*& ptr) { 15 | ptr = mxCreateDoubleScalar(0); 16 | return *mxGetPr(ptr); 17 | } 18 | 19 | // Function definitions for class MatlabScalar. 20 | // --------------------------------------------------------------- 21 | MatlabScalar::MatlabScalar (const mxArray* ptr) 22 | : x(getMatlabScalar(ptr)) { } 23 | 24 | MatlabScalar::MatlabScalar (mxArray*& ptr, double value) 25 | : x(createMatlabScalar(ptr)) { 26 | x = value; 27 | } 28 | 29 | MatlabScalar::MatlabScalar (MatlabScalar& source) 30 | : x(source.x) { } 31 | 32 | MatlabScalar& MatlabScalar::operator= (double value) { 33 | x = value; 34 | return *this; 35 | } 36 | -------------------------------------------------------------------------------- /functions/acc_eval.m: -------------------------------------------------------------------------------- 1 | 2 | function [f]=acc_eval(Params) 3 | dropout_In=Params(1); 4 | dropout_Hi=Params(2); 5 | momentum=Params(3); 6 | weightDecay=Params(4); 7 | maxWeight=Params(5); 8 | learningRate=Params(6); 9 | decayRate=Params(7); 10 | nodeNum_L1=round(Params(8)); 11 | nodeNum_L2=round(Params(9)); 12 | nodeNum_L3=round(Params(10)); 13 | active_choice=round(Params(11)); 14 | 15 | 16 | paramsString = sprintf('\''-dIn\'' %f \''-dHi\'' %f \''-mo\'' %f \''-wD\'' %f \''-wM\'' %f \''-lr\'' %f \''-dr\'' %f \''-nNum1\'' %d \''-nNum2\'' %d \''-nNum3\'' %d \''-actCh\'' %d \''-jobID\'' 1 ', dropout_In,dropout_Hi,momentum, weightDecay, maxWeight,learningRate,decayRate,nodeNum_L1,nodeNum_L2,nodeNum_L3,active_choice); 17 | 18 | commandString = 'python MINST_keras_remote_f.py'; 19 | 20 | fullCommand = sprintf('%s %s', commandString, paramsString); 21 | 22 | [~,cmdout] = system(fullCommand); 23 | 24 | splitOut = strsplit(cmdout, '\n'); 25 | 26 | output = splitOut{end-1}; 27 | A=[]; 28 | A =sscanf(output,'{ \"f\": %f}'); 29 | if isempty(A) 30 | A =sscanf(output,'{\"f\": %f}'); 31 | end 32 | 33 | f=A(1); 34 | 35 | end 36 | -------------------------------------------------------------------------------- /gpml/cov/covLIN.m: -------------------------------------------------------------------------------- 1 | function K = covLIN(hyp, x, z, i) 2 | 3 | % Linear covariance function. The covariance function is parameterized as: 4 | % 5 | % k(x^p,x^q) = x^p'*x^q 6 | % 7 | % The are no hyperparameters: 8 | % 9 | % hyp = [ ] 10 | % 11 | % Note that there is no bias or scale term; use covConst to add these. 12 | % 13 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 14 | % 15 | % See also COVFUNCTIONS.M. 16 | 17 | if nargin<2, K = '0'; return; end % report number of parameters 18 | if nargin<3, z = []; end % make sure, z exists 19 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 20 | 21 | % compute inner products 22 | if dg % vector kxx 23 | K = sum(x.*x,2); 24 | else 25 | if xeqz % symmetric matrix Kxx 26 | K = x*x'; 27 | else % cross covariances Kxz 28 | K = x*z'; 29 | end 30 | end 31 | 32 | if nargin>3 % derivatives 33 | error('Unknown hyperparameter') 34 | end -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covLIN.m: -------------------------------------------------------------------------------- 1 | function K = covLIN(hyp, x, z, i) 2 | 3 | % Linear covariance function. The covariance function is parameterized as: 4 | % 5 | % k(x^p,x^q) = x^p'*x^q 6 | % 7 | % The are no hyperparameters: 8 | % 9 | % hyp = [ ] 10 | % 11 | % Note that there is no bias or scale term; use covConst to add these. 12 | % 13 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 14 | % 15 | % See also COVFUNCTIONS.M. 16 | 17 | if nargin<2, K = '0'; return; end % report number of parameters 18 | if nargin<3, z = []; end % make sure, z exists 19 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 20 | 21 | % compute inner products 22 | if dg % vector kxx 23 | K = sum(x.*x,2); 24 | else 25 | if xeqz % symmetric matrix Kxx 26 | K = x*x'; 27 | else % cross covariances Kxz 28 | K = x*z'; 29 | end 30 | end 31 | 32 | if nargin>3 % derivatives 33 | error('Unknown hyperparameter') 34 | end -------------------------------------------------------------------------------- /functions/time_eval.m: -------------------------------------------------------------------------------- 1 | 2 | function [c]=time_eval(Params) 3 | dropout_In=Params(1); 4 | dropout_Hi=Params(2); 5 | momentum=Params(3); 6 | weightDecay=Params(4); 7 | maxWeight=Params(5); 8 | learningRate=Params(6); 9 | decayRate=Params(7); 10 | nodeNum_L1=round(Params(8)); 11 | nodeNum_L2=round(Params(9)); 12 | nodeNum_L3=round(Params(10)); 13 | active_choice=round(Params(11)); 14 | 15 | 16 | paramsString = sprintf('\''-dIn\'' %f \''-dHi\'' %f \''-mo\'' %f \''-wD\'' %f \''-wM\'' %f \''-lr\'' %f \''-dr\'' %f \''-nNum1\'' %d \''-nNum2\'' %d \''-nNum3\'' %d \''-actCh\'' %d \''-jobID\'' 1 ', dropout_In,dropout_Hi,momentum, weightDecay, maxWeight,learningRate,decayRate,nodeNum_L1,nodeNum_L2,nodeNum_L3,active_choice); 17 | 18 | commandString = 'python MINST_keras_remote_c.py'; 19 | 20 | fullCommand = sprintf('%s %s', commandString, paramsString); 21 | % fprintf(fullCommand) 22 | 23 | [~,cmdout] = system(fullCommand); 24 | 25 | splitOut = strsplit(cmdout, '\n'); 26 | 27 | output = splitOut{end-1}; 28 | A=[]; 29 | A =sscanf(output,'{\"c1\": %f}'); 30 | if isempty(A) 31 | A =sscanf(output,'{\"c1\": %f}'); 32 | end 33 | c =-A(1); 34 | end 35 | -------------------------------------------------------------------------------- /gpml/mean/meanMask.m: -------------------------------------------------------------------------------- 1 | function A = meanMask(mask, mean, hyp, x, i) 2 | 3 | % Apply a mean function to a subset of the dimensions only. The subset can 4 | % either be specified by a 0/1 mask by a boolean mask or by an index set. 5 | % 6 | % This function doesn't actually compute very much on its own, it merely does 7 | % some bookkeeping, and calls another mean function to do the actual work. 8 | % 9 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2013-11-02. 10 | % 11 | % See also MEANFUNCTIONS.M. 12 | 13 | nh_string = feval(mean{:}); % number of hyperparameters of the full mean 14 | 15 | if max(mask)<2 && length(mask)>1, mask = find(mask); end % convert 1/0->index 16 | D = length(mask); % masked dimension 17 | if nargin<4, A = num2str(eval(nh_string)); return; end % number of parameters 18 | 19 | if eval(nh_string)~=length(hyp) % check hyperparameters 20 | error('number of hyperparameters does not match size of masked data') 21 | end 22 | 23 | if nargin==4 24 | A = feval(mean{:}, hyp, x(:,mask)); 25 | else % compute derivative 26 | A = feval(mean{:}, hyp, x(:,mask), i); 27 | end 28 | -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanMask.m: -------------------------------------------------------------------------------- 1 | function A = meanMask(mask, mean, hyp, x, i) 2 | 3 | % Apply a mean function to a subset of the dimensions only. The subset can 4 | % either be specified by a 0/1 mask by a boolean mask or by an index set. 5 | % 6 | % This function doesn't actually compute very much on its own, it merely does 7 | % some bookkeeping, and calls another mean function to do the actual work. 8 | % 9 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2013-11-02. 10 | % 11 | % See also MEANFUNCTIONS.M. 12 | 13 | nh_string = feval(mean{:}); % number of hyperparameters of the full mean 14 | 15 | if max(mask)<2 && length(mask)>1, mask = find(mask); end % convert 1/0->index 16 | D = length(mask); % masked dimension 17 | if nargin<4, A = num2str(eval(nh_string)); return; end % number of parameters 18 | 19 | if eval(nh_string)~=length(hyp) % check hyperparameters 20 | error('number of hyperparameters does not match size of masked data') 21 | end 22 | 23 | if nargin==4 24 | A = feval(mean{:}, hyp, x(:,mask)); 25 | else % compute derivative 26 | A = feval(mean{:}, hyp, x(:,mask), i); 27 | end 28 | -------------------------------------------------------------------------------- /gpml/util/lbfgsb/matlabstring.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "matlabstring.h" 3 | #include "matlabexception.h" 4 | 5 | // Function definitions. 6 | // ----------------------------------------------------------------- 7 | char* copystring (const char* source) { 8 | int n = strlen(source); // The length of the string. 9 | char* dest = new char[n+1]; // The return value. 10 | strcpy(dest,source); 11 | return dest; 12 | } 13 | 14 | // Function definitions for class MatlabString. 15 | // ----------------------------------------------------------------- 16 | MatlabString::MatlabString (const mxArray* ptr) { 17 | s = 0; 18 | 19 | // Check to make sure the Matlab array is a string. 20 | if (!mxIsChar(ptr)) 21 | throw MatlabException("Matlab array must be a string (of type CHAR)"); 22 | 23 | // Get the string passed as a Matlab array. 24 | s = mxArrayToString(ptr); 25 | if (s == 0) 26 | throw MatlabException("Unable to obtain string from Matlab array"); 27 | } 28 | 29 | MatlabString::MatlabString (const MatlabString& source) { 30 | s = 0; 31 | 32 | // Copy the source string. 33 | s = copystring(source.s); 34 | } 35 | 36 | MatlabString::~MatlabString() { 37 | if (s) 38 | mxFree(s); 39 | } 40 | 41 | -------------------------------------------------------------------------------- /gpml/util/lbfgsb/timer.f: -------------------------------------------------------------------------------- 1 | c 2 | c L-BFGS-B is released under the “New BSD License” (aka “Modified BSD 3 | c License” or “3-clause license”) 4 | c Please read attached file License.txt 5 | c 6 | subroutine timer(ttime) 7 | double precision ttime 8 | c 9 | real temp 10 | c 11 | c This routine computes cpu time in double precision; it makes use of 12 | c the intrinsic f90 cpu_time therefore a conversion type is 13 | c needed. 14 | c 15 | c J.L Morales Departamento de Matematicas, 16 | c Instituto Tecnologico Autonomo de Mexico 17 | c Mexico D.F. 18 | c 19 | c J.L Nocedal Department of Electrical Engineering and 20 | c Computer Science. 21 | c Northwestern University. Evanston, IL. USA 22 | c 23 | c January 21, 2011 24 | c 25 | temp = sngl(ttime) 26 | call cpu_time(temp) 27 | ttime = dble(temp) 28 | 29 | return 30 | 31 | end 32 | 33 | -------------------------------------------------------------------------------- /bayesopt/gpml/util/lbfgsb/matlabstring.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "matlabstring.h" 3 | #include "matlabexception.h" 4 | 5 | // Function definitions. 6 | // ----------------------------------------------------------------- 7 | char* copystring (const char* source) { 8 | int n = strlen(source); // The length of the string. 9 | char* dest = new char[n+1]; // The return value. 10 | strcpy(dest,source); 11 | return dest; 12 | } 13 | 14 | // Function definitions for class MatlabString. 15 | // ----------------------------------------------------------------- 16 | MatlabString::MatlabString (const mxArray* ptr) { 17 | s = 0; 18 | 19 | // Check to make sure the Matlab array is a string. 20 | if (!mxIsChar(ptr)) 21 | throw MatlabException("Matlab array must be a string (of type CHAR)"); 22 | 23 | // Get the string passed as a Matlab array. 24 | s = mxArrayToString(ptr); 25 | if (s == 0) 26 | throw MatlabException("Unable to obtain string from Matlab array"); 27 | } 28 | 29 | MatlabString::MatlabString (const MatlabString& source) { 30 | s = 0; 31 | 32 | // Copy the source string. 33 | s = copystring(source.s); 34 | } 35 | 36 | MatlabString::~MatlabString() { 37 | if (s) 38 | mxFree(s); 39 | } 40 | 41 | -------------------------------------------------------------------------------- /bayesopt/gpml/util/lbfgsb/timer.f: -------------------------------------------------------------------------------- 1 | c 2 | c L-BFGS-B is released under the “New BSD License” (aka “Modified BSD 3 | c License” or “3-clause license”) 4 | c Please read attached file License.txt 5 | c 6 | subroutine timer(ttime) 7 | double precision ttime 8 | c 9 | real temp 10 | c 11 | c This routine computes cpu time in double precision; it makes use of 12 | c the intrinsic f90 cpu_time therefore a conversion type is 13 | c needed. 14 | c 15 | c J.L Morales Departamento de Matematicas, 16 | c Instituto Tecnologico Autonomo de Mexico 17 | c Mexico D.F. 18 | c 19 | c J.L Nocedal Department of Electrical Engineering and 20 | c Computer Science. 21 | c Northwestern University. Evanston, IL. USA 22 | c 23 | c January 21, 2011 24 | c 25 | temp = sngl(ttime) 26 | call cpu_time(temp) 27 | ttime = dble(temp) 28 | 29 | return 30 | 31 | end 32 | 33 | -------------------------------------------------------------------------------- /gpml/prior/priorT.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorT(mu,s2,nu,x) 2 | 3 | % Univariate Student's t hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = ( 1 + (x-mu)^2/(s2*(nu-2)) )^(-(nu+1)/2) / Z, where 8 | % Z = gamma(nu/2) * sqrt(pi*(nu-2)*s2) / gamma((nu+1)/2), 9 | % 10 | % mu(1x1) is the mean parameter, s2(1x1) is the variance parameter, 11 | % nu(1x1) > 2 is the degrees of freedom parameter and x(1xN) contains query 12 | % hyperparameters for prior evaluation. 13 | % 14 | % For more help on design of priors, try "help priorDistributions". 15 | % 16 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-10-16. 17 | % 18 | % See also PRIORDISTRIBUTIONS.M. 19 | 20 | if nargin<3, error('mu, s2 and nu parameters need to be provided'), end 21 | if ~(isscalar(mu)&&isscalar(s2)&&isscalar(nu)) 22 | error('mu, s2 and nu parameters need to be scalars') 23 | end 24 | if nargin<4, lp = mu + sqrt(s2*(nu-2)/priorGamma(nu/2,2))*randn; return, end 25 | 26 | lZ = gammaln((nu+1)/2)-gammaln(nu/2)-log(pi*(nu-2)*s2)/2; 27 | lp = -(nu+1)/2*log(1+(x-mu).^2/(s2*(nu-2))) + lZ; 28 | dlp = (nu+1)*(mu-x)./(mu^2-2*mu*x+s2*(nu-2)+x.^2); -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorT.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorT(mu,s2,nu,x) 2 | 3 | % Univariate Student's t hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = ( 1 + (x-mu)^2/(s2*(nu-2)) )^(-(nu+1)/2) / Z, where 8 | % Z = gamma(nu/2) * sqrt(pi*(nu-2)*s2) / gamma((nu+1)/2), 9 | % 10 | % mu(1x1) is the mean parameter, s2(1x1) is the variance parameter, 11 | % nu(1x1) > 2 is the degrees of freedom parameter and x(1xN) contains query 12 | % hyperparameters for prior evaluation. 13 | % 14 | % For more help on design of priors, try "help priorDistributions". 15 | % 16 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-10-16. 17 | % 18 | % See also PRIORDISTRIBUTIONS.M. 19 | 20 | if nargin<3, error('mu, s2 and nu parameters need to be provided'), end 21 | if ~(isscalar(mu)&&isscalar(s2)&&isscalar(nu)) 22 | error('mu, s2 and nu parameters need to be scalars') 23 | end 24 | if nargin<4, lp = mu + sqrt(s2*(nu-2)/priorGamma(nu/2,2))*randn; return, end 25 | 26 | lZ = gammaln((nu+1)/2)-gammaln(nu/2)-log(pi*(nu-2)*s2)/2; 27 | lp = -(nu+1)/2*log(1+(x-mu).^2/(s2*(nu-2))) + lZ; 28 | dlp = (nu+1)*(mu-x)./(mu^2-2*mu*x+s2*(nu-2)+x.^2); -------------------------------------------------------------------------------- /gpml/doc/gpml_randn.m: -------------------------------------------------------------------------------- 1 | function x = gpml_randn(seed,varargin) 2 | 3 | % Generate pseudo-random numbers in a quick and dirty way. 4 | % The function makes sure, we obtain the same random numbers using Octave and 5 | % Matlab for the demo scripts. 6 | % 7 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch 2010-07-07. 8 | 9 | if nargin==2 10 | sz = varargin{1}; 11 | else 12 | sz = zeros(1,nargin-1); 13 | for i=1:nargin-1 14 | sz(i) = varargin{i}; 15 | end 16 | end 17 | 18 | n = prod(sz); N = ceil(n/2)*2; 19 | 20 | % minimal uniform random number generator for uniform deviates from [0,1] 21 | % by Park and Miller 22 | a = 7^5; m = 2^31-1; 23 | % using Schrage's algorithm 24 | q = fix(m/a); r = mod(m,a); % m = a*q+r 25 | u = zeros(N+1,1); u(1) = fix(seed*2^31); 26 | for i=2:N+1 27 | % Schrage's algorithm for mod(a*u(i),m) 28 | u(i) = a*mod(u(i-1),q) - r*fix(u(i-1)/q); 29 | if u(i)<0, u(i) = u(i)+m; end 30 | end 31 | u = u(2:N+1)/2^31; 32 | 33 | % Box-Muller transform: Numerical Recipies, 2nd Edition, $7.2.8 34 | % http://en.wikipedia.org/wiki/Box-Muller_transform 35 | w = sqrt(- 2*log(u(1:N/2))); % split into two groups 36 | x = [w.*cos(2*pi*u(N/2+1:N)); w.*sin(2*pi*u(N/2+1:N))]; 37 | x = reshape(x(1:n),sz); 38 | -------------------------------------------------------------------------------- /bayesopt/gpml/doc/gpml_randn.m: -------------------------------------------------------------------------------- 1 | function x = gpml_randn(seed,varargin) 2 | 3 | % Generate pseudo-random numbers in a quick and dirty way. 4 | % The function makes sure, we obtain the same random numbers using Octave and 5 | % Matlab for the demo scripts. 6 | % 7 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch 2010-07-07. 8 | 9 | if nargin==2 10 | sz = varargin{1}; 11 | else 12 | sz = zeros(1,nargin-1); 13 | for i=1:nargin-1 14 | sz(i) = varargin{i}; 15 | end 16 | end 17 | 18 | n = prod(sz); N = ceil(n/2)*2; 19 | 20 | % minimal uniform random number generator for uniform deviates from [0,1] 21 | % by Park and Miller 22 | a = 7^5; m = 2^31-1; 23 | % using Schrage's algorithm 24 | q = fix(m/a); r = mod(m,a); % m = a*q+r 25 | u = zeros(N+1,1); u(1) = fix(seed*2^31); 26 | for i=2:N+1 27 | % Schrage's algorithm for mod(a*u(i),m) 28 | u(i) = a*mod(u(i-1),q) - r*fix(u(i-1)/q); 29 | if u(i)<0, u(i) = u(i)+m; end 30 | end 31 | u = u(2:N+1)/2^31; 32 | 33 | % Box-Muller transform: Numerical Recipies, 2nd Edition, $7.2.8 34 | % http://en.wikipedia.org/wiki/Box-Muller_transform 35 | w = sqrt(- 2*log(u(1:N/2))); % split into two groups 36 | x = [w.*cos(2*pi*u(N/2+1:N)); w.*sin(2*pi*u(N/2+1:N))]; 37 | x = reshape(x(1:n),sz); 38 | -------------------------------------------------------------------------------- /gpml/prior/priorInvGauss.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorInvGauss(mu,lam,x) 2 | 3 | % Univariate Inverse Gaussian hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-lam*(x-mu)^2/(2*mu^2*x)) / sqrt(2*pi*x^3/lam) 8 | % 9 | % where mu(1x1) is the mean parameter, lam(1x1) is the scale parameter 10 | % and x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<2, error('mu and lam parameters need to be provided'), end 19 | if ~(isscalar(mu)&&isscalar(lam)) 20 | error('mu and lam parameters need to be scalars'),end 21 | if nargin<3 % return a sample 22 | n = randn; y = n*n; 23 | r = mu + mu*(mu*y-sqrt(4*mu*lam*y+mu^2*y^2))/(2*lam); 24 | z = rand; 25 | if z<=mu/(mu+r) 26 | lp = r; 27 | else 28 | lp = mu^2/r; 29 | end 30 | return 31 | end 32 | 33 | lp = -lam*(x-mu).^2./(2*mu^2*x) - log(2*pi*x.^3/lam)/2; 34 | q = (x-mu)./x; 35 | dlp = -lam*q.*(2-q)/(2*mu^2) - 3./(2*x); 36 | lp(x<0) = -inf; dlp(x<0) = 0; -------------------------------------------------------------------------------- /gpml/cov/covConst.m: -------------------------------------------------------------------------------- 1 | function K = covConst(hyp, x, z, i) 2 | 3 | % covariance function for a constant function. The covariance function is 4 | % parameterized as: 5 | % 6 | % k(x^p,x^q) = s2; 7 | % 8 | % The scalar hyperparameter is: 9 | % 10 | % hyp = [ log(sqrt(s2)) ] 11 | % 12 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 13 | % 14 | % See also COVFUNCTIONS.M. 15 | 16 | if nargin<2, K = '1'; return; end % report number of parameters 17 | if nargin<3, z = []; end % make sure, z exists 18 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 19 | 20 | s2 = exp(2*hyp); % s2 21 | n = size(x,1); 22 | 23 | if dg % vector kxx 24 | K = s2*ones(n,1); 25 | else 26 | if xeqz % symmetric matrix Kxx 27 | K = s2*ones(n); 28 | else % cross covariances Kxz 29 | K = s2*ones(n,size(z,1)); 30 | end 31 | end 32 | 33 | if nargin>3 % derivatives 34 | if i==1 35 | K = 2*K; 36 | else 37 | error('Unknown hyperparameter') 38 | end 39 | end -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorInvGauss.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorInvGauss(mu,lam,x) 2 | 3 | % Univariate Inverse Gaussian hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-lam*(x-mu)^2/(2*mu^2*x)) / sqrt(2*pi*x^3/lam) 8 | % 9 | % where mu(1x1) is the mean parameter, lam(1x1) is the scale parameter 10 | % and x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M. 17 | 18 | if nargin<2, error('mu and lam parameters need to be provided'), end 19 | if ~(isscalar(mu)&&isscalar(lam)) 20 | error('mu and lam parameters need to be scalars'),end 21 | if nargin<3 % return a sample 22 | n = randn; y = n*n; 23 | r = mu + mu*(mu*y-sqrt(4*mu*lam*y+mu^2*y^2))/(2*lam); 24 | z = rand; 25 | if z<=mu/(mu+r) 26 | lp = r; 27 | else 28 | lp = mu^2/r; 29 | end 30 | return 31 | end 32 | 33 | lp = -lam*(x-mu).^2./(2*mu^2*x) - log(2*pi*x.^3/lam)/2; 34 | q = (x-mu)./x; 35 | dlp = -lam*q.*(2-q)/(2*mu^2) - 3./(2*x); 36 | lp(x<0) = -inf; dlp(x<0) = 0; -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covConst.m: -------------------------------------------------------------------------------- 1 | function K = covConst(hyp, x, z, i) 2 | 3 | % covariance function for a constant function. The covariance function is 4 | % parameterized as: 5 | % 6 | % k(x^p,x^q) = s2; 7 | % 8 | % The scalar hyperparameter is: 9 | % 10 | % hyp = [ log(sqrt(s2)) ] 11 | % 12 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 13 | % 14 | % See also COVFUNCTIONS.M. 15 | 16 | if nargin<2, K = '1'; return; end % report number of parameters 17 | if nargin<3, z = []; end % make sure, z exists 18 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 19 | 20 | s2 = exp(2*hyp); % s2 21 | n = size(x,1); 22 | 23 | if dg % vector kxx 24 | K = s2*ones(n,1); 25 | else 26 | if xeqz % symmetric matrix Kxx 27 | K = s2*ones(n); 28 | else % cross covariances Kxz 29 | K = s2*ones(n,size(z,1)); 30 | end 31 | end 32 | 33 | if nargin>3 % derivatives 34 | if i==1 35 | K = 2*K; 36 | else 37 | error('Unknown hyperparameter') 38 | end 39 | end -------------------------------------------------------------------------------- /gpml/prior/priorTransform.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorTransform(g,dg,ig,prior,x) 2 | 3 | % Transformed univariate hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % prior(y), where y = g(x), 8 | % 9 | % g is the transformation function (monotonically increasing and invertable), 10 | % dg is its derivative, ig is its inverse, prior is the distribution to be 11 | % transformed and x(1xN) contains query hyperparameters for prior evaluation. 12 | % Note that p(x) = prior(g(x)) is not necessarily normalised w.r.t. x. 13 | % 14 | % For example, to put a nonnegative gamma prior on the lenthscale 15 | % parameter ell of covSEiso, one can use 16 | % {@priorTransform,@exp,@exp,@log,{@priorGamma,k,t}} 17 | % since ell is represented in the log domain. 18 | % 19 | % For more help on design of priors, try "help priorDistributions". 20 | % 21 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 22 | % 23 | % See also PRIORDISTRIBUTIONS.M. 24 | 25 | if nargin<4, error('g, dg, ig and prior parameters need to be provided'), end 26 | if nargin<5, lp = ig(feval(prior{:})); return, end % apply inverse sampling 27 | 28 | if ~iscell(prior), prior = {prior}; end 29 | [lp,dlp] = feval(prior{:},g(x)); dlp = dlp.*dg(x); -------------------------------------------------------------------------------- /gpml/cov/covPref.m: -------------------------------------------------------------------------------- 1 | function K = covPref(cov, hyp, x, z, varargin) 2 | 3 | % covPref - covariance function for preference learning. The covariance 4 | % function corresponds to a prior on f(x1) - f(x2). 5 | % 6 | % k(x,z) = k_0(x1,z1) + k_0(x2,z2) - k_0(x1,z2) - k_0(x2,z1). 7 | % 8 | % The hyperparameters are: 9 | % 10 | % hyp = [ hyp_k0 ] 11 | % 12 | % For more help on design of covariance functions, try "help covFunctions". 13 | % 14 | % See Collaborative Gaussian Processes for Preference Learning, NIPS 2014. 15 | % 16 | % Copyright (c) by Hannes Nickisch and Roman Garnett, 2014-11-01. 17 | % 18 | % See also COVFUNCTIONS.M. 19 | 20 | if nargin<3, K = strrep(feval(cov{:}),'D','D/2'); return; end % no of params 21 | 22 | x1 = x(:,1:end/2); x2 = x(:,1+end/2:end); 23 | if nargin<4 || isempty(z), z = x; end 24 | if strcmp(z,'diag') 25 | n = size(x,1); diag12 = zeros(n,1); 26 | for i=1:n, diag12(i) = feval(cov{:},hyp,x1(i,:),x2(i,:),varargin{:}); end 27 | K = feval(cov{:},hyp,x1,z,varargin{:}) ... 28 | + feval(cov{:},hyp,x2,z,varargin{:}) - 2*diag12; 29 | else 30 | z1 = z(:,1:end/2); z2 = z(:,1+end/2:end); 31 | K = feval(cov{:},hyp,x1,z1,varargin{:}) ... 32 | + feval(cov{:},hyp,x2,z2,varargin{:}) ... 33 | - feval(cov{:},hyp,x1,z2,varargin{:}) ... 34 | - feval(cov{:},hyp,x2,z1,varargin{:}); 35 | end 36 | -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorTransform.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorTransform(g,dg,ig,prior,x) 2 | 3 | % Transformed univariate hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % prior(y), where y = g(x), 8 | % 9 | % g is the transformation function (monotonically increasing and invertable), 10 | % dg is its derivative, ig is its inverse, prior is the distribution to be 11 | % transformed and x(1xN) contains query hyperparameters for prior evaluation. 12 | % Note that p(x) = prior(g(x)) is not necessarily normalised w.r.t. x. 13 | % 14 | % For example, to put a nonnegative gamma prior on the lenthscale 15 | % parameter ell of covSEiso, one can use 16 | % {@priorTransform,@exp,@exp,@log,{@priorGamma,k,t}} 17 | % since ell is represented in the log domain. 18 | % 19 | % For more help on design of priors, try "help priorDistributions". 20 | % 21 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 22 | % 23 | % See also PRIORDISTRIBUTIONS.M. 24 | 25 | if nargin<4, error('g, dg, ig and prior parameters need to be provided'), end 26 | if nargin<5, lp = ig(feval(prior{:})); return, end % apply inverse sampling 27 | 28 | if ~iscell(prior), prior = {prior}; end 29 | [lp,dlp] = feval(prior{:},g(x)); dlp = dlp.*dg(x); -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covPref.m: -------------------------------------------------------------------------------- 1 | function K = covPref(cov, hyp, x, z, varargin) 2 | 3 | % covPref - covariance function for preference learning. The covariance 4 | % function corresponds to a prior on f(x1) - f(x2). 5 | % 6 | % k(x,z) = k_0(x1,z1) + k_0(x2,z2) - k_0(x1,z2) - k_0(x2,z1). 7 | % 8 | % The hyperparameters are: 9 | % 10 | % hyp = [ hyp_k0 ] 11 | % 12 | % For more help on design of covariance functions, try "help covFunctions". 13 | % 14 | % See Collaborative Gaussian Processes for Preference Learning, NIPS 2014. 15 | % 16 | % Copyright (c) by Hannes Nickisch and Roman Garnett, 2014-11-01. 17 | % 18 | % See also COVFUNCTIONS.M. 19 | 20 | if nargin<3, K = strrep(feval(cov{:}),'D','D/2'); return; end % no of params 21 | 22 | x1 = x(:,1:end/2); x2 = x(:,1+end/2:end); 23 | if nargin<4 || isempty(z), z = x; end 24 | if strcmp(z,'diag') 25 | n = size(x,1); diag12 = zeros(n,1); 26 | for i=1:n, diag12(i) = feval(cov{:},hyp,x1(i,:),x2(i,:),varargin{:}); end 27 | K = feval(cov{:},hyp,x1,z,varargin{:}) ... 28 | + feval(cov{:},hyp,x2,z,varargin{:}) - 2*diag12; 29 | else 30 | z1 = z(:,1:end/2); z2 = z(:,1+end/2:end); 31 | K = feval(cov{:},hyp,x1,z1,varargin{:}) ... 32 | + feval(cov{:},hyp,x2,z2,varargin{:}) ... 33 | - feval(cov{:},hyp,x1,z2,varargin{:}) ... 34 | - feval(cov{:},hyp,x2,z1,varargin{:}); 35 | end 36 | -------------------------------------------------------------------------------- /gpml/util/lbfgsb/matlabstring.h: -------------------------------------------------------------------------------- 1 | #ifndef INCLUDE_MATLABSTRING 2 | #define INCLUDE_MATLABSTRING 3 | 4 | #include "mex.h" 5 | #include 6 | 7 | // Function declarations. 8 | // ----------------------------------------------------------------- 9 | // Copy a C-style string (i.e. a null-terminated character array). 10 | char* copystring (const char* source); 11 | 12 | // Class MatlabString. 13 | // ----------------------------------------------------------------- 14 | // This class encapsulates read-only access to a MATLAB character 15 | // array. 16 | class MatlabString { 17 | public: 18 | 19 | // The constructor accepts as input a pointer to a Matlab array, 20 | // which must be a valid string (array of type CHAR). 21 | explicit MatlabString (const mxArray* ptr); 22 | 23 | // The copy constructor makes a full copy of the source string. 24 | MatlabString (const MatlabString& source); 25 | 26 | // The destructor. 27 | ~MatlabString(); 28 | 29 | // Conversion operator for null-terminated string. 30 | operator const char* () const { return s; }; 31 | 32 | protected: 33 | char* s; // The null-terminated string. 34 | 35 | // The copy assignment operator is not proper, thus remains 36 | // protected. 37 | MatlabString& operator= (const MatlabString& source) 38 | { return *this; }; 39 | }; 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /bayesopt/gpml/util/lbfgsb/matlabstring.h: -------------------------------------------------------------------------------- 1 | #ifndef INCLUDE_MATLABSTRING 2 | #define INCLUDE_MATLABSTRING 3 | 4 | #include "mex.h" 5 | #include 6 | 7 | // Function declarations. 8 | // ----------------------------------------------------------------- 9 | // Copy a C-style string (i.e. a null-terminated character array). 10 | char* copystring (const char* source); 11 | 12 | // Class MatlabString. 13 | // ----------------------------------------------------------------- 14 | // This class encapsulates read-only access to a MATLAB character 15 | // array. 16 | class MatlabString { 17 | public: 18 | 19 | // The constructor accepts as input a pointer to a Matlab array, 20 | // which must be a valid string (array of type CHAR). 21 | explicit MatlabString (const mxArray* ptr); 22 | 23 | // The copy constructor makes a full copy of the source string. 24 | MatlabString (const MatlabString& source); 25 | 26 | // The destructor. 27 | ~MatlabString(); 28 | 29 | // Conversion operator for null-terminated string. 30 | operator const char* () const { return s; }; 31 | 32 | protected: 33 | char* s; // The null-terminated string. 34 | 35 | // The copy assignment operator is not proper, thus remains 36 | // protected. 37 | MatlabString& operator= (const MatlabString& source) 38 | { return *this; }; 39 | }; 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /gpml/prior/priorMix.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorMix(w,priors,x) 2 | 3 | % Mixture of Univariate or Multivariate hyperparameter prior distributions. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = sum_i w_i p_i(x), 8 | % 9 | % w(bx1) is the mixture weight vector (nonnegative, normalised to add up to 1), 10 | % priors(bx1) is a cell array containing the priors to be mixed and x(1xN) 11 | % contains query hyperparameters for prior evaluation. 12 | % 13 | % For more help on design of priors, try "help priorDistributions". 14 | % 15 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 16 | % 17 | % See also PRIORDISTRIBUTIONS.M. 18 | 19 | if nargin<2, error('specification of w and priors required'), end 20 | n = numel(w); w = w(:); 21 | if abs(sum(w)-1)>1e-9, error('w needs to sum up to 1'), end 22 | if any(w<0), error('w needs to be nonnegative'), end 23 | for i=1:n, if ~iscell(priors{i}), priors{i} = {priors{i}}; end, end % sample 24 | if nargin<3, i = 1+nnz(rand>cumsum(w)); lp = feval(priors{i}{:}); return, end 25 | 26 | lpi = zeros(numel(x),n); dlpi = zeros(numel(x),n); 27 | for i=1:n, [lpi(:,i),dlpi(:,i)] = feval(priors{i}{:},x(:)); end 28 | 29 | mx = max(lpi,[],2); lp = log(exp(lpi-mx*ones(1,n))*w) + mx; 30 | pi = exp(lpi); dlp = ((pi.*dlpi)*w)./(pi*w); -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorMix.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorMix(w,priors,x) 2 | 3 | % Mixture of Univariate or Multivariate hyperparameter prior distributions. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = sum_i w_i p_i(x), 8 | % 9 | % w(bx1) is the mixture weight vector (nonnegative, normalised to add up to 1), 10 | % priors(bx1) is a cell array containing the priors to be mixed and x(1xN) 11 | % contains query hyperparameters for prior evaluation. 12 | % 13 | % For more help on design of priors, try "help priorDistributions". 14 | % 15 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 16 | % 17 | % See also PRIORDISTRIBUTIONS.M. 18 | 19 | if nargin<2, error('specification of w and priors required'), end 20 | n = numel(w); w = w(:); 21 | if abs(sum(w)-1)>1e-9, error('w needs to sum up to 1'), end 22 | if any(w<0), error('w needs to be nonnegative'), end 23 | for i=1:n, if ~iscell(priors{i}), priors{i} = {priors{i}}; end, end % sample 24 | if nargin<3, i = 1+nnz(rand>cumsum(w)); lp = feval(priors{i}{:}); return, end 25 | 26 | lpi = zeros(numel(x),n); dlpi = zeros(numel(x),n); 27 | for i=1:n, [lpi(:,i),dlpi(:,i)] = feval(priors{i}{:},x(:)); end 28 | 29 | mx = max(lpi,[],2); lp = log(exp(lpi-mx*ones(1,n))*w) + mx; 30 | pi = exp(lpi); dlp = ((pi.*dlpi)*w)./(pi*w); -------------------------------------------------------------------------------- /gpml/doc/demoGrid.m: -------------------------------------------------------------------------------- 1 | disp('See http://www.gaussianprocess.org/gpml/code/matlab/doc/ for details.') 2 | disp('Hit any key to continue...'); pause 3 | 4 | clear all, close all 5 | write_fig = 0; 6 | 7 | xg = {linspace(-2,2,120)',linspace(-3,3,120)'}; % construct a grid 8 | [xe,ng] = covGrid('expand',xg); 9 | y = sin(xe(:,2)) + xe(:,1); x = (1:prod(ng))'; % generate training data 10 | xgs = {linspace(-4,4,100)',linspace(-6,6,110)'}; 11 | y = y + 0.01*gpml_randn(1,size(y)); % add some noise 12 | [xs,ns] = covGrid('expand',xgs); 13 | 14 | cov = {{@covSEiso},{@covSEiso}}; covg = {@covGrid,cov,xg}; % set up a GP 15 | hyp.cov = zeros(4,1); hyp.mean = []; hyp.lik = log(0.1); 16 | hyp = minimize(hyp,@gp,-50,@infGrid,[],covg,[],x,y); % optimise hyperparameters 17 | opt.cg_maxit = 200; opt.cg_tol = 5e-3; % perform inference and prediction 18 | post = infGrid(hyp,{@meanZero},covg,'likGauss',x,y,opt); post.L = @(a) a; 19 | ym = gp(hyp, @infGrid,[],covg,[],x,post,xs); 20 | 21 | cl = [min(ym(:)), max(ym(:))]; 22 | subplot(121), imagesc(xg{1}, xg{2}, reshape(y, ng'),cl), title('data y') 23 | grid on, set(gca,'xtick',-2:2), set(gca,'ytick',-3:3) 24 | subplot(122), imagesc(xgs{1},xgs{2},reshape(ym,ns'),cl), title('prediction ym') 25 | grid on, set(gca,'xtick',-4:4), set(gca,'ytick',-6:6) 26 | rectangle('Position',[-2,-3,4,6]) 27 | if write_fig, print -depsc f9.eps; end -------------------------------------------------------------------------------- /gpml/util/lbfgsb/matlabscalar.h: -------------------------------------------------------------------------------- 1 | #ifndef INCLUDE_MATLABSCALAR 2 | #define INCLUDE_MATLABSCALAR 3 | 4 | #include "mex.h" 5 | 6 | // Class MatlabScalar 7 | // ----------------------------------------------------------------- 8 | // The main appeal of this class is that one can create a scalar 9 | // object that accesses a MATLAB array. 10 | // 11 | // Note that the copy assignment operator is not valid for this class 12 | // because we cannot reassign a reference. 13 | class MatlabScalar { 14 | public: 15 | 16 | // This constructor accepts as input a pointer to a Matlab array 17 | // which must be a scalar in double precision. 18 | explicit MatlabScalar (const mxArray* ptr); 19 | 20 | // This constructor creates a new Matlab array which is a scalar 21 | // in double precision. 22 | MatlabScalar (mxArray*& ptr, double value); 23 | 24 | // The copy constructor. 25 | MatlabScalar (MatlabScalar& source); 26 | 27 | // The destructor. 28 | ~MatlabScalar() { }; 29 | 30 | // Access the value of the scalar. 31 | operator const double () const { return x; }; 32 | 33 | // Assign the value of the scalar. 34 | MatlabScalar& operator= (double value); 35 | 36 | protected: 37 | double& x; 38 | 39 | // The copy assignment operator is kept protected because it is 40 | // invalid. 41 | MatlabScalar& operator= (const MatlabScalar& source) { return *this; }; 42 | }; 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /bayesopt/gpml/doc/demoGrid.m: -------------------------------------------------------------------------------- 1 | disp('See http://www.gaussianprocess.org/gpml/code/matlab/doc/ for details.') 2 | disp('Hit any key to continue...'); pause 3 | 4 | clear all, close all 5 | write_fig = 0; 6 | 7 | xg = {linspace(-2,2,120)',linspace(-3,3,120)'}; % construct a grid 8 | [xe,ng] = covGrid('expand',xg); 9 | y = sin(xe(:,2)) + xe(:,1); x = (1:prod(ng))'; % generate training data 10 | xgs = {linspace(-4,4,100)',linspace(-6,6,110)'}; 11 | y = y + 0.01*gpml_randn(1,size(y)); % add some noise 12 | [xs,ns] = covGrid('expand',xgs); 13 | 14 | cov = {{@covSEiso},{@covSEiso}}; covg = {@covGrid,cov,xg}; % set up a GP 15 | hyp.cov = zeros(4,1); hyp.mean = []; hyp.lik = log(0.1); 16 | hyp = minimize(hyp,@gp,-50,@infGrid,[],covg,[],x,y); % optimise hyperparameters 17 | opt.cg_maxit = 200; opt.cg_tol = 5e-3; % perform inference and prediction 18 | post = infGrid(hyp,{@meanZero},covg,'likGauss',x,y,opt); post.L = @(a) a; 19 | ym = gp(hyp, @infGrid,[],covg,[],x,post,xs); 20 | 21 | cl = [min(ym(:)), max(ym(:))]; 22 | subplot(121), imagesc(xg{1}, xg{2}, reshape(y, ng'),cl), title('data y') 23 | grid on, set(gca,'xtick',-2:2), set(gca,'ytick',-3:3) 24 | subplot(122), imagesc(xgs{1},xgs{2},reshape(ym,ns'),cl), title('prediction ym') 25 | grid on, set(gca,'xtick',-4:4), set(gca,'ytick',-6:6) 26 | rectangle('Position',[-2,-3,4,6]) 27 | if write_fig, print -depsc f9.eps; end -------------------------------------------------------------------------------- /bayesopt/gpml/util/lbfgsb/matlabscalar.h: -------------------------------------------------------------------------------- 1 | #ifndef INCLUDE_MATLABSCALAR 2 | #define INCLUDE_MATLABSCALAR 3 | 4 | #include "mex.h" 5 | 6 | // Class MatlabScalar 7 | // ----------------------------------------------------------------- 8 | // The main appeal of this class is that one can create a scalar 9 | // object that accesses a MATLAB array. 10 | // 11 | // Note that the copy assignment operator is not valid for this class 12 | // because we cannot reassign a reference. 13 | class MatlabScalar { 14 | public: 15 | 16 | // This constructor accepts as input a pointer to a Matlab array 17 | // which must be a scalar in double precision. 18 | explicit MatlabScalar (const mxArray* ptr); 19 | 20 | // This constructor creates a new Matlab array which is a scalar 21 | // in double precision. 22 | MatlabScalar (mxArray*& ptr, double value); 23 | 24 | // The copy constructor. 25 | MatlabScalar (MatlabScalar& source); 26 | 27 | // The destructor. 28 | ~MatlabScalar() { }; 29 | 30 | // Access the value of the scalar. 31 | operator const double () const { return x; }; 32 | 33 | // Assign the value of the scalar. 34 | MatlabScalar& operator= (double value); 35 | 36 | protected: 37 | double& x; 38 | 39 | // The copy assignment operator is kept protected because it is 40 | // invalid. 41 | MatlabScalar& operator= (const MatlabScalar& source) { return *this; }; 42 | }; 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /gpml/mean/meanPoly.m: -------------------------------------------------------------------------------- 1 | function A = meanPoly(d, hyp, x, i) 2 | 3 | % meanPoly - compose a mean function as a polynomial. 4 | % 5 | % The degree d has to be a strictly positive integer. 6 | % 7 | % m(x) = sum_i=1..D sum_j=1..d a_ij * x_i^j 8 | % 9 | % The hyperparameter is: 10 | % 11 | % hyp = [ a_11 12 | % a_21 13 | % .. 14 | % a_D1 15 | % a_12 16 | % a_22 17 | % .. 18 | % a_Dd] 19 | % 20 | % This function doesn't actually compute very much on its own, it merely does 21 | % some bookkeeping, and calls other mean function to do the actual work. 22 | % 23 | % Copyright (c) by Hannes Nickisch 2013-11-02. 24 | % 25 | % See also MEANFUNCTIONS.M. 26 | 27 | d = max(abs(floor(d)),1); % positive integer degree 28 | if nargin<3, A = ['D*',int2str(d)]; return; end % report number of hyperparams 29 | 30 | [n,D] = size(x); 31 | a = reshape(hyp,D,d); 32 | 33 | A = zeros(n,1); % allocate memory 34 | if nargin==3 % compute mean vector 35 | for i=1:d, A = A + (x.^i)*a(:,i); end % evaluate mean 36 | else % compute derivative vector 37 | if i<=d*D 38 | j = mod(i-1,D)+1; % which dimension 39 | A = x(:,j).^((i-j)/D+1); % derivative 40 | else 41 | A = zeros(n,1); 42 | end 43 | end -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanPoly.m: -------------------------------------------------------------------------------- 1 | function A = meanPoly(d, hyp, x, i) 2 | 3 | % meanPoly - compose a mean function as a polynomial. 4 | % 5 | % The degree d has to be a strictly positive integer. 6 | % 7 | % m(x) = sum_i=1..D sum_j=1..d a_ij * x_i^j 8 | % 9 | % The hyperparameter is: 10 | % 11 | % hyp = [ a_11 12 | % a_21 13 | % .. 14 | % a_D1 15 | % a_12 16 | % a_22 17 | % .. 18 | % a_Dd] 19 | % 20 | % This function doesn't actually compute very much on its own, it merely does 21 | % some bookkeeping, and calls other mean function to do the actual work. 22 | % 23 | % Copyright (c) by Hannes Nickisch 2013-11-02. 24 | % 25 | % See also MEANFUNCTIONS.M. 26 | 27 | d = max(abs(floor(d)),1); % positive integer degree 28 | if nargin<3, A = ['D*',int2str(d)]; return; end % report number of hyperparams 29 | 30 | [n,D] = size(x); 31 | a = reshape(hyp,D,d); 32 | 33 | A = zeros(n,1); % allocate memory 34 | if nargin==3 % compute mean vector 35 | for i=1:d, A = A + (x.^i)*a(:,i); end % evaluate mean 36 | else % compute derivative vector 37 | if i<=d*D 38 | j = mod(i-1,D)+1; % which dimension 39 | A = x(:,j).^((i-j)/D+1); % derivative 40 | else 41 | A = zeros(n,1); 42 | end 43 | end -------------------------------------------------------------------------------- /gpml/cov/covLINone.m: -------------------------------------------------------------------------------- 1 | function K = covLINone(hyp, x, z, i) 2 | 3 | % Linear covariance function with a single hyperparameter. The covariance 4 | % function is parameterized as: 5 | % 6 | % k(x^p,x^q) = (x^p'*x^q + 1)/t^2; 7 | % 8 | % where the P matrix is t2 times the unit matrix. The second term plays the 9 | % role of the bias. The hyperparameter is: 10 | % 11 | % hyp = [ log(t) ] 12 | % 13 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 14 | % 15 | % See also COVFUNCTIONS.M. 16 | 17 | if nargin<2, K = '1'; return; end % report number of parameters 18 | if nargin<3, z = []; end % make sure, z exists 19 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 20 | 21 | it2 = exp(-2*hyp); % t2 inverse 22 | 23 | % precompute inner products 24 | if dg % vector kxx 25 | K = sum(x.*x,2); 26 | else 27 | if xeqz % symmetric matrix Kxx 28 | K = x*x'; 29 | else % cross covariances Kxz 30 | K = x*z'; 31 | end 32 | end 33 | 34 | if nargin<4 % covariances 35 | K = it2*(1+K); 36 | else % derivatives 37 | if i==1 38 | K = -2*it2*(1+K); 39 | else 40 | error('Unknown hyperparameter') 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /gpml/lik/likExp.m: -------------------------------------------------------------------------------- 1 | function [varargout] = likExp(link, hyp, varargin) 2 | 3 | % likExp - Exponential likelihood function for strictly positive data y. The 4 | % expression for the likelihood is 5 | % likExp(f) = exp(-y/mu) / mu with 6 | % mean=mu and variance=mu^2 where mu = g(f) is the intensity, f is a 7 | % Gaussian process, y is the strictly positive data. Hence, we have 8 | % llik(f) = log(likExp(f)) = -y/g(f) - log(g(f)). 9 | % 10 | % Internally, the function is in fact a wrapper around likGamma with hyper 11 | % parameter al=1. 12 | % 13 | % We provide two inverse link functions 'exp' and 'logistic': 14 | % g(f) = exp(f) and g(f) = log(1+exp(f))). 15 | % The link functions are located at util/glm_invlink_*.m. 16 | % Note that for the 'exp' intensity the likelihood lik(f) is log concave. 17 | % 18 | % The hyperparameters are: 19 | % 20 | % hyp = [ ] 21 | % 22 | % Several modes are provided, for computing likelihoods, derivatives and moments 23 | % respectively, see likFunctions.m for the details. In general, care is taken 24 | % to avoid numerical issues when the arguments are extreme. 25 | % 26 | % See also LIKFUNCTIONS.M, LIKGAMMA.M. 27 | % 28 | % Copyright (c) by Hannes Nickisch, 2013-10-29. 29 | 30 | if nargin<4, varargout = {'0'}; return; end % report number of hyperparameters 31 | varargout = cell(nargout, 1); % allocate the right number of output arguments 32 | for j=1:nargout, varargout{j} = []; end % derivative output 33 | if nargin<=6, [varargout{:}] = likGamma(link,0,varargin{:}); end % log(al) = 0 -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covLINone.m: -------------------------------------------------------------------------------- 1 | function K = covLINone(hyp, x, z, i) 2 | 3 | % Linear covariance function with a single hyperparameter. The covariance 4 | % function is parameterized as: 5 | % 6 | % k(x^p,x^q) = (x^p'*x^q + 1)/t^2; 7 | % 8 | % where the P matrix is t2 times the unit matrix. The second term plays the 9 | % role of the bias. The hyperparameter is: 10 | % 11 | % hyp = [ log(t) ] 12 | % 13 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 14 | % 15 | % See also COVFUNCTIONS.M. 16 | 17 | if nargin<2, K = '1'; return; end % report number of parameters 18 | if nargin<3, z = []; end % make sure, z exists 19 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 20 | 21 | it2 = exp(-2*hyp); % t2 inverse 22 | 23 | % precompute inner products 24 | if dg % vector kxx 25 | K = sum(x.*x,2); 26 | else 27 | if xeqz % symmetric matrix Kxx 28 | K = x*x'; 29 | else % cross covariances Kxz 30 | K = x*z'; 31 | end 32 | end 33 | 34 | if nargin<4 % covariances 35 | K = it2*(1+K); 36 | else % derivatives 37 | if i==1 38 | K = -2*it2*(1+K); 39 | else 40 | error('Unknown hyperparameter') 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /bayesopt/gpml/lik/likExp.m: -------------------------------------------------------------------------------- 1 | function [varargout] = likExp(link, hyp, varargin) 2 | 3 | % likExp - Exponential likelihood function for strictly positive data y. The 4 | % expression for the likelihood is 5 | % likExp(f) = exp(-y/mu) / mu with 6 | % mean=mu and variance=mu^2 where mu = g(f) is the intensity, f is a 7 | % Gaussian process, y is the strictly positive data. Hence, we have 8 | % llik(f) = log(likExp(f)) = -y/g(f) - log(g(f)). 9 | % 10 | % Internally, the function is in fact a wrapper around likGamma with hyper 11 | % parameter al=1. 12 | % 13 | % We provide two inverse link functions 'exp' and 'logistic': 14 | % g(f) = exp(f) and g(f) = log(1+exp(f))). 15 | % The link functions are located at util/glm_invlink_*.m. 16 | % Note that for the 'exp' intensity the likelihood lik(f) is log concave. 17 | % 18 | % The hyperparameters are: 19 | % 20 | % hyp = [ ] 21 | % 22 | % Several modes are provided, for computing likelihoods, derivatives and moments 23 | % respectively, see likFunctions.m for the details. In general, care is taken 24 | % to avoid numerical issues when the arguments are extreme. 25 | % 26 | % See also LIKFUNCTIONS.M, LIKGAMMA.M. 27 | % 28 | % Copyright (c) by Hannes Nickisch, 2013-10-29. 29 | 30 | if nargin<4, varargout = {'0'}; return; end % report number of hyperparameters 31 | varargout = cell(nargout, 1); % allocate the right number of output arguments 32 | for j=1:nargout, varargout{j} = []; end % derivative output 33 | if nargin<=6, [varargout{:}] = likGamma(link,0,varargin{:}); end % log(al) = 0 -------------------------------------------------------------------------------- /FMINSEARCHBND/demo/fminsearchbnd_demo.m: -------------------------------------------------------------------------------- 1 | %% Optimization of a simple (Rosenbrock) function, with no constraints 2 | rosen = @(x) (1-x(1)).^2 + 105*(x(2)-x(1).^2).^2; 3 | 4 | % With no constraints, operation simply passes through 5 | % directly to fminsearch. The solution should be [1 1] 6 | xsol = fminsearchbnd(rosen,[3 3]) 7 | 8 | %% Only lower bound constraints 9 | xsol = fminsearchbnd(rosen,[3 3],[2 2]) 10 | 11 | %% Only upper bound constraints 12 | xsol = fminsearchbnd(rosen,[-5 -5],[],[0 0]) 13 | 14 | %% Dual constraints 15 | xsol = fminsearchbnd(rosen,[2.5 2.5],[2 2],[3 3]) 16 | 17 | %% Mixed constraints 18 | xsol = fminsearchbnd(rosen,[0 0],[2 -inf],[inf 3]) 19 | 20 | %% Provide your own fminsearch options 21 | opts = optimset('fminsearch'); 22 | opts.Display = 'iter'; 23 | opts.TolX = 1.e-12; 24 | opts.MaxFunEvals = 100; 25 | 26 | n = [10,5]; 27 | H = randn(n); 28 | H=H'*H; 29 | Quadraticfun = @(x) x*H*x'; 30 | 31 | % Global minimizer is at [0 0 0 0 0]. 32 | % Set all lower bound constraints, all of which will 33 | % be active in this test. 34 | LB = [.5 .5 .5 .5 .5]; 35 | xsol = fminsearchbnd(Quadraticfun,[1 2 3 4 5],LB,[],opts) 36 | 37 | %% Exactly fix one variable, constrain some others, and set a tolerance 38 | opts = optimset('fminsearch'); 39 | opts.TolFun = 1.e-12; 40 | 41 | LB = [-inf 2 1 -10]; 42 | UB = [ inf inf 1 inf]; 43 | xsol = fminsearchbnd(@(x) norm(x),[1 3 1 1],LB,UB,opts) 44 | 45 | %% All the standard outputs from fminsearch are still returned 46 | [xsol,fval,exitflag,output] = fminsearchbnd(@(x) norm(x),[1 3 1 1],LB,UB) 47 | 48 | -------------------------------------------------------------------------------- /gpml/prior/priorGamma.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorGamma(k,t,x) 2 | 3 | % Univariate Gamma hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(x/t)/gamma(k)*x^(k-1)/t^k 8 | % 9 | % where k(1x1) is the shape parameter, t(1x1) is the scale parameter 10 | % and x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % Sampling is done using the algorithm from p. 53 of the paper Generating gamma 13 | % variates by a modified rejection technique by J.H. Ahrens and U. Dieter, 1982, 14 | % ACM, 25, 47–54. 15 | % 16 | % For more help on design of priors, try "help priorDistributions". 17 | % 18 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 19 | % 20 | % See also PRIORDISTRIBUTIONS.M. 21 | 22 | if nargin<2, error('k and t parameters need to be provided'), end 23 | if ~(isscalar(k)&&isscalar(t)) 24 | error('k and t parameters need to be scalars'), end 25 | if nargin<3 26 | m = 1; 27 | d = k-floor(k); % fractional part 28 | v0 = exp(1)/(exp(1)+d); 29 | while true 30 | v = rand(3,1); 31 | if v(1)<=v0 32 | r = v(2)^(1/d); s = v(3)*r^(d-1); 33 | else 34 | r = 1-log(v(2)); s = v(3)*exp(-r); 35 | end 36 | if s<=r^(d-1)*exp(-r), break, end 37 | m = m+1; 38 | end 39 | u = rand(floor(k),1); 40 | lp = t*(r-sum(log(u))); 41 | return 42 | end 43 | 44 | lx = log(x); 45 | lp = -gammaln(k) - k*log(t) + (k-1)*lx - x/t; 46 | dlp = (k-1)./x - 1/t; 47 | lp(x<0) = -inf; dlp(x<0) = 0; -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorGamma.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorGamma(k,t,x) 2 | 3 | % Univariate Gamma hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(x/t)/gamma(k)*x^(k-1)/t^k 8 | % 9 | % where k(1x1) is the shape parameter, t(1x1) is the scale parameter 10 | % and x(1xN) contains query hyperparameters for prior evaluation. 11 | % 12 | % Sampling is done using the algorithm from p. 53 of the paper Generating gamma 13 | % variates by a modified rejection technique by J.H. Ahrens and U. Dieter, 1982, 14 | % ACM, 25, 47–54. 15 | % 16 | % For more help on design of priors, try "help priorDistributions". 17 | % 18 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-08. 19 | % 20 | % See also PRIORDISTRIBUTIONS.M. 21 | 22 | if nargin<2, error('k and t parameters need to be provided'), end 23 | if ~(isscalar(k)&&isscalar(t)) 24 | error('k and t parameters need to be scalars'), end 25 | if nargin<3 26 | m = 1; 27 | d = k-floor(k); % fractional part 28 | v0 = exp(1)/(exp(1)+d); 29 | while true 30 | v = rand(3,1); 31 | if v(1)<=v0 32 | r = v(2)^(1/d); s = v(3)*r^(d-1); 33 | else 34 | r = 1-log(v(2)); s = v(3)*exp(-r); 35 | end 36 | if s<=r^(d-1)*exp(-r), break, end 37 | m = m+1; 38 | end 39 | u = rand(floor(k),1); 40 | lp = t*(r-sum(log(u))); 41 | return 42 | end 43 | 44 | lx = log(x); 45 | lp = -gammaln(k) - k*log(t) + (k-1)*lx - x/t; 46 | dlp = (k-1)./x - 1/t; 47 | lp(x<0) = -inf; dlp(x<0) = 0; -------------------------------------------------------------------------------- /gpml/cov/covEye.m: -------------------------------------------------------------------------------- 1 | function K = covEye(hyp, x, z, i) 2 | 3 | % Independent covariance function, i.e. "white noise", with unit variance. 4 | % The covariance function is specified as: 5 | % 6 | % k(x^p,x^q) = \delta(p,q) 7 | % 8 | % \delta(p,q) is a Kronecker delta function which is 1 iff p=q and zero 9 | % otherwise in mode 1). 10 | % In cross covariance mode 2) two data points x_p and z_q are considered equal 11 | % if their difference norm |x_p-z_q| is less than eps, the machine precision. 12 | % The hyperparameters are: 13 | % 14 | % hyp = [ ] 15 | % 16 | % For more help on design of covariance functions, try "help covFunctions". 17 | % 18 | % Copyright (c) by Hannes Nickisch, 2014-09-05. 19 | % 20 | % See also COVFUNCTIONS.M. 21 | 22 | tol = eps; % threshold on the norm when two vectors are considered to be equal 23 | if nargin<2, K = '0'; return; end % report number of parameters 24 | if nargin<3, z = []; end % make sure, z exists 25 | dg = strcmp(z,'diag'); % determine mode 26 | 27 | n = size(x,1); 28 | 29 | % precompute raw 30 | if dg % vector kxx 31 | K = ones(n,1); 32 | else 33 | if isempty(z) % symmetric matrix Kxx 34 | K = eye(n); 35 | else % cross covariances Kxz 36 | K = double(sq_dist(x',z')3 % derivatives 38 | if i==1 39 | if dg 40 | K = -2*sum(x.*x,2); 41 | else 42 | if xeqz 43 | K = -2*x*x'; 44 | else 45 | K = -2*x*z'; 46 | end 47 | end 48 | else 49 | error('Unknown hyperparameter') 50 | end 51 | end -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covLINiso.m: -------------------------------------------------------------------------------- 1 | function K = covLINiso(hyp, x, z, i) 2 | 3 | % Linear covariance function with Automatic Relevance Determination (ARD). The 4 | % covariance function is parameterized as: 5 | % 6 | % k(x^p,x^q) = x^p'*inv(P)*x^q 7 | % 8 | % where the P matrix is ell^2 times the unit matrix. The hyperparameters are: 9 | % 10 | % hyp = [ log(ell) ] 11 | % 12 | % Note that there is no bias term; use covConst to add a bias. 13 | % 14 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2013-10-13. 15 | % 16 | % See also COVFUNCTIONS.M. 17 | 18 | if nargin<2, K = '1'; return; end % report number of parameters 19 | if nargin<3, z = []; end % make sure, z exists 20 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 21 | 22 | ell = exp(hyp(1)); 23 | x = x/ell; 24 | 25 | % precompute inner products 26 | if dg % vector kxx 27 | K = sum(x.*x,2); 28 | else 29 | if xeqz % symmetric matrix Kxx 30 | K = x*x'; 31 | else % cross covariances Kxz 32 | z = z/ell; 33 | K = x*z'; 34 | end 35 | end 36 | 37 | if nargin>3 % derivatives 38 | if i==1 39 | if dg 40 | K = -2*sum(x.*x,2); 41 | else 42 | if xeqz 43 | K = -2*x*x'; 44 | else 45 | K = -2*x*z'; 46 | end 47 | end 48 | else 49 | error('Unknown hyperparameter') 50 | end 51 | end -------------------------------------------------------------------------------- /gpml/cov/covCos.m: -------------------------------------------------------------------------------- 1 | function K = covCos(hyp, x, z, i) 2 | 3 | % Stationary covariance function for a sinusoid with period p in 1d: 4 | % 5 | % k(x,z) = sf^2*cos(2*pi*(x-z)/p) 6 | % 7 | % where the hyperparameters are: 8 | % 9 | % hyp = [ log(p) 10 | % log(sf) ] 11 | % 12 | % Note that covPeriodicNoDC converges to covCos as ell goes to infinity. 13 | % 14 | % Copyright (c) by James Robert Lloyd, 2013-08-05. 15 | % 16 | % See also COVFUNCTIONS.M, COVPERIODICNODC.M. 17 | 18 | if nargin<2, K = '2'; return; end % report number of parameters 19 | if nargin<3, z = []; end % make sure, z exists 20 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 21 | 22 | [n,D] = size(x); 23 | if D>1, error('Covariance is defined for 1d data only.'), end 24 | p = exp(hyp(1)); 25 | sf2 = exp(2*hyp(2)); 26 | 27 | % precompute distances 28 | if dg % vector kxx 29 | K = zeros(size(x,1),1); 30 | else 31 | if xeqz % symmetric matrix Kxx 32 | K = repmat(x,1,n) - repmat(x',n,1); 33 | else % cross covariances Kxz 34 | K = repmat(x,1,size(z,1)) - repmat(z',n,1); 35 | end 36 | end 37 | 38 | K = 2*pi*K/p; 39 | if nargin<4 % covariances 40 | K = sf2*cos(K); 41 | else % derivatives 42 | if i==1 43 | K = sf2*sin(K).*K; 44 | elseif i==2 45 | K = 2*sf2*cos(K); 46 | else 47 | error('Unknown hyperparameter') 48 | end 49 | end -------------------------------------------------------------------------------- /gpml/cov/covPoly.m: -------------------------------------------------------------------------------- 1 | function K = covPoly(d, hyp, x, z, i) 2 | 3 | % Polynomial covariance function. The covariance function is parameterized as: 4 | % 5 | % k(x^p,x^q) = sf^2 * ( c + (x^p)'*(x^q) )^d 6 | % 7 | % The hyperparameters are: 8 | % 9 | % hyp = [ log(c) 10 | % log(sf) ] 11 | % 12 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 13 | % 14 | % See also COVFUNCTIONS.M. 15 | 16 | if nargin<3, K = '2'; return; end % report number of parameters 17 | if nargin<4, z = []; end % make sure, z exists 18 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 19 | 20 | c = exp(hyp(1)); % inhomogeneous offset 21 | sf2 = exp(2*hyp(2)); % signal variance 22 | if d~=max(1,fix(d)), error('only nonzero integers allowed for d'), end % degree 23 | 24 | % precompute inner products 25 | if dg % vector kxx 26 | K = sum(x.*x,2); 27 | else 28 | if xeqz % symmetric matrix Kxx 29 | K = x*x'; 30 | else % cross covariances Kxz 31 | K = x*z'; 32 | end 33 | end 34 | 35 | if nargin<5 % covariances 36 | K = sf2*( c + K ).^d; 37 | else % derivatives 38 | if i==1 39 | K = c*d*sf2*( c + K ).^(d-1); 40 | elseif i==2 41 | K = 2*sf2*( c + K ).^d; 42 | else 43 | error('Unknown hyperparameter') 44 | end 45 | end -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covCos.m: -------------------------------------------------------------------------------- 1 | function K = covCos(hyp, x, z, i) 2 | 3 | % Stationary covariance function for a sinusoid with period p in 1d: 4 | % 5 | % k(x,z) = sf^2*cos(2*pi*(x-z)/p) 6 | % 7 | % where the hyperparameters are: 8 | % 9 | % hyp = [ log(p) 10 | % log(sf) ] 11 | % 12 | % Note that covPeriodicNoDC converges to covCos as ell goes to infinity. 13 | % 14 | % Copyright (c) by James Robert Lloyd, 2013-08-05. 15 | % 16 | % See also COVFUNCTIONS.M, COVPERIODICNODC.M. 17 | 18 | if nargin<2, K = '2'; return; end % report number of parameters 19 | if nargin<3, z = []; end % make sure, z exists 20 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 21 | 22 | [n,D] = size(x); 23 | if D>1, error('Covariance is defined for 1d data only.'), end 24 | p = exp(hyp(1)); 25 | sf2 = exp(2*hyp(2)); 26 | 27 | % precompute distances 28 | if dg % vector kxx 29 | K = zeros(size(x,1),1); 30 | else 31 | if xeqz % symmetric matrix Kxx 32 | K = repmat(x,1,n) - repmat(x',n,1); 33 | else % cross covariances Kxz 34 | K = repmat(x,1,size(z,1)) - repmat(z',n,1); 35 | end 36 | end 37 | 38 | K = 2*pi*K/p; 39 | if nargin<4 % covariances 40 | K = sf2*cos(K); 41 | else % derivatives 42 | if i==1 43 | K = sf2*sin(K).*K; 44 | elseif i==2 45 | K = 2*sf2*cos(K); 46 | else 47 | error('Unknown hyperparameter') 48 | end 49 | end -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covPoly.m: -------------------------------------------------------------------------------- 1 | function K = covPoly(d, hyp, x, z, i) 2 | 3 | % Polynomial covariance function. The covariance function is parameterized as: 4 | % 5 | % k(x^p,x^q) = sf^2 * ( c + (x^p)'*(x^q) )^d 6 | % 7 | % The hyperparameters are: 8 | % 9 | % hyp = [ log(c) 10 | % log(sf) ] 11 | % 12 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 13 | % 14 | % See also COVFUNCTIONS.M. 15 | 16 | if nargin<3, K = '2'; return; end % report number of parameters 17 | if nargin<4, z = []; end % make sure, z exists 18 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 19 | 20 | c = exp(hyp(1)); % inhomogeneous offset 21 | sf2 = exp(2*hyp(2)); % signal variance 22 | if d~=max(1,fix(d)), error('only nonzero integers allowed for d'), end % degree 23 | 24 | % precompute inner products 25 | if dg % vector kxx 26 | K = sum(x.*x,2); 27 | else 28 | if xeqz % symmetric matrix Kxx 29 | K = x*x'; 30 | else % cross covariances Kxz 31 | K = x*z'; 32 | end 33 | end 34 | 35 | if nargin<5 % covariances 36 | K = sf2*( c + K ).^d; 37 | else % derivatives 38 | if i==1 39 | K = c*d*sf2*( c + K ).^(d-1); 40 | elseif i==2 41 | K = 2*sf2*( c + K ).^d; 42 | else 43 | error('Unknown hyperparameter') 44 | end 45 | end -------------------------------------------------------------------------------- /gpml/cov/covSEisoU.m: -------------------------------------------------------------------------------- 1 | function K = covSEisoU(hyp, x, z, i) 2 | 3 | % Squared Exponential covariance function with isotropic distance measure with 4 | % unit magnitude. The covariance function is parameterized as: 5 | % 6 | % k(x^p,x^q) = exp(-(x^p - x^q)'*inv(P)*(x^p - x^q)/2) 7 | % 8 | % where the P matrix is ell^2 times the unit matrix and sf2 is the signal 9 | % variance. The hyperparameters are: 10 | % 11 | % hyp = [ log(ell) ] 12 | % 13 | % For more help on design of covariance functions, try "help covFunctions". 14 | % 15 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 16 | % 17 | % See also COVFUNCTIONS.M. 18 | 19 | if nargin<2, K = '1'; return; end % report number of parameters 20 | if nargin<3, z = []; end % make sure, z exists 21 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 22 | 23 | ell = exp(hyp(1)); % characteristic length scale 24 | 25 | % precompute squared distances 26 | if dg % vector kxx 27 | K = zeros(size(x,1),1); 28 | else 29 | if xeqz % symmetric matrix Kxx 30 | K = sq_dist(x'/ell); 31 | else % cross covariances Kxz 32 | K = sq_dist(x'/ell,z'/ell); 33 | end 34 | end 35 | 36 | if nargin<4 % covariances 37 | K = exp(-K/2); 38 | else % derivatives 39 | if i==1 40 | K = exp(-K/2).*K; 41 | else 42 | error('Unknown hyperparameter') 43 | end 44 | end -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covSEisoU.m: -------------------------------------------------------------------------------- 1 | function K = covSEisoU(hyp, x, z, i) 2 | 3 | % Squared Exponential covariance function with isotropic distance measure with 4 | % unit magnitude. The covariance function is parameterized as: 5 | % 6 | % k(x^p,x^q) = exp(-(x^p - x^q)'*inv(P)*(x^p - x^q)/2) 7 | % 8 | % where the P matrix is ell^2 times the unit matrix and sf2 is the signal 9 | % variance. The hyperparameters are: 10 | % 11 | % hyp = [ log(ell) ] 12 | % 13 | % For more help on design of covariance functions, try "help covFunctions". 14 | % 15 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 16 | % 17 | % See also COVFUNCTIONS.M. 18 | 19 | if nargin<2, K = '1'; return; end % report number of parameters 20 | if nargin<3, z = []; end % make sure, z exists 21 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 22 | 23 | ell = exp(hyp(1)); % characteristic length scale 24 | 25 | % precompute squared distances 26 | if dg % vector kxx 27 | K = zeros(size(x,1),1); 28 | else 29 | if xeqz % symmetric matrix Kxx 30 | K = sq_dist(x'/ell); 31 | else % cross covariances Kxz 32 | K = sq_dist(x'/ell,z'/ell); 33 | end 34 | end 35 | 36 | if nargin<4 % covariances 37 | K = exp(-K/2); 38 | else % derivatives 39 | if i==1 40 | K = exp(-K/2).*K; 41 | else 42 | error('Unknown hyperparameter') 43 | end 44 | end -------------------------------------------------------------------------------- /gpml/util/lik_epquad.m: -------------------------------------------------------------------------------- 1 | % Compute infEP part of a likelihood function based on the infLaplace part using 2 | % Gaussian-Hermite quadrature. 3 | % 4 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 5 | % and likInvGauss. 6 | % 7 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 8 | 9 | function varargout = lik_epquad(lik,hyp,y,mu,s2) 10 | n = max([length(y),length(mu),length(s2)]); on = ones(n,1); 11 | N = 20; [t,w] = gauher(N); oN = ones(1,N); lw = ones(n,1)*log(w'); 12 | y = y(:).*on; mu = mu(:).*on; sig = sqrt(s2(:)).*on; % vectors only 13 | [lpi,dlpi,d2lpi] = feval(lik{:},hyp,y*oN,sig*t'+mu*oN,[],'infLaplace'); 14 | lZ = s(lpi+lw); 15 | dlZ = {}; d2lZ = {}; 16 | if nargout>1 % 1st derivative wrt mean 17 | % Using p*dlp=dp, p=exp(lp), Z=sum_i wi*pi, dZ = sum_i wi*dpi we obtain 18 | % dlZ = sum_i exp(lpi-lZ+lwi)*dlpi = sum_i ai*dlpi. 19 | a = exp(lpi - lZ*oN + lw); 20 | dlZ = sum(a.*dlpi,2); 21 | if nargout>2 % 2nd derivative wrt mean 22 | % Using d2lZ=(d2Z*Z-dZ^2)/Z^2 <=> d2Z=Z*(d2lZ+dlZ^2) and 23 | % d2Z = sum_i wi*d2Zi, we get d2lZ = sum_i ai*(d2lpi+dlpi^2)-dlZ^2. 24 | d2lZ = sum(a.*(d2lpi+dlpi.*dlpi),2) - dlZ.*dlZ; 25 | end 26 | end 27 | varargout = {lZ,dlZ,d2lZ}; 28 | 29 | % computes y = log( sum(exp(x),2) ), the softmax in a numerically safe way by 30 | % subtracting the row maximum to avoid cancelation after taking the exp 31 | % the sum is done along the rows 32 | function [y,x] = s(logx) 33 | N = size(logx,2); max_logx = max(logx,[],2); 34 | % we have all values in the log domain, and want to calculate a sum 35 | x = exp(logx-max_logx*ones(1,N)); 36 | y = log(sum(x,2)) + max_logx; -------------------------------------------------------------------------------- /bayesopt/gpml/util/lik_epquad.m: -------------------------------------------------------------------------------- 1 | % Compute infEP part of a likelihood function based on the infLaplace part using 2 | % Gaussian-Hermite quadrature. 3 | % 4 | % The function is used in GLM likelihoods such as likPoisson, likGamma, likBeta 5 | % and likInvGauss. 6 | % 7 | % Copyright (c) by Hannes Nickisch, 2013-10-16. 8 | 9 | function varargout = lik_epquad(lik,hyp,y,mu,s2) 10 | n = max([length(y),length(mu),length(s2)]); on = ones(n,1); 11 | N = 20; [t,w] = gauher(N); oN = ones(1,N); lw = ones(n,1)*log(w'); 12 | y = y(:).*on; mu = mu(:).*on; sig = sqrt(s2(:)).*on; % vectors only 13 | [lpi,dlpi,d2lpi] = feval(lik{:},hyp,y*oN,sig*t'+mu*oN,[],'infLaplace'); 14 | lZ = s(lpi+lw); 15 | dlZ = {}; d2lZ = {}; 16 | if nargout>1 % 1st derivative wrt mean 17 | % Using p*dlp=dp, p=exp(lp), Z=sum_i wi*pi, dZ = sum_i wi*dpi we obtain 18 | % dlZ = sum_i exp(lpi-lZ+lwi)*dlpi = sum_i ai*dlpi. 19 | a = exp(lpi - lZ*oN + lw); 20 | dlZ = sum(a.*dlpi,2); 21 | if nargout>2 % 2nd derivative wrt mean 22 | % Using d2lZ=(d2Z*Z-dZ^2)/Z^2 <=> d2Z=Z*(d2lZ+dlZ^2) and 23 | % d2Z = sum_i wi*d2Zi, we get d2lZ = sum_i ai*(d2lpi+dlpi^2)-dlZ^2. 24 | d2lZ = sum(a.*(d2lpi+dlpi.*dlpi),2) - dlZ.*dlZ; 25 | end 26 | end 27 | varargout = {lZ,dlZ,d2lZ}; 28 | 29 | % computes y = log( sum(exp(x),2) ), the softmax in a numerically safe way by 30 | % subtracting the row maximum to avoid cancelation after taking the exp 31 | % the sum is done along the rows 32 | function [y,x] = s(logx) 33 | N = size(logx,2); max_logx = max(logx,[],2); 34 | % we have all values in the log domain, and want to calculate a sum 35 | x = exp(logx-max_logx*ones(1,N)); 36 | y = log(sum(x,2)) + max_logx; -------------------------------------------------------------------------------- /gpml/cov/covLINard.m: -------------------------------------------------------------------------------- 1 | function K = covLINard(hyp, x, z, i) 2 | 3 | % Linear covariance function with Automatic Relevance Determination (ARD). The 4 | % covariance function is parameterized as: 5 | % 6 | % k(x^p,x^q) = x^p'*inv(P)*x^q 7 | % 8 | % where the P matrix is diagonal with ARD parameters ell_1^2,...,ell_D^2, where 9 | % D is the dimension of the input space. The hyperparameters are: 10 | % 11 | % hyp = [ log(ell_1) 12 | % log(ell_2) 13 | % .. 14 | % log(ell_D) ] 15 | % 16 | % Note that there is no bias term; use covConst to add a bias. 17 | % 18 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 19 | % 20 | % See also COVFUNCTIONS.M. 21 | 22 | if nargin<2, K = 'D'; return; end % report number of parameters 23 | if nargin<3, z = []; end % make sure, z exists 24 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 25 | 26 | ell = exp(hyp); 27 | [n,D] = size(x); 28 | x = x*diag(1./ell); 29 | 30 | % precompute inner products 31 | if dg % vector kxx 32 | K = sum(x.*x,2); 33 | else 34 | if xeqz % symmetric matrix Kxx 35 | K = x*x'; 36 | else % cross covariances Kxz 37 | z = z*diag(1./ell); 38 | K = x*z'; 39 | end 40 | end 41 | 42 | if nargin>3 % derivatives 43 | if i<=D 44 | if dg 45 | K = -2*x(:,i).*x(:,i); 46 | else 47 | if xeqz 48 | K = -2*x(:,i)*x(:,i)'; 49 | else 50 | K = -2*x(:,i)*z(:,i)'; 51 | end 52 | end 53 | else 54 | error('Unknown hyperparameter') 55 | end 56 | end -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covLINard.m: -------------------------------------------------------------------------------- 1 | function K = covLINard(hyp, x, z, i) 2 | 3 | % Linear covariance function with Automatic Relevance Determination (ARD). The 4 | % covariance function is parameterized as: 5 | % 6 | % k(x^p,x^q) = x^p'*inv(P)*x^q 7 | % 8 | % where the P matrix is diagonal with ARD parameters ell_1^2,...,ell_D^2, where 9 | % D is the dimension of the input space. The hyperparameters are: 10 | % 11 | % hyp = [ log(ell_1) 12 | % log(ell_2) 13 | % .. 14 | % log(ell_D) ] 15 | % 16 | % Note that there is no bias term; use covConst to add a bias. 17 | % 18 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 19 | % 20 | % See also COVFUNCTIONS.M. 21 | 22 | if nargin<2, K = 'D'; return; end % report number of parameters 23 | if nargin<3, z = []; end % make sure, z exists 24 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 25 | 26 | ell = exp(hyp); 27 | [n,D] = size(x); 28 | x = x*diag(1./ell); 29 | 30 | % precompute inner products 31 | if dg % vector kxx 32 | K = sum(x.*x,2); 33 | else 34 | if xeqz % symmetric matrix Kxx 35 | K = x*x'; 36 | else % cross covariances Kxz 37 | z = z*diag(1./ell); 38 | K = x*z'; 39 | end 40 | end 41 | 42 | if nargin>3 % derivatives 43 | if i<=D 44 | if dg 45 | K = -2*x(:,i).*x(:,i); 46 | else 47 | if xeqz 48 | K = -2*x(:,i)*x(:,i)'; 49 | else 50 | K = -2*x(:,i)*z(:,i)'; 51 | end 52 | end 53 | else 54 | error('Unknown hyperparameter') 55 | end 56 | end -------------------------------------------------------------------------------- /FMINSEARCHBND/test/test_main.m: -------------------------------------------------------------------------------- 1 | %% Optimization of a simple (Rosenbrock) function, with no constraints 2 | % The unconstrained solution is at [1,1] 3 | rosen = @(x) (1-x(1)).^2 + 105*(x(2)-x(1).^2).^2; 4 | 5 | % With no constraints, operation simply passes through 6 | % directly to fminsearch. The solution should be [1 1] 7 | xsol = fminsearchbnd(rosen,[3 3]) 8 | 9 | %% Full lower and upper bound constraints which will all be inactive 10 | xsol = fminsearchbnd(rosen,[3 3],[-1 -1],[4 4]) 11 | 12 | %% Only lower bound constraints 13 | xsol = fminsearchbnd(rosen,[3 3],[2 2]) 14 | 15 | %% Only upper bound constraints 16 | xsol = fminsearchbnd(rosen,[-5 -5],[],[0 0]) 17 | 18 | %% Dual constraints 19 | xsol = fminsearchbnd(rosen,[2.5 2.5],[2 2],[3 3]) 20 | 21 | %% Dual constraints, with an infeasible starting guess 22 | xsol = fminsearchbnd(rosen,[0 0],[2 2],[3 3]) 23 | 24 | %% Mixed constraints 25 | xsol = fminsearchbnd(rosen,[0 0],[2 -inf],[inf 3]) 26 | 27 | %% Provide your own fminsearch options 28 | opts = optimset('fminsearch'); 29 | opts.Display = 'iter'; 30 | opts.TolX = 1.e-12; 31 | 32 | n = [10,5]; 33 | H = randn(n); 34 | H=H'*H; 35 | Quadraticfun = @(x) x*H*x'; 36 | 37 | % Global minimizer is at [0 0 0 0 0]. 38 | % Set all lower bound constraints, all of which will 39 | % be active in this test. 40 | LB = [.5 .5 .5 .5 .5]; 41 | xsol = fminsearchbnd(Quadraticfun,[1 2 3 4 5],LB,[],opts) 42 | 43 | %% Exactly fix one variable, constrain some others, and set a tolerance 44 | opts = optimset('fminsearch'); 45 | opts.TolFun = 1.e-12; 46 | 47 | LB = [-inf 2 1 -10]; 48 | UB = [ inf inf 1 inf]; 49 | xsol = fminsearchbnd(@(x) norm(x),[1 3 1 1],LB,UB,opts) 50 | 51 | %% All the standard outputs from fminsearch are still returned 52 | [xsol,fval,exitflag,output] = fminsearchbnd(@(x) norm(x),[1 3 1 1],LB,UB) 53 | 54 | -------------------------------------------------------------------------------- /gpml/prior/priorSmoothBox1.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorSmoothBox1(a,b,eta,x) 2 | 3 | % Univariate smoothed box prior distribution with linear decay in the log domain 4 | % and infinite support over the whole real axis. 5 | % Compute log-likelihood and its derivative or draw a random sample. 6 | % The prior distribution is parameterized as: 7 | % 8 | % p(x) = sigmoid(eta*(x-a))*(1-sigmoid(eta*(x-b))), 9 | % where sigmoid(z) = 1/(1+exp(-z)) 10 | % 11 | % a(1x1) is the lower bound parameter, b(1x1) is the upper bound parameter, 12 | % eta(1x1)>0 is the slope parameter and x(1xN) contains query hyperparameters 13 | % for prior evaluation. Larger values of eta make the distribution more 14 | % box-like. 15 | % 16 | % /------------\ 17 | % / \ 18 | % -------- | | --------> x 19 | % a b 20 | % 21 | % For more help on design of priors, try "help priorDistributions". 22 | % 23 | % Copyright (c) by Jose Vallet and Hannes Nickisch, 2014-09-08. 24 | % 25 | % See also PRIORDISTRIBUTIONS.M. 26 | 27 | if nargin<3, error('a, b and eta parameters need to be provided'), end 28 | if b<=a, error('b must be greater than a.'), end 29 | if ~(isscalar(a)&&isscalar(b)&&isscalar(eta)) 30 | error('a, b and eta parameters need to be scalar values') 31 | end 32 | 33 | if nargin<4 % inverse sampling 34 | u = exp((b-a)*eta*rand()); 35 | lp = log((u-1)/(exp(-eta*a)-u*exp(-eta*b)))/eta; 36 | return 37 | end 38 | 39 | [lpa,dlpa] = logr(eta*(x-a)); [lpb,dlpb] = logr(-eta*(x-b)); 40 | lp = lpa + lpb - log(b-a) + log(1-exp((a-b)*eta)); 41 | dlp = eta*(dlpa - dlpb); 42 | 43 | % r(z) = 1/(1+exp(-z)), log(r(z)) = -log(1+exp(-z)) 44 | function [lr,dlr] = logr(z) 45 | lr = z; ok = -350 is the slope parameter and x(1xN) contains query hyperparameters 13 | % for prior evaluation. Larger values of eta make the distribution more 14 | % box-like. 15 | % 16 | % /------------\ 17 | % / \ 18 | % -------- | | --------> x 19 | % a b 20 | % 21 | % For more help on design of priors, try "help priorDistributions". 22 | % 23 | % Copyright (c) by Jose Vallet and Hannes Nickisch, 2014-09-08. 24 | % 25 | % See also PRIORDISTRIBUTIONS.M. 26 | 27 | if nargin<3, error('a, b and eta parameters need to be provided'), end 28 | if b<=a, error('b must be greater than a.'), end 29 | if ~(isscalar(a)&&isscalar(b)&&isscalar(eta)) 30 | error('a, b and eta parameters need to be scalar values') 31 | end 32 | 33 | if nargin<4 % inverse sampling 34 | u = exp((b-a)*eta*rand()); 35 | lp = log((u-1)/(exp(-eta*a)-u*exp(-eta*b)))/eta; 36 | return 37 | end 38 | 39 | [lpa,dlpa] = logr(eta*(x-a)); [lpb,dlpb] = logr(-eta*(x-b)); 40 | lp = lpa + lpb - log(b-a) + log(1-exp((a-b)*eta)); 41 | dlp = eta*(dlpa - dlpb); 42 | 43 | % r(z) = 1/(1+exp(-z)), log(r(z)) = -log(1+exp(-z)) 44 | function [lr,dlr] = logr(z) 45 | lr = z; ok = -351, error('Covariance is defined for 1d data only.'), end 24 | ell = exp(hyp(1)); 25 | p = exp(hyp(2)); 26 | sf2 = exp(2*hyp(3)); 27 | 28 | % precompute distances 29 | if dg % vector kxx 30 | K = zeros(size(x,1),1); 31 | else 32 | if xeqz % symmetric matrix Kxx 33 | K = sqrt(sq_dist(x')); 34 | else % cross covariances Kxz 35 | K = sqrt(sq_dist(x',z')); 36 | end 37 | end 38 | 39 | K = pi*K/p; 40 | if nargin<4 % covariances 41 | K = sin(K)/ell; K = K.*K; K = sf2*exp(-2*K); 42 | else % derivatives 43 | if i==1 44 | K = sin(K)/ell; K = K.*K; K = 4*sf2*exp(-2*K).*K; 45 | elseif i==2 46 | R = sin(K)/ell; K = 4*sf2/ell*exp(-2*R.*R).*R.*cos(K).*K; 47 | elseif i==3 48 | K = sin(K)/ell; K = K.*K; K = 2*sf2*exp(-2*K); 49 | else 50 | error('Unknown hyperparameter') 51 | end 52 | end -------------------------------------------------------------------------------- /bayesopt/gpml/Copyright: -------------------------------------------------------------------------------- 1 | GAUSSIAN PROCESS REGRESSION AND CLASSIFICATION Toolbox version 3.5 2 | for GNU Octave 3.2.x and Matlab 7.x 3 | 4 | The code is released under the FreeBSD License. 5 | 6 | Copyright (c) 2005-2015 Carl Edward Rasmussen & Hannes Nickisch. All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 9 | 10 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY CARL EDWARD RASMUSSEN & HANNES NICKISCH ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARL EDWARD RASMUSSEN & HANNES NICKISCH OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 15 | 16 | The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of Carl Edward Rasmussen & Hannes Nickisch. 17 | 18 | The code and associated documentation is available from 19 | http://gaussianprocess.org/gpml/code. 20 | -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covPeriodic.m: -------------------------------------------------------------------------------- 1 | function K = covPeriodic(hyp, x, z, i) 2 | 3 | % Stationary covariance function for a smooth periodic function, with period p 4 | % in 1d (see covPERiso and covPERard for multivariate data): 5 | % 6 | % k(x,z) = sf^2 * exp( -2*sin^2( pi*||x-z||/p )/ell^2 ) 7 | % 8 | % where the hyperparameters are: 9 | % 10 | % hyp = [ log(ell) 11 | % log(p) 12 | % log(sf) ] 13 | % 14 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2011-01-05. 15 | % 16 | % See also COVFUNCTIONS.M. 17 | 18 | if nargin<2, K = '3'; return; end % report number of parameters 19 | if nargin<3, z = []; end % make sure, z exists 20 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 21 | 22 | [n,D] = size(x); 23 | if D>1, error('Covariance is defined for 1d data only.'), end 24 | ell = exp(hyp(1)); 25 | p = exp(hyp(2)); 26 | sf2 = exp(2*hyp(3)); 27 | 28 | % precompute distances 29 | if dg % vector kxx 30 | K = zeros(size(x,1),1); 31 | else 32 | if xeqz % symmetric matrix Kxx 33 | K = sqrt(sq_dist(x')); 34 | else % cross covariances Kxz 35 | K = sqrt(sq_dist(x',z')); 36 | end 37 | end 38 | 39 | K = pi*K/p; 40 | if nargin<4 % covariances 41 | K = sin(K)/ell; K = K.*K; K = sf2*exp(-2*K); 42 | else % derivatives 43 | if i==1 44 | K = sin(K)/ell; K = K.*K; K = 4*sf2*exp(-2*K).*K; 45 | elseif i==2 46 | R = sin(K)/ell; K = 4*sf2/ell*exp(-2*R.*R).*R.*cos(K).*K; 47 | elseif i==3 48 | K = sin(K)/ell; K = K.*K; K = 2*sf2*exp(-2*K); 49 | else 50 | error('Unknown hyperparameter') 51 | end 52 | end -------------------------------------------------------------------------------- /gpml/cov/covNoise.m: -------------------------------------------------------------------------------- 1 | function K = covNoise(hyp, x, z, i) 2 | 3 | % Independent covariance function, i.e. "white noise", with specified variance. 4 | % The covariance function is specified as: 5 | % 6 | % k(x^p,x^q) = s2 * \delta(p,q) 7 | % 8 | % where s2 is the noise variance and \delta(p,q) is a Kronecker delta function 9 | % which is 1 iff p=q and zero otherwise in mode 1). 10 | % In cross covariance mode 2) two data points x_p and z_q are considered equal 11 | % if their difference norm |x_p-z_q| is less than eps, the machine precision. 12 | % The hyperparameter is: 13 | % 14 | % hyp = [ log(sqrt(s2)) ] 15 | % 16 | % For more help on design of covariance functions, try "help covFunctions". 17 | % 18 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2013-05-15. 19 | % 20 | % See also COVFUNCTIONS.M. 21 | 22 | tol = eps; % threshold on the norm when two vectors are considered to be equal 23 | if nargin<2, K = '1'; return; end % report number of parameters 24 | if nargin<3, z = []; end % make sure, z exists 25 | dg = strcmp(z,'diag'); % determine mode 26 | 27 | n = size(x,1); 28 | s2 = exp(2*hyp); % noise variance 29 | 30 | % precompute raw 31 | if dg % vector kxx 32 | K = ones(n,1); 33 | else 34 | if numel(z)==0 % symmetric matrix Kxx 35 | K = eye(n); 36 | else % cross covariances Kxz 37 | K = double(sq_dist(x',z')1, Kuu = feval(cov{:},hyp,xu); end 31 | if nargout>2, Ku = feval(cov{:},hyp,xu,x); end 32 | else 33 | K = feval(cov{:},hyp,xu,z); 34 | end 35 | end 36 | else % derivatives 37 | if dg 38 | K = feval(cov{:},hyp,x,'diag',i); 39 | else 40 | if xeqz 41 | K = feval(cov{:},hyp,x,'diag',i); 42 | if nargout>1, Kuu = feval(cov{:},hyp,xu,[],i); end 43 | if nargout>2, Ku = feval(cov{:},hyp,xu,x,i); end 44 | else 45 | K = feval(cov{:},hyp,xu,z,i); 46 | end 47 | end 48 | end -------------------------------------------------------------------------------- /gpml/util/lbfgsb/matlabmatrix.cpp: -------------------------------------------------------------------------------- 1 | #include "matlabmatrix.h" 2 | #include "matlabexception.h" 3 | 4 | // Function definitions. 5 | // ----------------------------------------------------------------- 6 | double* getMatlabMatrixDouble (const mxArray* ptr) { 7 | if (mxGetNumberOfDimensions(ptr) != 2) 8 | throw MatlabException("Matlab array must be a matrix"); 9 | if (!mxIsDouble(ptr)) 10 | throw MatlabException("Matlab array must be of type double"); 11 | return mxGetPr(ptr); 12 | } 13 | 14 | double* createMatlabMatrixDouble (mxArray*& ptr, int height, int width) { 15 | ptr = mxCreateDoubleMatrix(height,width,mxREAL); 16 | return mxGetPr(ptr); 17 | } 18 | 19 | // Function definitions for class Matrix. 20 | // ----------------------------------------------------------------- 21 | Matrix::Matrix (int height, int width) 22 | : Array(height*width) { 23 | h = height; 24 | w = width; 25 | } 26 | 27 | Matrix::Matrix (double* data, int height, int width) 28 | : Array(data,height*width) { 29 | h = height; 30 | w = width; 31 | } 32 | 33 | Matrix::Matrix (const mxArray* ptr) 34 | : Array(getMatlabMatrixDouble(ptr),mxGetNumberOfElements(ptr)) { 35 | h = mxGetM(ptr); 36 | w = mxGetN(ptr); 37 | } 38 | 39 | Matrix::Matrix (mxArray*& ptr, int height, int width) 40 | : Array(createMatlabMatrixDouble(ptr,height,width), 41 | height*width) { 42 | h = height; 43 | w = width; 44 | } 45 | 46 | Matrix::Matrix (const Matrix& source) 47 | : Array(source) { 48 | h = source.h; 49 | w = source.w; 50 | } 51 | 52 | Matrix& Matrix::operator= (const Matrix& source) { 53 | inject(source); 54 | return *this; 55 | } 56 | 57 | bool Matrix::operator== (const Matrix& X) const { 58 | return (h == X.h) && (w == X.w); 59 | } 60 | 61 | double& Matrix::entry (int r, int c) { 62 | return elems[h*c + r]; 63 | } 64 | 65 | double Matrix::entry (int r, int c) const { 66 | return elems[h*c + r]; 67 | } 68 | -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covNoise.m: -------------------------------------------------------------------------------- 1 | function K = covNoise(hyp, x, z, i) 2 | 3 | % Independent covariance function, i.e. "white noise", with specified variance. 4 | % The covariance function is specified as: 5 | % 6 | % k(x^p,x^q) = s2 * \delta(p,q) 7 | % 8 | % where s2 is the noise variance and \delta(p,q) is a Kronecker delta function 9 | % which is 1 iff p=q and zero otherwise in mode 1). 10 | % In cross covariance mode 2) two data points x_p and z_q are considered equal 11 | % if their difference norm |x_p-z_q| is less than eps, the machine precision. 12 | % The hyperparameter is: 13 | % 14 | % hyp = [ log(sqrt(s2)) ] 15 | % 16 | % For more help on design of covariance functions, try "help covFunctions". 17 | % 18 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2013-05-15. 19 | % 20 | % See also COVFUNCTIONS.M. 21 | 22 | tol = eps; % threshold on the norm when two vectors are considered to be equal 23 | if nargin<2, K = '1'; return; end % report number of parameters 24 | if nargin<3, z = []; end % make sure, z exists 25 | dg = strcmp(z,'diag'); % determine mode 26 | 27 | n = size(x,1); 28 | s2 = exp(2*hyp); % noise variance 29 | 30 | % precompute raw 31 | if dg % vector kxx 32 | K = ones(n,1); 33 | else 34 | if numel(z)==0 % symmetric matrix Kxx 35 | K = eye(n); 36 | else % cross covariances Kxz 37 | K = double(sq_dist(x',z')1, Kuu = feval(cov{:},hyp,xu); end 31 | if nargout>2, Ku = feval(cov{:},hyp,xu,x); end 32 | else 33 | K = feval(cov{:},hyp,xu,z); 34 | end 35 | end 36 | else % derivatives 37 | if dg 38 | K = feval(cov{:},hyp,x,'diag',i); 39 | else 40 | if xeqz 41 | K = feval(cov{:},hyp,x,'diag',i); 42 | if nargout>1, Kuu = feval(cov{:},hyp,xu,[],i); end 43 | if nargout>2, Ku = feval(cov{:},hyp,xu,x,i); end 44 | else 45 | K = feval(cov{:},hyp,xu,z,i); 46 | end 47 | end 48 | end -------------------------------------------------------------------------------- /bayesopt/gpml/util/lbfgsb/matlabmatrix.cpp: -------------------------------------------------------------------------------- 1 | #include "matlabmatrix.h" 2 | #include "matlabexception.h" 3 | 4 | // Function definitions. 5 | // ----------------------------------------------------------------- 6 | double* getMatlabMatrixDouble (const mxArray* ptr) { 7 | if (mxGetNumberOfDimensions(ptr) != 2) 8 | throw MatlabException("Matlab array must be a matrix"); 9 | if (!mxIsDouble(ptr)) 10 | throw MatlabException("Matlab array must be of type double"); 11 | return mxGetPr(ptr); 12 | } 13 | 14 | double* createMatlabMatrixDouble (mxArray*& ptr, int height, int width) { 15 | ptr = mxCreateDoubleMatrix(height,width,mxREAL); 16 | return mxGetPr(ptr); 17 | } 18 | 19 | // Function definitions for class Matrix. 20 | // ----------------------------------------------------------------- 21 | Matrix::Matrix (int height, int width) 22 | : Array(height*width) { 23 | h = height; 24 | w = width; 25 | } 26 | 27 | Matrix::Matrix (double* data, int height, int width) 28 | : Array(data,height*width) { 29 | h = height; 30 | w = width; 31 | } 32 | 33 | Matrix::Matrix (const mxArray* ptr) 34 | : Array(getMatlabMatrixDouble(ptr),mxGetNumberOfElements(ptr)) { 35 | h = mxGetM(ptr); 36 | w = mxGetN(ptr); 37 | } 38 | 39 | Matrix::Matrix (mxArray*& ptr, int height, int width) 40 | : Array(createMatlabMatrixDouble(ptr,height,width), 41 | height*width) { 42 | h = height; 43 | w = width; 44 | } 45 | 46 | Matrix::Matrix (const Matrix& source) 47 | : Array(source) { 48 | h = source.h; 49 | w = source.w; 50 | } 51 | 52 | Matrix& Matrix::operator= (const Matrix& source) { 53 | inject(source); 54 | return *this; 55 | } 56 | 57 | bool Matrix::operator== (const Matrix& X) const { 58 | return (h == X.h) && (w == X.w); 59 | } 60 | 61 | double& Matrix::entry (int r, int c) { 62 | return elems[h*c + r]; 63 | } 64 | 65 | double Matrix::entry (int r, int c) const { 66 | return elems[h*c + r]; 67 | } 68 | -------------------------------------------------------------------------------- /gpml/prior/priorGaussMulti.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorGaussMulti(mu,s2,x) 2 | 3 | % Multivariate Gaussian hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-r2/2) / sqrt(det(2*pi*s2)) where r2(x) = (x-mu)'*inv(s2)*(x-mu), 8 | % 9 | % mu(Dx1) is the mean parameter, s2(Dx1) or s2(DxD) is the variance parameter 10 | % and x(DxN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-12. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M, PRIORGAUSS.M. 17 | 18 | if nargin<2, error('mu and s2 parameters need to be provided'), end 19 | if ndims(mu)~=2 || size(mu,2)~=1, error('mu needs to be (Dx1)'), end 20 | D = size(mu,1); 21 | s2_ok = ndims(s2)==2 && all(size(s2)==[D,1] | size(s2)==[D,D]); 22 | if ~s2_ok, error('s2 needs to be (DxD) or (Dx1)'), end 23 | if size(s2,2)==D % full multivariate case 24 | s = chol(s2)'; lds = sum(log(diag(s))); % lds = log(det(s2))/2 25 | else % diagonal covariance 26 | s = sqrt(s2); lds = sum(log(s)); 27 | end 28 | if nargin<3 % return a random sample 29 | lp = randn(D,1); % unit sample 30 | if size(s,2)==D, lp = s*lp+mu; else lp = s.*lp+mu; end % affine transformation 31 | return 32 | end 33 | if D==1 && size(x,1)>1 % both mu/s2 scalar => inflate 34 | D = size(x,1); mu = mu*ones(D,1); s = s*ones(D,1); lds = D*lds; 35 | end 36 | if ~(ndims(x)==2 && size(x,1)==D), error('x needs to be (Dxn)'), end 37 | 38 | oN = ones(1,size(x,2)); 39 | if size(s,2)==D 40 | xs = s\(x-mu*oN); dlp = -s'\xs; 41 | else 42 | xs = (x-mu*oN)./(s*oN); dlp = -xs./(s*oN); 43 | end 44 | lp = -sum(xs.^2,1)/2 - D*log(2*pi)/2 - lds; -------------------------------------------------------------------------------- /gpml/mean/meanSum.m: -------------------------------------------------------------------------------- 1 | function A = meanSum(mean, hyp, x, i) 2 | 3 | % meanSum - compose a mean function as the sum of other mean functions. 4 | % This function doesn't actually compute very much on its own, it merely does 5 | % some bookkeeping, and calls other mean functions to do the actual work. 6 | % 7 | % m(x) = \sum_i m_i(x) 8 | % 9 | % Copyright (c) by Carl Edward Rasmussen & Hannes Nickisch 2010-08-04. 10 | % 11 | % See also MEANFUNCTIONS.M. 12 | 13 | for ii = 1:numel(mean) % iterate over mean functions 14 | f = mean(ii); if iscell(f{:}), f = f{:}; end % expand cell array if necessary 15 | j(ii) = cellstr(feval(f{:})); % collect number hypers 16 | end 17 | 18 | if nargin<3 % report number of parameters 19 | A = char(j(1)); for ii=2:length(mean), A = [A, '+', char(j(ii))]; end; return 20 | end 21 | 22 | [n,D] = size(x); 23 | 24 | v = []; % v vector indicates to which mean parameters belong 25 | for ii = 1:length(mean), v = [v repmat(ii, 1, eval(char(j(ii))))]; end 26 | 27 | if nargin==3 % compute mean vector 28 | A = zeros(n,1); % allocate space 29 | for ii = 1:length(mean) % iteration over summand functions 30 | f = mean(ii); if iscell(f{:}), f = f{:}; end % expand cell array if needed 31 | A = A + feval(f{:}, hyp(v==ii), x); % accumulate means 32 | end 33 | else % compute derivative vector 34 | if i<=length(v) 35 | ii = v(i); % which mean function 36 | j = sum(v(1:i)==ii); % which parameter in that mean 37 | f = mean(ii); 38 | if iscell(f{:}), f = f{:}; end % dereference cell array if necessary 39 | A = feval(f{:}, hyp(v==ii), x, j); % compute derivative 40 | else 41 | A = zeros(n,1); 42 | end 43 | end -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorGaussMulti.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorGaussMulti(mu,s2,x) 2 | 3 | % Multivariate Gaussian hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-r2/2) / sqrt(det(2*pi*s2)) where r2(x) = (x-mu)'*inv(s2)*(x-mu), 8 | % 9 | % mu(Dx1) is the mean parameter, s2(Dx1) or s2(DxD) is the variance parameter 10 | % and x(DxN) contains query hyperparameters for prior evaluation. 11 | % 12 | % For more help on design of priors, try "help priorDistributions". 13 | % 14 | % Copyright (c) by Roman Garnett and Hannes Nickisch, 2014-09-12. 15 | % 16 | % See also PRIORDISTRIBUTIONS.M, PRIORGAUSS.M. 17 | 18 | if nargin<2, error('mu and s2 parameters need to be provided'), end 19 | if ndims(mu)~=2 || size(mu,2)~=1, error('mu needs to be (Dx1)'), end 20 | D = size(mu,1); 21 | s2_ok = ndims(s2)==2 && all(size(s2)==[D,1] | size(s2)==[D,D]); 22 | if ~s2_ok, error('s2 needs to be (DxD) or (Dx1)'), end 23 | if size(s2,2)==D % full multivariate case 24 | s = chol(s2)'; lds = sum(log(diag(s))); % lds = log(det(s2))/2 25 | else % diagonal covariance 26 | s = sqrt(s2); lds = sum(log(s)); 27 | end 28 | if nargin<3 % return a random sample 29 | lp = randn(D,1); % unit sample 30 | if size(s,2)==D, lp = s*lp+mu; else lp = s.*lp+mu; end % affine transformation 31 | return 32 | end 33 | if D==1 && size(x,1)>1 % both mu/s2 scalar => inflate 34 | D = size(x,1); mu = mu*ones(D,1); s = s*ones(D,1); lds = D*lds; 35 | end 36 | if ~(ndims(x)==2 && size(x,1)==D), error('x needs to be (Dxn)'), end 37 | 38 | oN = ones(1,size(x,2)); 39 | if size(s,2)==D 40 | xs = s\(x-mu*oN); dlp = -s'\xs; 41 | else 42 | xs = (x-mu*oN)./(s*oN); dlp = -xs./(s*oN); 43 | end 44 | lp = -sum(xs.^2,1)/2 - D*log(2*pi)/2 - lds; -------------------------------------------------------------------------------- /bayesopt/gpml/mean/meanSum.m: -------------------------------------------------------------------------------- 1 | function A = meanSum(mean, hyp, x, i) 2 | 3 | % meanSum - compose a mean function as the sum of other mean functions. 4 | % This function doesn't actually compute very much on its own, it merely does 5 | % some bookkeeping, and calls other mean functions to do the actual work. 6 | % 7 | % m(x) = \sum_i m_i(x) 8 | % 9 | % Copyright (c) by Carl Edward Rasmussen & Hannes Nickisch 2010-08-04. 10 | % 11 | % See also MEANFUNCTIONS.M. 12 | 13 | for ii = 1:numel(mean) % iterate over mean functions 14 | f = mean(ii); if iscell(f{:}), f = f{:}; end % expand cell array if necessary 15 | j(ii) = cellstr(feval(f{:})); % collect number hypers 16 | end 17 | 18 | if nargin<3 % report number of parameters 19 | A = char(j(1)); for ii=2:length(mean), A = [A, '+', char(j(ii))]; end; return 20 | end 21 | 22 | [n,D] = size(x); 23 | 24 | v = []; % v vector indicates to which mean parameters belong 25 | for ii = 1:length(mean), v = [v repmat(ii, 1, eval(char(j(ii))))]; end 26 | 27 | if nargin==3 % compute mean vector 28 | A = zeros(n,1); % allocate space 29 | for ii = 1:length(mean) % iteration over summand functions 30 | f = mean(ii); if iscell(f{:}), f = f{:}; end % expand cell array if needed 31 | A = A + feval(f{:}, hyp(v==ii), x); % accumulate means 32 | end 33 | else % compute derivative vector 34 | if i<=length(v) 35 | ii = v(i); % which mean function 36 | j = sum(v(1:i)==ii); % which parameter in that mean 37 | f = mean(ii); 38 | if iscell(f{:}), f = f{:}; end % dereference cell array if necessary 39 | A = feval(f{:}, hyp(v==ii), x, j); % compute derivative 40 | else 41 | A = zeros(n,1); 42 | end 43 | end -------------------------------------------------------------------------------- /FMINSEARCHBND/demo/fminsearchcon_demo.m: -------------------------------------------------------------------------------- 1 | %% Optimization of a simple (Rosenbrock) function 2 | rosen = @(x) (1-x(1)).^2 + 105*(x(2)-x(1).^2).^2; 3 | 4 | % With no constraints, operation simply passes through 5 | % directly to fminsearch. The solution should be [1 1] 6 | xsol = fminsearchcon(rosen,[3 3]) 7 | 8 | %% Only lower bound constraints 9 | xsol = fminsearchcon(rosen,[3 3],[2 2]) 10 | 11 | %% Only upper bound constraints 12 | xsol = fminsearchcon(rosen,[-5 -5],[],[0 0]) 13 | 14 | %% Dual constraints 15 | xsol = fminsearchcon(rosen,[2.5 2.5],[2 2],[3 3]) 16 | 17 | %% Mixed constraints 18 | xsol = fminsearchcon(rosen,[0 0],[2 -inf],[inf 3]) 19 | 20 | %% Fix a variable as constant, x(2) == 3 21 | fminsearchcon(rosen,[3 3],[-inf 3],[inf,3]) 22 | 23 | %% Linear inequality, x(1) + x(2) <= 1 24 | fminsearchcon(rosen,[0 0],[],[],[1 1],1) 25 | 26 | %% Nonlinear inequality, norm(x) <= 1 27 | fminsearchcon(rosen,[0 0],[],[],[],[],@(x) norm(x) - 1) 28 | 29 | %% Minimize a linear objective, subject to a nonlinear constraints. 30 | fun = @(x) x*[-2;1]; 31 | nonlcon = @(x) [norm(x) - 1;sin(sum(x))]; 32 | fminsearchcon(fun,[0 0],[],[],[],[],nonlcon) 33 | 34 | %% Provide your own fminsearch options 35 | opts = optimset('fminsearch'); 36 | opts.Display = 'iter'; 37 | opts.TolX = 1.e-12; 38 | opts.MaxFunEvals = 100; 39 | 40 | n = [10,5]; 41 | H = randn(n); 42 | H=H'*H; 43 | Quadraticfun = @(x) x*H*x'; 44 | 45 | % Global minimizer is at [0 0 0 0 0]. 46 | % Set all lower bound constraints, all of which will 47 | % be active in this test. 48 | LB = [.5 .5 .5 .5 .5]; 49 | xsol = fminsearchcon(Quadraticfun,[1 2 3 4 5],LB,[],[],[],[],opts) 50 | 51 | %% Exactly fix one variable, constrain some others, and set a tolerance 52 | opts = optimset('fminsearch'); 53 | opts.TolFun = 1.e-12; 54 | 55 | LB = [-inf 2 1 -10]; 56 | UB = [ inf inf 1 inf]; 57 | xsol = fminsearchcon(@(x) norm(x),[1 3 1 1],LB,UB,[],[],[],opts) 58 | 59 | %% All the standard outputs from fminsearch are still returned 60 | [xsol,fval,exitflag,output] = fminsearchcon(@(x) norm(x),[1 3 1 1],LB,UB) 61 | 62 | -------------------------------------------------------------------------------- /gpml/cov/covRQiso.m: -------------------------------------------------------------------------------- 1 | function K = covRQiso(hyp, x, z, i) 2 | 3 | % Rational Quadratic covariance function with isotropic distance measure. The 4 | % covariance function is parameterized as: 5 | % 6 | % k(x^p,x^q) = sf^2 * [1 + (x^p - x^q)'*inv(P)*(x^p - x^q)/(2*alpha)]^(-alpha) 7 | % 8 | % where the P matrix is ell^2 times the unit matrix, sf2 is the signal 9 | % variance and alpha is the shape parameter for the RQ covariance. The 10 | % hyperparameters are: 11 | % 12 | % hyp = [ log(ell) 13 | % log(sf) 14 | % log(alpha) ] 15 | % 16 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 17 | % 18 | % See also COVFUNCTIONS.M. 19 | 20 | if nargin<2, K = '3'; return; end % report number of parameters 21 | if nargin<3, z = []; end % make sure, z exists 22 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 23 | 24 | ell = exp(hyp(1)); 25 | sf2 = exp(2*hyp(2)); 26 | alpha = exp(hyp(3)); 27 | 28 | % precompute squared distances 29 | if dg % vector kxx 30 | D2 = zeros(size(x,1),1); 31 | else 32 | if xeqz % symmetric matrix Kxx 33 | D2 = sq_dist(x'/ell); 34 | else % cross covariances Kxz 35 | D2 = sq_dist(x'/ell,z'/ell); 36 | end 37 | end 38 | 39 | if nargin<4 % covariances 40 | K = sf2*(1+0.5*D2/alpha).^(-alpha); 41 | else % derivatives 42 | if i==1 % length scale parameter 43 | K = sf2*(1+0.5*D2/alpha).^(-alpha-1).*D2; 44 | elseif i==2 % magnitude parameter 45 | K = 2*sf2*(1+0.5*D2/alpha).^(-alpha); 46 | elseif i==3 47 | K = (1+0.5*D2/alpha); 48 | K = sf2*K.^(-alpha).*(0.5*D2./K - alpha*log(K)); 49 | else 50 | error('Unknown hyperparameter') 51 | end 52 | end -------------------------------------------------------------------------------- /bayesopt/gpml/cov/covRQiso.m: -------------------------------------------------------------------------------- 1 | function K = covRQiso(hyp, x, z, i) 2 | 3 | % Rational Quadratic covariance function with isotropic distance measure. The 4 | % covariance function is parameterized as: 5 | % 6 | % k(x^p,x^q) = sf^2 * [1 + (x^p - x^q)'*inv(P)*(x^p - x^q)/(2*alpha)]^(-alpha) 7 | % 8 | % where the P matrix is ell^2 times the unit matrix, sf2 is the signal 9 | % variance and alpha is the shape parameter for the RQ covariance. The 10 | % hyperparameters are: 11 | % 12 | % hyp = [ log(ell) 13 | % log(sf) 14 | % log(alpha) ] 15 | % 16 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-09-10. 17 | % 18 | % See also COVFUNCTIONS.M. 19 | 20 | if nargin<2, K = '3'; return; end % report number of parameters 21 | if nargin<3, z = []; end % make sure, z exists 22 | xeqz = isempty(z); dg = strcmp(z,'diag'); % determine mode 23 | 24 | ell = exp(hyp(1)); 25 | sf2 = exp(2*hyp(2)); 26 | alpha = exp(hyp(3)); 27 | 28 | % precompute squared distances 29 | if dg % vector kxx 30 | D2 = zeros(size(x,1),1); 31 | else 32 | if xeqz % symmetric matrix Kxx 33 | D2 = sq_dist(x'/ell); 34 | else % cross covariances Kxz 35 | D2 = sq_dist(x'/ell,z'/ell); 36 | end 37 | end 38 | 39 | if nargin<4 % covariances 40 | K = sf2*(1+0.5*D2/alpha).^(-alpha); 41 | else % derivatives 42 | if i==1 % length scale parameter 43 | K = sf2*(1+0.5*D2/alpha).^(-alpha-1).*D2; 44 | elseif i==2 % magnitude parameter 45 | K = 2*sf2*(1+0.5*D2/alpha).^(-alpha); 46 | elseif i==3 47 | K = (1+0.5*D2/alpha); 48 | K = sf2*K.^(-alpha).*(0.5*D2./K - alpha*log(K)); 49 | else 50 | error('Unknown hyperparameter') 51 | end 52 | end -------------------------------------------------------------------------------- /gpml/util/cov_deriv_sq_dist.m: -------------------------------------------------------------------------------- 1 | % Compute derivative k'(x^p,x^q) of a stationary covariance k(d2) (ard or iso) 2 | % w.r.t. to squared distance d2 = (x^p - x^q)'*inv(P)*(x^p - x^q) measure. Here 3 | % P is either diagonal with ARD parameters ell_1^2,...,ell_D^2 where D is the 4 | % dimension of the input space or ell^2 times the unit matrix for isotropic 5 | % covariance. 6 | % The derivatives can only be computed for one of the following eight 7 | % covariance functions: cov{Matern|PP|RQ|SE}{iso|ard}. 8 | % 9 | % Copyright (c) by Hannes Nickisch, 2013-10-28. 10 | % 11 | % See also INFFITC.M, COVFITC.M. 12 | 13 | function Kp = cov_deriv_sq_dist(cov,hyp,x,z) 14 | if nargin<4, z = []; end % make sure, z exists 15 | xeqz = numel(z)==0; dg = strcmp(z,'diag') && numel(z)>0; % determine mode 16 | 17 | if iscell(cov), covstr = cov{1}; else covstr = cov; end 18 | if ~ischar(covstr), covstr = func2str(covstr); end 19 | if numel([strfind(covstr,'iso'),strfind(covstr,'ard')])==0 20 | error('Only iso|ard covariances allowed for derivatives w.r.t. xu.') 21 | elseif numel([strfind(covstr,'covLIN'); 22 | strfind(covstr,'covGabor'); 23 | strfind(covstr,'covPER')])>0 24 | error('Gabor|LIN|PER covariances not allowed for derivatives w.r.t. xu.') 25 | end 26 | 27 | [n,D] = size(x); 28 | if numel(strfind(covstr,'iso')), id = 1:D; else id = 1; end % *iso covariance 29 | ell1 = exp(hyp(1)); % first characteristic length scale 30 | 31 | Kp = feval(cov{:},hyp,x,z,1); % use derivative w.r.t. log(ell(1)) 32 | % precompute squared distances 33 | if dg % vector kxx 34 | d2 = zeros(n,1); 35 | else 36 | if xeqz % symmetric matrix Kxx 37 | d2 = sq_dist(x(:,id)'/ell1); 38 | else % cross covariances Kxz 39 | d2 = sq_dist(x(:,id)'/ell1,z(:,id)'/ell1); 40 | end 41 | end 42 | Kp = -1/2*Kp./d2; Kp(d2==0) = 0; -------------------------------------------------------------------------------- /bayesopt/gpml/util/cov_deriv_sq_dist.m: -------------------------------------------------------------------------------- 1 | % Compute derivative k'(x^p,x^q) of a stationary covariance k(d2) (ard or iso) 2 | % w.r.t. to squared distance d2 = (x^p - x^q)'*inv(P)*(x^p - x^q) measure. Here 3 | % P is either diagonal with ARD parameters ell_1^2,...,ell_D^2 where D is the 4 | % dimension of the input space or ell^2 times the unit matrix for isotropic 5 | % covariance. 6 | % The derivatives can only be computed for one of the following eight 7 | % covariance functions: cov{Matern|PP|RQ|SE}{iso|ard}. 8 | % 9 | % Copyright (c) by Hannes Nickisch, 2013-10-28. 10 | % 11 | % See also INFFITC.M, COVFITC.M. 12 | 13 | function Kp = cov_deriv_sq_dist(cov,hyp,x,z) 14 | if nargin<4, z = []; end % make sure, z exists 15 | xeqz = numel(z)==0; dg = strcmp(z,'diag') && numel(z)>0; % determine mode 16 | 17 | if iscell(cov), covstr = cov{1}; else covstr = cov; end 18 | if ~ischar(covstr), covstr = func2str(covstr); end 19 | if numel([strfind(covstr,'iso'),strfind(covstr,'ard')])==0 20 | error('Only iso|ard covariances allowed for derivatives w.r.t. xu.') 21 | elseif numel([strfind(covstr,'covLIN'); 22 | strfind(covstr,'covGabor'); 23 | strfind(covstr,'covPER')])>0 24 | error('Gabor|LIN|PER covariances not allowed for derivatives w.r.t. xu.') 25 | end 26 | 27 | [n,D] = size(x); 28 | if numel(strfind(covstr,'iso')), id = 1:D; else id = 1; end % *iso covariance 29 | ell1 = exp(hyp(1)); % first characteristic length scale 30 | 31 | Kp = feval(cov{:},hyp,x,z,1); % use derivative w.r.t. log(ell(1)) 32 | % precompute squared distances 33 | if dg % vector kxx 34 | d2 = zeros(n,1); 35 | else 36 | if xeqz % symmetric matrix Kxx 37 | d2 = sq_dist(x(:,id)'/ell1); 38 | else % cross covariances Kxz 39 | d2 = sq_dist(x(:,id)'/ell1,z(:,id)'/ell1); 40 | end 41 | end 42 | Kp = -1/2*Kp./d2; Kp(d2==0) = 0; -------------------------------------------------------------------------------- /gpml/prior/priorSmoothBox2.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorSmoothBox2(a,b,eta,x) 2 | 3 | % Univariate smoothed box prior distribution with quadratic decay in the log 4 | % domain and infinite support over the whole real axis. 5 | % The plateau pdf is built by cutting a Gaussian into two parts and 6 | % inserting a uniform distribution from a to b. 7 | % Compute log-likelihood and its derivative or draw a random sample. 8 | % The prior distribution is parameterized as: 9 | % 10 | % / N(x|a,s^2), x<=a, 11 | % p(x) = 1/(w*(1/eta+1)) * | 1 , a0 is the slope parameter and x(1xN) contains query hyperparameters 17 | % for prior evaluation. Larger values of eta make the distribution more 18 | % box-like. 19 | % 20 | % /------------\ 21 | % / \ 22 | % -------- | | --------> x 23 | % a b 24 | % 25 | % For more help on design of priors, try "help priorDistributions". 26 | % 27 | % Copyright (c) by Jose Vallet and Hannes Nickisch, 2014-09-08. 28 | % 29 | % See also PRIORDISTRIBUTIONS.M. 30 | 31 | if nargin<3, error('a, b and eta parameters need to be provided'), end 32 | if b<=a, error('b must be greater than a.'), end 33 | if ~(isscalar(a)&&isscalar(b)&&isscalar(eta)) 34 | error('a, b and eta parameters need to be scalar values') 35 | end 36 | 37 | w = abs(b-a); sab = w/(eta*sqrt(2*pi)); % box width and boundary slope 38 | 39 | if nargin<4 % return a sample 40 | if randb; 54 | lp(i) = lp(i) - (x(i)-b).^2/(2*sab^2); 55 | dlp(i) = (b-x(i))/sab^2; -------------------------------------------------------------------------------- /FMINSEARCHBND/doc/Understanding_fminsearchcon.rtf: -------------------------------------------------------------------------------- 1 | {\rtf1\mac\ansicpg10000\cocoartf102 2 | {\fonttbl\f0\fswiss\fcharset77 Helvetica;\f1\fswiss\fcharset77 Helvetica-Bold;} 3 | {\colortbl;\red255\green255\blue255;} 4 | \margl1440\margr1440\vieww12760\viewh17780\viewkind0 5 | \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\qc 6 | 7 | \f0\fs24 \cf0 \ 8 | \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\qc 9 | 10 | \f1\b \cf0 Understanding fminsearchcon\ 11 | \ 12 | John D'Errico\ 13 | woodchips@rochester.rr.com 14 | \f0\b0 \ 15 | \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\ql\qnatural 16 | \cf0 \ 17 | \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\qj 18 | \cf0 \ 19 | Fminsearchcon is a simple extension of fminsearchbnd, which implemented simple bound constraints on top of fminsearch. (See Understanding_fminsearchbnd.rtf for an explanation of bound constraints.) Fminsearchcon goes one step further, allowing general linear inequality constraints and nonlinear inequality constraints. These constraints are implemented as penalty functions, thus if fminsearchbnd would have tried to evaluate your objective function outside the bounds, then I intercept the call BEFORE I allow it to evaluate the objective function, I then return inf to fminsearch. The expectation is that your objective function will never be evaluated outside of the supplied constraints.\ 20 | \ 21 | \ 22 | 23 | \f1\b Limitations of fminsearchcon\ 24 | 25 | \f0\b0 \ 26 | What does fminsearchcon NOT do? The only class of constraint that I have not implemented are general linear or nonlinear EQUALITY constraints. These coonstraints are incompatible with the bound transformations used. As well, fminsearch would not in general work well when confined to a nonlinear manifold.\ 27 | \ 28 | Do NOT attempt to implement a nonlinear equality constraint using a pair of matched inequalities. Thus\ 29 | \ 30 | c(x) <= 0\ 31 | -c(x) <= 0\ 32 | \ 33 | You will not be happy with the results in general.\ 34 | } -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorSmoothBox2.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorSmoothBox2(a,b,eta,x) 2 | 3 | % Univariate smoothed box prior distribution with quadratic decay in the log 4 | % domain and infinite support over the whole real axis. 5 | % The plateau pdf is built by cutting a Gaussian into two parts and 6 | % inserting a uniform distribution from a to b. 7 | % Compute log-likelihood and its derivative or draw a random sample. 8 | % The prior distribution is parameterized as: 9 | % 10 | % / N(x|a,s^2), x<=a, 11 | % p(x) = 1/(w*(1/eta+1)) * | 1 , a0 is the slope parameter and x(1xN) contains query hyperparameters 17 | % for prior evaluation. Larger values of eta make the distribution more 18 | % box-like. 19 | % 20 | % /------------\ 21 | % / \ 22 | % -------- | | --------> x 23 | % a b 24 | % 25 | % For more help on design of priors, try "help priorDistributions". 26 | % 27 | % Copyright (c) by Jose Vallet and Hannes Nickisch, 2014-09-08. 28 | % 29 | % See also PRIORDISTRIBUTIONS.M. 30 | 31 | if nargin<3, error('a, b and eta parameters need to be provided'), end 32 | if b<=a, error('b must be greater than a.'), end 33 | if ~(isscalar(a)&&isscalar(b)&&isscalar(eta)) 34 | error('a, b and eta parameters need to be scalar values') 35 | end 36 | 37 | w = abs(b-a); sab = w/(eta*sqrt(2*pi)); % box width and boundary slope 38 | 39 | if nargin<4 % return a sample 40 | if randb; 54 | lp(i) = lp(i) - (x(i)-b).^2/(2*sab^2); 55 | dlp(i) = (b-x(i))/sab^2; -------------------------------------------------------------------------------- /gpml/prior/priorLaplaceMulti.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorLaplaceMulti(mu,s2,x) 2 | 3 | % Multivariate Laplace hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-sqrt(2)*sqrt(r2))/sqrt(det(2*s2)), 8 | % where r2(x) = (x-mu)'*inv(s2)*(x-mu), 9 | % 10 | % further mu(Dx1) is the mean parameter, s2(Dx1) or s2(DxD) is the variance 11 | % parameter and x(DxN) contains query hyperparameters for prior evaluation. 12 | % 13 | % For more help on design of priors, try "help priorDistributions". 14 | % 15 | % Copyright (c) by Hannes Nickisch, 2014-10-15. 16 | % 17 | % See also PRIORDISTRIBUTIONS.M, PRIORLAPLACE.M. 18 | 19 | if nargin<2, error('mu and s2 parameters need to be provided'), end 20 | if ndims(mu)~=2 || size(mu,2)~=1, error('mu needs to be (Dx1)'), end 21 | D = size(mu,1); 22 | s2_ok = ndims(s2)==2 && all(size(s2)==[D,1] | size(s2)==[D,D]); 23 | if ~s2_ok, error('s2 needs to be (DxD) or (Dx1)'), end 24 | if size(s2,2)==D % full multivariate case 25 | s = chol(s2)'; lds = sum(log(diag(s))); % lds = log(det(s2))/2 26 | else % diagonal covariance 27 | s = sqrt(s2); lds = sum(log(s)); 28 | end 29 | if nargin<3 % return a random sample 30 | u = rand(D,1)-1/2; lp = sign(u).*log(1-2*abs(u))/sqrt(2); % unit sample 31 | if size(s,2)==D, lp = s*lp+mu; else lp = s.*lp+mu; end % affine transformation 32 | return 33 | end 34 | if D==1 && size(x,1)>1 % both mu/s2 scalar => inflate 35 | D = size(x,1); mu = mu*ones(D,1); s = s*ones(D,1); lds = D*lds; 36 | end 37 | if ~(ndims(x)==2 && size(x,1)==D), error('x needs to be (Dxn)'), end 38 | 39 | oN = ones(1,size(x,2)); 40 | if size(s,2)==D 41 | xm = x-mu*oN; xs = s\xm; 42 | else 43 | xm = x-mu*oN; xs = xm./(s*oN); 44 | end 45 | 46 | dlp = -sqrt(2)*sign(xs); 47 | lp = -sqrt(2)*sum(abs(xs),1) - D*log(2)/2 - lds; 48 | if size(s,2)==D, dlp = s'\dlp; else dlp = dlp./(s*oN); end -------------------------------------------------------------------------------- /bayesopt/gpml/prior/priorLaplaceMulti.m: -------------------------------------------------------------------------------- 1 | function [lp,dlp] = priorLaplaceMulti(mu,s2,x) 2 | 3 | % Multivariate Laplace hyperparameter prior distribution. 4 | % Compute log-likelihood and its derivative or draw a random sample. 5 | % The prior distribution is parameterized as: 6 | % 7 | % p(x) = exp(-sqrt(2)*sqrt(r2))/sqrt(det(2*s2)), 8 | % where r2(x) = (x-mu)'*inv(s2)*(x-mu), 9 | % 10 | % further mu(Dx1) is the mean parameter, s2(Dx1) or s2(DxD) is the variance 11 | % parameter and x(DxN) contains query hyperparameters for prior evaluation. 12 | % 13 | % For more help on design of priors, try "help priorDistributions". 14 | % 15 | % Copyright (c) by Hannes Nickisch, 2014-10-15. 16 | % 17 | % See also PRIORDISTRIBUTIONS.M, PRIORLAPLACE.M. 18 | 19 | if nargin<2, error('mu and s2 parameters need to be provided'), end 20 | if ndims(mu)~=2 || size(mu,2)~=1, error('mu needs to be (Dx1)'), end 21 | D = size(mu,1); 22 | s2_ok = ndims(s2)==2 && all(size(s2)==[D,1] | size(s2)==[D,D]); 23 | if ~s2_ok, error('s2 needs to be (DxD) or (Dx1)'), end 24 | if size(s2,2)==D % full multivariate case 25 | s = chol(s2)'; lds = sum(log(diag(s))); % lds = log(det(s2))/2 26 | else % diagonal covariance 27 | s = sqrt(s2); lds = sum(log(s)); 28 | end 29 | if nargin<3 % return a random sample 30 | u = rand(D,1)-1/2; lp = sign(u).*log(1-2*abs(u))/sqrt(2); % unit sample 31 | if size(s,2)==D, lp = s*lp+mu; else lp = s.*lp+mu; end % affine transformation 32 | return 33 | end 34 | if D==1 && size(x,1)>1 % both mu/s2 scalar => inflate 35 | D = size(x,1); mu = mu*ones(D,1); s = s*ones(D,1); lds = D*lds; 36 | end 37 | if ~(ndims(x)==2 && size(x,1)==D), error('x needs to be (Dxn)'), end 38 | 39 | oN = ones(1,size(x,2)); 40 | if size(s,2)==D 41 | xm = x-mu*oN; xs = s\xm; 42 | else 43 | xm = x-mu*oN; xs = xm./(s*oN); 44 | end 45 | 46 | dlp = -sqrt(2)*sign(xs); 47 | lp = -sqrt(2)*sum(abs(xs),1) - D*log(2)/2 - lds; 48 | if size(s,2)==D, dlp = s'\dlp; else dlp = dlp./(s*oN); end -------------------------------------------------------------------------------- /gpml/util/sq_dist.m: -------------------------------------------------------------------------------- 1 | % sq_dist - a function to compute a matrix of all pairwise squared distances 2 | % between two sets of vectors, stored in the columns of the two matrices, a 3 | % (of size D by n) and b (of size D by m). If only a single argument is given 4 | % or the second matrix is empty, the missing matrix is taken to be identical 5 | % to the first. 6 | % 7 | % Usage: C = sq_dist(a, b) 8 | % or: C = sq_dist(a) or equiv.: C = sq_dist(a, []) 9 | % 10 | % Where a is of size Dxn, b is of size Dxm (or empty), C is of size nxm. 11 | % 12 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-12-13. 13 | 14 | function C = sq_dist(a, b) 15 | 16 | if nargin<1 || nargin>3 || nargout>1, error('Wrong number of arguments.'); end 17 | bsx = exist('bsxfun','builtin'); % since Matlab R2007a 7.4.0 and Octave 3.0 18 | if ~bsx, bsx = exist('bsxfun'); end % bsxfun is not yet "builtin" in Octave 19 | [D, n] = size(a); 20 | 21 | % Computation of a^2 - 2*a*b + b^2 is less stable than (a-b)^2 because numerical 22 | % precision can be lost when both a and b have very large absolute value and the 23 | % same sign. For that reason, we subtract the mean from the data beforehand to 24 | % stabilise the computations. This is OK because the squared error is 25 | % independent of the mean. 26 | if nargin==1 % subtract mean 27 | mu = mean(a,2); 28 | if bsx 29 | a = bsxfun(@minus,a,mu); 30 | else 31 | a = a - repmat(mu,1,size(a,2)); 32 | end 33 | b = a; m = n; 34 | else 35 | [d, m] = size(b); 36 | if d ~= D, error('Error: column lengths must agree.'); end 37 | mu = (m/(n+m))*mean(b,2) + (n/(n+m))*mean(a,2); 38 | if bsx 39 | a = bsxfun(@minus,a,mu); b = bsxfun(@minus,b,mu); 40 | else 41 | a = a - repmat(mu,1,n); b = b - repmat(mu,1,m); 42 | end 43 | end 44 | 45 | if bsx % compute squared distances 46 | C = bsxfun(@plus,sum(a.*a,1)',bsxfun(@minus,sum(b.*b,1),2*a'*b)); 47 | else 48 | C = repmat(sum(a.*a,1)',1,m) + repmat(sum(b.*b,1),n,1) - 2*a'*b; 49 | end 50 | C = max(C,0); % numerical noise can cause C to negative i.e. C > -1e-14 51 | -------------------------------------------------------------------------------- /bayesopt/gpml/util/sq_dist.m: -------------------------------------------------------------------------------- 1 | % sq_dist - a function to compute a matrix of all pairwise squared distances 2 | % between two sets of vectors, stored in the columns of the two matrices, a 3 | % (of size D by n) and b (of size D by m). If only a single argument is given 4 | % or the second matrix is empty, the missing matrix is taken to be identical 5 | % to the first. 6 | % 7 | % Usage: C = sq_dist(a, b) 8 | % or: C = sq_dist(a) or equiv.: C = sq_dist(a, []) 9 | % 10 | % Where a is of size Dxn, b is of size Dxm (or empty), C is of size nxm. 11 | % 12 | % Copyright (c) by Carl Edward Rasmussen and Hannes Nickisch, 2010-12-13. 13 | 14 | function C = sq_dist(a, b) 15 | 16 | if nargin<1 || nargin>3 || nargout>1, error('Wrong number of arguments.'); end 17 | bsx = exist('bsxfun','builtin'); % since Matlab R2007a 7.4.0 and Octave 3.0 18 | if ~bsx, bsx = exist('bsxfun'); end % bsxfun is not yet "builtin" in Octave 19 | [D, n] = size(a); 20 | 21 | % Computation of a^2 - 2*a*b + b^2 is less stable than (a-b)^2 because numerical 22 | % precision can be lost when both a and b have very large absolute value and the 23 | % same sign. For that reason, we subtract the mean from the data beforehand to 24 | % stabilise the computations. This is OK because the squared error is 25 | % independent of the mean. 26 | if nargin==1 % subtract mean 27 | mu = mean(a,2); 28 | if bsx 29 | a = bsxfun(@minus,a,mu); 30 | else 31 | a = a - repmat(mu,1,size(a,2)); 32 | end 33 | b = a; m = n; 34 | else 35 | [d, m] = size(b); 36 | if d ~= D, error('Error: column lengths must agree.'); end 37 | mu = (m/(n+m))*mean(b,2) + (n/(n+m))*mean(a,2); 38 | if bsx 39 | a = bsxfun(@minus,a,mu); b = bsxfun(@minus,b,mu); 40 | else 41 | a = a - repmat(mu,1,n); b = b - repmat(mu,1,m); 42 | end 43 | end 44 | 45 | if bsx % compute squared distances 46 | C = bsxfun(@plus,sum(a.*a,1)',bsxfun(@minus,sum(b.*b,1),2*a'*b)); 47 | else 48 | C = repmat(sum(a.*a,1)',1,m) + repmat(sum(b.*b,1),n,1) - 2*a'*b; 49 | end 50 | C = max(C,0); % numerical noise can cause C to negative i.e. C > -1e-14 51 | -------------------------------------------------------------------------------- /gpml/cov/covSum.m: -------------------------------------------------------------------------------- 1 | function K = covSum(cov, hyp, x, z, i) 2 | 3 | % covSum - compose a covariance function as the sum of other covariance 4 | % functions. This function doesn't actually compute very much on its own, it 5 | % merely does some bookkeeping, and calls other covariance functions to do the 6 | % actual work. 7 | % 8 | % Copyright (c) by Carl Edward Rasmussen & Hannes Nickisch 2010-09-10. 9 | % 10 | % See also COVFUNCTIONS.M. 11 | 12 | if ~isempty(cov)==0, error('We require at least one summand.'), end 13 | for ii = 1:numel(cov) % iterate over covariance functions 14 | f = cov(ii); if iscell(f{:}), f = f{:}; end % expand cell array if necessary 15 | j(ii) = cellstr(feval(f{:})); % collect number hypers 16 | end 17 | 18 | if nargin<3 % report number of parameters 19 | K = char(j(1)); for ii=2:length(cov), K = [K, '+', char(j(ii))]; end, return 20 | end 21 | if nargin<4, z = []; end % make sure, z exists 22 | [n,D] = size(x); 23 | 24 | v = []; % v vector indicates to which covariance parameters belong 25 | for ii = 1:length(cov), v = [v repmat(ii, 1, eval(char(j(ii))))]; end 26 | 27 | if nargin<5 % covariances 28 | K = 0; if nargin==3, z = []; end % set default 29 | for ii = 1:length(cov) % iteration over summand functions 30 | f = cov(ii); if iscell(f{:}), f = f{:}; end % expand cell array if necessary 31 | K = K + feval(f{:}, hyp(v==ii), x, z); % accumulate covariances 32 | end 33 | else % derivatives 34 | if i<=length(v) 35 | vi = v(i); % which covariance function 36 | j = sum(v(1:i)==vi); % which parameter in that covariance 37 | f = cov(vi); 38 | if iscell(f{:}), f = f{:}; end % dereference cell array if necessary 39 | K = feval(f{:}, hyp(v==vi), x, z, j); % compute derivative 40 | else 41 | error('Unknown hyperparameter') 42 | end 43 | end --------------------------------------------------------------------------------