├── .gitattributes ├── Constrained_CEI.m ├── Constrained_PCEI.m ├── DTLZ2.m ├── GP_Predict.m ├── GP_Train.m ├── GP_Train_Cooperative.m ├── HighDimension_CoEGO.m ├── HighDimension_Dropout.m ├── HighDimension_ECI.m ├── Hypervolume.mexw64 ├── Infill_CEI.m ├── Infill_EHVI.m ├── Infill_EI.m ├── Infill_EIM.m ├── Infill_EIM_Matrix.asv ├── Infill_ESSI.m ├── Infill_FqEI.m ├── Infill_PEI.m ├── Infill_PoF.m ├── Infill_Pseudo_CEI.m ├── Infill_Pseudo_PoF.m ├── Infill_qEI.m ├── Infill_qEI_MC.m ├── MultiObjective_EHVI.m ├── MultiObjective_EIM.asv ├── MultiObjective_EIM.m ├── MultiObjective_MOEAD_EGO.m ├── MultiObjective_ParEGO.m ├── Optimizer_GA.m ├── Parallel_CL.m ├── Parallel_FqEI.m ├── Parallel_KB.m ├── Parallel_PEI.m ├── Parallel_qEI.m ├── Paretoset.m ├── README.md ├── Rosenbrock.m ├── Standard_BO.m ├── Tchebycheff_EI.m ├── UniformPoint.m └── Welded_Beam.m /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /Constrained_CEI.m: -------------------------------------------------------------------------------- 1 | clearvars;close all; 2 | % settings of the problem 3 | fun_name = 'Fun_Welded_Beam'; 4 | num_vari = 4; 5 | num_con = 6; 6 | lower_bound = [0.1, 0.1, 0.1, 0.1]; 7 | upper_bound = [2.0, 10.0, 10.0, 2.0]; 8 | % the number of initial design points 9 | num_initial = 20; 10 | % maximum number of evaluations 11 | max_evaluation = 100; 12 | % the 0th iteration 13 | % initial design points using Latin hypercube sampling method 14 | sample_x = lower_bound + (upper_bound-lower_bound).*lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000); 15 | [sample_y, sample_g] = feval(fun_name, sample_x); 16 | % the number of total evaluations 17 | evaluation = size(sample_x,1); 18 | iteration = 0; 19 | % check is there is at least one feasible solution 20 | index = sum(sample_g <= 0, 2) == num_con; 21 | if sum(index) ~= 0 22 | fmin = min(sample_y(index, :)); 23 | fprintf('iteration: %d, evaluation: %d, best solution: %f\n', 0, evaluation, fmin); 24 | else 25 | fmin = 1E6; 26 | fprintf('iteration: %d, evaluation: %d, best solution: no feasiable solution\n', 0, evaluation); 27 | end 28 | % beginning of the iteration 29 | while evaluation < max_evaluation 30 | % if there is no feasiable solution, there is no need to build the 31 | % kriging model for the objective function (build anyway) 32 | kriging_con = cell(1,num_con); 33 | for ii = 1: num_con 34 | kriging_con{ii} = Kriging_Train(sample_x,sample_g(:, ii),lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 35 | end 36 | kriging_obj = Kriging_Train(sample_x,sample_y,lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 37 | % select the infill samples 38 | if sum(index) ~= 0 39 | infill_criterion = @(x)-Infill_CEI(x,kriging_obj,kriging_con,fmin); 40 | else 41 | infill_criterion = @(x)-Infill_PoF(x,kriging_con); 42 | end 43 | % find the candidate by maximizing the infill criterion 44 | [best_x,max_EI] = Optimizer_GA(infill_criterion,num_vari,lower_bound,upper_bound,50,100); 45 | % evalaute the candidate points in parallel 46 | [best_y, best_g] = feval(fun_name,best_x); 47 | % add the new point to design set 48 | sample_x = [sample_x;best_x]; 49 | sample_y = [sample_y;best_y]; 50 | sample_g = [sample_g;best_g]; 51 | % update the best solution 52 | % check is there is at least one feasible solution 53 | evaluation = size(sample_x,1); 54 | iteration = iteration + 1; 55 | index = sum(sample_g <= 0, 2) == num_con; 56 | if sum(index) ~= 0 57 | fmin = min(sample_y(index, :)); 58 | fprintf('iteration: %d, evaluation: %d, best solution: %f\n',iteration,evaluation,fmin); 59 | else 60 | fmin = 1E6; 61 | fprintf('iteration: %d, evaluation: %d, best solution: no feasiable solution\n',iteration,evaluation); 62 | end 63 | end 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /Constrained_PCEI.m: -------------------------------------------------------------------------------- 1 | % the EGO algorithm using Constrained EI is implemented [1]. The GA [2] is used 2 | % to maximized to infill criterion. 3 | % [1] M. Schonlau, Computer experiments and global optimization. 1997, 4 | % University of Waterloo. 5 | % [2] K. Deb. An efficient constraint handling method for genetic 6 | % algorithms. Computer Methods in Applied Mechanics and Engineering, 2002, 7 | % 186(2): 311-338. 8 | % This program is free software; you can redistribute it and/or 9 | % modify it. This program is distributed in the hope that it will be useful, 10 | % but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 12 | %-------------------------------------------------------------------------- 13 | clearvars;close all; 14 | % settings of the problem 15 | fun_name = 'Fun_Welded_Beam'; 16 | num_vari = 4; 17 | num_con = 6; 18 | lower_bound = [0.1, 0.1, 0.1, 0.1]; 19 | upper_bound = [2.0, 10.0, 10.0, 2.0]; 20 | % the number of initial design points 21 | num_initial = 20; 22 | % maximum number of evaluations 23 | max_evaluation = 100; 24 | % the number of points selected in each iteration 25 | num_q = 2; 26 | % initial design points using Latin hypercube sampling method 27 | sample_x = lower_bound + (upper_bound-lower_bound).*lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000); 28 | [sample_y, sample_g] = feval(fun_name, sample_x); 29 | % the number of total evaluations 30 | evaluation = size(sample_x,1); 31 | iteration = 0; 32 | % check is there is at least one feasible solution 33 | index = sum(sample_g <= 0, 2) == num_con; 34 | if sum(index) ~= 0 35 | fmin = min(sample_y(index, :)); 36 | fprintf('iteration: %d, evaluation: %d, best solution: %f\n', 0, evaluation, fmin); 37 | else 38 | fmin = 1E6; 39 | fprintf('iteration: %d, evaluation: %d, best solution: no feasiable solution\n', 0, evaluation); 40 | end 41 | % beginning of the iteration 42 | while evaluation < max_evaluation 43 | % if there is no feasiable solution, there is no need to build the 44 | % kriging model for the objective function (build anyway) 45 | kriging_con = cell(1,num_con); 46 | for ii = 1: num_con 47 | kriging_con{ii} = Kriging_Train(sample_x,sample_g(:, ii),lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 48 | end 49 | kriging_obj = Kriging_Train(sample_x,sample_y,lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 50 | 51 | % select the infill samples 52 | num_k = min(num_q,max_evaluation - evaluation); 53 | best_x = zeros(num_q, num_vari); 54 | point_added = []; 55 | for ii = 1 : num_q 56 | if sum(index) ~= 0 57 | infill_criterion = @(x)-Infill_Pseudo_CEI(x,kriging_obj,kriging_con,fmin,point_added); 58 | else 59 | infill_criterion = @(x)-Infill_Pseudo_PoF(x,kriging_con,point_added); 60 | end 61 | % find the candidate by maximizing the infill criterion 62 | [best_x(ii,:),best_EI] = Optimizer_GA(infill_criterion,num_vari,lower_bound,upper_bound,50,100); 63 | point_added = best_x(1:ii,:); 64 | end 65 | % evalaute the candidate points in parallel 66 | [best_y, best_g] = feval(fun_name,best_x); 67 | % add the new point to design set 68 | sample_x = [sample_x;best_x]; 69 | sample_y = [sample_y;best_y]; 70 | sample_g = [sample_g;best_g]; 71 | % update the best solution 72 | % check is there is at least one feasible solution 73 | evaluation = size(sample_x,1); 74 | iteration = iteration + 1; 75 | index = sum(sample_g <= 0, 2) == num_con; 76 | if sum(index) ~= 0 77 | fmin = min(sample_y(index, :)); 78 | fprintf('iteration: %d, evaluation: %d, best solution: %f\n',iteration,evaluation,fmin); 79 | else 80 | fmin = 1E6; 81 | fprintf('iteration: %d, evaluation: %d, best solution: no feasiable solution\n',iteration,evaluation); 82 | end 83 | end 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /DTLZ2.m: -------------------------------------------------------------------------------- 1 | function y = Fun_DTLZ2(x,m) 2 | % 02 9 | for ii=2:m-1 10 | y(:,ii)=(1+g).*prod(cos(0.5*pi*x(:,1:m-ii)),2).*sin(0.5*pi*x(:,m-ii+1)); 11 | end 12 | end 13 | y(:,m)=(1+g).*sin(0.5*pi*x(:,1)); 14 | 15 | 16 | end -------------------------------------------------------------------------------- /GP_Predict.m: -------------------------------------------------------------------------------- 1 | function [u,s,Corr,Cov] = GP_Predict(test_x,model) 2 | % parameters of Kriging model 3 | theta = model.theta; 4 | mu = model.mu; 5 | sigma2 = model.sigma2; 6 | L = model.L; 7 | sample_x = model.sample_x; 8 | sample_y = model.sample_y; 9 | lower_bound = model.lower_bound; 10 | upper_bound = model.upper_bound; 11 | % normalize data 12 | X = (sample_x - lower_bound)./(upper_bound - lower_bound); 13 | x = (test_x - lower_bound)./(upper_bound- lower_bound); 14 | % initialize the prediction and variance 15 | one = ones(size(sample_x,1),1); 16 | % point-wise calculation 17 | temp1 = sum(x.^2.*theta,2)*ones(1,size(X,1)); 18 | temp2 = sum(X.^2.*theta,2)*ones(1,size(x,1)); 19 | R = exp(-(temp1 + temp2'-2.*(x.*theta)*X'))'; 20 | u = mu + R' *(L'\(L\(sample_y - mu))); 21 | mse = sigma2*(1 + (1-one'*(L'\(L\R)))'.^2/(one'*(L'\(L\one))) - sum((L\R).^2,1)'); 22 | s = sqrt(max(mse,0)); 23 | % the correlation matrix 24 | temp1 = sum(x.^2.*theta,2)*ones(1,size(test_x,1)); 25 | temp2 = x.*sqrt(theta); 26 | Corr = exp(-(temp1 + temp1'-2.*(temp2*temp2'))) + eye(size(test_x,1)).*(10+size(test_x,1))*eps; 27 | Cov = Corr.*(s*s'); 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /GP_Train.m: -------------------------------------------------------------------------------- 1 | function model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,theta0,theta_lower,theta_upper) 2 | [n,num_vari]= size(sample_x); 3 | X = (sample_x - lower_bound)./(upper_bound - lower_bound); 4 | Y = sample_y; 5 | % optimize the theta values with in [10^a,10^b] 6 | if nargin > 5 7 | theta0 = log10(theta0); 8 | theta_lower = log10(theta_lower); 9 | theta_upper = log10(theta_upper); 10 | options = optimoptions('fmincon','Algorithm','sqp','MaxFunctionEvaluations',20*num_vari,'OptimalityTolerance',1E-20,'StepTolerance',1E-20,'Display','off'); 11 | theta = fmincon(@(theta)Concentrated_lnLikelihood(theta,X,Y),theta0,[],[],[],[],theta_lower,theta_upper,[],options); 12 | theta = 10.^theta; 13 | else 14 | theta = theta0; 15 | end 16 | % calculate the correlation matrix 17 | one = ones(n,1); 18 | temp1 = sum(X.^2.*theta,2)*one'; 19 | temp2 = X.*sqrt(theta); 20 | R = exp(-(temp1 + temp1'-2.*(temp2*temp2'))) + eye(n).*(10+n)*eps; 21 | % use the Cholesky factorization 22 | L = chol(R,'lower'); 23 | % calculate mu and sigma 24 | mu = (one'*(L'\(L\Y)))/(one'*(L'\(L\one))); 25 | sigma2 = ((Y-mu)'*(L'\(L\(Y-mu))))/n; 26 | lnL = -0.5*n*log(sigma2)-sum(log(abs(diag(L)))); 27 | % output the results of the model 28 | model.theta = theta; 29 | model.mu = mu; 30 | model.sigma2 = sigma2; 31 | model.L = L; 32 | model.lnL = lnL; 33 | model.sample_x = sample_x; 34 | model.sample_y = sample_y; 35 | model.lower_bound = lower_bound; 36 | model.upper_bound = upper_bound; 37 | end 38 | 39 | 40 | 41 | 42 | function obj = Concentrated_lnLikelihood(theta,X,Y) 43 | theta = 10.^theta; 44 | % the concentrated ln-likelihood function 45 | n = size(X,1); 46 | one = ones(n,1); 47 | % calculate the correlation matrix 48 | temp1 = sum(X.^2.*theta,2)*one'; 49 | temp2 = X.*sqrt(theta); 50 | R = exp(-(temp1 + temp1'-2.*(temp2*temp2'))) + eye(n).*(10+n)*eps; 51 | % use the Cholesky factorization 52 | [L,p] = chol(R,'lower'); 53 | if p>0 54 | lnL = -1e8; 55 | else 56 | mu = (one'*(L'\(L\Y)))/(one'*(L'\(L\one))); 57 | sigma2 = ((Y-mu)'*(L'\(L\(Y-mu))))/n; 58 | lnL = -0.5*n*log(sigma2)-sum(log(abs(diag(L)))); 59 | end 60 | obj = -lnL; 61 | end 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /GP_Train_Cooperative.m: -------------------------------------------------------------------------------- 1 | function model = GP_Train_Cooperative(sample_x,sample_y,lower_bound,upper_bound,best_theta,theta_lower,theta_upper,index) 2 | n = size(sample_x,1); 3 | sub_vari = length(index); 4 | X = (sample_x - lower_bound)./(upper_bound - lower_bound); 5 | Y = sample_y; 6 | % optimize the theta values with in [10^a,10^b] 7 | theta0 = log10(best_theta(index)); 8 | theta_lower = log10(theta_lower); 9 | theta_upper = log10(theta_upper); 10 | options = optimoptions('fmincon','Algorithm','sqp','MaxFunctionEvaluations',20*sub_vari,'OptimalityTolerance',1E-20,'StepTolerance',1E-20,'Display','off'); 11 | sub_theta = fmincon(@(t)-concentrated_lnLikelihood_co(t,X,Y,best_theta,index),theta0,[],[],[],[],theta_lower,theta_upper,[],options); 12 | sub_theta = 10.^sub_theta; 13 | theta = best_theta; 14 | theta(index) = sub_theta; 15 | % calculate the correlation matrix 16 | one = ones(n,1); 17 | temp1 = sum(X.^2.*theta,2)*one'; 18 | temp2 = X.*sqrt(theta); 19 | R = exp(-(temp1 + temp1'-2.*(temp2*temp2'))) + eye(n).*(10+n)*eps; 20 | % use the Cholesky factorization 21 | L = chol(R,'lower'); 22 | % calculate mu and sigma 23 | mu = (one'*(L'\(L\Y)))/(one'*(L'\(L\one))); 24 | sigma2 = ((Y-mu)'*(L'\(L\(Y-mu))))/n; 25 | lnL = -0.5*n*log(sigma2)-sum(log(abs(diag(L)))); 26 | % output the results of the model 27 | model.theta = theta; 28 | model.mu = mu; 29 | model.sigma2 = sigma2; 30 | model.L = L; 31 | model.lnL = lnL; 32 | model.sample_x = sample_x; 33 | model.sample_y = sample_y; 34 | model.lower_bound = lower_bound; 35 | model.upper_bound = upper_bound; 36 | end 37 | 38 | 39 | 40 | 41 | 42 | function obj = concentrated_lnLikelihood_co(t,X,Y,best_theta,index) 43 | theta = best_theta; 44 | theta(index) = 10.^t; 45 | % the concentrated ln-likelihood function 46 | n = size(X,1); 47 | one = ones(n,1); 48 | % calculate the correlation matrix 49 | temp1 = sum(X.^2.*theta,2)*one'; 50 | temp2 = X.*sqrt(theta); 51 | R = exp(-(temp1 + temp1'-2.*(temp2*temp2'))) + eye(n).*(10+n)*eps; 52 | % use the Cholesky factorization 53 | [L,p] = chol(R,'lower'); 54 | if p>0 55 | lnL = -1e8; 56 | else 57 | mu = (one'*(L'\(L\Y)))/(one'*(L'\(L\one))); 58 | sigma2 = ((Y-mu)'*(L'\(L\(Y-mu))))/n; 59 | lnL = -0.5*n*log(sigma2)-sum(log(abs(diag(L)))); 60 | end 61 | obj = lnL; 62 | end 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /HighDimension_CoEGO.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of the cooperative efficient global 3 | % optimization algorithm according to the following work. 4 | % Reference: 5 | % D. Zhan, J. Wu, H. Xing, and T. Li. A cooperative approach to efficient 6 | % global optimization. Journal of Global Optimization. 2024, 88: 327-357. 7 | % Author: Dawei Zhan 8 | % Date: 2024/12/30 9 | %------------------------------------------------------------------------- 10 | clearvars; close all; 11 | % setting of the problem 12 | fun_name = 'Rosenbrock'; 13 | num_vari = 50; 14 | lower_bound = -2.048*ones(1,num_vari); 15 | upper_bound = 2.048*ones(1,num_vari); 16 | % number of variables to be optimized in each iteration 17 | sub_vari = 5; 18 | % the number of initial design points 19 | num_initial = 200; 20 | % maximum number of evaluations 21 | max_evaluation = 500; 22 | % initial design points using Latin hypercube sampling method 23 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 24 | sample_y = feval(fun_name,sample_x); 25 | % the current iteration and evaluation 26 | evaluation = size(sample_x,1); 27 | iteration = 0; 28 | % the current best solution 29 | [fmin,index] = min(sample_y); 30 | xmin = sample_x(index,:); 31 | % print the current information to the screen 32 | fprintf('CoEGO on %d-D %s function, iteration: %d, evaluation: %d, current found minimum: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 33 | % the iteration 34 | while evaluation < max_evaluation 35 | % randomly divide the variables into num_vari/sub_vari groups 36 | rand_ind = randperm(num_vari); 37 | unsolved = min(max_evaluation-evaluation,num_vari/sub_vari); 38 | best_theta = ones(1,num_vari); 39 | for ii = 1:unsolved 40 | optimized_index = rand_ind((ii-1)*sub_vari+1:ii*sub_vari); 41 | % build the GP model cooperatively 42 | train_x = sample_x(:,optimized_index); 43 | GP_model = GP_Train_Cooperative(sample_x,sample_y,lower_bound,upper_bound,best_theta,0.001*ones(1,sub_vari),1000*ones(1,sub_vari),optimized_index); 44 | best_theta(optimized_index) = GP_model.theta(optimized_index); 45 | % optimize the EI cooperatively 46 | optimized_x = Optimizer_GA(@(x)-Infill_ESSI(x,GP_model,fmin,xmin,optimized_index),sub_vari,lower_bound(optimized_index),upper_bound(optimized_index),10*sub_vari,200); 47 | infill_x = xmin; 48 | infill_x(optimized_index) = optimized_x; 49 | % evaluate the query point with the real function 50 | infill_y = feval(fun_name,infill_x); 51 | % add the new point to design set 52 | sample_x = [sample_x;infill_x]; 53 | sample_y = [sample_y;infill_y]; 54 | % update some parameters 55 | evaluation = size(sample_x,1); 56 | iteration = iteration + 1; 57 | [fmin,index] = min(sample_y); 58 | xmin = sample_x(index,:); 59 | % print the current information to the screen 60 | fprintf('CoEGO on %d-D %s function, iteration: %d, evaluation: %d, current found minimum: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 61 | end 62 | end 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /HighDimension_Dropout.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of the Dropout approach according to 3 | % the following work. The Dropout approach randomly select a subset 4 | % variables to optimize in each BO iteration. Instead of training the GP 5 | % model in the subspace, I train the GP in the original space in the 6 | % implemetation since I found training GP in the original high-dimensional 7 | % space can yield better results. 8 | % Reference: 9 | % C. Li, S. Gupta, S. Rana, T. V. Nguyen, S. Venkatesh, and A. Shilton, 10 | % High dimensional Bayesian optimization using dropout. International Joint 11 | % Conference on Artifical Intelligence, 2017, 2096-2102. 12 | % Author: Dawei Zhan 13 | % Date: 2024/12/30 14 | %------------------------------------------------------------------------- 15 | clearvars; close all; 16 | % setting of the problem 17 | fun_name = 'Rosenbrock'; 18 | num_vari = 50; 19 | lower_bound = -2.048*ones(1,num_vari); 20 | upper_bound = 2.048*ones(1,num_vari); 21 | % number of variables to be optimized in each iteration 22 | sub_vari = 5; 23 | % the number of initial design points 24 | num_initial = 200; 25 | % maximum number of evaluations 26 | max_evaluation = 500; 27 | % initial design points using Latin hypercube sampling method 28 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 29 | sample_y = feval(fun_name,sample_x); 30 | % the current iteration and evaluation 31 | evaluation = size(sample_x,1); 32 | iteration = 0; 33 | % the current best solution 34 | [fmin,index] = min(sample_y); 35 | xmin = sample_x(index,:); 36 | % print the current information to the screen 37 | fprintf('Dropout on %d-D %s function, iteration: %d, evaluation: %d, current found minimum: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 38 | % the iteration 39 | while evaluation < max_evaluation 40 | optimized_index = randperm(num_vari,sub_vari); 41 | % build the GP model 42 | train_x = sample_x(:,optimized_index); 43 | GP_model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 44 | % find the point with the highest EI value using GA algorithm 45 | optimized_x = Optimizer_GA(@(x)-Infill_ESSI(x,GP_model,fmin,xmin,optimized_index),sub_vari,lower_bound(optimized_index),upper_bound(optimized_index),10*sub_vari,200); 46 | infill_x = xmin; 47 | infill_x(optimized_index) = optimized_x; 48 | % evaluate the query point with the real function 49 | infill_y = feval(fun_name,infill_x); 50 | % add the new point to design set 51 | sample_x = [sample_x;infill_x]; 52 | sample_y = [sample_y;infill_y]; 53 | % update some parameters 54 | evaluation = size(sample_x,1); 55 | iteration = iteration + 1; 56 | [fmin,index] = min(sample_y); 57 | xmin = sample_x(index,:); 58 | % print the current information to the screen 59 | fprintf('Dropout on %d-D %s function, iteration: %d, evaluation: %d, current found minimum: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 60 | end 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /HighDimension_ECI.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of the expected coordinate improvement 3 | % approach according to the following work. 4 | % Reference: 5 | % D. Zhan. Expected coordinate improvement for high-dimensional Bayesian 6 | % optimization. Swarm and Evolutionary Computation. 2024, 91: 101745. 7 | % Date: 2024/12/30 8 | %------------------------------------------------------------------------- 9 | clearvars; close all; 10 | % setting of the problem 11 | fun_name = 'Rosenbrock'; 12 | num_vari = 50; 13 | lower_bound = -2.048*ones(1,num_vari); 14 | upper_bound = 2.048*ones(1,num_vari); 15 | % number of variables to be optimized in each iteration 16 | sub_vari = 5; 17 | % the number of initial design points 18 | num_initial = 200; 19 | % maximum number of evaluations 20 | max_evaluation = 500; 21 | % initial design points using Latin hypercube sampling method 22 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 23 | sample_y = feval(fun_name,sample_x); 24 | % the current iteration and evaluation 25 | evaluation = size(sample_x,1); 26 | iteration = 0; 27 | % the current best solution 28 | [fmin,index] = min(sample_y); 29 | xmin = sample_x(index,:); 30 | % print the current information to the screen 31 | fprintf('ECI on %d-D %s function, iteration: %d, evaluation: %d, current found minimum: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 32 | % the iteration 33 | while evaluation < max_evaluation 34 | % train the GP model 35 | GP_model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 36 | % find the maximum ECI values for all the coordinates 37 | max_ECI = zeros(1,num_vari); 38 | max_x = zeros(1,num_vari); 39 | % you can use parfor to compute this in parallel 40 | for ii = 1:num_vari 41 | [max_x(ii),max_ECI(ii)] = Optimizer_GA(@(x)-Infill_ESSI(x,GP_model,fmin,xmin,ii),1,lower_bound(ii),upper_bound(ii),10,20); 42 | end 43 | % get the coordinate optimization order based on the ECI values 44 | [sort_EI,sort_dim] = sort(-max_ECI,'descend'); 45 | % optimize one coordinate at a time 46 | for ii = 1:num_vari 47 | optimized_index = sort_dim(ii); 48 | if ii == 1 49 | optimized_x = max_x(optimized_index); 50 | else 51 | GP_model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 52 | [optimized_x,EI] = Optimizer_GA(@(x)-Infill_ESSI(x,GP_model,fmin,xmin,optimized_index),1,lower_bound(optimized_index),upper_bound(optimized_index),10,20); 53 | end 54 | infill_x = xmin; 55 | infill_x(optimized_index) = optimized_x; 56 | % evaluate the query point with the real function 57 | infill_y = feval(fun_name,infill_x); 58 | % add the new point to design set 59 | sample_x = [sample_x;infill_x]; 60 | sample_y = [sample_y;infill_y]; 61 | % update some parameters 62 | evaluation = size(sample_x,1); 63 | iteration = iteration + 1; 64 | [fmin,index] = min(sample_y); 65 | xmin = sample_x(index,:); 66 | % print the current information to the screen 67 | fprintf('ECI on %d-D %s function, iteration: %d, evaluation: %d, current found minimum: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 68 | end 69 | end 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /Hypervolume.mexw64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhandawei/Bayesian_Optimization_Algorithms/2da2376d845ac071424b4a5528202ab0aecd4191/Hypervolume.mexw64 -------------------------------------------------------------------------------- /Infill_CEI.m: -------------------------------------------------------------------------------- 1 | function obj = Infill_CEI(x, kriging_obj,kriging_con,fmin) 2 | % the kriging prediction and varince 3 | [u,s] = Kriging_Predictor(x,kriging_obj); 4 | % the EI value 5 | EI = (fmin-u).*normcdf((fmin-u)./s)+s.*normpdf((fmin-u)./s); 6 | % the number of constraints 7 | num_con = length(kriging_con); 8 | % the kriging prediction and varince 9 | u_g = zeros(size(x,1), num_con); 10 | s_g = zeros(size(x,1), num_con); 11 | for ii = 1: num_con 12 | [u_g(:, ii), s_g(:, ii)] = Kriging_Predictor(x, kriging_con{ii}); 13 | end 14 | % the PoF value 15 | PoF = prod(normpdf((0-u_g)./s_g),2); 16 | CEI = EI.*PoF; 17 | obj = CEI; 18 | end 19 | -------------------------------------------------------------------------------- /Infill_EHVI.m: -------------------------------------------------------------------------------- 1 | function EHVI = Infill_EHVI(x,model,f) 2 | num_x = size(x,1); 3 | % number of objectives 4 | num_obj = size(f,2); 5 | r = 1.1*ones(1,num_obj); 6 | % the kriging prediction and varince 7 | u = zeros(num_x,num_obj); 8 | s = zeros(num_x,num_obj); 9 | for ii = 1:num_obj 10 | [u(:, ii),s(:, ii)] = GP_Predict(x,model{ii}); 11 | end 12 | EHVI = zeros(num_x,1); 13 | for ii = 1:size(x,1) 14 | % EHVI calculated using Monte Carlo simulation 15 | num_simluation_point = 50; 16 | num_simultion_HV = 50; 17 | hvi = zeros(num_simluation_point,1); 18 | rand_sample = mvnrnd(u(ii,:),diag(s(ii,:).^2),num_simluation_point); 19 | for jj = 1:num_simluation_point 20 | if any(all(f <= rand_sample(jj,:),2)) % dominated solution 21 | hvi(jj) = 0; 22 | else % non dominated solution 23 | new_front = [f;rand_sample(jj,:)]; 24 | upper_bound = r; 25 | lower_bound = min(new_front); 26 | sim_point = rand(num_simultion_HV,num_obj).*(upper_bound-lower_bound)+lower_bound; 27 | num_front_point = size(new_front,1); 28 | simulated_point_matrix = repelem(sim_point,num_front_point,1); 29 | new_front_matrix = repmat(new_front,num_simultion_HV,1); 30 | is_dominated = reshape(all(new_front_matrix <= simulated_point_matrix,2),num_front_point,num_simultion_HV)'; 31 | num_improvement = sum(any(is_dominated,2)) - sum(any(is_dominated(:,1:size(f,1)),2)); 32 | hvi(jj) = prod(upper_bound-lower_bound)*num_improvement/num_simultion_HV; 33 | end 34 | end 35 | EHVI(ii,:) = mean(hvi); 36 | end 37 | -------------------------------------------------------------------------------- /Infill_EI.m: -------------------------------------------------------------------------------- 1 | function obj = Infill_EI(x,model,fmin) 2 | % get the GP prediction and variance 3 | [u,s] = GP_Predict(x,model); 4 | % calcuate the EI value 5 | EI = (fmin-u).*normcdf((fmin-u)./s)+s.*normpdf((fmin-u)./s); 6 | % this EI needs to be maximized 7 | obj = EI; 8 | 9 | end 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Infill_EIM.m: -------------------------------------------------------------------------------- 1 | function y = Infill_EIM(x,kriging_obj,non_dominated_front,criterion) 2 | % you can choose criterion as 'Euclidean', 'Maximin', or 'Hypervolume' 3 | num_x = size(x,1); 4 | % number of non-dominated points,number of objectives 5 | [num_pareto,num_obj] = size(non_dominated_front); 6 | % the kriging prediction and varince 7 | u = zeros(num_x,num_obj); 8 | s = zeros(num_x,num_obj); 9 | for ii = 1 : num_obj 10 | [u(:, ii),s(:, ii)] = GP_Predict(x, kriging_obj{ii}); 11 | end 12 | u_matrix = repelem(u,num_pareto,1); 13 | s_matrix = repelem(s,num_pareto,1); 14 | f_matrix = repmat(non_dominated_front,num_x,1); 15 | EIM = (f_matrix - u_matrix).*normcdf((f_matrix - u_matrix)./s_matrix) + s_matrix.*normpdf((f_matrix - u_matrix)./s_matrix); 16 | switch criterion 17 | case 'Euclidean' 18 | y = min(reshape(sqrt(sum(EIM.^2,2)),[num_pareto,num_x]))'; 19 | case 'Maximin' 20 | y = min(reshape(max(EIM,[],2),[num_pareto,num_x]))'; 21 | case 'Hypervolume' 22 | ref_point = 1.1*ones(1, num_obj); 23 | y = min(reshape(prod(ref_point-f_matrix+EIM,2)-prod(ref_point-f_matrix,2),[num_pareto,num_x]))'; 24 | end 25 | -------------------------------------------------------------------------------- /Infill_EIM_Matrix.asv: -------------------------------------------------------------------------------- 1 | function obj = Infill_EIM_Matrix(x,kriging_obj,non_dominated_front) 2 | num_x = size(x,1); 3 | % number of non-dominated points,number of objectives 4 | [num_pareto,num_obj] = size(non_dominated_front); 5 | % the kriging prediction and varince 6 | u = zeros(num_x,num_obj); 7 | s = zeros(num_x,num_obj); 8 | for ii = 1 : num_obj 9 | [u(:, ii),s(:, ii)] = GP_Predict(x, kriging_obj{ii}); 10 | end 11 | 12 | 13 | f_matrix = repelem(non_dominated_front,num_x,1); 14 | 15 | 16 | 17 | 18 | u_matrix = repmat(u,num_pareto,1); 19 | s_matrix = repmat(s,num_pareto,1); 20 | 21 | 22 | EIM = (f_matrix - u_matrix).*normcdf((f_matrix - u_matrix)./s_matrix) + s_matrix.*normpdf((f_matrix - u_matrix)./s_matrix); 23 | y = min(reshape(sqrt(sum(EIM.^2,2)))); 24 | obj = y; 25 | end 26 | -------------------------------------------------------------------------------- /Infill_ESSI.m: -------------------------------------------------------------------------------- 1 | function y = Infill_ESSI(x,model,fmin,best_x,subspace) 2 | 3 | n = size(x,1); 4 | new_x = repmat(best_x,n,1); 5 | new_x(:,subspace) = x; 6 | [u,s] = GP_Predict(new_x,model); 7 | y = (fmin-u).*normcdf((fmin-u)./s)+s.*normpdf((fmin-u)./s); 8 | end 9 | -------------------------------------------------------------------------------- /Infill_FqEI.m: -------------------------------------------------------------------------------- 1 | function y = Infill_FqEI(x,model,fmin) 2 | y = zeros(size(x,1),1); 3 | for ii = 1:size(x,1) 4 | new_x = x(ii,:); 5 | % reshape the input 6 | new_x = reshape(new_x,size(model.sample_x,2),[])'; 7 | % remove reduplicative points 8 | new_x = unique(new_x,'rows'); 9 | % remove sampled points 10 | Lia = ismember(new_x,model.sample_x,'rows'); 11 | new_x = new_x(~Lia,:); 12 | % number of infill samples 13 | q = size(new_x,1); 14 | % predictions and covarince matrix 15 | [u,s,Corr] = GP_Predict(new_x,model); 16 | EI = (fmin-u).*normcdf((fmin-u)./s)+s.*normpdf((fmin-u)./s); 17 | y(ii) = sum(EI) - sum((sum(Corr.*min(EI*ones(1,q),ones(q,1)*EI'),2) - EI)/q); 18 | end -------------------------------------------------------------------------------- /Infill_PEI.m: -------------------------------------------------------------------------------- 1 | function obj = Infill_PEI(x,model,fmin,point_added) 2 | % the pseudo EI criterion 3 | % get the GP prediction and variance 4 | [u,s] = GP_Predict(x,model); 5 | % calcuate the EI value 6 | EI = (fmin-u).*normcdf((fmin-u)./s)+s.*normpdf((fmin-u)./s); 7 | lower_bound = model.lower_bound; 8 | upper_bound = model.upper_bound; 9 | theta = model.theta; 10 | % if this is not the first infill point 11 | if ~isempty(point_added) 12 | % the scaling of x 13 | x1 = (x - lower_bound)./(upper_bound- lower_bound); 14 | x2 = (point_added - lower_bound)./(upper_bound - lower_bound); 15 | % calculate the correlation 16 | temp1 = sum(x1.^2.*theta,2)*ones(1,size(x2,1)); 17 | temp2 = sum(x2.^2.*theta,2)*ones(1,size(x1,1)); 18 | correlation = exp(-(temp1 + temp2'-2.*(x1.*theta)*x2')); 19 | % the Pseudo EI matrix 20 | EI = EI.*prod(1-correlation,2); 21 | end 22 | % the objective needs to be maximized 23 | obj = EI; 24 | end 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /Infill_PoF.m: -------------------------------------------------------------------------------- 1 | function obj = Infill_PoF(x, kriging_con) 2 | % the number of constraints 3 | num_con = length(kriging_con); 4 | % the kriging prediction and varince 5 | u = zeros(size(x,1), num_con); 6 | s = zeros(size(x,1), num_con); 7 | for ii = 1: num_con 8 | [u(:, ii), s(:, ii)] = Kriging_Predictor(x, kriging_con{ii}); 9 | end 10 | % the PoF value 11 | PoF = prod(normcdf((0-u)./s),2); 12 | obj = PoF; 13 | end 14 | -------------------------------------------------------------------------------- /Infill_Pseudo_CEI.m: -------------------------------------------------------------------------------- 1 | function obj = Infill_Pseudo_CEI(x,kriging_obj,kriging_con,fmin,point_added) 2 | % the kriging prediction and varince 3 | [u,s] = Kriging_Predictor(x,kriging_obj); 4 | % the EI value 5 | EI=(fmin-u).*normcdf((fmin-u)./s)+s.*normpdf((fmin-u)./s); 6 | % the number of constraints 7 | num_con = length(kriging_con); 8 | % the kriging prediction and varince 9 | u_g = zeros(size(x,1), num_con); 10 | s_g = zeros(size(x,1), num_con); 11 | for ii = 1: num_con 12 | [u_g(:, ii), s_g(:, ii)] = Kriging_Predictor(x, kriging_con{ii}); 13 | end 14 | % the PoF value 15 | PoF = prod(normcdf((0-u_g)./s_g), 2); 16 | CEI = EI.*PoF; 17 | lower_bound = kriging_obj.lower_bound; 18 | upper_bound = kriging_obj.upper_bound; 19 | theta = kriging_obj.theta; 20 | % if this is the first infill point 21 | if ~isempty(point_added) 22 | % the scaling of x is the same for different objectives 23 | x1 = (x - lower_bound)./(upper_bound- lower_bound); 24 | x2 = (point_added - lower_bound)./(upper_bound - lower_bound); 25 | % calculate the correlation 26 | temp1 = sum(x1.^2.*theta,2)*ones(1,size(x2,1)); 27 | temp2 = sum(x2.^2.*theta,2)*ones(1,size(x1,1)); 28 | correlation = exp(-(temp1 + temp2'-2.*(x1.*theta)*x2')); 29 | % the Pseudo EI matrix 30 | CEI = CEI.*prod(1-correlation,2); 31 | end 32 | % the objective is maximized 33 | obj = CEI; 34 | end 35 | -------------------------------------------------------------------------------- /Infill_Pseudo_PoF.m: -------------------------------------------------------------------------------- 1 | function obj = Infill_Pseudo_PoF(x,kriging_con,point_added) 2 | % the number of constraints 3 | num_con = length(kriging_con); 4 | % the kriging prediction and varince 5 | u = zeros(size(x,1), num_con); 6 | s = zeros(size(x,1), num_con); 7 | for ii = 1: num_con 8 | [u(:, ii), s(:, ii)] = Kriging_Predictor(x, kriging_con{ii}); 9 | end 10 | % the PoF value 11 | PoF = prod(normcdf((0-u)./s), 2); 12 | lower_bound = kriging_con{1}.lower_bound; 13 | upper_bound = kriging_con{1}.upper_bound; 14 | theta = kriging_con{1}.theta; 15 | % if this is the first infill point 16 | if ~isempty(point_added) 17 | % the scaling of x 18 | x1 = (x - lower_bound)./(upper_bound- lower_bound); 19 | x2 = (point_added - lower_bound)./(upper_bound - lower_bound); 20 | % calculate the correlation 21 | temp1 = sum(x1.^2.*theta,2)*ones(1,size(x2,1)); 22 | temp2 = sum(x2.^2.*theta,2)*ones(1,size(x1,1)); 23 | correlation = exp(-(temp1 + temp2'-2.*(x1.*theta)*x2')); 24 | % the Pseudo EI matrix 25 | PoF=PoF.*prod(1-correlation,2); 26 | end 27 | % the objective is maximized 28 | obj = PoF; 29 | end 30 | -------------------------------------------------------------------------------- /Infill_qEI.m: -------------------------------------------------------------------------------- 1 | function y = Infill_qEI(x,model,fmin) 2 | warning off; 3 | y = zeros(size(x,1),1); 4 | for ii = 1:size(x,1) 5 | new_x = x(ii,:); 6 | % reshape the input 7 | new_x = reshape(new_x,size(model.sample_x,2),[])'; 8 | % delete duplicated rows 9 | new_x = unique(new_x,'rows'); 10 | % number of infill samples 11 | q = size(new_x,1); 12 | % predictions and covarince matrix 13 | [u,s,~,Cov] = GP_Predict(new_x,model); 14 | mu = u'; 15 | sigma = Cov; 16 | symetric_term = zeros(q,q); 17 | nonsymetric_term = zeros(q,q); 18 | pk = zeros(1,q); 19 | first_term = zeros(1,q); 20 | second_term = zeros(1,q); 21 | if q == 1 22 | y = (fmin-u).*normcdf((fmin-u)./s)+s.*normpdf((fmin-u)./s); 23 | else 24 | for k = 1:q 25 | % Sigma_k: covariance matrix of vector Z^{k} 26 | Sigma_k = sigma; 27 | for i = 1:q 28 | if i ~= k 29 | Sigma_k(i,k) = sigma(k,k)-sigma(k,i); 30 | Sigma_k(k,i) = Sigma_k(i,k); 31 | end 32 | end 33 | for i = 1:q 34 | for j = 1:q 35 | if i~=k && j~=k 36 | if j>=i 37 | Sigma_k(i,j) = sigma(i,j) + sigma(k,k) - sigma(k,i) - sigma(k,j); 38 | else 39 | Sigma_k(i,j) = Sigma_k(j,i); 40 | end 41 | end 42 | end 43 | end 44 | % mu_k: mean of vector Z^{k} 45 | mu_k = mu(k) - mu; 46 | mu_k(k) = mu(k); 47 | % vector b_k 48 | b_k = zeros(1,q); 49 | b_k(k) = fmin; 50 | % calcualte pk 51 | Sigma_k = Sigma_k + diag(ones(1,q)*1E-20); 52 | [~,p] = chol(Sigma_k); 53 | if p > 0 54 | [L, DMC, P] = modchol_ldlt(Sigma_k); 55 | Sigma_k = P'*L*DMC*L'*P; 56 | end 57 | %pk(k) = mvncdf(b_k - mu_k,zeros(1,q),Sigma_k); 58 | pk(k) = qsimvnv(200*q, Sigma_k, -inf*ones(q,1), (b_k - mu_k)'); 59 | % replace NaN by 0 60 | if isnan(pk(k)) 61 | pk(k) = 0; 62 | end 63 | first_term(k) = (fmin - mu(k))*pk(k); 64 | 65 | nonsymetric_term(k,:) = Sigma_k(:,k)'; 66 | for i = 1:q 67 | if i >= k 68 | mik = mu_k(i); 69 | sigma_ii_k = Sigma_k(i,i); 70 | bik = b_k(i); 71 | phi_ik = normpdf(bik,mik,sqrt(sigma_ii_k)); 72 | % calculate c.i^(k) 73 | sigmai = Sigma_k(i,:)/Sigma_k(i,i); 74 | cik = (b_k - mu_k) - (b_k(i) - mu_k(i))*sigmai; 75 | cik(i) = []; 76 | % calculate sigma.i^(k) 77 | sigmaik = Sigma_k; 78 | for uu = 1:q 79 | for vv = 1:q 80 | if uu~=i && vv~=i 81 | if Sigma_k(i,i) == 0 82 | sigmaik(uu,vv) = Sigma_k(uu,vv); 83 | else 84 | sigmaik(uu,vv) = Sigma_k(uu,vv) - Sigma_k(uu,i)*Sigma_k(vv,i)/Sigma_k(i,i); 85 | end 86 | else 87 | sigmaik(uu,vv) = 0; 88 | end 89 | end 90 | end 91 | sigmaik(i,:)=[]; 92 | sigmaik(:,i)=[]; 93 | % calculate phi_ik 94 | [~,p] = chol(sigmaik); 95 | if p > 0 96 | [L, DMC, P] = modchol_ldlt(sigmaik); 97 | sigmaik = P'*L*DMC*L'*P; 98 | end 99 | %Phi_ik = mvncdf(cik,zeros(1,q-1),sigmaik); 100 | Phi_ik = qsimvnv(200*(q-1), sigmaik, -inf*ones(q-1,1),cik'); 101 | % replace NaN by 0 102 | if isnan(Phi_ik) 103 | Phi_ik = 0; 104 | end 105 | symetric_term(k,i) = phi_ik*Phi_ik; 106 | else 107 | symetric_term(k,i) = symetric_term(i,k); 108 | end 109 | end 110 | second_term(k) = sum(nonsymetric_term(k,:).*symetric_term(k,:)); 111 | 112 | end 113 | y(ii) = sum(first_term + second_term); 114 | end 115 | end 116 | end 117 | 118 | 119 | 120 | function [L, DMC, P, D] = modchol_ldlt(A,delta) 121 | % modchol_ldlt Modified Cholesky algorithm based on LDL' factorization. 122 | % [L D,P,D0] = modchol_ldlt(A,delta) computes a modified 123 | % Cholesky factorization P*(A + E)*P' = L*D*L', where 124 | % P is a permutation matrix, L is unit lower triangular, 125 | % and D is block diagonal and positive definite with 1-by-1 and 2-by-2 126 | % diagonal blocks. Thus A+E is symmetric positive definite, but E is 127 | % not explicitly computed. Also returned is a block diagonal D0 such 128 | % that P*A*P' = L*D0*L'. If A is sufficiently positive definite then 129 | % E = 0 and D = D0. 130 | % The algorithm sets the smallest eigenvalue of D to the tolerance 131 | % delta, which defaults to sqrt(eps)*norm(A,'fro'). 132 | % The LDL' factorization is compute using a symmetric form of rook 133 | % pivoting proposed by Ashcraft, Grimes and Lewis. 134 | 135 | % Reference: 136 | % S. H. Cheng and N. J. Higham. A modified Cholesky algorithm based 137 | % on a symmetric indefinite factorization. SIAM J. Matrix Anal. Appl., 138 | % 19(4):1097-1110, 1998. doi:10.1137/S0895479896302898, 139 | 140 | % Authors: Bobby Cheng and Nick Higham, 1996; revised 2015. 141 | 142 | if ~ishermitian(A) 143 | error('Must supply symmetric matrix.') 144 | end 145 | if nargin < 2, delta = sqrt(eps)*norm(A,'fro'); end 146 | 147 | n = max(size(A)); 148 | 149 | [L,D,p] = ldl(A,'vector'); 150 | DMC = eye(n); 151 | 152 | % Modified Cholesky perturbations. 153 | k = 1; 154 | while k <= n 155 | 156 | if k == n || D(k,k+1) == 0 % 1-by-1 block 157 | 158 | if D(k,k) <= delta 159 | DMC(k,k) = delta; 160 | else 161 | DMC(k,k) = D(k,k); 162 | end 163 | k = k+1; 164 | 165 | else % 2-by-2 block 166 | 167 | E = D(k:k+1,k:k+1); 168 | [U,T] = eig(E); 169 | for ii = 1:2 170 | if T(ii,ii) <= delta 171 | T(ii,ii) = delta; 172 | end 173 | end 174 | temp = U*T*U'; 175 | DMC(k:k+1,k:k+1) = (temp + temp')/2; % Ensure symmetric. 176 | k = k + 2; 177 | 178 | end 179 | 180 | end 181 | 182 | if nargout >= 3 183 | P = eye(n); 184 | P = P(p,:); 185 | end 186 | 187 | end 188 | 189 | 190 | 191 | 192 | function [ p, e ] = qsimvnv( m, r, a, b ) 193 | % 194 | % [ P E ] = QSIMVNV( M, R, A, B ) 195 | % uses a randomized quasi-random rule with m points to estimate an 196 | % MVN probability for positive definite covariance matrix r, 197 | % with lower integration limit column vector a and upper 198 | % integration limit column vector b. 199 | % Probability p is output with error estimate e. 200 | % Example: 201 | % r = [4 3 2 1;3 5 -1 1;2 -1 4 2;1 1 2 5]; 202 | % a = -inf*[1 1 1 1 ]'; b = [ 1 2 3 4 ]'; 203 | % [ p e ] = qsimvnv( 5000, r, a, b ); disp([ p e ]) 204 | % 205 | % This function uses an algorithm given in the paper 206 | % "Numerical Computation of Multivariate Normal Probabilities", in 207 | % J. of Computational and Graphical Stat., 1(1992), pp. 141-149, by 208 | % Alan Genz, WSU Math, PO Box 643113, Pullman, WA 99164-3113 209 | % Email : alangenz@wsu.edu 210 | % The primary references for the numerical integration are 211 | % "On a Number-Theoretical Integration Method" 212 | % H. Niederreiter, Aequationes Mathematicae, 8(1972), pp. 304-11, and 213 | % "Randomization of Number Theoretic Methods for Multiple Integration" 214 | % R. Cranley and T.N.L. Patterson, SIAM J Numer Anal, 13(1976), pp. 904-14. 215 | % 216 | % Alan Genz is the author of this function and following Matlab functions. 217 | % 218 | % 219 | % Copyright (C) 2013, Alan Genz, All rights reserved. 220 | % 221 | % Redistribution and use in source and binary forms, with or without 222 | % modification, are permitted provided the following conditions are met: 223 | % 1. Redistributions of source code must retain the above copyright 224 | % notice, this list of conditions and the following disclaimer. 225 | % 2. Redistributions in binary form must reproduce the above copyright 226 | % notice, this list of conditions and the following disclaimer in 227 | % the documentation and/or other materials provided with the 228 | % distribution. 229 | % 3. The contributor name(s) may not be used to endorse or promote 230 | % products derived from this software without specific prior 231 | % written permission. 232 | % THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 233 | % "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 234 | % LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 235 | % FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 236 | % COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 237 | % INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 238 | % BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 239 | % OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 240 | % ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 241 | % TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF USE 242 | % OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 243 | % 244 | % Initialization 245 | % 246 | [ch as bs] = chlrdr(r,a,b); ct = ch(1,1); ai = as(1); bi = bs(1); 247 | if ai > -9*ct, if ai < 9*ct, c = Phi(ai/ct); else, c=1; end, else c=0; end 248 | if bi > -9*ct, if bi < 9*ct, d = Phi(bi/ct); else, d=1; end, else d=0; end 249 | [n, n] = size(r); ci = c; dci = d - ci; p = 0; e = 0; 250 | ps = sqrt(primes(5*n*log(n+1)/4)); q = ps(1:n-1)'; % Richtmyer generators 251 | ns = 12; nv = fix( max( [ m/ns 1 ] ) ); on = ones(1,nv); y = zeros(n-1,nv); 252 | % 253 | % Randomization loop for ns samples 254 | % 255 | for j = 1 : ns, c = ci*on; dc = dci*on; pv = dc; 256 | for i = 2 : n, x = abs( 2*mod( q(i-1)*[1:nv] + rand, 1 ) - 1 ); 257 | y(i-1,:) = Phinv( c + x.*dc ); s = ch(i,1:i-1)*y(1:i-1,:); 258 | ct = ch(i,i); ai = as(i) - s; bi = bs(i) - s; c = on; d = c; 259 | c(find( ai < -9*ct )) = 0; d(find( bi < -9*ct )) = 0; 260 | tstl = find( abs(ai) < 9*ct ); c(tstl) = Phi( ai(tstl)/ct ); 261 | tstl = find( abs(bi) < 9*ct ); d(tstl) = Phi( bi(tstl)/ct ); 262 | dc = d - c; pv = pv.*dc; 263 | end, d = ( mean(pv) - p )/j; p = p + d; e = ( j - 2 )*e/j + d^2; 264 | end, e = 3*sqrt(e); % error estimate is 3 x standard error with ns samples. 265 | % 266 | end 267 | % end qsimvnv 268 | % 269 | % 270 | % Standard statistical normal distribution functions 271 | % 272 | function p = Phi(z) 273 | p = erfc( -z/sqrt(2) )/2; 274 | end 275 | %function z = Phinv(p), z = norminv( p ); 276 | function z = Phinv(p) 277 | z = -sqrt(2)*erfcinv( 2*p ); 278 | end% use if no norminv 279 | % 280 | function [ c, ap, bp ] = chlrdr( R, a, b ) 281 | % 282 | % Computes permuted lower Cholesky factor c for R which may be singular, 283 | % also permuting integration limit vectors a and b. 284 | % 285 | ep = 1e-10; % singularity tolerance; 286 | % 287 | [n,n] = size(R); c = R; ap = a; bp = b; d = sqrt(max(diag(c),0)); 288 | for i = 1 : n 289 | if d(i) > 0, c(:,i) = c(:,i)/d(i); c(i,:) = c(i,:)/d(i); 290 | ap(i) = ap(i)/d(i); bp(i) = bp(i)/d(i); 291 | end 292 | end 293 | y = zeros(n,1); sqtp = sqrt(2*pi); 294 | for k = 1 : n, im = k; ckk = 0; dem = 1; s = 0; 295 | for i = k : n 296 | if c(i,i) > eps, cii = sqrt( max( [c(i,i) 0] ) ); 297 | if i > 1, s = c(i,1:k-1)*y(1:k-1); end 298 | ai = ( ap(i)-s )/cii; bi = ( bp(i)-s )/cii; de = Phi(bi) - Phi(ai); 299 | if de <= dem, ckk = cii; dem = de; am = ai; bm = bi; im = i; end 300 | end 301 | end 302 | if im > k, c(im,im) = c(k,k); 303 | ap([im k]) = ap([k im]); bp([im k]) = bp([k im]); 304 | c([im;k],1:k-1) = c([k;im],1:k-1); c(im+1:n,[im k]) = c(im+1:n,[k im]); 305 | t = c(k+1:im-1,k); c(k+1:im-1,k) = c(im,k+1:im-1)'; c(im,k+1:im-1) = t'; 306 | end, c(k,k+1:n) = 0; 307 | if ckk > ep*k, c(k,k) = ckk; 308 | for i = k+1 : n 309 | c(i,k) = c(i,k)/ckk; c(i,k+1:i) = c(i,k+1:i) - c(i,k)*c(k+1:i,k)'; 310 | end 311 | if abs(dem) > ep, y(k) = ( exp(-am^2/2) - exp(-bm^2/2) )/(sqtp*dem); 312 | else 313 | if am < -10, y(k) = bm; 314 | elseif bm > 10, y(k) = am; 315 | else, y(k) = ( am + bm )/2; 316 | end 317 | end 318 | else, c(k:n,k) = 0; y(k) = 0; 319 | end 320 | end 321 | end 322 | % 323 | % end chlrdr 324 | 325 | -------------------------------------------------------------------------------- /Infill_qEI_MC.m: -------------------------------------------------------------------------------- 1 | function EI = Infill_qEI_MC(new_x,model,fmin) 2 | n = 1E5; 3 | % reshape the input 4 | new_x = reshape(new_x,size(model.S,2),[])'; 5 | % predictions and errors 6 | [u,~,~,Cov] = predictor(new_x,model); 7 | mu = u'; 8 | sigma = Cov; 9 | sample = mvnrnd(mu,sigma,n); 10 | EI = mean(max(fmin - min(sample,[],2),0)); -------------------------------------------------------------------------------- /MultiObjective_EHVI.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of EHVI algorithm according to 3 | % the following work. The EHVI is calcuated using Monte Carlo 4 | % approximation. 5 | % Reference: 6 | % M. T. M. Emmerich, K. C. Giannakoglou, and B. Naujoks. Sinlge- and 7 | % Multiobjective Evolutionary Optimization assited by Gaussian random field 8 | % metamodels. IEEE Transactions on Evolutionary Computation. 2006, 10: 9 | % 421-439. 10 | % Author: Dawei Zhan 11 | % Date: 2024/12/30 12 | %------------------------------------------------------------------------- 13 | clearvars;close all; 14 | % settings of the problem 15 | fun_name = 'DTLZ2'; 16 | % number of objectives 17 | num_obj = 2; 18 | % number of design variables 19 | num_vari = 10; 20 | lower_bound = zeros(1,num_vari); 21 | upper_bound = ones(1,num_vari); 22 | % reference point for calculating hypervolume 23 | ref_point = 2.5*ones(1,num_obj); 24 | % number of initial design points 25 | num_initial = 100; 26 | % maximum number of evaluations 27 | max_evaluation = 200; 28 | % the intial design points, points sampled all at once 29 | sample_x = lower_bound + (upper_bound-lower_bound).*lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000); 30 | sample_y = feval(fun_name, sample_x, num_obj); 31 | % scale the objectives to [0,1] 32 | sample_y_scaled =(sample_y - min(sample_y))./(max(sample_y)-min(sample_y)); 33 | % initialize some parameters 34 | iteration = 0; 35 | evaluation = size(sample_x,1); 36 | GP_obj = cell(1,num_obj); 37 | % calculate the initial hypervolume values 38 | index = Paretoset(sample_y); 39 | non_dominated_front = sample_y(index,:); 40 | non_dominated_front_scaled = sample_y_scaled(index,:); 41 | hypervolume = Hypervolume(non_dominated_front,ref_point); 42 | % print the hypervolume information 43 | fprintf('EHVI on %d-D %s function, iteration: %d, evaluation: %d, hypervolume: %0.2f \n',num_vari,fun_name,iteration,evaluation,hypervolume); 44 | % beginning of the iteration 45 | while evaluation < max_evaluation 46 | % build the GP models for each objective 47 | for ii = 1:num_obj 48 | GP_obj{ii} = GP_Train(sample_x,sample_y_scaled(:,ii),lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 49 | end 50 | % select updating points using the EHVI criterion 51 | infill_x = Optimizer_GA(@(x)-Infill_EHVI(x,GP_obj,non_dominated_front_scaled),num_vari,lower_bound,upper_bound,10*num_vari,200); 52 | infill_y = feval(fun_name,infill_x, num_obj); 53 | % add the new points to the design set 54 | sample_x = [sample_x;infill_x]; 55 | sample_y = [sample_y;infill_y]; 56 | sample_y_scaled = (sample_y - min(sample_y))./(max(sample_y)-min(sample_y)); 57 | evaluation = evaluation + size(infill_x,1); 58 | iteration = iteration + 1; 59 | % calculate the hypervolume values 60 | index = Paretoset(sample_y); 61 | non_dominated_front = sample_y(index,:); 62 | non_dominated_front_scaled = sample_y_scaled(index,:); 63 | hypervolume = Hypervolume(non_dominated_front,ref_point); 64 | % print the hypervolume information 65 | fprintf('EHVI on %d-D %s function, iteration: %d, evaluation: %d, hypervolume: %0.2f\n',num_vari,fun_name,iteration,evaluation,hypervolume); 66 | end 67 | 68 | -------------------------------------------------------------------------------- /MultiObjective_EIM.asv: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of EIM algorithm according to 3 | % the following work. For the improvement function in EIM, you can choose 4 | % '' 5 | % Reference: 6 | % D. Zhan, Y. Cheng, and J. Liu. Expected Improvement Matrix-Based Infill 7 | % Criteria for Expensive Multiobjective Optimization. IEEE Transactions on 8 | % Evolutionary Computation. 2017, 21: 956-975. 9 | % Author: Dawei Zhan 10 | % Date: 2024/12/30 11 | %------------------------------------------------------------------------- 12 | clearvars;close all; 13 | % settings of the problem 14 | fun_name = 'DTLZ2'; 15 | % number of objectives 16 | num_obj = 2; 17 | % number of design variables 18 | num_vari = 10; 19 | lower_bound = zeros(1,num_vari); 20 | upper_bound = ones(1,num_vari); 21 | % reference point for calculating hypervolume 22 | ref_point = 2.5*ones(1,num_obj); 23 | % number of initial design points 24 | num_initial = 100; 25 | % maximum number of evaluations 26 | max_evaluation = 200; 27 | % the intial design points, points sampled all at once 28 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 29 | sample_y = feval(fun_name, sample_x, num_obj); 30 | % scale the objectives to [0,1] 31 | sample_y_scaled =(sample_y - min(sample_y))./(max(sample_y)-min(sample_y)); 32 | % initialize some parameters 33 | iteration = 0; 34 | evaluation = size(sample_x,1); 35 | GP_obj = cell(1,num_obj); 36 | % calculate the initial hypervolume values 37 | index = Paretoset(sample_y); 38 | non_dominated_front = sample_y(index,:); 39 | non_dominated_front_scaled = sample_y_scaled(index,:); 40 | hypervolume = Hypervolume(non_dominated_front,ref_point); 41 | % print the hypervolume information 42 | fprintf('EIM on %d-D %s function, iteration: %d, evaluation: %d, hypervolume: %0.2f \n',num_vari,fun_name,iteration,evaluation,hypervolume); 43 | % beginning of the iteration 44 | while evaluation < max_evaluation 45 | % build the kriging model for each objective 46 | for ii = 1:num_obj 47 | GP_obj{ii} = GP_Train(sample_x,sample_y_scaled(:,ii),lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 48 | end 49 | % select updating points using the EIM criteria 50 | % you can choose 'Euclidean', 'Maximin', and 'Hypervolume' 51 | criterion = 'Euclidean'; 52 | infill_x = Optimizer_GA(@(x)-Infill_EIM(x,GP_obj,non_dominated_front_scaled,'Maximin'),num_vari,lower_bound,upper_bound,10*num_vari,200); 53 | infill_y = feval(fun_name,infill_x, num_obj); 54 | % add the new points to the design set 55 | sample_x = [sample_x;infill_x]; 56 | sample_y = [sample_y;infill_y]; 57 | sample_y_scaled = (sample_y - min(sample_y))./(max(sample_y)-min(sample_y)); 58 | evaluation = evaluation + size(infill_x,1); 59 | iteration = iteration + 1; 60 | % calculate the hypervolume values 61 | index = Paretoset(sample_y); 62 | non_dominated_front = sample_y(index,:); 63 | non_dominated_front_scaled = sample_y_scaled(index,:); 64 | hypervolume = Hypervolume(non_dominated_front,ref_point); 65 | % print the hypervolume information 66 | fprintf('EIM on %d-D %s function, iteration: %d, evaluation: %d, hypervolume: %0.2f\n',num_vari,fun_name,iteration,evaluation,hypervolume); 67 | end 68 | 69 | -------------------------------------------------------------------------------- /MultiObjective_EIM.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of EIM algorithm according to 3 | % the following work. For the improvement function in EIM, you can choose 4 | % 'Euclidean', 'Maximin', or 'Hypervolume'. 5 | % Reference: 6 | % D. Zhan, Y. Cheng, and J. Liu. Expected Improvement Matrix-Based Infill 7 | % Criteria for Expensive Multiobjective Optimization. IEEE Transactions on 8 | % Evolutionary Computation. 2017, 21: 956-975. 9 | % Author: Dawei Zhan 10 | % Date: 2024/12/30 11 | %------------------------------------------------------------------------- 12 | clearvars;close all; 13 | % settings of the problem 14 | fun_name = 'DTLZ2'; 15 | % number of objectives 16 | num_obj = 2; 17 | % number of design variables 18 | num_vari = 10; 19 | lower_bound = zeros(1,num_vari); 20 | upper_bound = ones(1,num_vari); 21 | % reference point for calculating hypervolume 22 | ref_point = 2.5*ones(1,num_obj); 23 | % number of initial design points 24 | num_initial = 100; 25 | % maximum number of evaluations 26 | max_evaluation = 200; 27 | % the intial design points, points sampled all at once 28 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 29 | sample_y = feval(fun_name, sample_x, num_obj); 30 | % scale the objectives to [0,1] 31 | sample_y_scaled =(sample_y - min(sample_y))./(max(sample_y)-min(sample_y)); 32 | % initialize some parameters 33 | iteration = 0; 34 | evaluation = size(sample_x,1); 35 | GP_obj = cell(1,num_obj); 36 | % calculate the initial hypervolume values 37 | index = Paretoset(sample_y); 38 | non_dominated_front = sample_y(index,:); 39 | non_dominated_front_scaled = sample_y_scaled(index,:); 40 | hypervolume = Hypervolume(non_dominated_front,ref_point); 41 | % print the hypervolume information 42 | fprintf('EIM on %d-D %s function, iteration: %d, evaluation: %d, hypervolume: %0.2f \n',num_vari,fun_name,iteration,evaluation,hypervolume); 43 | % beginning of the iteration 44 | while evaluation < max_evaluation 45 | % build the kriging model for each objective 46 | for ii = 1:num_obj 47 | GP_obj{ii} = GP_Train(sample_x,sample_y_scaled(:,ii),lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 48 | end 49 | % select updating points using the EIM criteria 50 | % you can choose 'Euclidean', 'Maximin', and 'Hypervolume' 51 | criterion = 'Euclidean'; 52 | infill_x = Optimizer_GA(@(x)-Infill_EIM(x,GP_obj,non_dominated_front_scaled,'Euclidean'),num_vari,lower_bound,upper_bound,10*num_vari,200); 53 | infill_y = feval(fun_name,infill_x,num_obj); 54 | % add the new points to the design set 55 | sample_x = [sample_x;infill_x]; 56 | sample_y = [sample_y;infill_y]; 57 | sample_y_scaled = (sample_y - min(sample_y))./(max(sample_y)-min(sample_y)); 58 | evaluation = evaluation + size(infill_x,1); 59 | iteration = iteration + 1; 60 | % calculate the hypervolume values 61 | index = Paretoset(sample_y); 62 | non_dominated_front = sample_y(index,:); 63 | non_dominated_front_scaled = sample_y_scaled(index,:); 64 | hypervolume = Hypervolume(non_dominated_front,ref_point); 65 | % print the hypervolume information 66 | fprintf('EIM on %d-D %s function, iteration: %d, evaluation: %d, hypervolume: %0.2f\n',num_vari,fun_name,iteration,evaluation,hypervolume); 67 | end 68 | 69 | -------------------------------------------------------------------------------- /MultiObjective_MOEAD_EGO.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of MOEA/D-EGO algorithm according to 3 | % the following work. 4 | % Reference: 5 | % Q. Zhang, W. Liu, E. Tsang, and B. Virginas. Expensive Multiobjective 6 | % Optimization by MOEA/D with Gaussian Process model. IEEE Transactions on 7 | % Evolutionary Computation. 2010, 14, 456-474. 8 | % Author: Dawei Zhan 9 | % Date: 2024/12/30 10 | %------------------------------------------------------------------------- 11 | clearvars;close all; 12 | % settings of the problem 13 | fun_name = 'DTLZ2'; 14 | % number of objectives 15 | num_obj = 2; 16 | % number of design variables 17 | num_vari = 10; 18 | lower_bound = zeros(1,num_vari); 19 | upper_bound = ones(1,num_vari); 20 | % reference point for calculating hypervolume 21 | ref_point = 2.5*ones(1,num_obj); 22 | % number of points evaluated in each iteration 23 | Ke = 5; 24 | % number of initial designs 25 | num_initial = 100; 26 | % maximum number of evaluations 27 | max_evaluation = 200; 28 | % initial sampling 29 | sample_x = lower_bound + (upper_bound-lower_bound).*lhsdesign(num_initial, num_vari,'criterion','maximin','iterations',1000); 30 | sample_y = feval(fun_name,sample_x,num_obj); 31 | z = min(sample_y); 32 | % the number of weight vectors 33 | if num_obj == 2 34 | n = 300; 35 | elseif num_obj == 3 36 | n = 595; 37 | else 38 | n = 800; 39 | end 40 | % generate weights 41 | weight = UniformPoint(n,num_obj); 42 | n = size(weight,1); 43 | iteration = 0; 44 | evaluation = size(sample_x,1); 45 | GP_obj = cell(1,num_obj); 46 | index = Paretoset(sample_y); 47 | non_dominated_front = sample_y(index,:); 48 | hypervolume = Hypervolume(non_dominated_front,ref_point); 49 | fprintf('MOEA/D-EGO on %d-D %s function, iteration: %d, evaluation: %d, hypervolume: %0.2f\n',num_vari,fun_name,iteration,evaluation,hypervolume); 50 | while evaluation < max_evaluation 51 | q = min(max_evaluation-evaluation,Ke); 52 | for ii = 1:num_obj 53 | GP_obj{ii} = GP_Train(sample_x,sample_y(:,ii),lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 54 | end 55 | % optimize n EIs using MOEA/D 56 | T = 20; 57 | B = pdist2(weight,weight); 58 | [~,B] = sort(B,2); 59 | B = B(:,1:T); 60 | pop_vari = rand(n,num_vari).*(upper_bound-lower_bound) + lower_bound; 61 | pop_EI = zeros(n,1); 62 | pop_y = zeros(n,num_obj); 63 | pop_s = zeros(n,num_obj); 64 | for ii = 1:n 65 | for jj = 1:num_obj 66 | [pop_y(ii,jj),pop_s(ii,jj)] = GP_Predict(pop_vari(ii,:),GP_obj{jj}); 67 | end 68 | pop_EI(ii) = Tchebycheff_EI(pop_y(ii,:),pop_s(ii,:),weight(ii,:),z,sample_y); 69 | end 70 | for gen = 2:round(10000/n) 71 | for ii = 1:n 72 | parent = pop_vari(ii,:); 73 | if rand < 0.9 74 | P = B(ii,randperm(T)); 75 | else 76 | P = randperm(n); 77 | end 78 | F = 0.5; 79 | CR = 0.8; 80 | mutation = parent + F*(pop_vari(P(1),:)-pop_vari(P(2),:)); 81 | if any(mutation upper_bound) 82 | mutation = lower_bound + rand(1,num_vari).*(upper_bound-lower_bound); 83 | end 84 | rand_vector = rand(1,num_vari); 85 | rand_vector(1,randi(num_vari)) = 0; 86 | mui = rand_vector < CR; 87 | offspring = mutation.*mui + parent.*(1-mui); 88 | 89 | offspring_y = zeros(1,num_obj); 90 | offspring_s = zeros(1,num_obj); 91 | for jj = 1:num_obj 92 | [offspring_y(1,jj),offspring_s(1,jj)] = GP_Predict(offspring,GP_obj{jj}); 93 | end 94 | offspring_EI = Tchebycheff_EI(offspring_y,offspring_s,weight(P,:),z,sample_y); 95 | replace_index = P(offspring_EI > pop_EI(P,:)); 96 | c = min(2,length(replace_index)); 97 | replace_index = replace_index(1:c); 98 | if ~isempty(replace_index) 99 | pop_vari(replace_index,:) = repmat(offspring,c,1); 100 | pop_y(replace_index,:) = repmat(offspring_y,c,1); 101 | pop_s(replace_index,:) = repmat(offspring_s,c,1); 102 | pop_EI(replace_index) = offspring_EI(1:c); 103 | end 104 | end 105 | end 106 | % delete points that are too close to each other 107 | temp_point = sample_x; 108 | candi_index = []; 109 | for ii = 1:n 110 | if min(pdist2(pop_vari(ii,:),temp_point)) > 1E-5 111 | candi_index = [candi_index;ii]; 112 | end 113 | end 114 | candi_point = pop_vari(candi_index,:); 115 | candi_weight = weight(candi_index,:); 116 | candi_y = pop_y(candi_index,:); 117 | candi_s = pop_s(candi_index,:); 118 | % clustering 119 | idx = kmeans(candi_weight,q); 120 | infill_x = zeros(q,num_vari); 121 | for ii = 1:q 122 | group_point = candi_point(idx==ii,:); 123 | group_weight = candi_weight(idx==ii,:); 124 | group_y = candi_y(idx==ii,:); 125 | group_s = candi_s(idx==ii,:); 126 | EI = zeros(size(group_point,1),1); 127 | for jj = 1:size(group_point,1) 128 | EI(jj) = Tchebycheff_EI(group_y(jj,:),group_s(jj,:),group_weight(jj,:),z,sample_y); 129 | end 130 | [~,select_index] = max(EI); 131 | infill_x(ii,:) = group_point(select_index,:); 132 | end 133 | infill_y = feval(fun_name,infill_x,num_obj); 134 | evaluation = evaluation + size(infill_y,1); 135 | iteration = iteration + 1; 136 | sample_x = [sample_x;infill_x]; 137 | sample_y = [sample_y;infill_y]; 138 | % update z 139 | z = min(sample_y); 140 | index = Paretoset(sample_y); 141 | non_dominated_front = sample_y(index,:); 142 | hypervolume = Hypervolume(non_dominated_front,ref_point); 143 | fprintf('MOEA/D-EGO on %d-D %s function, iteration: %d, evaluation: %d, hypervolume: %0.2f\n',num_vari,fun_name,iteration,evaluation,hypervolume); 144 | end 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /MultiObjective_ParEGO.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of Pareto Efficient Global Optimization 3 | % algorithm according to the following work. 4 | % Reference: 5 | % J. Knowles. ParEGO: A Hybrid Algorithm With On-line Landscape Approximation 6 | % for Expensive Multiobjective Optimization Problems. IEEE Transactions on 7 | % Evolutionary Computation. 2006, 10: 50-66. 8 | % Author: Dawei Zhan 9 | % Date: 2024/12/30 10 | %------------------------------------------------------------------------- 11 | clearvars;close all; 12 | % settings of the problem 13 | fun_name = 'DTLZ2'; 14 | % number of objectives 15 | num_obj = 2; 16 | % number of design variables 17 | num_vari = 10; 18 | lower_bound = zeros(1,num_vari); 19 | upper_bound = ones(1,num_vari); 20 | % reference point for calculating hypervolume 21 | ref_point = 2.5*ones(1,num_obj); 22 | % number of initial design points 23 | num_initial = 100; 24 | % the maximum allowed evaluations 25 | max_evaluation = 200; 26 | % generate weight vectors 27 | if num_obj == 2 28 | num_weight = 11; 29 | elseif num_obj == 3 30 | num_weight = 15; 31 | elseif num_obj == 4 32 | num_weight = 20; 33 | else 34 | num_weight = 56; 35 | end 36 | weight = UniformPoint(num_weight,num_obj); 37 | % the intial design points, points sampled all at once 38 | sample_x = lower_bound + (upper_bound-lower_bound).*lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000); 39 | sample_y = feval(fun_name,sample_x,num_obj); 40 | iteration = 0; 41 | evaluation = size(sample_x,1); 42 | % calculate the initial hypervolume value 43 | index = Paretoset(sample_y); 44 | non_dominated_front = sample_y(index,:); 45 | hypervolume = Hypervolume(non_dominated_front,ref_point); 46 | % print the hypervolume information 47 | fprintf('ParEGO on %d-D %s problem, iteration: %d, evaluation: %d, hypervolume: %0.2f \n',num_vari,fun_name,iteration,evaluation,hypervolume); 48 | % beginning of the iteration 49 | while evaluation < max_evaluation 50 | % randomly select a weight vector 51 | lamda = weight(randi(size(weight,1)),:); 52 | % build the weighted objective function 53 | sample_y_scaled = (sample_y-min(sample_y))./(max(sample_y)-min(sample_y)); 54 | sample_y_pcheby = max(sample_y_scaled.*lamda,[],2) + 0.05*sum(sample_y_scaled.*lamda,2); 55 | % build the initial GP model 56 | GP_model = GP_Train(sample_x,sample_y_pcheby,lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 57 | % GA is used for the maximization problem 58 | [infill_x,best_EI] = Optimizer_GA(@(x)-Infill_EI(x,GP_model,min(sample_y_pcheby)),num_vari,lower_bound,upper_bound,10*num_vari,200); 59 | % do the expensive evaluations 60 | infill_y = feval(fun_name,infill_x,num_obj); 61 | evaluation = evaluation + size(infill_y,1); 62 | iteration = iteration + 1; 63 | % add the evaluated points to design set 64 | sample_x = [sample_x;infill_x]; 65 | sample_y = [sample_y;infill_y]; 66 | % plot current non-dominated front points 67 | index = Paretoset(sample_y); 68 | non_dominated_front = sample_y(index,:); 69 | hypervolume = Hypervolume(non_dominated_front,ref_point); 70 | fprintf('ParEGO on %d-D %s problem, iteration: %d, evaluation: %d, hypervolume: %0.2f \n',num_vari,fun_name,iteration, evaluation, hypervolume); 71 | end -------------------------------------------------------------------------------- /Optimizer_GA.m: -------------------------------------------------------------------------------- 1 | function [best_x,best_y] = Optimizer_GA(obj_fun,num_vari,lower_bound,upper_bound,pop_size,max_gen) 2 | % the internal optimization using genetic algorithm 3 | % the number of current generation 4 | generation = 1; 5 | % the initial generation 6 | pop_vari = lhsdesign(pop_size, num_vari).*(upper_bound - lower_bound) + lower_bound; 7 | % calculate the objective values 8 | pop_fitness = feval(obj_fun,pop_vari); 9 | % the evoluation of the generation 10 | while generation < max_gen 11 | % parent selection using k-tournament (default k=2) selection 12 | k = 2; 13 | temp = randi(pop_size,pop_size,k); 14 | [~,index] = min(pop_fitness(temp),[],2); 15 | pop_parent = pop_vari(sum(temp.*(index == 1:k),2),:); 16 | % crossover (simulated binary crossover) 17 | % dic_c is the distribution index of crossover 18 | dis_c = 10; 19 | mu = rand(pop_size/2,num_vari); 20 | parent1 = pop_parent(1:2:pop_size,:); 21 | parent2 = pop_parent(2:2:pop_size,:); 22 | beta = 1 + 2*min(min(parent1,parent2)-lower_bound,upper_bound-max(parent1,parent2))./max(abs(parent2-parent1),1E-6); 23 | alpha = 2 - beta.^(-dis_c-1); 24 | betaq = (alpha.*mu).^(1/(dis_c+1)).*(mu <= 1./alpha) + (1./(2-alpha.*mu)).^(1/(dis_c+1)).*(mu > 1./alpha); 25 | % the crossover is performed randomly on each variable 26 | betaq = betaq.*(-1).^randi([0,1],pop_size/2,num_vari); 27 | betaq(rand(pop_size/2,num_vari)>0.5) = 1; 28 | offspring1 = 0.5*((1+betaq).*parent1 + (1-betaq).*parent2); 29 | offspring2 = 0.5*((1-betaq).*parent1 + (1+betaq).*parent2); 30 | pop_crossover = [offspring1;offspring2]; 31 | % mutation (ploynomial mutation) 32 | % dis_m is the distribution index of polynomial mutation 33 | dis_m = 20; 34 | pro_m = 1/num_vari; 35 | rand_var = rand(pop_size,num_vari); 36 | mu = rand(pop_size,num_vari); 37 | deta = min(pop_crossover-lower_bound, upper_bound-pop_crossover)./(upper_bound-lower_bound); 38 | detaq = zeros(pop_size,num_vari); 39 | position1 = rand_var<=pro_m & mu<=0.5; 40 | position2 = rand_var<=pro_m & mu>0.5; 41 | detaq(position1) = ((2*mu(position1) + (1-2*mu(position1)).*(1-deta(position1)).^(dis_m+1)).^(1/(dis_m+1))-1); 42 | detaq(position2) = (1 - (2*(1-mu(position2))+2*(mu(position2)-0.5).*(1-deta(position2)).^(dis_m+1)).^(1/(dis_m+1))); 43 | pop_mutation = pop_crossover + detaq.*(upper_bound-lower_bound); 44 | pop_mutation = max(min(pop_mutation,upper_bound),lower_bound); 45 | % fitness calculation 46 | pop_mutation_fitness = feval(obj_fun, pop_mutation); 47 | % environment selection 48 | pop_vari_iter = [pop_vari;pop_mutation]; 49 | pop_fitness_iter = [pop_fitness;pop_mutation_fitness]; 50 | [~,win_num] = sort(pop_fitness_iter); 51 | pop_vari = pop_vari_iter(win_num(1:pop_size),:); 52 | pop_fitness = pop_fitness_iter(win_num(1:pop_size),:); 53 | % update the evaluation number and generation number 54 | generation = generation + 1; 55 | end 56 | 57 | [best_y,index] = min(pop_fitness); 58 | best_x = pop_vari(index,:); 59 | -------------------------------------------------------------------------------- /Parallel_CL.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------------------------------------- 2 | % This is the constant liar approach which always uses the current minimum 3 | % objective value as the fake objective to update the GP model to produce 4 | % multiple query points for parallel function evaluations. It is coded 5 | % based on the following work. 6 | % Reference: 7 | % D. Ginsbourger, R. Le Riche, and L. Carraro. Kriging is 8 | % well-suited to parallelize optimization. Computational intelligence in 9 | % expensive optimization problems. 2010, 131-162. 10 | % Author: Dawei Zhan 11 | % Date: 2024.11.27 12 | %-------------------------------------------------------------------------- 13 | clearvars; close all; 14 | % setting of the problem 15 | fun_name = 'Rosenbrock'; 16 | num_vari = 10; 17 | lower_bound = -2.048*ones(1,num_vari); 18 | upper_bound = 2.048*ones(1,num_vari); 19 | % the number of initial design points 20 | num_initial = 20; 21 | % maximum number of evaluations 22 | max_evaluation = 120; 23 | % batch size 24 | num_q = 4; 25 | % initial design points using Latin hypercube sampling method 26 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 27 | sample_y = feval(fun_name,sample_x); 28 | % the current iteration and evaluation 29 | evaluation = size(sample_x,1); 30 | iteration = 0; 31 | % the current best solution 32 | fmin = min(sample_y); 33 | % print the current information to the screen 34 | fprintf('Constant Liar on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %f\n',num_vari,fun_name,iteration,evaluation,fmin); 35 | % iterations 36 | while evaluation < max_evaluation 37 | % build the GP model 38 | GP_model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 39 | % maximize the EI function 40 | infill_x = zeros(num_q,num_vari); 41 | for ii = 1: num_q 42 | [infill_x(ii,:),max_EI] = Optimizer_GA(@(x)-Infill_EI(x,GP_model,fmin),num_vari,lower_bound,upper_bound,10*num_vari,200); 43 | % use fmin as the fake objective 44 | GP_model = GP_Train([sample_x;infill_x(1:ii,:)],[sample_y;fmin*ones(ii,1)],lower_bound,upper_bound,GP_model.theta); 45 | end 46 | % evaluate the query points with the real function 47 | infill_y = feval(fun_name,infill_x); 48 | % add the new points to design set 49 | sample_x = [sample_x;infill_x]; 50 | sample_y = [sample_y;infill_y]; 51 | % update some parameters 52 | evaluation = size(sample_x,1); 53 | iteration = iteration + 1; 54 | fmin = min(sample_y); 55 | % print the current information to the screen 56 | fprintf('Constant Liar on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %f\n',num_vari,fun_name,iteration,evaluation,fmin); 57 | end 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /Parallel_FqEI.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of the fast multi-point expected 3 | % improvement approach. The FqEI criterion has very similar properties as 4 | % the qEI, but is significantly faster than qEI. Therefore, it can be used 5 | % for large batch size. 6 | % Reference: 7 | % D. Zhan, Y. Meng, and H. Xing. A Fast Multipoint Expected Improvement 8 | % for Parallel Expensive Optimization. IEEE Transactions on Evolutionary 9 | % Computation, 2023, 27(1): 170:184. 10 | % Author: Dawei Zhan 11 | % Date: 2024.11.27 12 | %------------------------------------------------------------------------- 13 | clearvars; close all; 14 | % setting of the problem 15 | fun_name = 'Rosenbrock'; 16 | num_vari = 10; 17 | lower_bound = -2.048*ones(1,num_vari); 18 | upper_bound = 2.048*ones(1,num_vari); 19 | % number of initial design points 20 | num_initial = 20; 21 | % maximum number of evaluations 22 | max_evaluation = 120; 23 | % batch size 24 | num_q = 4; 25 | % initial design points using Latin hypercube sampling method 26 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 27 | sample_y = feval(fun_name,sample_x); 28 | evaluation = size(sample_x,1); 29 | iteration = 0; 30 | % current best solution 31 | fmin = min(sample_y); 32 | % print the current information to screen 33 | fprintf('FqEI on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 34 | % the iteration 35 | while evaluation < max_evaluation 36 | % build the GP model 37 | GP_model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 38 | % maximize the FqEI function using GA 39 | [best_x,max_EI] = Optimizer_GA(@(x)-Infill_FqEI(x,GP_model,fmin),num_vari*num_q,repmat(lower_bound,1,num_q),repmat(upper_bound,1,num_q),4*num_vari*num_q,200); 40 | infill_x = reshape(best_x,num_vari,[])'; 41 | % evaluate the query points with the real function 42 | infill_y = feval(fun_name,infill_x); 43 | % add the new points to design set 44 | sample_x = [sample_x;infill_x]; 45 | sample_y = [sample_y;infill_y]; 46 | % update some parameters 47 | evaluation = size(sample_x,1); 48 | iteration = iteration + 1; 49 | fmin = min(sample_y); 50 | % print the current information to screen 51 | fprintf('FqEI on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 52 | end 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /Parallel_KB.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------------------------------------- 2 | % This is the Kriging believe approach which always use the Kriging 3 | % prediction value as the fake objective to update the GP model to produce 4 | % multiple query points for parallel function evaluations. It is coded 5 | % based on the following work. 6 | % Reference: 7 | % D. Ginsbourger, R. Le Riche, and L. Carraro. Kriging is 8 | % well-suited to parallelize optimization. Computational intelligence in 9 | % expensive optimization problems. 2010, 131-162. 10 | % Author: Dawei Zhan 11 | % Date: 2024.11.27 12 | %-------------------------------------------------------------------------- 13 | clearvars; close all; 14 | % setting of the problem 15 | fun_name = 'Rosenbrock'; 16 | num_vari = 10; 17 | lower_bound = -2.048*ones(1,num_vari); 18 | upper_bound = 2.048*ones(1,num_vari); 19 | % the number of initial design points 20 | num_initial = 20; 21 | % maximum number of evaluations 22 | max_evaluation = 120; 23 | % batch size 24 | num_q = 4; 25 | % initial design points using Latin hypercube sampling method 26 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 27 | sample_y = feval(fun_name,sample_x); 28 | % the current iteration and evaluation 29 | evaluation = size(sample_x,1); 30 | iteration = 0; 31 | % the current best solution 32 | fmin = min(sample_y); 33 | % print the current information to the screen 34 | fprintf('Kriging Believer on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 35 | % the iteration 36 | while evaluation < max_evaluation 37 | % train the GP model 38 | GP_model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 39 | infill_x = zeros(num_q,num_vari); 40 | for ii = 1: num_q 41 | [infill_x(ii,:),max_EI] = Optimizer_GA(@(x)-Infill_EI(x,GP_model,fmin),num_vari,lower_bound,upper_bound,10*num_vari,200); 42 | % use GP prediction as the fake objective 43 | GP_model = GP_Train([sample_x;infill_x(1:ii,:)],[sample_y;GP_Predict(infill_x(1:ii,:),GP_model)],lower_bound,upper_bound,GP_model.theta); 44 | end 45 | % evaluate the query points with the real function 46 | infill_y = feval(fun_name,infill_x); 47 | % add the new points to design set 48 | sample_x = [sample_x;infill_x]; 49 | sample_y = [sample_y;infill_y]; 50 | % update some parameters 51 | evaluation = size(sample_x,1); 52 | iteration = iteration + 1; 53 | fmin = min(sample_y); 54 | % print the current information to the screen 55 | fprintf('Kriging Believer on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 56 | end 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /Parallel_PEI.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------------------------------------- 2 | % This is the pseudo expected improvement approach which uses an influence 3 | % function to simulate the sequential EI's selection behavior. It is coded 4 | % based on the following work. 5 | % Reference: 6 | % D. Zhan, J. Qian, and Y. Cheng. Pseudo expected improvement 7 | % criterion for parallel EGO algorithm. Journal of Global Optimization, 8 | % 2017, 68(3): 641-662. 9 | % Author: Dawei Zhan. 10 | % Date: 2024.11.27 11 | %-------------------------------------------------------------------------- 12 | clearvars; close all; 13 | % setting of the problem 14 | fun_name = 'Rosenbrock'; 15 | num_vari = 10; 16 | lower_bound = -2.048*ones(1,num_vari); 17 | upper_bound = 2.048*ones(1,num_vari); 18 | % the number of initial design points 19 | num_initial = 20; 20 | % maximum number of evaluations 21 | max_evaluation = 120; 22 | % the number of points selected in each iteration 23 | num_q = 4; 24 | % initial design points using Latin hypercube sampling method 25 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 26 | sample_y = feval(fun_name,sample_x); 27 | % the current iteration and evaluation 28 | evaluation = size(sample_x,1); 29 | iteration = 0; 30 | % the current best solution 31 | fmin = min(sample_y); 32 | % print the current information to the screen 33 | fprintf('Pseuso EI on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 34 | % the iteration 35 | while evaluation < max_evaluation 36 | % build the GP model 37 | GP_model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 38 | infill_x = zeros(num_q,num_vari); 39 | point_added = []; 40 | for ii = 1: num_q 41 | % find the point with the highest pseudo EI value using GA algorithm 42 | infill_x(ii,:) = Optimizer_GA(@(x)-Infill_PEI(x,GP_model,fmin,point_added),num_vari,lower_bound,upper_bound,10*num_vari,200); 43 | % update point_added 44 | point_added = infill_x(1:ii,:); 45 | end 46 | % evaluate the query points with the real function 47 | best_y = feval(fun_name,infill_x); 48 | % add the new points to design set 49 | sample_x = [sample_x;infill_x]; 50 | sample_y = [sample_y;best_y]; 51 | % update some parameters 52 | evaluation = size(sample_x,1); 53 | iteration = iteration + 1; 54 | fmin = min(sample_y); 55 | % print the current information to the screen 56 | fprintf('Pseuso EI on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 57 | end 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /Parallel_qEI.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------------------------------------- 2 | % This is the multi-point expected improvement approach for batch Bayesian 3 | % optimization. The qEI function is coded according to the R code in [1]. I 4 | % used the Modified Cholesky algorithm [2] and a quasi-random approach to 5 | % estimate an MVN probability [3] in the qEI implementation. 6 | % Reference: 7 | % [1] O. Roustant, D. Ginsbourger, and Y. Deville. DiceKriging, 8 | % DiceOptim: Two R Packages for the Analysis of Computer Experiments by 9 | % Kriging-Based Metamodeling and Optimization. Journal of Statistical 10 | % Software, 2012, 51(1): 1-55. 11 | % [2] S. H. Cheng and N. J. Higham. A modified Cholesky algorithm based 12 | % on a symmetric indefinite factorization. SIAM J. Matrix Anal. Appl., 13 | % 19(4):1097-1110, 1998. https://github.com/higham/modified-cholesky. 14 | % [3] Alan Genz. Numerical Computation of Multivariate Normal 15 | % Probabilities. J. of Computational and Graphical Stat., 1992 1: 141-149. 16 | %-------------------------------------------------------------------------- 17 | clearvars; close all; 18 | % setting of the problem 19 | fun_name = 'Rosenbrock'; 20 | num_vari = 10; 21 | lower_bound = -2.048*ones(1,num_vari); 22 | upper_bound = 2.048*ones(1,num_vari); 23 | % the number of initial design points 24 | num_initial = 20; 25 | % maximum number of evaluations 26 | max_evaluation = 120; 27 | % the number of points selected in each iteration 28 | num_q = 4; 29 | % initial design points using Latin hypercube sampling method 30 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 31 | sample_y = feval(fun_name,sample_x); 32 | % the current iteration and evaluation 33 | evaluation = size(sample_x,1); 34 | iteration = 0; 35 | % the current best solution 36 | fmin = min(sample_y); 37 | % print the current information to the screen 38 | fprintf('qEI on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 39 | % the iteration 40 | while evaluation < max_evaluation 41 | % build the GP model 42 | GP_model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 43 | % maximize the qEI function 44 | [best_x,max_EI] = Optimizer_GA(@(x)-Infill_qEI(x,GP_model,fmin),num_vari*num_q,repmat(lower_bound,1,num_q),repmat(upper_bound,1,num_q),num_vari*num_q,100); 45 | infill_x = reshape(best_x,num_vari,[])'; 46 | % evaluate the query points with the real function 47 | infill_y = feval(fun_name,infill_x); 48 | % add the new points to design set 49 | sample_x = [sample_x;infill_x]; 50 | sample_y = [sample_y;infill_y]; 51 | % update some parameters 52 | evaluation = size(sample_x,1); 53 | iteration = iteration + 1; 54 | fmin = min(sample_y); 55 | % print the current information to the screen 56 | fprintf('qEI on %d-D %s function, iteration: %d, evaluation: %d, current best solution: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 57 | end 58 | 59 | -------------------------------------------------------------------------------- /Paretoset.m: -------------------------------------------------------------------------------- 1 | function membership = Paretoset(X) 2 | % PARETOSET To get the Pareto set from a given set of points. 3 | % synopsis: membership =paretoset (objectiveMatrix) 4 | % where: 5 | % objectiveMatrix: [number of points X number of objectives] array 6 | % membership: [number of points X 1] logical vector to indicate if ith 7 | % point belongs to the Pareto set (true) or not (false). 8 | % 9 | % by Yi Cao, Cranfield University, 02 June 2007 10 | % Revised by Yi Cao on 17 October 2007 11 | % Version 3, 21 October 2007, new sorting scheme to improve speed. 12 | % Bugfix, 25 July 2008, devided by zero error is fixed. 13 | % 14 | % Examples: see paretoset_examples 15 | % 16 | 17 | m=size(X,1); 18 | Xmin=min(X); 19 | X1=X-Xmin(ones(m,1),:); %make sure X1>=0; 20 | Xmean=mean(X1); 21 | %sort X1 so that dominated points can be removed quickly 22 | [~,checklist]=sort(max(X1./(Xmean(ones(m,1),:)+max(Xmean)),[],2)); 23 | Y=X(checklist,:); 24 | membership=false(m,1); 25 | while numel(checklist)>1 26 | k=checklist(1); 27 | [membership(k),checklist,Y]=paretosub(Y,checklist); 28 | end 29 | membership(checklist)=true; 30 | 31 | function [ispareto,nondominated,X]=paretosub(X,checklist) 32 | 33 | Z=X-X(ones(size(X,1),1),:); 34 | nondominated=any(Z<0,2); %retain nondominated points from the check list 35 | ispareto=all(any(Z(nondominated,:)>0,2)); %check if current point belongs to pareto set 36 | X=X(nondominated,:); 37 | nondominated=checklist(nondominated); 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Matlab implementations of Bayesian Optimization algorithms 2 | Bayesian Optimization (BO) algorithms, also known as Efficient Global Optimization (EGO) algorithms are widely used to solve expensive optimization problems. I try to make a collection of different Bayesian optimization algorithms that we have proposed and implemented during my research. In these implementations, I try keep the codes as simple as possile. 3 | 4 | 5 | ## Table of Contents 6 | * [Requirements](#Requirements) 7 | * [Standard Algorithm](#Standard-algorithm) 8 | * [High-Dimensional Algorithms](#High-Dimensional-Algorithms) 9 | * [Parallel (Batch) Algorithms](#Parallel-Efficient-Global-Optimization-algorithms) 10 | * [Multiobjecitve Algorithms](#Multiobjective-Efficient-Global-Optimization-algorithms) 11 | * [Constrained Algorithms](#Constrained-Efficient-Global-Optimization-algorithms) 12 | * [References](#References) 13 | 14 | 15 | ## Requirements 16 | 1. **Windows system.** I have not tested other operating systems, but the codes should also work as Matlab is cross-platformed. 17 | 2. **MATLAB 2016b and above**. I used a lot of ```.*``` to multiply vector and matrix. This multiplication uses implicit expansion, which was introduced in MATLAB 2016b. 18 | 19 | ## Standard Bayesian Optimization algorithm 20 | **The standard BO algorithm**[^1] ```Standard_BO.m```. For the Kriging modeling, the Gaussian correlation function is used as the corrlation function and the constant mean is used as the trend function. 21 | I refered some codes in the book *Engineering design via surrogate modelling: a practical guide* [^2] for the Kriging model. 22 | The MATLAB ```fmincon``` function is used for maximizing the likehihood function to get the estimated hyperparameters when training the Kriging model. 23 | The expected improvement function is maximized by a real-coded genetic algorithm [^3]. 24 | 25 | 26 | ## High-Dimensional Bayesian Optimization Algorithms 27 | **The Dropout Approach**[^4] ```HD_Dropout.m``` 28 | 29 | 30 | 31 | ## Parallel Bayesian Optimization Algorithms 32 | **The Kriging Believer Approach**[^5] ```Parallel_KB.m```. The Kriging believer approach always uses the Kriging prediction value as the fake objective to update the GP model to produce multiple query points for parallel function evaluations. 33 | 34 | **The Constant Liar Approach**[^5] ```Parallel_CL.m```. The constant liar approach always uses the current minimum objective value as the fake objective to update the GP model to produce multiple query points for parallel function evaluations. 35 | 36 | **The Peseudo Expected Improvement**[^6] ```Parallel_PEI.m```. The pseudo expected improvement approach uses an influence function to simulate the sequential EI's selection behavior. It uses the influence function to update the EI function to prduce multiple query points. 37 | 38 | **The Multipoint Expected Improvement**[^7] ```Parallel_qEI.m```. The qEI function is coded according to the R code in [^8]. I used the Modified Cholesky algorithm [^9] and a quasi-random approach to estimate an MVN probability [^10] in the qEI implementation. 39 | 40 | **The Fast Multipoint Expected Improvement**[^11] ```Parallel_FqEI.m```. The FqEI criterion has very similar properties as the qEI, but is significantly faster than qEI. Therefore, it can be used for large batch size. 41 | 42 | 43 | ## Multiobjective Bayesian Optimization Algorithms 44 | 1. **The ParEGO (Pareto EGO)** (*Multiobjective_ParEGO.m*) [^12]. 45 | 2. **The Expected Improvement Matrix** (*Multiobjective_EIM.m*) [^13]. 46 | 3. **The Expected Hypervolume Improvement** (*Multiobjective_EHVI.m*)[^14]. The *EHVI* criterion is calculated using Monte Carlo approximation. 47 | 4. **The MOEA/D-EGO** (*Multiobjective_MOEAD_EGO.m*)[^15]. We use all the samples to train the Kriging models instead of using the fuzzy clusting based modeling method. 48 | 49 | 50 | ## Constrained Bayesian Optimization Algorithms 51 | 1. **The Constrained Expected Improvemment** (*Constrained_CEI.m*) [^16]. 52 | 2. **The Pseudo Constrained Expected Improvement** (*Constrained_PCEI.m*) [^17]. 53 | 54 | ## References 55 | [^1]: D. R. Jones, M. Schonlau, and W. J. Welch. Efficient global optimization of expensive black-box functions. Journal of Global Optimization, 1998. 13(4): 455-492. 56 | [^2]: A. Forrester and A. Keane. Engineering design via surrogate modelling: a practical guide. 2008, John Wiley & Sons. 57 | [^3]: K. Deb. An efficient constraint handling method for genetic algorithms. Computer Methods in Applied Mechanics and Engineering, 2000. 186(2): 311-338. 58 | [^4]: C. Li, S. Gupta, S. Rana, T. V. Nguyen, S. Venkatesh, and A. Shilton. High dimensional bayesian optimization using dropout, in International Joint Conference on Artificial Intelligence, 2017, 2096-2102. 59 | [^5]: D. Ginsbourger, R. Le Riche, and L. Carraro. Kriging Is Well-Suited to Parallelize Optimization, in Computational Intelligence in Expensive Optimization Problems, Y. Tenne and C.-K. Goh, Editors. 2010, 131-162. 60 | [^6]: D. Zhan, J. Qian, and Y. Cheng. Pseudo expected improvement criterion for parallel EGO algorithm. Journal of Global Optimization, 2017. 68(3): 641-662. 61 | [^7]: C. Chevalier, and D. Ginsbourger. Fast computation of the multi-points expected improvement with applications in batch selection, in Learning and Intelligent Optimization, G. Nicosia and P. Pardalos, Editors. 2013, 59-69. 62 | [^8]: O. Roustant, D. Ginsbourger, and Y. Deville. DiceKriging, DiceOptim: Two R Packages for the Analysis of Computer Experiments by Kriging-Based Metamodeling and Optimization. Journal of Statistical Software, 2012. 51(1): 1-55. 63 | [^9]: S. H. Cheng and N. J. Higham. A modified Cholesky algorithm based on a symmetric indefinite factorization. SIAM J. Matrix Anal. Appl., 19(4):1097-1110, 1998. https://github.com/higham/modified-cholesky. 64 | [^10]: Alan Genz. Numerical Computation of Multivariate Normal Probabilities. J. of Computational and Graphical Stat., 1992 1: 141-149. 65 | [^11]: D. Zhan, Y. Meng and H. Xing. A fast multi-point expected improvement for parallel expensive optimization. IEEE Transactions on Evolutionary Computation, 2023, 27(1): 170:184. 66 | [^12]: J. Knowles. ParEGO: A hybrid algorithm with on-line landscape approximation for expensive multiobjective optimization problems. IEEE Transactions on Evolutionary Computation, 2006. 10(1): 50-66. 67 | [^13]: D. Zhan, Y. Cheng, and J. Liu. Expected improvement matrix-based infill criteria for expensive multiobjective optimization. IEEE Transactions on Evolutionary Computation, 2017. 21(6): 956-975. 68 | [^14]: M. T. M. Emmerich, K. C. Giannakoglou, and B. Naujoks. Single- and multiobjective evolutionary optimization assisted by Gaussian random field metamodels. IEEE Transactions on Evolutionary Computation, 2006, 10(4): 421-439. 69 | [^15]: Q. Zhang, W. Liu, E. Tsang, and B. Virginas. Expensive Multiobjective Optimization by MOEA/D With Gaussian Process Model. IEEE Transactions on Evolutionary Computation, 2010, 14(3): 456-474. 70 | [^16]: M. Schonlau. Computer experiments and global optimization. 1997, University of Waterloo. 71 | [^17]: J. Qian, Y. Cheng, J. zhang, J. Liu, and D. Zhan. A parallel constrained efficient global optimization algorithm for expensive constrained optimization problems. Engineering Optimization, 2021. 53(2): 300-320. 72 | -------------------------------------------------------------------------------- /Rosenbrock.m: -------------------------------------------------------------------------------- 1 | function f = Rosenbrock(x) 2 | % the Rosenbrock function 3 | % xi = [-2.048,2.048] 4 | f = sum(100*(x(:,2:end) - x(:,1:end-1).^2).^2 + (x(:,1:end-1)-1).^2,2); 5 | end -------------------------------------------------------------------------------- /Standard_BO.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------------------------- 2 | % This is the matlab implementation of the standard Bayesian optimization 3 | % algorithm according to the following work. But I use a genetic algorithm 4 | % to maximize the EI function. 5 | % Reference: 6 | % D. R. Jones, M. Schonlau, and W. J. Welch. Efficient global 7 | % optimization of expensive black-box functions. Journal of Global 8 | % Optimization, 1998, 13(4): 455-192. 9 | % Author: Dawei Zhan 10 | % Date: 2024/11/26 11 | %------------------------------------------------------------------------- 12 | clearvars; close all; 13 | % setting of the problem 14 | fun_name = 'Rosenbrock'; 15 | num_vari = 50; 16 | lower_bound = -2.048*ones(1,num_vari); 17 | upper_bound = 2.048*ones(1,num_vari); 18 | % the number of initial design points 19 | num_initial = 200; 20 | % maximum number of evaluations 21 | max_evaluation = 500; 22 | % initial design points using Latin hypercube sampling method 23 | sample_x = lhsdesign(num_initial,num_vari,'criterion','maximin','iterations',1000).*(upper_bound-lower_bound) + lower_bound; 24 | sample_y = feval(fun_name,sample_x); 25 | % the current iteration and evaluation 26 | evaluation = size(sample_x,1); 27 | iteration = 0; 28 | % the current best solution 29 | fmin = min(sample_y); 30 | % print the current information to screen 31 | fprintf('BO on %d-D %s function, iteration: %d, evaluation: %d, current found minimum: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 32 | % the BO iteration 33 | while evaluation < max_evaluation 34 | % build the GP model 35 | GP_model = GP_Train(sample_x,sample_y,lower_bound,upper_bound,1*ones(1,num_vari),0.001*ones(1,num_vari),1000*ones(1,num_vari)); 36 | % find the point with the highest EI value using GA algorithm 37 | infill_x = Optimizer_GA(@(x)-Infill_EI(x,GP_model,fmin),num_vari,lower_bound,upper_bound,10*num_vari,200); 38 | % evaluate the query point with the real function 39 | infill_y = feval(fun_name,infill_x); 40 | % add the new point to design set 41 | sample_x = [sample_x;infill_x]; 42 | sample_y = [sample_y;infill_y]; 43 | % update some parameters 44 | evaluation = size(sample_x,1); 45 | iteration = iteration + 1; 46 | fmin = min(sample_y); 47 | % print the current information to the screen 48 | fprintf('BO on %d-D %s function, iteration: %d, evaluation: %d, current found minimum: %0.2f\n',num_vari,fun_name,iteration,evaluation,fmin); 49 | end 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /Tchebycheff_EI.m: -------------------------------------------------------------------------------- 1 | function EI = Tchebycheff_EI(y,s,lamda,z,sample_y) 2 | 3 | m = size(y,2); 4 | new_y = lamda(:,1).*(y(1)-z(1)); 5 | new_s = lamda(:,1).*s(1); 6 | for ii = 2:m 7 | mu1 = new_y; 8 | sigma1_square = new_s.^2; 9 | mu2 = lamda(:,ii).*(y(ii)-z(ii)); 10 | sigma2_square = lamda(:,ii).^2.*s(ii)^2; 11 | tao = sqrt(sigma1_square+sigma2_square); 12 | aphla = (mu1-mu2)./tao; 13 | new_y = mu1.*gausscdf(aphla) + mu2.*gausscdf(-aphla) + tao.*gausspdf(aphla); 14 | new_sigma2 = (mu1.^2+sigma1_square).*gausscdf(aphla) + (mu2.^2+sigma2_square).*gausscdf(-aphla) + (mu1+mu2).*gausspdf(aphla) - new_y.^2; 15 | new_s = sqrt(max(new_sigma2,0)); 16 | end 17 | 18 | gmin = min(reshape(max(repelem(lamda,size(sample_y,1),1).*repmat(sample_y-z,size(lamda,1),1),[],2),size(sample_y,1),size(lamda,1)))'; 19 | EI = (gmin-new_y).*gausscdf((gmin-new_y)./new_s)+new_s.*gausspdf((gmin-new_y)./new_s); 20 | end 21 | 22 | 23 | function y = gausscdf(x) 24 | y = 0.5*(1+erf(x/sqrt(2))); 25 | end 26 | 27 | function res = gausspdf(x) 28 | res = 1/sqrt(2*pi)*exp(-x.^2/2); 29 | end 30 | 31 | 32 | -------------------------------------------------------------------------------- /UniformPoint.m: -------------------------------------------------------------------------------- 1 | function [W,N] = UniformPoint(N,M) 2 | %UniformPoint - Generate a set of uniformly distributed points on the unit 3 | %hyperplane 4 | % 5 | % [W,N] = UniformPoint(N,M) returns approximate N uniformly distributed 6 | % points with M objectives. 7 | % 8 | % Example: 9 | % [W,N] = UniformPoint(275,10) 10 | 11 | %-------------------------------------------------------------------------- 12 | % Copyright (c) 2016-2017 BIMK Group. You are free to use the PlatEMO for 13 | % research purposes. All publications which use this platform or any code 14 | % in the platform should acknowledge the use of "PlatEMO" and reference "Ye 15 | % Tian, Ran Cheng, Xingyi Zhang, and Yaochu Jin, PlatEMO: A MATLAB Platform 16 | % for Evolutionary Multi-Objective Optimization [Educational Forum], IEEE 17 | % Computational Intelligence Magazine, 2017, 12(4): 73-87". 18 | %-------------------------------------------------------------------------- 19 | 20 | H1 = 1; 21 | while nchoosek(H1+M,M-1) <= N 22 | H1 = H1 + 1; 23 | end 24 | W = nchoosek(1:H1+M-1,M-1) - repmat(0:M-2,nchoosek(H1+M-1,M-1),1) - 1; 25 | W = ([W,zeros(size(W,1),1)+H1]-[zeros(size(W,1),1),W])/H1; 26 | if H1 < M 27 | H2 = 0; 28 | while nchoosek(H1+M-1,M-1)+nchoosek(H2+M,M-1) <= N 29 | H2 = H2 + 1; 30 | end 31 | if H2 > 0 32 | W2 = nchoosek(1:H2+M-1,M-1) - repmat(0:M-2,nchoosek(H2+M-1,M-1),1) - 1; 33 | W2 = ([W2,zeros(size(W2,1),1)+H2]-[zeros(size(W2,1),1),W2])/H2; 34 | W = [W;W2/2+1/(2*M)]; 35 | end 36 | end 37 | W = max(W,1e-6); 38 | N = size(W,1); 39 | end -------------------------------------------------------------------------------- /Welded_Beam.m: -------------------------------------------------------------------------------- 1 | function [obj, con] = Fun_Welded_Beam(x) 2 | % ---------------------------------------------------------------------------- 3 | % the welded beam design problem 4 | % Rao, S.S. Engineering Optimization. Wiley, New York. 1996. 5 | % fmin = 1.7250 6 | % ---------------------------------------------------------------------------- 7 | x1 = x(:, 1); x2 = x(:, 2); x3 = x(:, 3); x4 = x(:, 4); 8 | 9 | P = 6000; 10 | L = 14; 11 | E = 30*10^6; 12 | G = 12 * 10^6; 13 | tao_max = 13600; 14 | sigma_max = 30000; 15 | deta_max = 0.25; 16 | 17 | M = P*(L + 0.5*x2); 18 | R = sqrt(x2.^2/4 + ((x1+x3)/2).^2); 19 | J = 2*sqrt(2)*x1.*x2.*(x2.^2/12 + ((x1+x3)/2).^2); 20 | tao_1 = P./(sqrt(2)*x1.*x2); 21 | tao_2 = M.*R./J; 22 | 23 | tao = sqrt(tao_1.^2 + 2*tao_1.*tao_2.*x2./(2*R) + tao_2.^2); 24 | sigma = 6*P*L./(x4.*x3.^2); 25 | deta = 4*P*L.^3./(E*x3.^3.*x4); 26 | Pc = 4.013*E*sqrt(x3.^2.*x4.^6/36).*(1-x3.*sqrt(E/(4*G))/(2*L))/L^2; 27 | 28 | obj = 1.10471*x1.^2.*x2 + 0.04811*x3.*x4.*(x2 + 14); 29 | con(:, 1) = tao./tao_max - 1; 30 | con(:, 2) = sigma./sigma_max - 1; 31 | con(:, 3) = (x1 - x4)/10; 32 | con(:, 4) = 0.10471*x1.^2 +0.04811*x3.*x4.*(14.0+x2) -5; 33 | con(:, 5) = deta./deta_max - 1; 34 | con(:, 6) = 1 - Pc./P; 35 | 36 | 37 | 38 | end --------------------------------------------------------------------------------