├── Non convex ├── acc_prox_grad.m ├── accmon_prox_grad.m ├── backtrack_PG.m ├── cap_l1.m ├── line_search_PG.m ├── mat_fun.m ├── nc_monotone_APG.m ├── nc_non_monotone_APG.m ├── non_convex.m ├── prox_grad.m ├── prox_minimize.m ├── prox_op.m ├── soft_th_capped_l1.m └── test_non_APG.asv ├── README.md └── test_non_APG.m /Non convex/acc_prox_grad.m: -------------------------------------------------------------------------------- 1 | function [x,apg] = acc_prox_grad(obj,iter) 2 | 3 | f = obj.f; 4 | g = obj.g; 5 | gamma = obj.gamma; 6 | grad_f = obj.grad_f; 7 | prox = obj.prox; 8 | x = obj.init_x; 9 | lambda = 1; 10 | x_prev = x; 11 | if isempty(iter) 12 | max_iter = 10000; 13 | else 14 | max_iter = iter; 15 | end 16 | 17 | for i=1:max_iter 18 | y = x + (i/(i+3))*(x - x_prev); 19 | [lambda,z] = line_search_PG(f,grad_f,y,prox,lambda,gamma); 20 | x_prev = x; 21 | x = z; 22 | apg.prox_val(i) = f(x) + g(x); 23 | if i>1 && abs(apg.prox_val(i) - apg.prox_val(i-1)) < 1e-6 24 | break; 25 | end 26 | end 27 | apg.x_prox = x; 28 | apg.p_prox = apg.prox_val(end); 29 | apg.time = toc; 30 | end -------------------------------------------------------------------------------- /Non convex/accmon_prox_grad.m: -------------------------------------------------------------------------------- 1 | function [x,apg] = accmon_prox_grad(obj,iter) 2 | f = obj.f; 3 | g = obj.g; 4 | gamma = obj.gamma; 5 | grad_f = obj.grad_f; 6 | prox = obj.prox; 7 | x = obj.init_x; 8 | lambda = 1; 9 | x_prev = x; 10 | t_pr = 1; 11 | t = 1; 12 | z = x; 13 | F = @(x)(f(x) + g(x)); 14 | if isempty(iter) 15 | max_iter = 10000; 16 | else 17 | max_iter = iter; 18 | end 19 | 20 | for i=1:max_iter 21 | y = x + ((t_pr-1)/t)*(z - x) + ((t_pr-1)/t)*(x - x_prev); 22 | [lambda,z] = line_search_PG(f,grad_f,y,prox,lambda,gamma); 23 | t_pr = t; 24 | t = (sqrt(4*(t)^2+1) + 1)/2; 25 | x_prev = x; 26 | if F(z) <= F(x) 27 | x = z; 28 | end 29 | apg.prox_val(i) = f(x) + g(x); 30 | if i>1 && abs(apg.prox_val(i) - apg.prox_val(i-1)) < 1e-6 31 | break; 32 | end 33 | end 34 | apg.x_prox = x; 35 | apg.p_prox = apg.prox_val(end); 36 | apg.time = toc; 37 | end -------------------------------------------------------------------------------- /Non convex/backtrack_PG.m: -------------------------------------------------------------------------------- 1 | function t = backtrack_PG(prox,grad_f,f,x) 2 | beta = 0.5; 3 | t = 1; 4 | G = @(x)((x-prox(x-t*grad_f(x),t))/t); 5 | 6 | while f(x - t*G(x)) > f(x) - t*grad_f(x)'*G(x) + 0.5*t*norm(G(x))^2 7 | t = beta*t; 8 | end 9 | 10 | end -------------------------------------------------------------------------------- /Non convex/cap_l1.m: -------------------------------------------------------------------------------- 1 | function r = cap_l1(x,lambda) 2 | n = length(x); 3 | theta = 0.1*lambda; 4 | r = 0; 5 | for i=1:n 6 | r = r + lambda*min([abs(x(i));theta]); 7 | end 8 | 9 | end -------------------------------------------------------------------------------- /Non convex/line_search_PG.m: -------------------------------------------------------------------------------- 1 | function [lambda,z] = line_search_PG(f,grad_f,x,prox,lambda,g) 2 | 3 | beta = 0.5; 4 | z = prox(x - lambda*grad_f(x),lambda*g); 5 | while f(z) >= f(x) + grad_f(x)'*z + (0.5/lambda)*norm(z,2)^2 6 | lambda = beta*lambda; 7 | z = prox(x - lambda*grad_f(x),lambda*g); 8 | end 9 | 10 | end -------------------------------------------------------------------------------- /Non convex/mat_fun.m: -------------------------------------------------------------------------------- 1 | function [f,g] = mat_fun(A,b,gamma,x) 2 | f = 0.5*norm(A*x-b,2)^2 + gamma*norm(x,1); 3 | % f = 0.5*norm(A*x-b,2)^2 + cap_l1(x,gamma); 4 | 5 | if nargout >1 6 | g = A'*(A*x-b); 7 | end 8 | end -------------------------------------------------------------------------------- /Non convex/nc_monotone_APG.m: -------------------------------------------------------------------------------- 1 | function [x_opt,apg] = nc_monotone_APG(obj,iter) 2 | 3 | f = obj.f; 4 | g = obj.g; 5 | gamma = obj.gamma; 6 | grad_f = obj.grad_f; 7 | prox = obj.prox; 8 | x_0 = obj.init_x; 9 | n = size(x_0,1); 10 | if isempty(iter) 11 | max_iter = 10000; 12 | else 13 | max_iter = iter; 14 | end 15 | x = zeros(n,max_iter); 16 | y = zeros(n,max_iter); 17 | z = zeros(n,max_iter); 18 | v = zeros(n,max_iter); 19 | t = zeros(1,max_iter); 20 | t(:,2) = 1; 21 | lambda_z = 1; 22 | lambda_v = 1; 23 | 24 | for i=2:max_iter-1 25 | y(:,i) = x(:,i) + (t(i-1)/t(i))*(z(:,i) - x(:,i)) +... 26 | (t(i-1)-1)/(t(i))*(x(:,i) - x(:,i-1)); 27 | [lambda_z,z(:,i+1)] = line_search_PG(f,grad_f,y(:,i),prox,lambda_z,gamma); 28 | [lambda_v,v(:,i+1)] = line_search_PG(f,grad_f,x(:,i),prox,lambda_v,gamma); 29 | 30 | t(:,i+1) = 0.5*(sqrt(4*t(:,i)^2+1)+1); 31 | if (f(z(:,i+1)) + g(z(:,i+1))) <= (f(x(:,i+1)) + g(x(:,i+1))) 32 | x(:,i+1)=z(:,i+1); 33 | else 34 | x(:,i+1) = v(:,i+1); 35 | end 36 | 37 | apg.prox_val(i) = f(x(:,i+1)) + g(x(:,i+1)); 38 | if i>2 && abs(apg.prox_val(i) - apg.prox_val(i-1)) < 1e-6 39 | break; 40 | end 41 | end 42 | x_opt = x(:,i); 43 | apg.x_prox = x_opt; 44 | apg.p_prox = apg.prox_val(end); 45 | apg.time = toc; 46 | end -------------------------------------------------------------------------------- /Non convex/nc_non_monotone_APG.m: -------------------------------------------------------------------------------- 1 | function [x_opt,apg] = nc_non_monotone_APG(obj,iter) 2 | 3 | f = obj.f; 4 | g = obj.g; 5 | gamma = obj.gamma; 6 | grad_f = obj.grad_f; 7 | prox = obj.prox; 8 | x_0 = obj.init_x; 9 | n = size(x_0,1); 10 | if isempty(iter) 11 | max_iter = 10000; 12 | else 13 | max_iter = iter; 14 | end 15 | x = zeros(n,max_iter); 16 | y = zeros(n,max_iter); 17 | z = zeros(n,max_iter); 18 | v = zeros(n,max_iter); 19 | t = zeros(1,max_iter); 20 | t(:,2) = 1; 21 | eta = 0.5; 22 | del = 0.5; 23 | lambda_z = 1; 24 | lambda_v = 1; 25 | c1 = zeros(1,max_iter); 26 | c1(:,2) = f(x(:,2)) + g(x(:,2)); 27 | q = zeros(1,max_iter); 28 | q(:,2) = 1; 29 | F = @(x)(f(x) + g(x)); 30 | 31 | for i=2:max_iter-1 32 | y(:,i) = x(:,i) + (t(i-1)/t(i))*(z(:,i) - x(:,i)) +... 33 | (t(i-1)-1)/(t(i))*(x(:,i) - x(:,i-1)); 34 | [lambda_z,z(:,i+1)] = line_search_PG(f,grad_f,y(:,i),prox,lambda_z,gamma); 35 | 36 | if F(z(:,i+1)) <= c1(:,i) - del*norm(z(:,i+1)-y(:,i)) 37 | x(:,i+1) = z(:,i+1); 38 | else 39 | [lambda_v,v(:,i+1)] = line_search_PG(f,grad_f,x(:,i),prox,lambda_v,gamma); 40 | if (f(z(:,i+1)) + g(z(:,i+1))) <= (f(x(:,i+1)) + g(x(:,i+1))) 41 | x(:,i+1)=z(:,i+1); 42 | else 43 | x(:,i+1) = v(:,i+1); 44 | end 45 | end 46 | t(:,i+1) = 0.5*(sqrt(4*t(:,i)^2+1)+1); 47 | q(:,i+1) = eta*q(:,i) + 1; 48 | c1(:,i+1) = (eta*q(:,i)*c1(:,i) + F(x(:,i+1)))/q(:,i+1); 49 | apg.prox_val(i) = f(x(:,i+1)) + g(x(:,i+1)); 50 | if i>2 && abs(apg.prox_val(i) - apg.prox_val(i-1)) < 1e-6 51 | break; 52 | end 53 | end 54 | x_opt = x(:,i); 55 | apg.x_prox = x_opt; 56 | apg.p_prox = apg.prox_val(end); 57 | apg.time = toc; 58 | 59 | end -------------------------------------------------------------------------------- /Non convex/non_convex.m: -------------------------------------------------------------------------------- 1 | classdef non_convex 2 | properties 3 | f 4 | g 5 | gamma 6 | grad_f 7 | prox 8 | init_x 9 | end 10 | methods 11 | % Constructor 12 | function n_c = non_convex(f,g,gamma,grad_f,prox,init_x) 13 | if nargin > 1 14 | n_c.f = f; % Objective function 15 | n_c.g = g; % Non-convex function 16 | n_c.gamma = gamma; % Regularizer weight 17 | n_c.grad_f = grad_f; % Convex function gradient 18 | n_c.prox = prox; % Proximal opertaor 19 | n_c.init_x = init_x; % Initialization 20 | end 21 | end 22 | % Proximal gradient 23 | function [x_opt,h] = PG(obj,iter) 24 | tic; 25 | [x_opt,h] = prox_grad(obj,iter); 26 | end 27 | % Accelerated proximal gradient 28 | function [x_opt,h] = APG(obj,iter) 29 | tic; 30 | [x_opt,h] = acc_prox_grad(obj,iter); 31 | end 32 | % Monotonic accelerated proximal gradient 33 | function [x_opt,h] = APG_MON(obj,iter) 34 | tic; 35 | [x_opt,h] = accmon_prox_grad(obj,iter); 36 | end 37 | % Monotonic accelerated non convex proxial gradient descent 38 | function [x_opt,h] = nc_mon_APG(obj,iter) 39 | tic; 40 | [x_opt,h] = nc_monotone_APG(obj,iter); 41 | end 42 | % Non Monotonic accelerated non convex proxial gradient descent 43 | function [x_opt,h] = nc_nmon_APG(obj,iter) 44 | tic; 45 | [x_opt,h] = nc_non_monotone_APG(obj,iter); 46 | end 47 | end 48 | end -------------------------------------------------------------------------------- /Non convex/prox_grad.m: -------------------------------------------------------------------------------- 1 | function [x,h] = prox_grad(obj,iter) 2 | 3 | f = obj.f; 4 | g = obj.g; 5 | gamma = obj.gamma; 6 | grad_f = obj.grad_f; 7 | prox = obj.prox; 8 | x = obj.init_x; 9 | if isempty(iter) 10 | max_iter = 10000; 11 | else 12 | max_iter = iter; 13 | end 14 | lambda = 1; 15 | 16 | for i=1:max_iter 17 | [lambda,x] = line_search_PG(f,grad_f,x,prox,lambda,gamma); 18 | h.prox_val(i) = f(x) + g(x); 19 | if i>1 && abs(h.prox_val(i) - h.prox_val(i-1)) < 1e-6 20 | break; 21 | end 22 | end 23 | h.x_prox = x; 24 | h.p_prox = h.prox_val(end); 25 | h.time = toc; 26 | end 27 | -------------------------------------------------------------------------------- /Non convex/prox_minimize.m: -------------------------------------------------------------------------------- 1 | function prox = prox_minimize(obj) 2 | opts1= optimset('display','off'); 3 | g = obj.g; 4 | prox = @(x,t)fminunc(@(z)((1/(2*t))*norm(x-z)^2 + g(z)),zeros(size(x)),opts1); 5 | 6 | end -------------------------------------------------------------------------------- /Non convex/prox_op.m: -------------------------------------------------------------------------------- 1 | function x = prox_op(beta,lambda) 2 | n = length(beta); 3 | x = zeros(n,1); 4 | for i=1:n 5 | if beta(i) > lambda 6 | x(i) = beta(i) - lambda; 7 | elseif (-lambda<=beta(i)) && (beta(i)<=lambda) 8 | x(i) = 0; 9 | elseif beta(i) < -lambda 10 | x(i) = beta(i) + lambda; 11 | end 12 | end 13 | end -------------------------------------------------------------------------------- /Non convex/soft_th_capped_l1.m: -------------------------------------------------------------------------------- 1 | function x = soft_th_capped_l1(beta,lambda) 2 | n = length(beta); 3 | x = zeros(n,1); 4 | for i=1:n 5 | if beta(i) > lambda 6 | x(i) = beta(i) - lambda; 7 | elseif (-lambda<=beta(i)) && (beta(i)<=lambda) 8 | x(i) = 0; 9 | elseif beta(i) < -lambda 10 | x(i) = beta(i) + lambda; 11 | end 12 | end 13 | end -------------------------------------------------------------------------------- /Non convex/test_non_APG.asv: -------------------------------------------------------------------------------- 1 | % Test non-convex APG 2 | clear; 3 | close all; 4 | clc; 5 | 6 | %% Parameters 7 | m = 100; % Data dimension 8 | n = 500; % Feature dimension 9 | x0 = sprandn(n,1,0.05); % Generate sparse vector 10 | A = rand(m,n); % A matrix (weight matrix) 11 | A = A*spdiags(1./sqrt(sum(A.^2))',0,n,n); % Make the norm of the column = 1 12 | v = sqrt(0.001)*randn(m,1); % Generate noise 13 | b = A*x0 + v; % Measurement vector 14 | init_x = zeros(n,1); % Initialization 15 | 16 | %% Lasso 17 | f = @(x)(0.5*norm(A*x-b,2)^2); % Convex function 18 | gamma_max = norm(A'*b,'inf'); % Regularization weight 19 | gamma = 0.1*gamma_max; % Scaling 20 | g = @(x)(gamma*norm(x,1)); % Non-convex function 21 | 22 | grad_f =@(x)(A'*(A*x-b)); % Gradient of the convex function 23 | prox = @(x,lambda)(prox_op(x,lambda)); % Proximal operator 24 | 25 | prob = non_convex(f,g,gamma,grad_f,prox,init_x); % Setting the class parameters 26 | 27 | %% Solving 28 | % Proximal gradient method 29 | [x_PG,PG] = prob.PG(1000); 30 | % Accelerated proximal gradient 31 | [x_APG,APG] = prob.APG(1000); 32 | % Monotonic accelerated proximal gradient 33 | [x_APG_MON,APG_MON] = prob.APG_MON(1000); 34 | % Monotonic accelerated non convex proximal gradient 35 | [x_nc_APG,nc_APG] = prob.nc_mon_APG(1000); 36 | % Non monotonic accelerated non convex proximal gradient 37 | [x_nc_nAPG,nc_nAPG] = prob.nc_nmon_APG(1000); 38 | % Matlab optimization 39 | tic; 40 | fun = @(x)(mat_fun(A,b,gamma,x)); 41 | [x,fval] = fminunc(fun,init_x); 42 | mt_time = toc; 43 | 44 | %% Plotting 45 | t = max([length(PG.prox_val);length(APG.prox_val);length(APG_MON.prox_val);length(nc_APG.prox_val);length(nc_nAPG.prox_val)]); 46 | figure; 47 | plot(PG.prox_val,'b'); 48 | hold on; 49 | plot(APG.prox_val,'r'); 50 | plot(APG_MON.prox_val,'g'); 51 | plot(nc_APG.prox_val,'c'); 52 | plot(nc_nAPG.prox_val,'m'); 53 | legend('Prox gradient','Acc proxgradient') -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Non-convex optimization 2 | This repository contains matlab interface to use proximal algorithms and different variants for composite funtions containing convex and non-convex functions. 3 | 4 | ## Description 5 | Non convex and non smooth probelms have been observed widely in the field of signal processing and machine learning. Solving non convex non smooth problems is a big challenge involving saddle point localization, non differentialibilty, escape from saddle point and others. Proximal algorithms and its variants provides a good framework for non convex problems.Here a non convex for proximal algorithms have been built. 6 | 7 | Given a composite function 8 | 9 | ![](http://latex.codecogs.com/gif.latex?min_%7Bx%5Cin%20%5Cmathbb%7BR%7D%5E%7Bn%7D%7D%20%5C%20F%28x%29%20%3D%20f%28x%29%20+%20g%28x%29) 10 | 11 | ![](http://latex.codecogs.com/gif.latex?where) 12 | 13 | ![](http://latex.codecogs.com/gif.latex?f%28x%29%20-%20convex%20%5C%20function) 14 | ![](http://latex.codecogs.com/gif.latex?g%28x%29%20-%20nonconvex%20%5C%20function) 15 | 16 | The non convex class tries to solve the optimization probelm given the proximal operator and the gradient of the convex function. 17 | 18 | The algorithms present are: 19 | - Proximal gradient method 20 | - Accelerated proximal gradient method 21 | - Monotonic accelerated proximal gradient method 22 | - Monotonic non convex proximal gradient method 23 | - Nonmonotonic non convex proximal gradient method 24 | 25 | The references for the algorithms stated above are provided in the reference below 26 | 27 | ## Getting Started 28 | 29 | Clone or download the repository. Various inputs are needed to the non convex class which are explained as follows. 30 | f - convex function 31 | 32 | g - non-convex function 33 | 34 | grad_f - gradient of convex function 35 | 36 | prox - proximal operator 37 | 38 | init_x - initialization 39 | 40 | A constructor is also present for the class and the cronological order is as mentioned above. 41 | 42 | ### Prerequisites 43 | 44 | Matlab 2010 or higher 45 | 46 | 47 | ## Running the tests 48 | 49 | The test_non_APG.m contains a random system. With the objective of a lasso with l1 regularization. 50 | 51 | ## Authors 52 | 53 | * **Sandeep Banik** - [Projects](https://github.com/sandeepbanik) 54 | 55 | ## Reference 56 | 57 | **[Proximal algorithm]** by Neal Parikh and Stephen Boyd 58 | 59 | **[Fast Gradient-Based Algorithms for Constrained Total Variation Image Denoising and Deblurring Problems]** by Amir Beck and Marc Teboulle 60 | 61 | **[Accelerated Proximal Gradient Methods for Nonconvex Programming]** by Huan Li and Zhouchen Lin 62 | 63 | [Proximal algorithm]: 64 | [Fast Gradient-Based Algorithms for Constrained Total Variation Image Denoising and Deblurring Problems]: 65 | [Accelerated Proximal Gradient Methods for Nonconvex Programming]: 66 | -------------------------------------------------------------------------------- /test_non_APG.m: -------------------------------------------------------------------------------- 1 | % Test non-convex APG 2 | clear; 3 | close all; 4 | clc; 5 | addpath('Non convex'); 6 | %% Parameters 7 | m = 500; % Data dimension 8 | n = 2500; % Feature dimension 9 | x0 = sprandn(n,1,0.05); % Generate sparse vector 10 | A = rand(m,n); % A matrix (weight matrix) 11 | A = A*spdiags(1./sqrt(sum(A.^2))',0,n,n); % Make the norm of the column = 1 12 | v = sqrt(0.001)*randn(m,1); % Generate noise 13 | b = A*x0 + v; % Measurement vector 14 | init_x = rand(n,1); % Initialization 15 | 16 | %% Lasso 17 | f = @(x)(0.5*norm(A*x-b,2)^2); % Convex function 18 | gamma_max = norm(A'*b,'inf'); % Regularization weight 19 | gamma = 0.1*gamma_max; % Scaling 20 | g = @(x)(gamma*norm(x,1)); % Non-convex function 21 | 22 | grad_f =@(x)(A'*(A*x-b)); % Gradient of the convex function 23 | prox = @(x,lambda)(prox_op(x,lambda)); % Proximal operator 24 | 25 | prob = non_convex(f,g,gamma,grad_f,prox,init_x); % Setting the class parameters 26 | 27 | %% Solving 28 | % Proximal gradient method 29 | [x_PG,PG] = prob.PG(1000); 30 | 31 | % Accelerated proximal gradient 32 | [x_APG,APG] = prob.APG(1000); 33 | 34 | % Monotonic accelerated proximal gradient 35 | [x_APG_MON,APG_MON] = prob.APG_MON(1000); 36 | 37 | % Monotonic accelerated non convex proximal gradient 38 | [x_nc_APG,nc_APG] = prob.nc_mon_APG(1000); 39 | 40 | % Non monotonic accelerated non convex proximal gradient 41 | [x_nc_nAPG,nc_nAPG] = prob.nc_nmon_APG(1000); 42 | 43 | % Matlab optimization 44 | % tic; 45 | % fun = @(x)(mat_fun(A,b,gamma,x)); 46 | % [x,fval] = fminunc(fun,init_x); 47 | % mt_time = toc; 48 | 49 | %% Plotting 50 | t = max([length(PG.prox_val);length(APG.prox_val);length(APG_MON.prox_val);length(nc_APG.prox_val);length(nc_nAPG.prox_val)]); 51 | figure; 52 | plot(PG.prox_val,'b'); 53 | hold on; 54 | plot(APG.prox_val,'r'); 55 | plot(APG_MON.prox_val,'g'); 56 | plot(nc_APG.prox_val(2:end),'c'); 57 | plot(nc_nAPG.prox_val(2:end),'m'); 58 | legend(['Prox gradient ' num2str(round(PG.time,2)) ' sec'],['Accelerated ' num2str(round(APG.time,2)) ' sec'],... 59 | ['Monotonic Accelerated ' num2str(round(APG_MON.time,2)) ' sec'],... 60 | ['Non convex monotonic prox ' num2str(round(nc_APG.time,2)) ' sec'],... 61 | ['Non convex non monotonic prox ' num2str(round(nc_nAPG.time,2)) ' sec']); 62 | axis tight; 63 | --------------------------------------------------------------------------------