├── FP_LSTM.m ├── ISOMP_LSTM.m ├── LSTM_Grad_Clip.m ├── LSTM_Grads.m ├── LSTM_TrainCE.m ├── MNIST_Data ├── ImsTest.mat ├── ImsTr.mat ├── ImsVal.mat ├── Prepare_MNIST_Data.m ├── loadMNISTImages.m └── loadMNISTLabels.m ├── Main.m ├── NISOMPnew.m ├── OneLayerBackprop_new.m ├── OneLayerBackprop_new_Biased.m ├── Other_Methods ├── BCS_demo │ ├── BCS_fast_rvm.m │ ├── Fig2.m │ ├── Fig4_ab.m │ ├── approx_results.mat │ ├── l1qc_logbarrier.m │ ├── l1qc_newton.m │ ├── multi_approx_measures.m │ ├── multi_optimized_measures.m │ ├── multi_random_measures.m │ ├── optimized_results.mat │ └── random_results.mat ├── MT_CS_demo │ ├── Fig2.m │ ├── Fig3.m │ ├── mt_CS.m │ ├── multi_results_25.mat │ ├── multi_results_50.mat │ ├── multi_results_75.mat │ ├── multi_runs_25.m │ ├── multi_runs_50.m │ └── multi_runs_75.m └── TMSBL_code │ ├── MFOCUSS.m │ ├── MSBL.m │ ├── ReadMe.txt │ ├── TMSBL.m │ ├── TSBL.m │ ├── demo.m │ ├── demo_fig3.m │ ├── demo_fig6_SNR10.m │ ├── demo_fig8.m │ ├── demo_identicalVector.m │ ├── demo_time_varying.m │ └── perfSupp.m ├── README.md ├── README_LSTM.txt ├── SOMP.m ├── TgenerateISOMP_MNIST_MMV.m ├── TgenerateISOMP_MNIST_MMV_CE.m └── mnist_fun.m /FP_LSTM.m: -------------------------------------------------------------------------------- 1 | function [Fs] = FP_LSTM(r,Ws,y_prev,c_prev) 2 | % LSTM forward pass 3 | %% Contact: hamidp@ece.ubc.com 4 | 5 | Fs.yg = tanh( Ws.W4*r + Ws.Wrec4*y_prev ); 6 | Fs.i = Ws.W3*r + Ws.Wrec3*y_prev; Fs.i = sigmf(Fs.i,[1 0]);% Note: sigmf(x,[a c])=1/(1+exp( -a*(x-c) )) 7 | Fs.c = c_prev + Fs.i .* Fs.yg; 8 | Fs.o = Ws.W1*r + Ws.Wrec1*y_prev; Fs.o = sigmf(Fs.o,[1 0]); 9 | Fs.y = Fs.o .* tanh( Fs.c ); 10 | Fs.z = Ws.U*Fs.y; 11 | den = sum( exp( Fs.z ),1 ); % den is a 1xnTr vector. 12 | nTr = size(den,2); 13 | for j=1:nTr 14 | Fs.s(:,j) = exp( Fs.z(:,j) ) ./ den(1,j); 15 | end -------------------------------------------------------------------------------- /ISOMP_LSTM.m: -------------------------------------------------------------------------------- 1 | function S_OMP = ISOMP_LSTM(phi_m,Y,res_min,Ws,winSize,input_zero_mean) 2 | 3 | %% Contact: Hamid Palangi, Email: hamidp@ece.ubc.ca 4 | % Distributed Compressed Sensing with Long Short Term Memory. 5 | m = size(phi_m,1); 6 | n = size(phi_m,2); 7 | ncell = size(Ws.Wrec1,1); 8 | L = size(Y,2); 9 | iter_max = n; 10 | done=0; 11 | shiftLen = floor(winSize/2); 12 | Rinit = Y; 13 | R = zeros(m*winSize,L); 14 | % Input data preparation for CNN: START** 15 | temp1 = Rinit; 16 | temp2 = zeros(m,L+2*shiftLen); % one shift for left and one for right. 17 | temp2( : , shiftLen+1 : shiftLen+L ) = temp1; 18 | for iL=1:L 19 | temp3 = temp2( : , iL:iL-1+winSize ); 20 | R(:,iL) = temp3(:); 21 | end 22 | % Input data preparation for CNN: END** 23 | S_OMP = zeros(n,L); 24 | cell_I = cell(L,1); 25 | j = 0; 26 | Rtemp = zeros(m,L); 27 | Stemp = cell(L,1); 28 | y_init = zeros(ncell,1); 29 | c_init = zeros(ncell,1); 30 | while done == 0 31 | j = j+1; 32 | normR=0; 33 | for iL=1:L 34 | r1 = R(:,iL);y1 = Y(:,iL);I = cell_I{iL,1}; 35 | if input_zero_mean == 1 36 | r1 = r1 - mean(r1); 37 | r1 = r1/max( abs( r1 ) ); 38 | else 39 | r1 = r1/max( abs( r1 ) ); 40 | end 41 | % LSTM forward pass: START 42 | if iL==1 43 | Fs(iL,1) = FP_LSTM(r1,Ws,y_init,c_init); 44 | else 45 | Fs(iL,1) = FP_LSTM(r1,Ws,Fs(iL-1,1).y,Fs(iL-1,1).c); 46 | end 47 | % LSTM forward pass: END 48 | s_hat = Fs(iL,1).s; 49 | [val1,idx1] = max( abs(s_hat) ); 50 | if length( find( I == idx1 ) ) == 0 51 | I = [I;idx1]; 52 | phiT = phi_m(:,I); 53 | s_temp = pinv(phiT)*y1; % Least squares 54 | r1temp = y1-phiT*s_temp; 55 | normR = normR + norm(r1temp); 56 | else 57 | continue; 58 | end 59 | cell_I{iL,1} = I; 60 | Rtemp(:,iL) = r1temp; 61 | Stemp{iL,1} = s_temp; 62 | end 63 | if (j > iter_max) || (normR < res_min ) 64 | done = 1; 65 | end 66 | if done == 1 67 | for iL=1:L 68 | I = cell_I{iL,1}; 69 | S_OMP(I,iL) = Stemp{iL,1}; 70 | end 71 | else 72 | % Input data preparation for CNN: START** 73 | temp1 = Rtemp; 74 | temp2 = zeros(m,L+2*shiftLen); % one shift for left and one for right. 75 | temp2( : , shiftLen+1 : shiftLen+L ) = temp1; 76 | for iL=1:L 77 | temp3 = temp2( : , iL:iL-1+winSize ); 78 | R(:,iL) = temp3(:); 79 | end 80 | % Input data preparation for CNN: END** 81 | end 82 | 83 | end 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /LSTM_Grad_Clip.m: -------------------------------------------------------------------------------- 1 | function [gs] = LSTM_Grad_Clip(gs,th) 2 | % Gradient clipping for LSTM 3 | %% Contact: hamidp@ece.ubc.ca 4 | 5 | I = find(gs.W1>th); 6 | gs.W1(I) = th; 7 | I = find(gs.Wrec1>th); 8 | gs.Wrec1(I) = th; 9 | I = find(gs.W3>th); 10 | gs.W3(I) = th; 11 | I = find(gs.Wrec3>th); 12 | gs.Wrec3(I) = th; 13 | I = find(gs.W4>th); 14 | gs.W4(I) = th; 15 | I = find(gs.Wrec4>th); 16 | gs.Wrec4(I) = th; 17 | I = find(gs.U>th); 18 | gs.U(I) = th; 19 | -------------------------------------------------------------------------------- /LSTM_Grads.m: -------------------------------------------------------------------------------- 1 | function [gs] = LSTM_Grads(In,T,ncell,Ws) 2 | % This function trains a LSTM network with following parameters: 3 | % In: input train date. 4 | % T: tartget train data 5 | % gs: a structure contatining all LSTM gradient matrices. 6 | %% Contact: hamidp@ece.ubc.ca 7 | 8 | n = size(T,1); 9 | m = size(In,1); 10 | L = size(In,2); % Memory length. 11 | nTr = size(In,3); % Number of training samples at each time step. 12 | % Pre-allocation: START 13 | for iL=1:L 14 | Fs(iL,1) = struct('yg',zeros(ncell,nTr),'i',zeros(ncell,nTr),'c',zeros(ncell,nTr),... 15 | 'o',zeros(ncell,nTr),'y',zeros(ncell,nTr),'z',zeros(n,nTr),'s',zeros(n,nTr)); 16 | deltas(iL,1) = struct('rec1',zeros(ncell,nTr),'rec3',zeros(ncell,nTr),'rec4',zeros(ncell,nTr),... 17 | 'u',zeros(n,nTr)); 18 | end 19 | % Pre-allocation: END 20 | % Setting gradients to zero 21 | gs.W1 = zeros(ncell,m); 22 | gs.Wrec1 = zeros(ncell); 23 | gs.W3 = zeros(ncell,m); 24 | gs.Wrec3 = zeros(ncell); 25 | gs.W4 = zeros(ncell,m); 26 | gs.Wrec4 = zeros(ncell); 27 | gs.U = zeros(n,ncell); 28 | 29 | gW3c = zeros(ncell,m); 30 | gWrec3c = zeros(ncell); 31 | gW4c = zeros(ncell,m); 32 | gWrec4c = zeros(ncell); 33 | for iL=1:L 34 | deltas(iL,1).rec1 = zeros(ncell,nTr); 35 | deltas(iL,1).rec3 = zeros(ncell,nTr); 36 | deltas(iL,1).rec4 = zeros(ncell,nTr); 37 | deltas(iL,1).u = zeros(n,nTr); 38 | end 39 | % Forward pass: START 40 | y_init = zeros(ncell,nTr); 41 | c_init = zeros(ncell,nTr); 42 | for iL=1:L 43 | r = reshape(In(:,iL,:),[m nTr]); 44 | if iL==1 45 | Fs(iL,1) = FP_LSTM(r,Ws,y_init,c_init); 46 | else 47 | Fs(iL,1) = FP_LSTM(r,Ws,Fs(iL-1,1).y,Fs(iL-1,1).c); 48 | end 49 | end 50 | % Forward pass: END 51 | % Backward pass: START 52 | s0 = reshape(T(:,L,:),[n nTr]); 53 | diff = Fs(L,1).s - s0; 54 | e = Ws.U' * diff; 55 | % U 56 | deltas(L,1).u = diff; 57 | % Output Gate 58 | deltas(L,1).rec1 = Fs(L,1).o .* ( 1 - Fs(L,1).o ) .* tanh( Fs(L,1).c ) .* e; 59 | % Input Gate 60 | deltas(L,1).rec3 = ( 1 - tanh( Fs(L,1).c ) ).*( 1 + tanh( Fs(L,1).c ) ) .*... 61 | Fs(L,1).o .* e; 62 | % yg 63 | deltas(L,1).rec4 = deltas(L,1).rec3; 64 | for iL=L-1:-1:1 65 | s0 = reshape(T(:,iL,:),[n nTr]); 66 | diff = Fs(iL,1).s - s0; 67 | e = Ws.U' * diff; 68 | % U 69 | deltas(iL,1).u = diff; 70 | % Output Gate 71 | deltas(iL,1).rec1 = ( Fs(iL,1).o .* (1-Fs(iL,1).o) .* tanh( Fs(iL,1).c ) )... 72 | .* (Ws.Wrec1' * deltas(iL+1,1).rec1 + e); 73 | % Input Gate 74 | temp3_4 = ( 1 - tanh( Fs(iL,1).c ) ).*( 1 + tanh( Fs(iL,1).c ) ).* Fs(iL,1).o; 75 | deltas(iL,1).rec3 = temp3_4 .* (Ws.Wrec3' * deltas(iL+1,1).rec3 + e); 76 | % yg 77 | deltas(iL,1).rec4 = temp3_4 .* (Ws.Wrec4' * deltas(iL+1,1).rec4 + e); 78 | end 79 | % Backward pass: END 80 | % Gradients: START 81 | % U gradient 82 | for iL=1:L 83 | gs.U = gs.U + deltas(iL,1).u * Fs(iL,1).y'; 84 | end 85 | % Output Gate, Input Gate, yg 86 | for iL=1:L 87 | r = reshape(In(:,iL,:),[m nTr]); 88 | % Output Gate 89 | gs.W1 = gs.W1 + deltas(iL,1).rec1 * r'; 90 | if iL>1 91 | gs.Wrec1 = gs.Wrec1 + deltas(iL,1).rec1 * Fs(iL-1,1).y'; 92 | end 93 | % Input Gate and yg for L=1. 94 | if iL==1 95 | % for iL=1, dc(iL-1)/dW is zero. 96 | bi = Fs(iL,1).yg .* Fs(iL,1).i .* ( 1 - Fs(iL,1).i ); 97 | bg = Fs(iL,1).i .* ( 1 - Fs(iL,1).yg ) .* ( 1 + Fs(iL,1).yg ); 98 | gs.W3 = gs.W3 + ( deltas(iL,1).rec3 .* bi ) * r'; 99 | gs.W4 = gs.W4 + ( deltas(iL,1).rec4 .* bg ) * r'; 100 | end 101 | end 102 | % Input Gate and yg for rest of L. 103 | if L>1 104 | for ir=1:nTr 105 | gW3c_prev = zeros(ncell,m); 106 | gWrec3c_prev = zeros(ncell); 107 | gW4c_prev = zeros(ncell,m); 108 | gWrec4c_prev = zeros(ncell); 109 | for iL=2:L 110 | r = reshape(In(:,iL,ir),[m 1]); 111 | % Input Gate 112 | bi_r = Fs(iL,1).yg(:,ir) .* Fs(iL,1).i(:,ir) .* ( 1 - Fs(iL,1).i(:,ir) ); 113 | gWrec3c = gWrec3c_prev + bi_r * Fs(iL-1,1).y(:,ir)'; 114 | gs.Wrec3 = gs.Wrec3 + sparse( diag( deltas(iL,1).rec3(:,ir) ) ) * gWrec3c; 115 | gWrec3c_prev = gWrec3c; 116 | gW3c = gW3c_prev + bi_r * r'; 117 | gs.W3 = gs.W3 + sparse( diag( deltas(iL,1).rec3(:,ir) ) ) * gW3c; 118 | gW3c_prev = gW3c; 119 | % yg 120 | bg_r = Fs(iL,1).i(:,ir) .* ( 1 - Fs(iL,1).yg(:,ir) ) .* ( 1 + Fs(iL,1).yg(:,ir) ); 121 | gWrec4c = gWrec4c_prev + bg_r * Fs(iL-1,1).y(:,ir)'; 122 | gs.Wrec4 = gs.Wrec4 + sparse( diag( deltas(iL,1).rec4(:,ir) ) ) * gWrec4c; 123 | gWrec4c_prev = gWrec4c; 124 | gW4c = gW4c_prev + bg_r * r'; 125 | gs.W4 = gs.W4 + sparse( diag( deltas(iL,1).rec4(:,ir) ) ) * gW4c; 126 | gW4c_prev = gW4c; 127 | end 128 | end 129 | end 130 | gs.W1 = gs.W1 ./ nTr; 131 | gs.W3 = gs.W3 ./ nTr; 132 | gs.W4 = gs.W4 ./ nTr; 133 | gs.Wrec1 = gs.Wrec1 ./ nTr; 134 | gs.Wrec3 = gs.Wrec3 ./ nTr; 135 | gs.Wrec4 = gs.Wrec4 ./ nTr; 136 | gs.U = gs.U ./ nTr; 137 | % Gradients: END 138 | -------------------------------------------------------------------------------- /LSTM_TrainCE.m: -------------------------------------------------------------------------------- 1 | function [Ws,ce_lstm,ce_lstm_dev] = LSTM_TrainCE(varargin) 2 | % This function trains a LSTM network with following parameters: 3 | % In: input train date. 4 | % T: tartget train data 5 | % Ws: a structure contatining all LSTM weight matrices. 6 | %% Contact: hamidp@ece.ubc.ca 7 | args = varargin; 8 | nargs= length(args); 9 | random_init = 1; 10 | nBatch = -1; 11 | Bsize = -1; 12 | for i=1:2:nargs 13 | switch args{i}, 14 | case 'input train', In = args{i+1}; 15 | case 'target train', T = args{i+1}; 16 | case 'input validation', In_Dev = args{i+1}; 17 | case 'target validation', T_Dev = args{i+1}; 18 | case 'number of cells', ncell = args{i+1}; 19 | case 'step size', StepSize = args{i+1}; 20 | case 'epoch max', maxEpochLSTM = args{i+1}; 21 | case 'number of mini-batches', nBatch = args{i+1}; 22 | case 'initial weight matrices', Ws = args{i+1};random_init = 0; 23 | case 'trained network path', NetPath = args{i+1}; 24 | case 'mini-batch size', Bsize = args{i+1}; 25 | otherwise 26 | error('The option does not exist'); 27 | end 28 | end 29 | n = size(T,1); 30 | m = size(In,1); 31 | L = size(In,2); % Memory length. 32 | nTr = size(In,3); % Number of training samples at each time step. 33 | nTr_Dev = size(In_Dev,3); 34 | if nBatch ~= -1 35 | Bsize = floor(nTr/nBatch); % mini-batch size. 36 | elseif Bsize ~= -1 37 | nBatch = floor(nTr/Bsize); % Number of mini-batches 38 | end 39 | % Random Initialization: START 40 | if random_init 41 | Ws.W1 = 0.1*randn(ncell,m); 42 | Ws.W3 = 0.1*randn(ncell,m); 43 | Ws.W4 = 0.1*randn(ncell,m); 44 | Ws.Wrec1 = 0.1*randn(ncell); 45 | Ws.Wrec3 = 0.1*randn(ncell); 46 | Ws.Wrec4 = 0.1*randn(ncell); 47 | Ws.U = 0.1*randn(n,ncell); 48 | end 49 | % Random Initialization: END 50 | % Random Initialization: START 51 | diffWs.W1 = zeros(ncell,m); 52 | diffWs.W3 = zeros(ncell,m); 53 | diffWs.W4 = zeros(ncell,m); 54 | diffWs.Wrec1 = zeros(ncell); 55 | diffWs.Wrec3 = zeros(ncell); 56 | diffWs.Wrec4 = zeros(ncell); 57 | diffWs.U = zeros(n,ncell); 58 | % Random Initialization: END 59 | epoch = 0; 60 | nupdate = 0; 61 | disp('LSTM Training: START'); 62 | while epoch < maxEpochLSTM 63 | tic; 64 | % Nesterov method 65 | if ( nupdate < floor(0.1*maxEpochLSTM*nBatch) ) ||... 66 | ( nupdate >= floor(0.9*maxEpochLSTM*nBatch) ) 67 | mu = 0.9; 68 | else 69 | mu = 0.995; 70 | end 71 | err = 0; 72 | for iB=1:nBatch 73 | In1 = In(:,:, (iB-1)*Bsize+1 : iB*Bsize ); 74 | T1 = T(:,:, (iB-1)*Bsize+1 : iB*Bsize ); 75 | % New input 76 | Ws_grad.W1 = Ws.W1 + mu * diffWs.W1; 77 | Ws_grad.Wrec1 = Ws.Wrec1 + mu * diffWs.Wrec1; 78 | Ws_grad.W3 = Ws.W3 + mu * diffWs.W3; 79 | Ws_grad.Wrec3 = Ws.Wrec3 + mu * diffWs.Wrec3; 80 | Ws_grad.W4 = Ws.W4 + mu * diffWs.W4; 81 | Ws_grad.Wrec4 = Ws.Wrec4 + mu * diffWs.Wrec4; 82 | Ws_grad.U = Ws.U + mu * diffWs.U; 83 | % Gradients 84 | gs = LSTM_Grads(In1,T1,ncell,Ws_grad); 85 | % Clipping 86 | gs = LSTM_Grad_Clip(gs,100); % Clipping gradients to prevent gradient explosion. 87 | % Parameter update 88 | diffWs.W1 = mu*diffWs.W1 - StepSize*gs.W1; 89 | diffWs.Wrec1 = mu*diffWs.Wrec1 - StepSize*gs.Wrec1; 90 | diffWs.W3 = mu*diffWs.W3 - StepSize*gs.W3; 91 | diffWs.Wrec3 = mu*diffWs.Wrec3 - StepSize*gs.Wrec3; 92 | diffWs.W4 = mu*diffWs.W4 - StepSize*gs.W4; 93 | diffWs.Wrec4 = mu*diffWs.Wrec4 - StepSize*gs.Wrec4; 94 | diffWs.U = mu*diffWs.U - StepSize*gs.U; 95 | % Recover original parameters 96 | Ws.W1 = Ws.W1 + diffWs.W1; 97 | Ws.Wrec1 = Ws.Wrec1 + diffWs.Wrec1; 98 | Ws.W3 = Ws.W3 + diffWs.W3; 99 | Ws.Wrec3 = Ws.Wrec3 + diffWs.Wrec3; 100 | Ws.W4 = Ws.W4 + diffWs.W4; 101 | Ws.Wrec4 = Ws.Wrec4 + diffWs.Wrec4; 102 | Ws.U = Ws.U + diffWs.U; 103 | % Value of cost function: Training set 104 | % Forward pass: START 105 | y_init = zeros(ncell,Bsize); 106 | c_init = zeros(ncell,Bsize); 107 | for iL=1:L 108 | r = reshape(In1(:,iL,:),[m Bsize]); 109 | s0 = reshape(T1(:,iL,:),[n Bsize]); 110 | if iL==1 111 | Fs(iL,1) = FP_LSTM(r,Ws,y_init,c_init); 112 | else 113 | Fs(iL,1) = FP_LSTM(r,Ws,Fs(iL-1,1).y,Fs(iL-1,1).c); 114 | end 115 | CE = -sum( s0 .* log( Fs(iL,1).s ) , 1 ); 116 | err = err + sum( CE , 2 ); 117 | end 118 | % Forward pass: END 119 | nupdate = nupdate + 1; 120 | end 121 | % Value of cost function: Validation set 122 | % Forward pass: START 123 | err_dev = 0; 124 | y_init = zeros(ncell,nTr_Dev); 125 | c_init = zeros(ncell,nTr_Dev); 126 | for iL=1:L 127 | r = reshape(In_Dev(:,iL,:),[m nTr_Dev]); 128 | s0 = reshape(T_Dev(:,iL,:),[n nTr_Dev]); 129 | if iL==1 130 | Fs(iL,1) = FP_LSTM(r,Ws,y_init,c_init); 131 | else 132 | Fs(iL,1) = FP_LSTM(r,Ws,Fs(iL-1,1).y,Fs(iL-1,1).c); 133 | end 134 | CE = -sum( s0 .* log( Fs(iL,1).s ) , 1 ); 135 | err_dev = err_dev + sum( CE , 2 ); 136 | end 137 | epoch = epoch + 1; 138 | epochTime = toc; 139 | save(strcat([NetPath '_Ws_Epoch' num2str(epoch)]),'Ws'); 140 | ce_lstm(epoch,1) = err; 141 | ce_lstm_dev(epoch,1) = err_dev; 142 | disp(strcat('epoch = ',num2str(epoch),', ce_lstm = ',num2str(err),... 143 | ', ce_lstm_dev = ', num2str(err_dev),', epochTime = ', num2str(floor(epochTime)),' sec.')); 144 | save(strcat([NetPath '_CEvecTr']),'ce_lstm'); 145 | save(strcat([NetPath '_CEvecDev']),'ce_lstm_dev'); 146 | end 147 | disp('LSTM Training: END'); 148 | 149 | 150 | -------------------------------------------------------------------------------- /MNIST_Data/ImsTest.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidpalangi/Distributed-Compressive-Sensing-A-Deep-Learning-Approach/2eedf4c8b87f67156acafe713a912780841d7423/MNIST_Data/ImsTest.mat -------------------------------------------------------------------------------- /MNIST_Data/ImsTr.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidpalangi/Distributed-Compressive-Sensing-A-Deep-Learning-Approach/2eedf4c8b87f67156acafe713a912780841d7423/MNIST_Data/ImsTr.mat -------------------------------------------------------------------------------- /MNIST_Data/ImsVal.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidpalangi/Distributed-Compressive-Sensing-A-Deep-Learning-Approach/2eedf4c8b87f67156acafe713a912780841d7423/MNIST_Data/ImsVal.mat -------------------------------------------------------------------------------- /MNIST_Data/Prepare_MNIST_Data.m: -------------------------------------------------------------------------------- 1 | % This Mfile prepares the MNIST data set to feed into LSTM-CS algorithm. 2 | % It creates 10 channels each one including samples of one of the numbers 3 | % from the set {0,1,2,..,9}. 4 | % Contact: Hamid Palangi, hamidp@ece.ubc.ca 5 | 6 | clear all;close all;clc; 7 | % You need to download following files from: 8 | % "http://yann.lecun.com/exdb/mnist/" and edit following paths accordingly. 9 | DataPathTr = '..\train-images.idx3-ubyte'; 10 | LabelPathTr = '..\train-labels.idx1-ubyte'; 11 | DataPathTest = '..\t10k-images.idx3-ubyte'; 12 | LabelPathTest = '..\t10k-labels.idx1-ubyte'; 13 | Ims1 = loadMNISTImages(DataPathTr); 14 | Labels = loadMNISTLabels(LabelPathTr); 15 | % Reference for above two functions: 16 | % "http://ufldl.stanford.edu/wiki/index.php/Main_Page". 17 | ImsTr = cell(10,1); 18 | for i=0:9 19 | idx = find( Labels == i); 20 | Ims = zeros(28,28,length(idx)); 21 | temp = Ims1(:,idx); 22 | for j=1:length(idx) 23 | Ims(:,:,j) = col2im( temp(:,j), [28 28], [28 28], 'distinct' ); 24 | end 25 | ImsTr{i+1,1} = Ims; 26 | end 27 | clear temp; 28 | clear Ims; 29 | clear idx; 30 | ImsVal = cell(10,1); 31 | for i=1:10 32 | temp = ImsTr{i,1}; 33 | ImsVal{i,1} = temp(:,:,1:100); 34 | temp(:,:,1:100) = []; 35 | ImsTr{i,1} = temp; 36 | end 37 | clear temp; 38 | Ims1 = loadMNISTImages(DataPathTest); 39 | Labels = loadMNISTLabels(LabelPathTest); 40 | ImsTest = cell(10,1); 41 | for i=0:9 42 | idx = find( Labels == i); 43 | Ims = zeros(28,28,length(idx)); 44 | temp = Ims1(:,idx); 45 | for j=1:length(idx) 46 | Ims(:,:,j) = col2im( temp(:,j), [28 28], [28 28], 'distinct' ); 47 | end 48 | ImsTest{i+1,1} = Ims; 49 | end 50 | % Add the path that you want to save the prepared files: 51 | save('..\ImsTr.mat','ImsTr'); 52 | save('..\ImsVal.mat','ImsVal'); 53 | save('..\ImsTest.mat','ImsTest'); 54 | 55 | -------------------------------------------------------------------------------- /MNIST_Data/loadMNISTImages.m: -------------------------------------------------------------------------------- 1 | function images = loadMNISTImages(filename) 2 | %loadMNISTImages returns a 28x28x[number of MNIST images] matrix containing 3 | %the raw MNIST images 4 | 5 | fp = fopen(filename, 'rb'); 6 | assert(fp ~= -1, ['Could not open ', filename, '']); 7 | 8 | magic = fread(fp, 1, 'int32', 0, 'ieee-be'); 9 | assert(magic == 2051, ['Bad magic number in ', filename, '']); 10 | 11 | numImages = fread(fp, 1, 'int32', 0, 'ieee-be'); 12 | numRows = fread(fp, 1, 'int32', 0, 'ieee-be'); 13 | numCols = fread(fp, 1, 'int32', 0, 'ieee-be'); 14 | 15 | images = fread(fp, inf, 'unsigned char'); 16 | images = reshape(images, numCols, numRows, numImages); 17 | images = permute(images,[2 1 3]); 18 | 19 | fclose(fp); 20 | 21 | % Reshape to #pixels x #examples 22 | images = reshape(images, size(images, 1) * size(images, 2), size(images, 3)); 23 | % Convert to double and rescale to [0,1] 24 | images = double(images) / 255; 25 | 26 | end 27 | -------------------------------------------------------------------------------- /MNIST_Data/loadMNISTLabels.m: -------------------------------------------------------------------------------- 1 | function labels = loadMNISTLabels(filename) 2 | %loadMNISTLabels returns a [number of MNIST images]x1 matrix containing 3 | %the labels for the MNIST images 4 | 5 | fp = fopen(filename, 'rb'); 6 | assert(fp ~= -1, ['Could not open ', filename, '']); 7 | 8 | magic = fread(fp, 1, 'int32', 0, 'ieee-be'); 9 | assert(magic == 2049, ['Bad magic number in ', filename, '']); 10 | 11 | numLabels = fread(fp, 1, 'int32', 0, 'ieee-be'); 12 | 13 | labels = fread(fp, inf, 'unsigned char'); 14 | 15 | assert(size(labels,1) == numLabels, 'Mismatch in label count'); 16 | 17 | fclose(fp); 18 | 19 | end 20 | -------------------------------------------------------------------------------- /Main.m: -------------------------------------------------------------------------------- 1 | % Copy right: 2 | % Permission is granted for anyone to copy, use, modify, or distribute 3 | % this program and accompanying programs and documents for any purpose, 4 | % provided this copyright notice is retained and prominently displayed, 5 | % along with a note saying that the original programs are available from 6 | % Hamid Palangi, "hamidp@ece.ubc.ca". 7 | % The programs and documents are distributed without any warranty, 8 | % express or implied. As the programs were written for research purposes only, 9 | % they have not been tested to the degree that would be advisable in any important 10 | % application. All use of these programs is entirely at the user's own risk. 11 | 12 | % This Mfile implements the LSTM-CS method and generates Fig.4 of the 13 | % paper. To refer please use the following: 14 | % @ARTICLE{hp_LSTM_CS, 15 | % author={Hamid Palangi and Rabab Ward and Li Deng}, 16 | % journal={IEEE Transactions on Signal Processing}, 17 | % title={Distributed Compressive Sensing: A Deep Learning Approach}, 18 | % year={2016}, 19 | % volume={64}, 20 | % number={17}, 21 | % pages={4504-4518}, 22 | % month={Sept}, 23 | % } 24 | 25 | % In generating Fig.4, the codes for Bayesian Compressive Sensing from 26 | % following references are used. Please refer to their website for copy 27 | % right terms and conditions. 28 | % [1] Bayesian and Multitask Compressive Sensing: "http://www.ece.duke.edu/~lcarin/bcs_ver0.1.zip" 29 | % [2] Sparse Signal Recovery with Temporally Correlated Source Vectors 30 | % Using Sparse Bayesian Learning: "http://sccn.ucsd.edu/~zhang/TMSBL_code.zip" 31 | 32 | %% Contact: Hamid Palangi, Email: hamidp@ece.ubc.ca 33 | 34 | mkdir('Results') % to save results of experiment 35 | mkdir('Trained_Network') % to save the trained models 36 | 37 | close all;clear all;clc; 38 | rand('state', 1); 39 | randn('state', 2); 40 | addpath('Other_Methods\MT_CS_demo'); 41 | addpath('Other_Methods\BCS_demo'); 42 | addpath('Other_Methods\TMSBL_code'); 43 | % Signal definition and setting: START 44 | Cost_Function = 'CE';% Cross Entropy. 45 | CodeType = strcat(['MMV_MNIST_LSTM_Demo_' Cost_Function]); 46 | trained_net_path = 'Trained_Network\'; 47 | DataPathTr = 'MNIST_Data\ImsTr.mat'; 48 | DataPathVal = 'MNIST_Data\ImsVal.mat'; 49 | DataPathTest = 'MNIST_Data\ImsTest.mat'; 50 | ncell = 512; 51 | NetPath = strcat([trained_net_path CodeType '_ncell_' num2str(ncell)]); 52 | StepSize = 0.0003; 53 | maxEpochLSTM = 25; 54 | maxEpochGeneral = 25; 55 | numhidGeneral = 512; 56 | Idim = 24;% image dimension 57 | Bdim = 12;% image block dimension 58 | n = Bdim*Bdim; % length of the signal 59 | m = floor(n/2); % # of measurements 60 | nsparseTrain = floor(m/4); % # of sparsity levels generated in train data. 61 | nsparse = n; % # of sparsity levels to evaluate performance 62 | nBatch = 300; % # of minibatches for LSTM training. 63 | L = 4; % number of sparse vectors in MMV 64 | nImageTrain = 200; % # of images per channel for training. 65 | nImageVal = 3; % # of images per channel for validation set. 66 | nImageTest = 10; % # of images per channel for test set. 67 | winSize = 1; 68 | input_zero_mean = 0; % if 1 the input (r), is normalized to have zero mean. 69 | input_random_permutation = 1; 70 | NOISE = 1; % NOISE = 0 means that we don't have noise 71 | sigma_noise = 0.005; 72 | Biased = 0; % Biased = 1 means that a network with bias is used 73 | % % Biased = 0 means that a network without bias is used 74 | phi1 = randn(n,n); 75 | phi = orth(phi1); % orthogonalizing the columns 76 | % Randomly select m lines of phi and construct phi_m 77 | sel = randperm(n); 78 | phi_m = phi(sel(1:m),:); 79 | for i1=1:size(phi_m,2) 80 | phi_m(:,i1) = phi_m(:,i1)/norm(phi_m(:,i1)); 81 | end 82 | % Minimum residual to stop solver. 83 | % Without noise 84 | if (NOISE == 0) 85 | res_min = L*(1e-5); 86 | noise = zeros(m,L); 87 | else 88 | % With noise 89 | noise = sigma_noise*randn(m,L); 90 | res_min = L*norm(noise(:)); 91 | end 92 | TransformProps.noise = noise; 93 | % Signal definition and setting: END 94 | % Training and Validation set generation: START 95 | TransformProps.DataType = 'Tr'; 96 | [In,T] = TgenerateISOMP_MNIST_MMV_CE(phi_m,L,DataPathTr,nImageTrain,Idim,nsparseTrain,Bdim,TransformProps); 97 | TransformProps.DataType = 'Val'; 98 | [In_Dev,T_Dev] = TgenerateISOMP_MNIST_MMV_CE(phi_m,L,DataPathVal,nImageVal,Idim,nsparseTrain,Bdim,TransformProps); 99 | % Training and Validation set generation: END 100 | % Random permutation: START 101 | perm_In = randperm( size(In,3) ); 102 | In = In(:,:,perm_In); 103 | T = T(:,:,perm_In); 104 | perm_In = randperm( size(In_Dev,3) ); 105 | In_Dev = In_Dev(:,:,perm_In); 106 | T_Dev = T_Dev(:,:,perm_In); 107 | % Random permutation: END 108 | % LSTM Training: START 109 | [Ws,ce_lstm,ce_lstm_dev] = LSTM_TrainCE('input train',In,'target train',T... 110 | ,'input validation',In_Dev,'target validation',T_Dev,'number of cells',ncell... 111 | ,'step size',StepSize,'epoch max',maxEpochLSTM,'number of mini-batches',nBatch,... 112 | 'trained network path',NetPath);%,'initial weight matrices',Ws_init); 113 | % LSTM Training: END 114 | % Because the cost function for NWSOMP is mean square error not cross 115 | % entropy. 116 | %%%%%%%%%%%%%%%%%%%% 117 | TransformProps.DataType = 'Tr'; 118 | [In,T] = TgenerateISOMP_MNIST_MMV(phi_m,L,DataPathTr,nImageTrain,Idim,nsparseTrain,Bdim,TransformProps); 119 | TransformProps.DataType = 'Val'; 120 | [In_Dev,T_Dev] = TgenerateISOMP_MNIST_MMV(phi_m,L,DataPathVal,nImageVal,Idim,nsparseTrain,Bdim,TransformProps); 121 | % Random permutation: START 122 | perm_In = randperm( size(In,3) ); 123 | In = In(:,:,perm_In); 124 | T = T(:,:,perm_In); 125 | perm_In = randperm( size(In_Dev,3) ); 126 | In_Dev = In_Dev(:,:,perm_In); 127 | T_Dev = T_Dev(:,:,perm_In); 128 | % Random permutation: END 129 | %%%%%%%%%%%%%%%%%%%% 130 | In = reshape(In,[size(In,1) size(In,2)*size(In,3)]); 131 | T = reshape(T,[size(T,1) size(T,2)*size(T,3)]); 132 | In_Dev = reshape(In_Dev,[size(In_Dev,1) size(In_Dev,2)*size(In_Dev,3)]); 133 | T_Dev = reshape(T_Dev,[size(T_Dev,1) size(T_Dev,2)*size(T_Dev,3)]); 134 | if Biased == 1 135 | In = [In;ones(1,size(In,2))]; % input with bias 136 | In_Dev = [In_Dev;ones(1,size(In_Dev,2))]; % input with bias 137 | end 138 | NumberofInputNeurons = size(In,1); 139 | MaxEpoch = maxEpochGeneral; 140 | if Biased == 1 141 | NumberofHiddenNeurons = numhidGeneral; 142 | InputDim = NumberofInputNeurons-1; 143 | Winit = zeros(NumberofHiddenNeurons, InputDim +1); 144 | Winit(:,1:InputDim)=1 - 2*rand(NumberofHiddenNeurons,InputDim); 145 | Winit(:,InputDim+1:InputDim+1) = rand(NumberofHiddenNeurons,1); % initial weights for bias 146 | elseif Biased == 0 147 | NumberofHiddenNeurons = numhidGeneral; 148 | InputDim = NumberofInputNeurons; 149 | Winit = zeros(NumberofHiddenNeurons, InputDim); 150 | Winit(:,1:InputDim)=1 - 2*rand(NumberofHiddenNeurons,InputDim); 151 | end 152 | Reg = 100; 153 | disp('NWSOMP Training: START'); 154 | if Biased == 1 155 | [MSE_Tot, MSE_Tg, MSE_Test,Wnew,Unew] = OneLayerBackprop_new_Biased(.5,.09,StepSize,In,T,numhidGeneral,Reg,MaxEpoch,NumberofInputNeurons,Winit,In_Dev,T_Dev); 156 | elseif Biased == 0 157 | [MSE_Tot, MSE_Tg, MSE_Test,Wnew,Unew] = OneLayerBackprop_new(.5,.09,StepSize,In,T,numhidGeneral,Reg,MaxEpoch,NumberofInputNeurons,Winit,In_Dev,T_Dev); 158 | end 159 | disp('NWSOMP Training: END'); 160 | figure; 161 | subplot(3,1,1); 162 | plot(MSE_Tot);ylabel('MSE Training and Rec');xlabel('epoch'); 163 | subplot(3,1,2); 164 | plot(MSE_Tg);ylabel('MSE Training');xlabel('epoch'); 165 | subplot(3,1,3); 166 | plot(MSE_Test);ylabel('MSE Test Data');xlabel('epoch'); 167 | title(strcat([' MaxEpoch = ',num2str(MaxEpoch),', Reg = ',num2str(Reg),', Hidden units = ',num2str(numhidGeneral)])); 168 | if (NOISE == 0 && Biased == 0) 169 | saveas(gcf,strcat(['Results\MSE_with_Regularization_' num2str(Reg) '_num_' num2str(numhidGeneral) '_StepSize_' num2str(StepSize) '_NoBias_n' num2str(n) CodeType '_MultiChannel_Complete.fig'])); 170 | elseif (NOISE == 0 && Biased == 1) 171 | saveas(gcf,strcat(['Results\MSE_with_Regularization_' num2str(Reg) '_num_' num2str(numhidGeneral) '_StepSize_' num2str(StepSize) '_n' num2str(n) CodeType '_MultiChannel_Complete.fig'])); 172 | elseif (NOISE == 1 && Biased == 0) 173 | saveas(gcf,strcat(['Results\MSE_with_Regularization_' num2str(Reg) '_num_' num2str(numhidGeneral) '_StepSize_' num2str(StepSize) '_Noisy_NoBias_n' num2str(n) CodeType '_MultiChannel_Complete.fig'])); 174 | elseif (NOISE == 1 && Biased == 1) 175 | saveas(gcf,strcat(['Results\MSE_with_Regularization_' num2str(Reg) '_num_' num2str(numhidGeneral) '_StepSize_' num2str(StepSize) '_Noisy_n' num2str(n) CodeType '_MultiChannel_Complete.fig'])); 176 | end 177 | close all; 178 | % Performance Evaluation of Different Solvers: START 179 | disp('Performance Evaluation: START'); 180 | disp('"K" is the number of non-zero entries in each sparse vector.'); 181 | k_vec = 1:nsparse; 182 | k_vec = k_vec'; 183 | errNISOMP = zeros(length(k_vec),1); 184 | errSOMP = zeros(length(k_vec),1); 185 | errBCS = zeros(length(k_vec),1); 186 | errMTBCS = zeros(length(k_vec),1); 187 | errRao = zeros(length(k_vec),1); 188 | errLSTM = zeros(length(k_vec),1); 189 | 190 | B1=cell(nImageTest,L); 191 | load(DataPathTest); 192 | for iL=1:L 193 | Im1 = ImsTest{iL,1}; 194 | for i33=1:nImageTest % number of test images per channel 195 | Im=Im1(:,:,i33); 196 | Im=imresize(Im,[Idim,Idim]); 197 | B0 = im2col(Im,[Bdim Bdim],'distinct'); 198 | B1{i33,iL} = B0; 199 | end 200 | end 201 | nBlockPerImage = Idim*Idim / (Bdim*Bdim); 202 | nBlockPerChannel = nBlockPerImage * nImageTest; 203 | cell_SOMP = cell(L,1); 204 | cell_Y = cell(L,1); 205 | cell_NISOMP = cell(L,1); 206 | cell_BCS = {L,1}; 207 | cell_MTBCS = cell(L,1); 208 | cell_Rao = cell(L,1); 209 | cell_LSTM = cell(L,1); 210 | B0_SOMP = zeros(size(B0)); 211 | B0_Y = zeros(m,L); 212 | B0_NISOMP = zeros(size(B0)); 213 | B0_BCS = zeros(size(B0)); 214 | B0_MTBCS = zeros(size(B0)); 215 | B0_Rao = zeros(size(B0)); 216 | B0_LSTM = zeros(size(B0)); 217 | for iL=1:L 218 | cell_SOMP{iL,1} = B0_SOMP; 219 | cell_Y{iL,1} = B0_Y; 220 | cell_NISOMP{iL,1} = B0_NISOMP; 221 | cell_BCS{iL,1} = B0_BCS; 222 | cell_MTBCS{iL,1} = B0_MTBCS; 223 | cell_Rao{iL,1} = B0_Rao; 224 | cell_LSTM{iL,1} = B0_LSTM; 225 | end 226 | for i2=1:length(k_vec) 227 | errSOMPavg = 0; 228 | errNISOMPavg = 0; 229 | errBCSavg = 0; 230 | errMTBCSavg = 0; 231 | errRaoavg = 0; 232 | errLSTMavg = 0; 233 | k = k_vec(i2,1); % sparsity of signal 234 | v=[]; 235 | vY = []; 236 | vSOMP = []; 237 | vNISOMP = []; 238 | vBCS = []; 239 | vMTBCS = []; 240 | vRao = []; 241 | vLSTM = []; 242 | for i33=1:nImageTest 243 | snr_val = 0; 244 | for iB0 = 1 : nBlockPerImage % for the all number of blocks in image 245 | S0 = zeros(n,L); 246 | for iL=1:L 247 | temp1 = B1{i33,iL}; 248 | S0(:,iL) = mnist_fun(temp1( : , iB0 ),n,k); % amplitudes of sparse signal 249 | end 250 | if (NOISE == 0) 251 | % without noise 252 | Y = phi_m*S0; 253 | else 254 | % with noise 255 | Y = phi_m*S0; 256 | Y = Y + noise; 257 | end 258 | % output of NISOMP 259 | S_NISOMP = NISOMPnew(phi_m,Y,res_min,Wnew,Unew,numhidGeneral); 260 | % output of SOMP 261 | s_SOMP = SOMP(phi_m,Y,res_min); 262 | % LSTM-CS 263 | S_LSTM = ISOMP_LSTM(phi_m,Y,res_min,Ws,winSize,input_zero_mean); 264 | % BCS 265 | s_BCS = zeros(n,L); 266 | for iL=1:L 267 | y1 = Y(:,iL); 268 | initsigma2 = std(y1(:))^2/1e2; 269 | [weights,used,sigma2,errbars] = BCS_fast_rvm(phi_m,y1,initsigma2,1e-8); 270 | s_BCS1 = zeros(n,1); 271 | s_BCS1(used) = weights; 272 | s_BCS(:,iL) = s_BCS1; 273 | end 274 | % MT_BCS % Multitask BCS 275 | a = 1e2/0.1; b = 1; 276 | for iMT=1:L 277 | phiMT{1,iMT} = phi_m; 278 | yMT{1,iMT} = Y(:,iMT); 279 | end 280 | S_MT = mt_CS(phiMT,yMT,a,b,1e-8); 281 | % T-SBL: Sparse Baysian learning 282 | S_Rao = TSBL(phi_m, Y); 283 | for iL=1:L 284 | rec_imblock = reshape(Y(:,iL),[Bdim/2 Bdim]); 285 | temp = cell_Y{iL,1}; 286 | temp(:,iB0) = rec_imblock(:); 287 | cell_Y{iL,1} = temp; 288 | 289 | rec_imblock = reshape(s_SOMP(:,iL),[Bdim Bdim]); 290 | temp = cell_SOMP{iL,1}; 291 | temp(:,iB0) = rec_imblock(:); 292 | cell_SOMP{iL,1} = temp; 293 | 294 | rec_imblock = reshape(S_NISOMP(:,iL),[Bdim Bdim]); 295 | temp = cell_NISOMP{iL,1}; 296 | temp(:,iB0) = rec_imblock(:); 297 | cell_NISOMP{iL,1} = temp; 298 | 299 | rec_imblock = reshape(s_BCS(:,iL),[Bdim Bdim]); 300 | temp = cell_BCS{iL,1}; 301 | temp(:,iB0) = rec_imblock(:); 302 | cell_BCS{iL,1} = temp; 303 | 304 | rec_imblock = reshape(S_MT(:,iL),[Bdim Bdim]); 305 | temp = cell_MTBCS{iL,1}; 306 | temp(:,iB0) = rec_imblock(:); 307 | cell_MTBCS{iL,1} = temp; 308 | 309 | rec_imblock = reshape(S_Rao(:,iL),[Bdim Bdim]); 310 | temp = cell_Rao{iL,1}; 311 | temp(:,iB0) = rec_imblock(:); 312 | cell_Rao{iL,1} = temp; 313 | 314 | rec_imblock = reshape(S_LSTM(:,iL),[Bdim Bdim]); 315 | temp = cell_LSTM{iL,1}; 316 | temp(:,iB0) = rec_imblock(:); 317 | cell_LSTM{iL,1} = temp; 318 | end 319 | end 320 | errSOMPavg2 = 0; 321 | errNISOMPavg2 = 0; 322 | errBCSavg2 = 0; 323 | errMTBCSavg2 = 0; 324 | errRaoavg2 = 0; 325 | errLSTMavg2 = 0; 326 | for iL=1:L 327 | Im = col2im(B1{i33,iL},[Bdim Bdim],[Idim Idim],'distinct'); 328 | Y = col2im(cell_Y{iL,1},[Bdim/2 Bdim],[Idim/2 Idim],'distinct'); 329 | Im_recSOMP = col2im(cell_SOMP{iL,1},[Bdim Bdim],[Idim Idim],'distinct'); 330 | Im_recNISOMP = col2im(cell_NISOMP{iL,1},[Bdim Bdim],[Idim Idim],'distinct'); 331 | Im_recBCS = col2im(cell_BCS{iL,1},[Bdim Bdim],[Idim Idim],'distinct'); 332 | Im_recMTBCS = col2im(cell_MTBCS{iL,1},[Bdim Bdim],[Idim Idim],'distinct'); 333 | Im_recRao = col2im(cell_Rao{iL,1},[Bdim Bdim],[Idim Idim],'distinct'); 334 | Im_recLSTM = col2im(cell_LSTM{iL,1},[Bdim Bdim],[Idim Idim],'distinct'); 335 | 336 | v=cat(3,v,Im); 337 | vY = cat(3,vY,Y); 338 | vSOMP = cat(3,vSOMP,Im_recSOMP); 339 | vNISOMP = cat(3,vNISOMP,Im_recNISOMP); 340 | vBCS = cat(3,vBCS,Im_recBCS); 341 | vMTBCS = cat(3,vMTBCS,Im_recMTBCS); 342 | vRao = cat(3,vRao,Im_recRao); 343 | vLSTM = cat(3,vLSTM,Im_recLSTM); 344 | 345 | errSOMP1=sqrt(sum(sum((Im_recSOMP-Im).^2,1),2)/sum(sum(Im.^2,1),2)); 346 | errNISOMP1=sqrt(sum(sum((Im_recNISOMP-Im).^2,1),2)/sum(sum(Im.^2,1),2)); 347 | errBCS1=sqrt(sum(sum((Im_recBCS-Im).^2,1),2)/sum(sum(Im.^2,1),2)); 348 | errMTBCS1=sqrt(sum(sum((Im_recMTBCS-Im).^2,1),2)/sum(sum(Im.^2,1),2)); 349 | errRao1=sqrt(sum(sum((Im_recRao-Im).^2,1),2)/sum(sum(Im.^2,1),2)); 350 | errLSTM1=sqrt(sum(sum((Im_recLSTM-Im).^2,1),2)/sum(sum(Im.^2,1),2)); 351 | 352 | errSOMPavg = errSOMPavg + errSOMP1; 353 | errNISOMPavg = errNISOMPavg + errNISOMP1; 354 | errBCSavg = errBCSavg + errBCS1; 355 | errMTBCSavg = errMTBCSavg + errMTBCS1; 356 | errRaoavg = errRaoavg + errRao1; 357 | errLSTMavg = errLSTMavg + errLSTM1; 358 | 359 | errSOMPavg2 = errSOMPavg2 + errSOMP1; 360 | errNISOMPavg2 = errNISOMPavg2 + errNISOMP1; 361 | errBCSavg2 = errBCSavg2 + errBCS1; 362 | errMTBCSavg2 = errMTBCSavg2 + errMTBCS1; 363 | errRaoavg2 = errRaoavg2 + errRao1; 364 | errLSTMavg2 = errLSTMavg2 + errLSTM1; 365 | end 366 | disp( strcat('K = ',num2str(k),', Realization = ',num2str(i33),', MSE of LSTM-CS method = ',num2str(errLSTMavg2/L) )); 367 | end 368 | 369 | errNISOMP(i2,1) = errNISOMPavg / (nImageTest*L); 370 | errSOMP(i2,1) = errSOMPavg / (nImageTest*L); 371 | errBCS(i2,1) = errBCSavg / nImageTest; 372 | errMTBCS(i2,1) = errMTBCSavg / (nImageTest*L); 373 | errRao(i2,1) = errRaoavg / (nImageTest*L); 374 | errLSTM(i2,1) = errLSTMavg / (nImageTest*L); 375 | 376 | end 377 | 378 | % Result representation starts***************************************** 379 | 380 | figure; 381 | plot(k_vec,errSOMP,':rs','LineWidth',2,'MarkerEdgeColor','k','MarkerFaceColor','g','MarkerSize',5); axis('tight'); 382 | xlabel('Sparsity');ylabel('Mean of Error'); 383 | hold on; 384 | plot(k_vec,errNISOMP,':gs','LineWidth',2,'MarkerEdgeColor','k','MarkerFaceColor','b','MarkerSize',5); axis('tight'); 385 | plot(k_vec,errBCS,':rs','LineWidth',2,'MarkerEdgeColor','g','MarkerFaceColor','b','MarkerSize',5); axis('tight'); 386 | plot(k_vec,errMTBCS,':gs','LineWidth',2,'MarkerEdgeColor','g','MarkerFaceColor','g','MarkerSize',5); axis('tight'); 387 | plot(k_vec,errRao,':ks','LineWidth',2,'MarkerEdgeColor','k','MarkerFaceColor','k','MarkerSize',5); axis('tight'); 388 | plot(k_vec,errLSTM,':cs','LineWidth',2,'MarkerEdgeColor','c','MarkerFaceColor','c','MarkerSize',5); axis('tight'); 389 | legend('SOMP','NWSOMP','BCS','MT-BCS','T-SBL','LSTM-CS',3); 390 | if (NOISE == 0 && Biased == 0) 391 | title(strcat([' MaxEpoch = ',num2str(maxEpochGeneral),', Reg = ',num2str(Reg),', Hidden units = ',num2str(numhidGeneral)])); 392 | saveas(gcf,strcat(['Results\Solver_with_Regularization_' num2str(Reg) '_num_' num2str(numhidGeneral) '_nreal_' num2str(nImageTest) '_StepSize_' num2str(StepSize) '_NoBias_n' num2str(n) CodeType '_Error_MultiChannel_Complete.fig'])); 393 | elseif (NOISE == 0 && Biased == 1) 394 | title(strcat([' MaxEpoch = ',num2str(maxEpochGeneral),', Reg = ',num2str(Reg),', Hidden units = ',num2str(numhidGeneral)])); 395 | saveas(gcf,strcat(['Results\Solver_with_Regularization_' num2str(Reg) '_num_' num2str(numhidGeneral) '_nreal_' num2str(nImageTest) '_StepSize_' num2str(StepSize) '_n' num2str(n) CodeType '_Error_MultiChannel_Complete.fig'])); 396 | elseif (NOISE == 1 && Biased == 1) 397 | title(strcat([' MaxEpoch = ',num2str(maxEpochGeneral),', Reg = ',num2str(Reg),', Hidden units = ',num2str(numhidGeneral),', Noisy'])); 398 | saveas(gcf,strcat(['Results\Solver_with_Regularization_' num2str(Reg) '_num_' num2str(numhidGeneral) '_nreal_' num2str(nImageTest) '_StepSize_' num2str(StepSize) '_Noisy_n' num2str(n) CodeType '_Error_MultiChannel_Complete.fig'])); 399 | elseif (NOISE == 1 && Biased == 0) 400 | title(strcat([' MaxEpoch = ',num2str(maxEpochGeneral),', Reg = ',num2str(Reg),', Hidden units = ',num2str(numhidGeneral),', Noisy'])); 401 | saveas(gcf,strcat(['Results\Solver_with_Regularization_' num2str(Reg) '_num_' num2str(numhidGeneral) '_nreal_' num2str(nImageTest) '_StepSize_' num2str(StepSize) '_Noisy_NoBias_n' num2str(n) CodeType '_Error_MultiChannel_Complete.fig'])); 402 | end 403 | disp('Performance Evaluation: END'); -------------------------------------------------------------------------------- /NISOMPnew.m: -------------------------------------------------------------------------------- 1 | function S_OMP = NISOMPnew(phi_m,Y,res_min,Wnew,Unew,num) 2 | 3 | 4 | %% Contact: Hamid Palangi, Email: hamidp@ece.ubc.ca 5 | % num: number of hidden units 6 | % Non-Linear Weighted Simaltaneous Orthogonal Matching Pursuit (NISOMP) 7 | 8 | m = size(phi_m,1); 9 | n = size(phi_m,2); 10 | L = size(Y,2); 11 | iter_max = L*m;done=0; 12 | 13 | j = 0; R = Y; S_OMP = zeros(n,L); 14 | I = []; 15 | while done == 0 16 | j = j+1; 17 | for i5=1:size(R,2) 18 | R(:,i5) = R(:,i5)/max( abs( R(:,i5) ) ); 19 | end 20 | H_test = ones(num, size(R,2)); 21 | H_test(1:num,:) = 1 ./ (1 + exp(-Wnew*R)); 22 | S_hatTemp=(H_test' * Unew)'; 23 | S_hat = S_hatTemp; 24 | row_norms = zeros(n,1); 25 | for inorm=1:size(S_hat,1) 26 | row_norms(inorm,1) = norm( S_hat(inorm,:) ); 27 | end 28 | [val1,idx1] = max( row_norms ); 29 | if length( find( I == idx1 ) ) == 0 30 | I = [I;idx1]; 31 | phiT = phi_m(:,I); 32 | S_temp = lscov(phiT,Y); % Least squares 33 | R = Y-phiT*S_temp; 34 | normR = 0; 35 | for iR=1:size(R,2) 36 | normR = normR + norm(R(:,iR)); 37 | end 38 | if (j > iter_max) || (normR < res_min) 39 | done = 1; 40 | end 41 | else 42 | done = 1; 43 | end 44 | if done == 1 45 | for iSOMP=1:size(Y,2) 46 | S_OMP(:,iSOMP) = zeros(n,1); 47 | S_OMP(I,iSOMP) = S_temp(:,iSOMP); 48 | end 49 | end 50 | end 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /OneLayerBackprop_new.m: -------------------------------------------------------------------------------- 1 | function [MSE_Tot, MSE_Tg, MSE_Test, InputWeight,OutputWeight] = OneLayerBackprop_new(s_ratio_per_7,gamma,stepSize,Data_Train,Target_Train,NumberofHiddenNeurons,C,maxEpoch,NumberofInputNeurons,Winit,Data_Test,Target_Test) 2 | 3 | % Dong Yu, Li Deng, "Efficient and effective algorithms for training 4 | % single-hidden-layer neural networks", Pattern Recognition Letters, 5 | % Volume 33, Issue 5, 1 April 2012, Pages 554-558. 6 | 7 | 8 | % I have just modified error calculation part to have the same criterion 9 | % for comparison. I have also added one initial random weight as input to 10 | % this function to initial weights of both functions at the same point. 11 | % Hamid Palangi, March 28. 2012, Email: hamidp@ece.ubc.ca 12 | % July 28, 2012 13 | 14 | 15 | % Data_Train = [Data_Train;ones(1,size(Data_Train,2))]; 16 | 17 | global NumberofTrainingData NumberofTestingData 18 | 19 | NumberofTrainingData = size(Data_Train,2); 20 | NumberofTestingData = size(Data_Test,2); 21 | InputDim = NumberofInputNeurons; 22 | 23 | InputWeight = Winit; 24 | 25 | 26 | %% 27 | 28 | 29 | 30 | H = ones(NumberofHiddenNeurons, NumberofTrainingData); 31 | 32 | IWT=InputWeight; 33 | 34 | t =1.0; told=t; 35 | 36 | %%%%%%Adding backfitting itereations; once for maxEpoch=1 %%%%%%%%%% 37 | 38 | fid1 = fopen(strcat(['Results\accuracy_Regularization_' num2str(C) '_num_' num2str(NumberofHiddenNeurons) '_StepSize_' num2str(stepSize) '_NoBias.txt']), 'w'); 39 | 40 | for epoch=1:maxEpoch 41 | 42 | 43 | 44 | % if (mod(epoch,6) ==0 ) 45 | % 46 | % stepSize= stepSize * s_ratio_per_7; 47 | % 48 | % end 49 | 50 | % **************************************************** 51 | 52 | H(1:NumberofHiddenNeurons, :) = 1 ./ (1 + exp(-InputWeight*Data_Train)); %always use sigmoid 53 | 54 | %%%%%%%%%%% Calculate output weights OutputWeight (beta_i)by pseudo-inverse 55 | 56 | OutputWeight=(eye(size(H,1))/C + H * H') \ H * Target_Train'; % faster implementation 57 | 58 | 59 | 60 | %figure ; mesh(OutputWeight) 61 | 62 | 63 | 64 | Output_Train =(H' * OutputWeight)'; 65 | 66 | 67 | 68 | MSE_Tg(epoch,1)=sqrt(mse(Target_Train - Output_Train)); % Calculate training accuracy (RMSE) for regression case 69 | 70 | 71 | 72 | % do not apply softmax to visible data at the output 73 | 74 | OutputWeight_Reconstruct =(eye(size(H,1))/C + H * H') \ H * Data_Train'; 75 | 76 | Output_Train_Reconstruct =(H' * OutputWeight_Reconstruct)'; 77 | 78 | MSE_Re(epoch,1)=sqrt(mse(Data_Train - Output_Train_Reconstruct)); 79 | 80 | MSE_Tot(epoch,1) = MSE_Tg(epoch,1) + gamma*MSE_Re(epoch,1); 81 | 82 | 83 | 84 | 85 | 86 | %%%%%%%%%%% Calculate the testing accuracy 87 | 88 | H_test = ones(NumberofHiddenNeurons, NumberofTestingData); 89 | 90 | H_test(1:NumberofHiddenNeurons,:) = 1 ./ (1 + exp(-InputWeight*Data_Test)); 91 | 92 | 93 | 94 | 95 | 96 | Output_Test=(H_test' * OutputWeight)'; 97 | 98 | MSE_Test(epoch,1)=sqrt(mse(Target_Test - Output_Test)); 99 | % ************************************************************* 100 | if epoch==1 101 | disp(strcat(['Epoch = 0' ', mse_tr = ' num2str(MSE_Tg(epoch,1))... 102 | ', mse_dev = ' num2str(MSE_Test(epoch,1)) ', initial mse, no training.'])); 103 | end 104 | 105 | 106 | if maxEpoch > 0, 107 | 108 | 109 | fprintf(fid1, '%f %f\n',MSE_Tg(epoch,1),MSE_Test(epoch,1)); 110 | 111 | 112 | 113 | 114 | %backward (replace the old long code for backfitting with the new one: 115 | 116 | DH=H .* (1-H); 117 | 118 | PInvH = H' / (eye(size(H,1))/C+H*H'); 119 | 120 | TPInvH=Target_Train*PInvH; 121 | 122 | DW1=((PInvH * (H*Target_Train') * TPInvH - Target_Train'*TPInvH)' .* DH) * Data_Train'; 123 | 124 | DW1 = DW1(1:NumberofHiddenNeurons,:); 125 | norm_grad = norm(DW1); 126 | disp(strcat(['Epoch = ' num2str(epoch) ', mse_tr = ' num2str(MSE_Tg(epoch,1))... 127 | ', mse_dev = ' num2str(MSE_Test(epoch,1)) ',norm_grad = ' num2str(norm_grad) '.'])); 128 | 129 | % NEW: add gradient DW2 for the reconstruction MSE term 130 | 131 | if gamma ~= 0 132 | 133 | TPInvH2=Data_Train*PInvH; 134 | 135 | DW2=((PInvH * (H*Data_Train') * TPInvH2 - Data_Train'*TPInvH2)' .* DH) * Data_Train'; 136 | 137 | DW2 = DW2(1:NumberofHiddenNeurons,:); 138 | 139 | end 140 | 141 | %%%%%%%%%%%%%%%%%%%% 142 | 143 | IWOld = InputWeight; 144 | 145 | 146 | 147 | if gamma ~= 0 148 | 149 | InputWeight = IWT - stepSize * (DW1 + gamma*DW2); 150 | 151 | else 152 | 153 | InputWeight = IWT - stepSize * DW1 ; 154 | 155 | end 156 | 157 | 158 | 159 | ttemp = t; 160 | 161 | t = (1+sqrt(1+4*t*t))/2.0; 162 | 163 | IWT = InputWeight + told/t*(InputWeight-IWOld); %IWT = InputWeight + (told-1)/t*(InputWeight-IWOld); 164 | 165 | told=ttemp; 166 | 167 | IWOld = IWT; 168 | 169 | 170 | 171 | end 172 | 173 | 174 | 175 | 176 | 177 | end 178 | 179 | 180 | clear H; 181 | fclose(fid1); 182 | 183 | 184 | 185 | 186 | end -------------------------------------------------------------------------------- /OneLayerBackprop_new_Biased.m: -------------------------------------------------------------------------------- 1 | function [MSE_Tot, MSE_Tg, MSE_Test, InputWeight,OutputWeight] = OneLayerBackprop_new_Biased(s_ratio_per_7,gamma,stepSize,Data_Train,Target_Train,NumberofHiddenNeurons,C,maxEpoch,NumberofInputNeurons,Winit,Data_Test,Target_Test) 2 | 3 | % Dong Yu, Li Deng, "Efficient and effective algorithms for training 4 | % single-hidden-layer neural networks", Pattern Recognition Letters, 5 | % Volume 33, Issue 5, 1 April 2012, Pages 554-558. 6 | 7 | 8 | % I have just modified error calculation part to have the same criterion 9 | % for comparison. I have also added one initial random weight as input to 10 | % this function to initial weights of both functions at the same point. 11 | % Hamid Palangi, March 28. 2012, Email: hamidp@ece.ubc.ca 12 | % July 28, 2012 13 | 14 | 15 | % Data_Train = [Data_Train;ones(1,size(Data_Train,2))]; 16 | 17 | global NumberofTrainingData NumberofTestingData 18 | 19 | NumberofTrainingData = size(Data_Train,2); 20 | NumberofTestingData = size(Data_Test,2); 21 | InputDim = NumberofInputNeurons-1; 22 | 23 | InputWeight = Winit; 24 | %% 25 | 26 | 27 | 28 | H = ones(NumberofHiddenNeurons+1, NumberofTrainingData); 29 | 30 | % H = ones(NumberofHiddenNeurons, NumberofTrainingData); % removing bias of hidden layer, August 01,2012 31 | 32 | IWT=InputWeight; 33 | 34 | t =1.0; told=t; 35 | 36 | %%%%%%Adding backfitting itereations; once for maxEpoch=1 %%%%%%%%%% 37 | 38 | fid1 = fopen(strcat(['Results\accuracy_Regularization_' num2str(C) '_num_' num2str(NumberofHiddenNeurons) '_StepSize_' num2str(stepSize) '.txt']), 'w'); 39 | 40 | for epoch=1:maxEpoch 41 | 42 | 43 | % if (mod(epoch,6) ==0 ) 44 | % 45 | % stepSize= stepSize * s_ratio_per_7; 46 | % 47 | % end 48 | 49 | 50 | 51 | % **************************************************** 52 | 53 | H(1:NumberofHiddenNeurons, :) = 1 ./ (1 + exp(-InputWeight*Data_Train)); %always use sigmoid 54 | 55 | %%%%%%%%%%% Calculate output weights OutputWeight (beta_i)by pseudo-inverse 56 | 57 | OutputWeight=(eye(size(H,1))/C + H * H') \ H * Target_Train'; % faster implementation 58 | 59 | 60 | 61 | %figure ; mesh(OutputWeight) 62 | 63 | 64 | 65 | Output_Train =(H' * OutputWeight)'; 66 | 67 | 68 | 69 | MSE_Tg(epoch,1)=sqrt(mse(Target_Train - Output_Train)); % Calculate training accuracy (RMSE) for regression case 70 | 71 | 72 | 73 | % do not apply softmax to visible data at the output 74 | 75 | OutputWeight_Reconstruct =(eye(size(H,1))/C + H * H') \ H * Data_Train'; 76 | 77 | 78 | Output_Train_Reconstruct =(H' * OutputWeight_Reconstruct)'; 79 | 80 | MSE_Re(epoch,1)=sqrt(mse(Data_Train - Output_Train_Reconstruct)); 81 | 82 | MSE_Tot(epoch,1) = MSE_Tg(epoch,1) + gamma*MSE_Re(epoch,1); 83 | 84 | 85 | 86 | 87 | 88 | %%%%%%%%%%% Calculate the testing accuracy 89 | 90 | H_test = ones(NumberofHiddenNeurons+1, NumberofTestingData); 91 | 92 | % H_test = ones(NumberofHiddenNeurons, NumberofTestingData); % removing bias of hidden layer, August 01,2012 93 | 94 | H_test(1:NumberofHiddenNeurons,:) = 1 ./ (1 + exp(-InputWeight*Data_Test)); 95 | 96 | 97 | 98 | 99 | 100 | Output_Test=(H_test' * OutputWeight)'; 101 | 102 | MSE_Test(epoch,1)=sqrt(mse(Target_Test - Output_Test)); 103 | % ************************************************************* 104 | if epoch==1 105 | disp(strcat(['Epoch = 0' ', mse_tr = ' num2str(MSE_Tg(epoch,1))... 106 | ', mse_dev = ' num2str(MSE_Test(epoch,1)) ', initial mse, no training.'])); 107 | end 108 | 109 | 110 | if maxEpoch > 0, 111 | 112 | 113 | fprintf(fid1, '%f %f\n',MSE_Tg(epoch,1),MSE_Test(epoch,1)); 114 | 115 | 116 | 117 | 118 | %backward (replace the old long code for backfitting with the new one: 119 | 120 | DH=H .* (1-H); 121 | 122 | PInvH = H' / (eye(size(H,1))/C+H*H'); 123 | 124 | TPInvH=Target_Train*PInvH; 125 | 126 | DW1=((PInvH * (H*Target_Train') * TPInvH - Target_Train'*TPInvH)' .* DH) * Data_Train'; 127 | 128 | DW1 = DW1(1:NumberofHiddenNeurons,:); 129 | 130 | norm_grad = norm(DW1); 131 | disp(strcat(['Epoch = ' num2str(epoch) ', mse_tr = ' num2str(MSE_Tg(epoch,1))... 132 | ', mse_dev = ' num2str(MSE_Test(epoch,1)) ',norm_grad = ' num2str(norm_grad) '.'])); 133 | 134 | % NEW: add gradient DW2 for the reconstruction MSE term 135 | 136 | if gamma ~= 0 137 | 138 | TPInvH2=Data_Train*PInvH; 139 | 140 | DW2=((PInvH * (H*Data_Train') * TPInvH2 - Data_Train'*TPInvH2)' .* DH) * Data_Train'; 141 | 142 | DW2 = DW2(1:NumberofHiddenNeurons,:); 143 | 144 | end 145 | 146 | %%%%%%%%%%%%%%%%%%%% 147 | 148 | IWOld = InputWeight; 149 | 150 | 151 | 152 | if gamma ~= 0 153 | 154 | InputWeight = IWT - stepSize * (DW1 + gamma*DW2); 155 | 156 | else 157 | 158 | InputWeight = IWT - stepSize * DW1 ; 159 | 160 | end 161 | 162 | 163 | 164 | ttemp = t; 165 | 166 | t = (1+sqrt(1+4*t*t))/2.0; 167 | 168 | IWT = InputWeight + told/t*(InputWeight-IWOld); %IWT = InputWeight + (told-1)/t*(InputWeight-IWOld); 169 | 170 | told=ttemp; 171 | 172 | IWOld = IWT; 173 | 174 | 175 | 176 | end 177 | 178 | 179 | 180 | 181 | 182 | end 183 | 184 | 185 | clear H; 186 | fclose(fid1); 187 | 188 | 189 | 190 | 191 | end -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/BCS_fast_rvm.m: -------------------------------------------------------------------------------- 1 | function [weights,used,sigma2,errbars,basis] = BCS_fast_rvm(PHI,t,sigma2,eta,adaptive,optimal,scale) 2 | %------------------------------------------------------------------ 3 | % The BCS algorithm for the following paper: 4 | % "Bayesian Compressive Sesning" (Preprint, 2007). The algorithm 5 | % adopts from the fast RVM algorithm [Tipping & Faul, 2003]. 6 | % Coded by: Shihao Ji, ECE, Duke University 7 | % last change: Jan. 2, 2007 8 | % You are suggested to use mt_CS.m for improved robustness 9 | %------------------------------------------------------------------ 10 | % Input for BCS: 11 | % PHI: projection matrix 12 | % t: CS measurements 13 | % sigma2: initial noise variance 14 | % If measurement noise exists and/or w is not truely sparse, 15 | % then sigma2 = std(t)^2/1e2 (suggested) 16 | % If no measurement noise and w is truely sparse, 17 | % then sigma2 = std(t)^2/1e6 (suggested) 18 | % This term is in fact not updated in the implementation to allow 19 | % the fast algorithm. For this reason, you are recommended to use 20 | % mt_CS.m, in which the noise variance is marginalized. 21 | % eta: threshold for stopping the algorithm (suggested value: 1e-8) 22 | % Input for Adaptive CS: 23 | % adaptive: generate basis for adpative CS? (default: 0) 24 | % optimal: use the rigorous implementation of adaptive CS? (default: 1) 25 | % scale: diagonal loading parameter (default: 0.1) 26 | % Output: 27 | % weights: sparse weights 28 | % used: the positions of sparse weights 29 | % sigma2: re-estimated noise variance 30 | % errbars: one standard deviation around the sparse weights 31 | % basis: if adaptive==1, then basis = the next projection vector 32 | % 33 | if nargin < 5 34 | adaptive = 0; 35 | end 36 | if nargin < 6 37 | optimal = 1; 38 | end 39 | if nargin < 7 40 | scale = 0.1; 41 | end 42 | 43 | % find initial alpha 44 | [N,M] = size(PHI); 45 | PHIt = PHI'*t; 46 | PHI2 = sum(PHI.^2)'; 47 | ratio = (PHIt.^2)./PHI2; 48 | [maxr,index] = max(ratio); 49 | alpha = PHI2(index)/(maxr-sigma2); 50 | % compute initial mu, Sig, S, Q 51 | phi = PHI(:,index); 52 | Hessian = alpha + phi'*phi/sigma2; 53 | Sig = 1/Hessian; 54 | mu = Sig*PHIt(index)/sigma2; 55 | left = PHI'*phi/sigma2; 56 | S = PHI2/sigma2-Sig*left.^2; 57 | Q = PHIt/sigma2-Sig*PHIt(index)/sigma2*left; 58 | % 59 | for count = 1:10000 60 | 61 | s = S; q = Q; 62 | s(index) = alpha.*S(index)./(alpha-S(index)); 63 | q(index) = alpha.*Q(index)./(alpha-S(index)); 64 | theta = q.^2-s; 65 | 66 | % choice the next alpha that maximizes marginal likelihood 67 | ml = -inf*ones(1,M); 68 | ig0 = find(theta>0); 69 | % index for re-estimate 70 | [ire,foo,which] = intersect(ig0,index); 71 | if ~isempty(ire) 72 | Alpha = s(ire).^2./theta(ire); 73 | delta = (alpha(which)-Alpha)./(Alpha.*alpha(which)); 74 | ml(ire) = Q(ire).^2.*delta./(S(ire).*delta+1)-log(1+S(ire).*delta); 75 | end 76 | % index for adding 77 | iad = setdiff(ig0,ire); 78 | if ~isempty(iad) 79 | ml(iad) = (Q(iad).^2-S(iad))./S(iad)+log(S(iad)./(Q(iad).^2)); 80 | end 81 | is0 = setdiff([1:M],ig0); 82 | % index for deleting 83 | [ide,foo,which] = intersect(is0,index); 84 | if ~isempty(ide) 85 | ml(ide) = Q(ide).^2./(S(ide)-alpha(which))-log(1-S(ide)./alpha(which)); 86 | end 87 | 88 | [ML(count),idx] = max(ml); 89 | % check if terminates? 90 | if count > 2 & abs(ML(count)-ML(count-1)) < abs(ML(count)-ML(1))*eta 91 | break; 92 | end 93 | 94 | % update alphas 95 | which = find(index==idx); 96 | if theta(idx) > 0 97 | if ~isempty(which) % re-estimate 98 | Alpha = s(idx)^2/theta(idx); 99 | Sigii = Sig(which,which); mui = mu(which); Sigi = Sig(:,which); 100 | delta = Alpha-alpha(which); 101 | ki = delta/(1+Sigii*delta); 102 | mu = mu-ki*mui*Sigi; 103 | Sig = Sig-ki*Sigi*Sigi'; 104 | comm = PHI'*(phi*Sigi)/sigma2; 105 | S = S + ki*comm.^2; 106 | Q = Q + ki*mui*comm; 107 | % 108 | alpha(which) = Alpha; 109 | else % adding 110 | Alpha = s(idx)^2/theta(idx); 111 | phii = PHI(:,idx); Sigii = 1/(Alpha+S(idx)); mui = Sigii*Q(idx); 112 | comm1 = Sig*(phi'*phii)/sigma2; 113 | ei = phii-phi*comm1; 114 | off = -Sigii*comm1; 115 | Sig = [Sig+Sigii*comm1*comm1', off; off', Sigii]; 116 | mu = [mu-mui*comm1; mui]; 117 | comm2 = PHI'*ei/sigma2; 118 | S = S - Sigii*comm2.^2; 119 | Q = Q - mui*comm2; 120 | % 121 | index = [index;idx]; 122 | alpha = [alpha;Alpha]; 123 | phi = [phi,phii]; 124 | end 125 | else 126 | if ~isempty(which) % deleting 127 | Sigii = Sig(which,which); mui = mu(which); Sigi = Sig(:,which); 128 | Sig = Sig-Sigi*Sigi'/Sigii; Sig(:,which) = []; Sig(which,:) = []; 129 | mu = mu-mui/Sigii*Sigi; mu(which) = []; 130 | comm = PHI'*(phi*Sigi)/sigma2; 131 | S = S + comm.^2/Sigii; 132 | Q = Q + mui/Sigii*comm; 133 | % 134 | index(which) = []; 135 | alpha(which) = []; 136 | phi(:,which) = []; 137 | end 138 | end 139 | 140 | end 141 | weights = mu; 142 | used = index; 143 | % re-estimated sigma2 144 | sigma2 = sum((t-phi*mu).^2)/(N-length(index)+alpha'*diag(Sig)); 145 | errbars = sqrt(diag(Sig)); 146 | 147 | % generate a basis for adaptive CS? 148 | if adaptive 149 | if optimal 150 | [V,D] = eig(Sig); 151 | [foo,idx] = max(diag(D)); 152 | basis = V(:,idx)'; 153 | else 154 | temp = phi'*phi/sigma2; 155 | Sig_inv = temp + scale*mean(diag(temp))*eye(length(used)); 156 | [V,D] = eig(Sig_inv); 157 | [foo,idx] = min(diag(D)); 158 | basis = V(:,idx)'; 159 | end 160 | end 161 | -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/Fig2.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------ 2 | % This code generates Figure 2 of the following paper: 3 | % "Bayesian Compressive Sensing" (Preprint, 2007) 4 | % This example is modified from l1qc_example.m, an example 5 | % from l1magic. 6 | % Coded by: Shihao Ji, ECE, Duke University 7 | % last change: Jan. 2, 2007 8 | %------------------------------------------------------ 9 | clear all 10 | rand('state', 1); 11 | randn('state', 2); 12 | % 13 | N = 512; % signal length 14 | T = 20; % number of spikes 15 | K = 100; % number of CS measurements 16 | % 17 | % random +/- 1 signal 18 | x = zeros(N,1); 19 | q = randperm(N); 20 | x(q(1:T)) = sign(randn(T,1)); 21 | 22 | % projection matrix 23 | A = randn(K,N); 24 | A = A./repmat(sqrt(sum(A.^2,2)),[1,N]); 25 | % noisy observations 26 | sigma = 0.005; 27 | e = sigma*randn(K,1); 28 | y = A*x + e; 29 | 30 | % solve by BP 31 | x0 = A'*inv(A*A')*y; 32 | % take epsilon a little bigger than sigma*sqrt(K) 33 | epsilon = sigma*sqrt(K)*sqrt(1 + 2*sqrt(2)/sqrt(K)); 34 | tic; 35 | x_BP = l1qc_logbarrier(x0, A, [], y, epsilon, 1e-3); 36 | t_BP = toc; 37 | fprintf(1,'BP number of nonzero weights: %d\n',sum(x_BP~=0)); 38 | 39 | % solve by BCS 40 | initsigma2 = std(y)^2/1e2; 41 | tic; 42 | [weights,used,sigma2,errbars] = BCS_fast_rvm(A,y,initsigma2,1e-8); 43 | t_BCS = toc; 44 | fprintf(1,'BCS number of nonzero weights: %d\n',length(used)); 45 | x_BCS = zeros(N,1); err = zeros(N,1); 46 | x_BCS(used) = weights; err(used) = errbars; 47 | 48 | % reconstruction error 49 | E_BP = norm(x-x_BP)/norm(x); 50 | E_BCS = norm(x-x_BCS)/norm(x); 51 | 52 | figure 53 | subplot(3,1,1); plot(x); axis([1 N -max(abs(x))-0.2 max(abs(x))+0.2]); title(['(a) Original Signal']); 54 | subplot(3,1,2); plot(x_BP); axis([1 N -max(abs(x))-0.2 max(abs(x))+0.2]); title(['(b) Reconstruction with BP, K=' num2str(K)]); 55 | subplot(3,1,3); errorbar(x_BCS,err); axis([1 N -max(abs(x))-0.2 max(abs(x))+0.2]); title(['(c) Reconstruction with BCS, K=' num2str(K)]); box on; 56 | 57 | disp(['BP: ||I_hat-I||/||I|| = ' num2str(E_BP) ', time = ' num2str(t_BP) ' secs']); 58 | disp(['BCS: ||I_hat-I||/||I|| = ' num2str(E_BCS) ', time = ' num2str(t_BCS) ' secs']); 59 | -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/Fig4_ab.m: -------------------------------------------------------------------------------- 1 | %------------------------------------------------------ 2 | % This code generates Figures 4(a,b) of the following paper: 3 | % "Bayesian Compressive Sensing" (Preprint, 2007) 4 | % This example is modified from l1qc_example.m, an example 5 | % from l1magic. 6 | % Coded by: Shihao Ji, ECE, Duke University 7 | % last change: Jan. 2, 2007 8 | %------------------------------------------------------ 9 | clear all 10 | % 11 | load random_results.mat 12 | rand_mean = mean(err); 13 | rand_std = std(err); 14 | load optimized_results.mat 15 | opt_mean = mean(err); 16 | opt_std = std(err); 17 | load approx_results.mat 18 | app_mean = mean(err); 19 | app_std = std(err); 20 | % 21 | base = 40; 22 | ns = 80; 23 | dN = 1; 24 | % plot the mean 25 | figure 26 | plot(base+(1:ns)*dN,rand_mean,'b-o'); 27 | hold on; 28 | plot(base+(1:ns)*dN,opt_mean,'r-*'); 29 | hold on; 30 | plot(base+(1:ns)*dN,app_mean,'k-s'); 31 | xlabel('Number of Measurements'); ylabel('Reconstruction Error'); 32 | box on; 33 | legend('Random','Optimized','Approx.',1); 34 | % plot the variance 35 | figure 36 | errorbar(base+(1:ns)*dN,rand_mean,rand_std,'b-o'); 37 | hold on; 38 | errorbar(base+(1:ns)*dN,opt_mean,opt_std,'r-*'); 39 | % hold on; 40 | % errorbar(base+(1:ns)*dN,app_mean,app_std,'k-s'); 41 | xlabel('Number of Measurements'); ylabel('Reconstruction Error'); 42 | box on; axis([39,121,-0.3,1.6]); 43 | legend('Random','Optimized',1); 44 | -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/approx_results.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidpalangi/Distributed-Compressive-Sensing-A-Deep-Learning-Approach/2eedf4c8b87f67156acafe713a912780841d7423/Other_Methods/BCS_demo/approx_results.mat -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/l1qc_logbarrier.m: -------------------------------------------------------------------------------- 1 | % l1qc_logbarrier.m 2 | % 3 | % Solve quadratically constrained l1 minimization: 4 | % min ||x||_1 s.t. ||Ax - b||_2 <= \epsilon 5 | % 6 | % Reformulate as the second-order cone program 7 | % min_{x,u} sum(u) s.t. x - u <= 0, 8 | % -x - u <= 0, 9 | % 1/2(||Ax-b||^2 - \epsilon^2) <= 0 10 | % and use a log barrier algorithm. 11 | % 12 | % Usage: xp = l1qc_logbarrier(x0, A, At, b, epsilon, lbtol, mu, cgtol, cgmaxiter) 13 | % 14 | % x0 - Nx1 vector, initial point. 15 | % 16 | % A - Either a handle to a function that takes a N vector and returns a K 17 | % vector , or a KxN matrix. If A is a function handle, the algorithm 18 | % operates in "largescale" mode, solving the Newton systems via the 19 | % Conjugate Gradients algorithm. 20 | % 21 | % At - Handle to a function that takes a K vector and returns an N vector. 22 | % If A is a KxN matrix, At is ignored. 23 | % 24 | % b - Kx1 vector of observations. 25 | % 26 | % epsilon - scalar, constraint relaxation parameter 27 | % 28 | % lbtol - The log barrier algorithm terminates when the duality gap <= lbtol. 29 | % Also, the number of log barrier iterations is completely 30 | % determined by lbtol. 31 | % Default = 1e-3. 32 | % 33 | % mu - Factor by which to increase the barrier constant at each iteration. 34 | % Default = 10. 35 | % 36 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 37 | % Default = 1e-8. 38 | % 39 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 40 | % if A is a matrix. 41 | % Default = 200. 42 | % 43 | % Written by: Justin Romberg, Caltech 44 | % Email: jrom@acm.caltech.edu 45 | % Created: October 2005 46 | % 47 | 48 | function xp = l1qc_logbarrier(x0, A, At, b, epsilon, lbtol, mu, cgtol, cgmaxiter) 49 | 50 | largescale = isa(A,'function_handle'); 51 | 52 | if (nargin < 6), lbtol = 1e-3; end 53 | if (nargin < 7), mu = 10; end 54 | if (nargin < 8), cgtol = 1e-8; end 55 | if (nargin < 9), cgmaxiter = 200; end 56 | 57 | newtontol = lbtol; 58 | newtonmaxiter = 50; 59 | 60 | N = length(x0); 61 | 62 | % starting point --- make sure that it is feasible 63 | if (largescale) 64 | if (norm(A(x0)-b) > epsilon) 65 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 66 | AAt = @(z) A(At(z)); 67 | [w, cgres] = cgsolve(AAt, b, cgtol, cgmaxiter, 0); 68 | if (cgres > 1/2) 69 | disp('A*At is ill-conditioned: cannot find starting point'); 70 | xp = x0; 71 | return; 72 | end 73 | x0 = At(w); 74 | end 75 | else 76 | if (norm(A*x0-b) > epsilon) 77 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 78 | opts.POSDEF = true; opts.SYM = true; 79 | [w, hcond] = linsolve(A*A', b, opts); 80 | if (hcond < 1e-14) 81 | disp('A*At is ill-conditioned: cannot find starting point'); 82 | xp = x0; 83 | return; 84 | end 85 | x0 = A'*w; 86 | end 87 | end 88 | x = x0; 89 | u = (0.95)*abs(x0) + (0.10)*max(abs(x0)); 90 | 91 | disp(sprintf('Original l1 norm = %.3f, original functional = %.3f', sum(abs(x0)), sum(u))); 92 | 93 | % choose initial value of tau so that the duality gap after the first 94 | % step will be about the origial norm 95 | tau = max((2*N+1)/sum(abs(x0)), 1); 96 | 97 | lbiter = ceil((log(2*N+1)-log(lbtol)-log(tau))/log(mu)); 98 | disp(sprintf('Number of log barrier iterations = %d\n', lbiter)); 99 | 100 | totaliter = 0; 101 | 102 | for ii = 1:lbiter 103 | 104 | [xp, up, ntiter] = l1qc_newton(x, u, A, At, b, epsilon, tau, newtontol, newtonmaxiter, cgtol, cgmaxiter); 105 | totaliter = totaliter + ntiter; 106 | 107 | disp(sprintf('\nLog barrier iter = %d, l1 = %.3f, functional = %8.3f, tau = %8.3e, total newton iter = %d\n', ... 108 | ii, sum(abs(xp)), sum(up), tau, totaliter)); 109 | 110 | x = xp; 111 | u = up; 112 | 113 | tau = mu*tau; 114 | 115 | end 116 | 117 | -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/l1qc_newton.m: -------------------------------------------------------------------------------- 1 | % l1qc_newton.m 2 | % 3 | % Newton algorithm for log-barrier subproblems for l1 minimization 4 | % with quadratic constraints. 5 | % 6 | % Usage: 7 | % [xp,up,niter] = l1qc_newton(x0, u0, A, At, b, epsilon, tau, 8 | % newtontol, newtonmaxiter, cgtol, cgmaxiter) 9 | % 10 | % x0,u0 - starting points 11 | % 12 | % A - Either a handle to a function that takes a N vector and returns a K 13 | % vector , or a KxN matrix. If A is a function handle, the algorithm 14 | % operates in "largescale" mode, solving the Newton systems via the 15 | % Conjugate Gradients algorithm. 16 | % 17 | % At - Handle to a function that takes a K vector and returns an N vector. 18 | % If A is a KxN matrix, At is ignored. 19 | % 20 | % b - Kx1 vector of observations. 21 | % 22 | % epsilon - scalar, constraint relaxation parameter 23 | % 24 | % tau - Log barrier parameter. 25 | % 26 | % newtontol - Terminate when the Newton decrement is <= newtontol. 27 | % Default = 1e-3. 28 | % 29 | % newtonmaxiter - Maximum number of iterations. 30 | % Default = 50. 31 | % 32 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 33 | % Default = 1e-8. 34 | % 35 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 36 | % if A is a matrix. 37 | % Default = 200. 38 | % 39 | % Written by: Justin Romberg, Caltech 40 | % Email: jrom@acm.caltech.edu 41 | % Created: October 2005 42 | % 43 | 44 | 45 | function [xp, up, niter] = l1qc_newton(x0, u0, A, At, b, epsilon, tau, newtontol, newtonmaxiter, cgtol, cgmaxiter) 46 | 47 | % check if the matrix A is implicit or explicit 48 | largescale = isa(A,'function_handle'); 49 | 50 | % line search parameters 51 | alpha = 0.01; 52 | beta = 0.5; 53 | 54 | if (~largescale), AtA = A'*A; end 55 | 56 | % initial point 57 | x = x0; 58 | u = u0; 59 | if (largescale), r = A(x) - b; else r = A*x - b; end 60 | fu1 = x - u; 61 | fu2 = -x - u; 62 | fe = 1/2*(r'*r - epsilon^2); 63 | f = sum(u) - (1/tau)*(sum(log(-fu1)) + sum(log(-fu2)) + log(-fe)); 64 | 65 | niter = 0; 66 | done = 0; 67 | while (~done) 68 | 69 | if (largescale), atr = At(r); else atr = A'*r; end 70 | 71 | ntgz = 1./fu1 - 1./fu2 + 1/fe*atr; 72 | ntgu = -tau - 1./fu1 - 1./fu2; 73 | gradf = -(1/tau)*[ntgz; ntgu]; 74 | 75 | sig11 = 1./fu1.^2 + 1./fu2.^2; 76 | sig12 = -1./fu1.^2 + 1./fu2.^2; 77 | sigx = sig11 - sig12.^2./sig11; 78 | 79 | w1p = ntgz - sig12./sig11.*ntgu; 80 | if (largescale) 81 | h11pfun = @(z) sigx.*z - (1/fe)*At(A(z)) + 1/fe^2*(atr'*z)*atr; 82 | [dx, cgres, cgiter] = cgsolve(h11pfun, w1p, cgtol, cgmaxiter, 0); 83 | if (cgres > 1/2) 84 | disp('Cannot solve system. Returning previous iterate. (See Section 4 of notes for more information.)'); 85 | xp = x; up = u; 86 | return 87 | end 88 | Adx = A(dx); 89 | else 90 | H11p = diag(sigx) - (1/fe)*AtA + (1/fe)^2*atr*atr'; 91 | opts.POSDEF = true; opts.SYM = true; 92 | [dx,hcond] = linsolve(H11p, w1p, opts); 93 | if (hcond < 1e-14) 94 | disp('Matrix ill-conditioned. Returning previous iterate. (See Section 4 of notes for more information.)'); 95 | xp = x; up = u; 96 | return 97 | end 98 | Adx = A*dx; 99 | end 100 | du = (1./sig11).*ntgu - (sig12./sig11).*dx; 101 | 102 | % minimum step size that stays in the interior 103 | ifu1 = find((dx-du) > 0); ifu2 = find((-dx-du) > 0); 104 | aqe = Adx'*Adx; bqe = 2*r'*Adx; cqe = r'*r - epsilon^2; 105 | smax = min(1,min([... 106 | -fu1(ifu1)./(dx(ifu1)-du(ifu1)); -fu2(ifu2)./(-dx(ifu2)-du(ifu2)); ... 107 | (-bqe+sqrt(bqe^2-4*aqe*cqe))/(2*aqe) 108 | ])); 109 | s = (0.99)*smax; 110 | 111 | % backtracking line search 112 | suffdec = 0; 113 | backiter = 0; 114 | while (~suffdec) 115 | xp = x + s*dx; up = u + s*du; rp = r + s*Adx; 116 | fu1p = xp - up; fu2p = -xp - up; fep = 1/2*(rp'*rp - epsilon^2); 117 | fp = sum(up) - (1/tau)*(sum(log(-fu1p)) + sum(log(-fu2p)) + log(-fep)); 118 | flin = f + alpha*s*(gradf'*[dx; du]); 119 | suffdec = (fp <= flin); 120 | s = beta*s; 121 | backiter = backiter + 1; 122 | if (backiter > 32) 123 | disp('Stuck on backtracking line search, returning previous iterate. (See Section 4 of notes for more information.)'); 124 | xp = x; up = u; 125 | return 126 | end 127 | end 128 | 129 | % set up for next iteration 130 | x = xp; u = up; r = rp; 131 | fu1 = fu1p; fu2 = fu2p; fe = fep; f = fp; 132 | 133 | lambda2 = -(gradf'*[dx; du]); 134 | stepsize = s*norm([dx; du]); 135 | niter = niter + 1; 136 | done = (lambda2/2 < newtontol) | (niter >= newtonmaxiter); 137 | 138 | disp(sprintf('Newton iter = %d, Functional = %8.3f, Newton decrement = %8.3f, Stepsize = %8.3e', ... 139 | niter, f, lambda2/2, stepsize)); 140 | if (largescale) 141 | disp(sprintf(' CG Res = %8.3e, CG Iter = %d', cgres, cgiter)); 142 | else 143 | disp(sprintf(' H11p condition number = %8.3e', hcond)); 144 | end 145 | 146 | end 147 | 148 | 149 | 150 | 151 | -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/multi_approx_measures.m: -------------------------------------------------------------------------------- 1 | %----------------------------------------------------------------- 2 | % This code generates Figure 4 (Approx.) of the following paper: 3 | % "Bayesian Compressive Sensing" (Preprint, 2007) 4 | % The dataset used is similar to l1qc_example.m from l1magic package 5 | % Coded by: Shihao Ji, ECE, Duke University 6 | % last change: Jan. 2, 2007 7 | %----------------------------------------------------------------- 8 | clear all 9 | % 10 | total_count = 100; 11 | N = 512; % signal length 12 | T = 20; % number of spikes to put down 13 | dN = 1; 14 | base = 40; 15 | ns = 80; 16 | sigma = 0.005; 17 | % 18 | for count = 1:total_count 19 | count 20 | rand('state', count); 21 | randn('state', 2*count); 22 | % 23 | % random +/- 1 signal 24 | x = zeros(N,1); 25 | q = randperm(N); 26 | x(q(1:T)) = sign(randn(T,1)); 27 | % noisy observations 28 | A = randn(base,N); 29 | A = 1.01*A./repmat(sqrt(sum(A.^2,2)),[1,N]); 30 | e = sigma*randn(base,1); 31 | y = A*x + e; 32 | 33 | initsigma2 = std(y)^2/1e2; 34 | [weights,used,sigma2,errbars,basis] = BCS_fast_rvm(A,y,initsigma2,1e-8,1,0,0.1); 35 | 36 | for i = 1:ns 37 | 38 | K = base+i*dN; 39 | a = randn(dN,N); 40 | unused = setdiff([1:N],used); 41 | a(unused) = sqrt(1.01^2-1)*a(unused)/sqrt(sum(a(unused).^2)); % noise imputation 42 | a(used) = basis; 43 | % noisy observations 44 | e = sigma*randn(dN,1); 45 | t = a*x + e; 46 | y = [y;t]; 47 | A = [A;a]; 48 | 49 | initsigma2 = std(y)^2/1e2; 50 | [weights,used,sigma2,errbars,basis] = BCS_fast_rvm(A,y,initsigma2,1e-8,1,0,0.1); 51 | % 52 | xp = zeros(N,1); 53 | xp(used) = weights; 54 | err(count,i) = norm(x-xp)/norm(x); 55 | 56 | end 57 | 58 | end 59 | save approx_results.mat err; 60 | beep; 61 | disp('Done!'); 62 | -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/multi_optimized_measures.m: -------------------------------------------------------------------------------- 1 | %----------------------------------------------------------------- 2 | % This code generates Figure 4 (optimized) of the following paper: 3 | % "Bayesian Compressive Sensing" (Preprint, 2007) 4 | % The dataset used is similar to l1qc_example.m from l1magic package 5 | % Coded by: Shihao Ji, ECE, Duke University 6 | % last change: Jan. 2, 2007 7 | %----------------------------------------------------------------- 8 | clear all 9 | % 10 | total_count = 100; 11 | N = 512; % signal length 12 | T = 20; % number of spikes 13 | dN = 1; 14 | base = 40; % number of initial random measurements 15 | ns = 80; % number of additional optimized measurements 16 | sigma = 0.005; 17 | % 18 | for count = 1:total_count 19 | count 20 | rand('state', count); 21 | randn('state', 2*count); 22 | % 23 | % random +/- 1 signal 24 | x = zeros(N,1); 25 | q = randperm(N); 26 | x(q(1:T)) = sign(randn(T,1)); 27 | % noisy observations 28 | A = randn(base,N); 29 | A = 1.01*A./repmat(sqrt(sum(A.^2,2)),[1,N]); 30 | e = sigma*randn(base,1); 31 | y = A*x + e; 32 | initsigma2 = std(y)^2/1e2; 33 | [weights,used,sigma2,errbars,basis] = BCS_fast_rvm(A,y,initsigma2,1e-8,1); 34 | 35 | for i = 1:ns 36 | 37 | K = base+i*dN; 38 | a = randn(dN,N); 39 | unused = setdiff([1:N],used); 40 | a(unused) = sqrt(1.01^2-1)*a(unused)/sqrt(sum(a(unused).^2)); % noise imputation 41 | a(used) = basis; 42 | % noisy observations 43 | e = sigma*randn(dN,1); 44 | t = a*x + e; 45 | y = [y;t]; 46 | A = [A;a]; 47 | 48 | initsigma2 = std(y)^2/1e2; 49 | [weights,used,sigma2,errbars,basis] = BCS_fast_rvm(A,y,initsigma2,1e-8,1); 50 | % 51 | xp = zeros(N,1); 52 | xp(used) = weights; 53 | err(count,i) = norm(x-xp)/norm(x); 54 | 55 | end 56 | 57 | end 58 | save optimized_results.mat err; 59 | beep; 60 | disp('Done!'); 61 | 62 | -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/multi_random_measures.m: -------------------------------------------------------------------------------- 1 | %----------------------------------------------------------------- 2 | % This code generates Figure 4 (random) of the following paper: 3 | % "Bayesian Compressive Sensing" (Preprint, 2007) 4 | % The dataset used is similar to l1qc_example.m from l1magic package 5 | % Coded by: Shihao Ji, ECE, Duke University 6 | % last change: Jan. 2, 2007 7 | %----------------------------------------------------------------- 8 | clear all 9 | % 10 | total_count = 100; 11 | N = 512; % signal length 12 | T = 20; % number of spikes 13 | dN = 1; 14 | base = 40; % number of initial random measurements 15 | ns = 80; % number of additional random measurements 16 | sigma = 0.005; 17 | % 18 | for count = 1:total_count 19 | count 20 | rand('state', count); 21 | randn('state', 2*count); 22 | % 23 | % random +/- 1 signal 24 | x = zeros(N,1); 25 | q = randperm(N); 26 | x(q(1:T)) = sign(randn(T,1)); 27 | % noisy observations 28 | A = randn(base,N); 29 | A = 1.01*A./repmat(sqrt(sum(A.^2,2)),[1,N]); 30 | e = sigma*randn(base,1); 31 | y = A*x + e; 32 | 33 | for i = 1:ns 34 | 35 | K = base+i*dN; 36 | a = randn(dN,N); 37 | a = 1.01*a/sqrt(sum(a.^2)); 38 | % noisy observations 39 | e = sigma*randn(dN,1); 40 | t = a*x + e; 41 | y = [y;t]; 42 | A = [A;a]; 43 | 44 | initsigma2 = std(y)^2/1e2; 45 | [weights,used] = BCS_fast_rvm(A,y,initsigma2,1e-8); 46 | % 47 | xp = zeros(N,1); 48 | xp(used) = weights; 49 | err(count,i) = norm(x-xp)/norm(x); 50 | 51 | end 52 | 53 | end 54 | % 55 | save random_results.mat err; 56 | beep; 57 | disp('Done!'); 58 | 59 | 60 | -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/optimized_results.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidpalangi/Distributed-Compressive-Sensing-A-Deep-Learning-Approach/2eedf4c8b87f67156acafe713a912780841d7423/Other_Methods/BCS_demo/optimized_results.mat -------------------------------------------------------------------------------- /Other_Methods/BCS_demo/random_results.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidpalangi/Distributed-Compressive-Sensing-A-Deep-Learning-Approach/2eedf4c8b87f67156acafe713a912780841d7423/Other_Methods/BCS_demo/random_results.mat -------------------------------------------------------------------------------- /Other_Methods/MT_CS_demo/Fig2.m: -------------------------------------------------------------------------------- 1 | %--------------------------------------------------------- 2 | % This code generates Figure 2 of the following paper: 3 | % "Multi-Task Compressive Sensing" (Preprint, 2007) 4 | % This example is modified from l1qc_example.m, an example 5 | % from l1magic. 6 | % Coded by: Shihao Ji, ECE, Duke University 7 | % last change: May. 15, 2007 8 | %--------------------------------------------------------- 9 | clear all 10 | rand('state', 1); 11 | randn('state', 2); 12 | % 13 | N = 512; % signal length 14 | T = 20; % number of spikes 15 | K = [90,70]; % number of CS measurements 16 | % 17 | % random +/- 1 signal 18 | x1 = zeros(N,1); 19 | x2 = zeros(N,1); 20 | q = randperm(N); 21 | x1(q(1:T)) = sign(randn(T,1)); 22 | x2(q([1:T-5,T+1:T+5])) = sign(randn(T,1)); % 75% similarity 23 | 24 | % projection matrix 25 | A1 = randn(K(1),N); 26 | A1 = A1./repmat(sqrt(sum(A1.^2,2)),[1,N]); 27 | A2 = randn(K(2),N); 28 | A2 = A2./repmat(sqrt(sum(A2.^2,2)),[1,N]); 29 | A{1} = A1; 30 | A{2} = A2; 31 | % noisy observations 32 | sigma = 0.005; 33 | e1 = sigma*randn(K(1),1); 34 | e2 = sigma*randn(K(2),1); 35 | y1 = A1*x1 + e1; 36 | y2 = A2*x2 + e2; 37 | y{1} = y1; 38 | y{2} = y2; 39 | 40 | %solve task 1 41 | a = 1e2/0.1; b = 1; 42 | weights = mt_CS(A1,y1,a,b,1e-8); 43 | fprintf(1,'Task 1 number of nonzero weights: %d\n',sum(weights~=0)); 44 | x1_BCS = weights; 45 | 46 | % solve task 2 47 | a = 1e2/0.1; b = 1; 48 | weights = mt_CS(A2,y2,a,b,1e-8); 49 | fprintf(1,'Task 2 number of nonzero weights: %d\n',sum(weights~=0)); 50 | x2_BCS = weights; 51 | 52 | % solve 1&2 by MT-BCS 53 | a = 1e2/0.1; b = 1; 54 | weights = mt_CS(A,y,a,b,1e-8); 55 | fprintf(1,'Task 1 number of nonzero weights: %d\n',sum(weights(:,1)~=0)); 56 | fprintf(1,'Task 2 number of nonzero weights: %d\n',sum(weights(:,2)~=0)); 57 | x1_MT = weights(:,1); 58 | x2_MT = weights(:,2); 59 | 60 | % reconstruction error 61 | E1_BCS = norm(x1-x1_BCS)/norm(x1); 62 | E2_BCS = norm(x2-x2_BCS)/norm(x2); 63 | E1_MT = norm(x1-x1_MT)/norm(x1); 64 | E2_MT = norm(x2-x2_MT)/norm(x2); 65 | 66 | figure 67 | subplot(3,2,1); plot(x1); axis([1 N -1.2 1.2]); title(['(a) Original Signal 1']); 68 | subplot(3,2,2); plot(x2); axis([1 N -1.2 1.2]); title(['(b) Original Signal 2']); 69 | subplot(3,2,3); plot(x1_BCS); axis([1 N -1.2 1.2]); title(['(c) ST Reconstruction 1, n=' num2str(K(1))]); box on; 70 | subplot(3,2,4); plot(x2_BCS); axis([1 N -1.2 1.2]); title(['(d) ST Reconstruction 2, n=' num2str(K(2))]); box on; 71 | subplot(3,2,5); plot(x1_MT); axis([1 N -1.2 1.2]); title(['(e) MT Reconstruction 1, n=' num2str(K(1))]); box on; 72 | subplot(3,2,6); plot(x2_MT); axis([1 N -1.2 1.2]); title(['(f) MT Reconstruction 2, n=' num2str(K(2))]); box on; 73 | 74 | disp(['ST1: ||I1_hat-I1||/||I1|| = ' num2str(E1_BCS)]); 75 | disp(['MT1: ||I1_hat-I1||/||I1|| = ' num2str(E1_MT)]); 76 | disp(['ST2: ||I2_hat-I2||/||I2|| = ' num2str(E2_BCS)]); 77 | disp(['MT2: ||I2_hat-I2||/||I2|| = ' num2str(E2_MT)]); 78 | -------------------------------------------------------------------------------- /Other_Methods/MT_CS_demo/Fig3.m: -------------------------------------------------------------------------------- 1 | %--------------------------------------------------------- 2 | % This code generates Figure 3 of the following paper: 3 | % "Multi-Task Compressive Sensing" (Preprint, 2007) 4 | % This example is modified from l1qc_example.m, an example 5 | % from l1magic. 6 | % Coded by: Shihao Ji, ECE, Duke University 7 | % last change: May. 15, 2007 8 | %--------------------------------------------------------- 9 | clear all 10 | % 11 | base = 40; % number of initial random measurements 12 | ns = 100; % number of additional random measurements 13 | load multi_results_75.mat E1_BCS E2_BCS E1_MT E2_MT; 14 | ST_mean_75 = mean((E1_BCS+E2_BCS)/2); 15 | ST_std_75 = std((E1_BCS+E2_BCS)/2); 16 | MT_mean_75 = mean((E1_MT+E2_MT)/2); 17 | MT_std_75 = std((E1_MT+E2_MT)/2); 18 | load multi_results_50.mat E1_BCS E2_BCS E1_MT E2_MT; 19 | ST_mean_50 = mean((E1_BCS+E2_BCS)/2); 20 | ST_std_50 = std((E1_BCS+E2_BCS)/2); 21 | MT_mean_50 = mean((E1_MT+E2_MT)/2); 22 | MT_std_50 = std((E1_MT+E2_MT)/2); 23 | load multi_results_25.mat E1_BCS E2_BCS E1_MT E2_MT; 24 | ST_mean_25 = mean((E1_BCS+E2_BCS)/2); 25 | ST_std_25 = std((E1_BCS+E2_BCS)/2); 26 | MT_mean_25 = mean((E1_MT+E2_MT)/2); 27 | MT_std_25 = std((E1_MT+E2_MT)/2); 28 | % 29 | ST_mean = (ST_mean_25+ST_mean_50+ST_mean_75)/3; 30 | ST_std = (ST_std_25+ST_std_50+ST_std_75)/3; 31 | 32 | figure; hold on; 33 | plot((1:ns)+base,ST_mean,'k-o'); 34 | plot((1:ns)+base,MT_mean_25,'b-+'); 35 | plot((1:ns)+base,MT_mean_50,'g-s'); 36 | plot((1:ns)+base,MT_mean_75,'r-*'); 37 | xlabel('Number of Measurements'); ylabel('Reconstruction Error'); 38 | legend('ST','MT 25%','MT 50%', 'MT 75%',1); box on; 39 | % 40 | figure; hold on; 41 | errorbar((1:ns)+base,ST_mean,ST_std,'k-o'); 42 | %errorbar((1:ns)+base,MT_mean_25,MT_std_25,'b-+'); 43 | %errorbar((1:ns)+base,MT_mean_50,MT_std_50,'g-s'); 44 | errorbar((1:ns)+base,MT_mean_75,MT_std_75,'r-*'); 45 | xlabel('Number of Measurements'); ylabel('Reconstruction Error'); 46 | %legend('ST','MT 25%','MT 50%', 'MT 75%',1); box on; 47 | box on; axis([40,140,-0.2,1.5]); 48 | legend('ST','MT 75%',1); 49 | -------------------------------------------------------------------------------- /Other_Methods/MT_CS_demo/mt_CS.m: -------------------------------------------------------------------------------- 1 | function [weights,ML] = mt_CS(PHI,t,a,b,eta) 2 | %--------------------------------------------------------------------------------------- 3 | % The MT-CS algorithm for the following paper: 4 | % "Multi-Task Compressive Sesning" (Preprint, 2007). The algorithm 5 | % is an extension of the fast RVM algorithm [Tipping & Faul, 2003] 6 | % in two-fold: (i) the noise variance is marginalized and (ii) it is for 7 | % multi-task CS, including single-task CS as a special case 8 | % Coded by: Shihao Ji, ECE, Duke University 9 | % last change: May. 15, 2007 10 | % A bug was fixed on Aug.03, 2008 for the cases where signals are dramatic undersampled 11 | %--------------------------------------------------------------------------------------- 12 | % Input: 13 | % PHI: projection matrix. Cell structure, One cell for one task. 14 | % t: CS measurements. Cell structure, One cell for one task. 15 | % a,b: parameters of Gamma prior on noise variance 16 | % eta: threshold for stopping the algorithm (suggested value: 1e-8) 17 | % Output: 18 | % weights: sparse weights for all the tasks. One column for one task 19 | % ML: the increase of the joint mariginal likelihood for each 20 | % iteration 21 | % 22 | if iscell(t) 23 | NT = length(t); 24 | else 25 | NT = 1; 26 | PHI = {PHI}; 27 | t = {t}; 28 | end 29 | % % fprintf(1,'This is a %d-task learning!\n',NT); 30 | % 31 | % find initial alpha 32 | for k = 1:NT 33 | [N(k),M(k)] = size(PHI{k}); 34 | end 35 | if sum(abs(M-M(1))) ~= 0 36 | error('Sorry! The sizes of the underlying signals should be the same!\n'); 37 | else 38 | M = M(1); 39 | end 40 | % find initial alpha 41 | K = repmat(N+2*a,[M,1]); 42 | % 43 | for k = 1:NT 44 | PHIt(:,k) = PHI{k}'*t{k}; 45 | PHI2(:,k) = sum(PHI{k}.^2)'; 46 | G2(k) = t{k}'*t{k}+2*b; 47 | end 48 | G2 = repmat(G2,[M,1]); 49 | X = G2.*PHI2./PHIt.^2; 50 | ml = K.*log(X./K)-(K-1).*log((X-1)./(K-1)); 51 | 52 | ml_sum = sum(ml,2); 53 | while 1 54 | [ML,index] = max(ml_sum); 55 | alpha = NT./sum((K(index,:).*PHIt(index,:).^2./G2(index,:)-PHI2(index,:))./(PHI2(index,:).*(PHI2(index,:)-PHIt(index,:).^2./G2(index,:))),2); 56 | if alpha > 0 % A bug was fixed here, Aug 03, 2008 (alpha should be greater than 0) 57 | break; 58 | else 59 | ml_sum(index) = 0; 60 | end 61 | end 62 | 63 | for k = 1:NT 64 | % compute initial mu, Sig, S, Q, G 65 | phi{k} = PHI{k}(:,index); 66 | Hessian = alpha+phi{k}'*phi{k}; 67 | Sig{k} = 1/Hessian; 68 | mu{k} = Sig{k}*PHIt(index,k); 69 | left = PHI{k}'*phi{k}; 70 | S(:,k) = PHI2(:,k)-Sig{k}*left.^2; 71 | Q(:,k) = PHIt(:,k)-Sig{k}*PHIt(index,k)*left; 72 | G(:,k) = G2(:,k)-Sig{k}*PHIt(index,k)^2; 73 | end 74 | clear PHI2 left; 75 | % 76 | for count = 2:10000 77 | 78 | s = S; q = Q; g = G; 79 | Alpha = repmat(alpha,[1,NT]); 80 | s(index,:) = Alpha.*S(index,:)./(Alpha-S(index,:)); 81 | q(index,:) = Alpha.*Q(index,:)./(Alpha-S(index,:)); 82 | g(index,:) = g(index,:)+Q(index,:).^2./(Alpha-S(index,:)); 83 | theta = NT./sum((K.*q.^2./g-s)./(s.*(s-q.^2./g)),2); 84 | 85 | % choice the next alpha that maximizes marginal likelihood 86 | ml = repmat(-inf,[M,NT]); 87 | ig0 = find(theta>0); 88 | % index for re-estimate 89 | [ire,foo,which] = intersect(ig0,index); 90 | if ~isempty(ire) 91 | Alpha1 = repmat(theta(ire),[1,NT]); 92 | Alpha0 = repmat(alpha(which),[1,NT]); 93 | delta = 1./Alpha1-1./Alpha0; 94 | X = G(ire,:).*S(ire,:)./Q(ire,:).^2; 95 | ml(ire,:) = (K(ire,:)-1).*log(1+S(ire,:).*delta)+K(ire,:).*log(((Alpha0+s(ire,:)).*g(ire,:)-q(ire,:).^2).*Alpha1./(((Alpha1+s(ire,:)).*g(ire,:)-q(ire,:).^2).*Alpha0)); 96 | end 97 | % index for adding 98 | iad = setdiff(ig0,ire); 99 | if ~isempty(iad) 100 | Alpha = repmat(theta(iad),[1,NT]); 101 | ml(iad,:) = log(Alpha./(Alpha+s(iad,:)))-K(iad,:).*log(1-(q(iad,:).^2./g(iad,:))./(Alpha+s(iad,:))); 102 | end 103 | is0 = setdiff([1:M],ig0); 104 | % index for deleting 105 | [ide,foo,which] = intersect(is0,index); 106 | if ~isempty(ide) 107 | Alpha = repmat(alpha(which),[1,NT]); 108 | ml(ide,:) = -log(1-S(ide,:)./Alpha)-K(ide,:).*log(1+Q(ide,:).^2./(G(ide,:).*(Alpha-S(ide,:)))); 109 | end 110 | [ML(count),idx] = max(sum(real(ml),2)); 111 | 112 | % check if terminates? 113 | if count > 2 & abs(ML(count)-ML(count-1)) < (max(ML)-ML(count))*eta 114 | break; 115 | end 116 | % update alphas 117 | which = find(index==idx); 118 | if theta(idx) > 0 119 | if ~isempty(which) % re-estimate 120 | Alpha = theta(idx); 121 | delta = Alpha-alpha(which); 122 | for k = 1:NT 123 | Sigii = Sig{k}(which,which); mui = mu{k}(which); Sigi = Sig{k}(:,which); 124 | ki = delta/(1+Sigii*delta); 125 | mu{k} = mu{k}-ki*mui*Sigi; 126 | Sig{k} = Sig{k}-ki*Sigi*Sigi'; 127 | comm = PHI{k}'*(phi{k}*Sigi); 128 | S(:,k) = S(:,k) + ki*(comm.^2); 129 | Q(:,k) = Q(:,k) + ki*mui*comm; 130 | G(:,k) = G(:,k) + ki*(Sigi'*PHIt(index,k))^2; 131 | end 132 | % 133 | alpha(which) = Alpha; 134 | else % adding 135 | Alpha = theta(idx); 136 | for k = 1:NT 137 | phii = PHI{k}(:,idx); Sigii = 1/(Alpha+S(idx,k)); mui = Sigii*Q(idx,k); 138 | comm1 = Sig{k}*(phi{k}'*phii); 139 | ei = phii-phi{k}*comm1; 140 | off = -Sigii*comm1; 141 | Sig{k} = [Sig{k}+Sigii*comm1*comm1', off; off', Sigii]; 142 | mu{k} = [mu{k}-mui*comm1; mui]; 143 | comm2 = PHI{k}'*ei; 144 | S(:,k) = S(:,k) - Sigii*(comm2.^2); 145 | Q(:,k) = Q(:,k) - mui*comm2; 146 | G(:,k) = G(:,k) - Sigii*(t{k}'*ei)^2; 147 | phi{k} = [phi{k},phii]; 148 | end 149 | % 150 | index = [index;idx]; 151 | alpha = [alpha;Alpha]; 152 | end 153 | else 154 | if ~isempty(which) % deleting 155 | for k = 1:NT 156 | Sigii = Sig{k}(which,which); mui = mu{k}(which); Sigi = Sig{k}(:,which); 157 | Sig{k} = Sig{k}-Sigi*Sigi'/Sigii; Sig{k}(:,which) = []; Sig{k}(which,:) = []; 158 | mu{k} = mu{k}-mui/Sigii*Sigi; mu{k}(which) = []; 159 | comm = PHI{k}'*(phi{k}*Sigi); 160 | S(:,k) = S(:,k) + (comm.^2)/Sigii; 161 | Q(:,k) = Q(:,k) + mui/Sigii*comm; 162 | G(:,k) = G(:,k) + (Sigi'*PHIt(index,k))^2/Sigii; 163 | phi{k}(:,which) = []; 164 | end 165 | % 166 | index(which) = []; 167 | alpha(which) = []; 168 | end 169 | end 170 | 171 | end 172 | % output 173 | weights = zeros(M,NT); 174 | for k = 1:NT 175 | weights(index,k) = mu{k}; 176 | end 177 | -------------------------------------------------------------------------------- /Other_Methods/MT_CS_demo/multi_results_25.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidpalangi/Distributed-Compressive-Sensing-A-Deep-Learning-Approach/2eedf4c8b87f67156acafe713a912780841d7423/Other_Methods/MT_CS_demo/multi_results_25.mat -------------------------------------------------------------------------------- /Other_Methods/MT_CS_demo/multi_results_50.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidpalangi/Distributed-Compressive-Sensing-A-Deep-Learning-Approach/2eedf4c8b87f67156acafe713a912780841d7423/Other_Methods/MT_CS_demo/multi_results_50.mat -------------------------------------------------------------------------------- /Other_Methods/MT_CS_demo/multi_results_75.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidpalangi/Distributed-Compressive-Sensing-A-Deep-Learning-Approach/2eedf4c8b87f67156acafe713a912780841d7423/Other_Methods/MT_CS_demo/multi_results_75.mat -------------------------------------------------------------------------------- /Other_Methods/MT_CS_demo/multi_runs_25.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------------------------- 2 | % This code generates Figure 3 (MT 25%) of the following paper: 3 | % "Multi-Task Compressive Sensing" (Preprint, 2007) 4 | % This example is modified from l1qc_example.m, an example 5 | % from l1magic. 6 | % Coded by: Shihao Ji, ECE, Duke University 7 | % last change: May. 15, 2007 8 | %-------------------------------------------------------------- 9 | clear all 10 | % 11 | total_count = 100; 12 | N = 512; % signal length 13 | T = 20; % number of spikes 14 | dN = 1; 15 | base = 40; % number of initial random measurements 16 | ns = 100; % number of additional random measurements 17 | sigma = 0.005; 18 | 19 | for count = 1:total_count 20 | count 21 | rand('state', count); 22 | randn('state', 2*count); 23 | % 24 | % random +/- 1 signal 25 | x1 = zeros(N,1); 26 | x2 = zeros(N,1); 27 | q = randperm(N); 28 | x1(q(1:T)) = sign(randn(T,1)); 29 | x2(q([1:5,T+1:T+15])) = sign(randn(T,1)); % 25% similarity 30 | 31 | % projection matrix 32 | A1 = randn(base,N); 33 | A1 = A1./repmat(sqrt(sum(A1.^2,2)),[1,N]); 34 | A2 = randn(base,N); 35 | A2 = A2./repmat(sqrt(sum(A2.^2,2)),[1,N]); 36 | A{1} = A1; 37 | A{2} = A2; 38 | % noisy observations 39 | e1 = sigma*randn(base,1); 40 | e2 = sigma*randn(base,1); 41 | y1 = A1*x1 + e1; 42 | y2 = A2*x2 + e2; 43 | y{1} = y1; 44 | y{2} = y2; 45 | 46 | for i = 1:ns 47 | 48 | K = base+i*dN; 49 | a1 = randn(dN,N); 50 | a1 = a1/sqrt(sum(a1.^2)); 51 | % noisy observations 52 | e1 = sigma*randn(dN,1); 53 | y1 = a1*x1 + e1; 54 | y{1} = [y{1};y1]; 55 | A{1} = [A{1};a1]; 56 | % 57 | a2 = randn(dN,N); 58 | a2 = a2/sqrt(sum(a2.^2)); 59 | % noisy observations 60 | e2 = sigma*randn(dN,1); 61 | y2 = a2*x2 + e2; 62 | y{2} = [y{2};y2]; 63 | A{2} = [A{2};a2]; 64 | 65 | % solve task 1 66 | a = 1e2/0.1; b = 1; 67 | weights = mt_CS(A{1},y{1},a,b,1e-8); 68 | x1_BCS = weights; 69 | 70 | % solve task 2 71 | a = 1e2/0.1; b = 1; 72 | weights = mt_CS(A{2},y{2},a,b,1e-8); 73 | x2_BCS = weights; 74 | 75 | % solve 1&2 by MT-CS 76 | a = 1e2/0.1; b = 1; 77 | weights = mt_CS(A,y,a,b,1e-8); 78 | x1_MT = weights(:,1); 79 | x2_MT = weights(:,2); 80 | 81 | % reconstruction error 82 | E1_BCS(count,i) = norm(x1-x1_BCS)/norm(x1); 83 | E2_BCS(count,i) = norm(x2-x2_BCS)/norm(x2); 84 | E1_MT(count,i) = norm(x1-x1_MT)/norm(x1); 85 | E2_MT(count,i) = norm(x2-x2_MT)/norm(x2); 86 | 87 | end 88 | save multi_results_25.mat E1_BCS E2_BCS E1_MT E2_MT; 89 | end 90 | save multi_results_25.mat E1_BCS E2_BCS E1_MT E2_MT; 91 | disp('Done!'); 92 | beep; 93 | -------------------------------------------------------------------------------- /Other_Methods/MT_CS_demo/multi_runs_50.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------------------------- 2 | % This code generates Figure 3 (MT 50%) of the following paper: 3 | % "Multi-Task Compressive Sensing" (Preprint, 2007) 4 | % This example is modified from l1qc_example.m, an example 5 | % from l1magic. 6 | % Coded by: Shihao Ji, ECE, Duke University 7 | % last change: May. 15, 2007 8 | %-------------------------------------------------------------- 9 | clear all 10 | % 11 | total_count = 100; 12 | N = 512; % signal length 13 | T = 20; % number of spikes 14 | dN = 1; 15 | base = 40; % number of initial random measurements 16 | ns = 100; % number of additional random measurements 17 | sigma = 0.005; 18 | 19 | for count = 1:total_count 20 | count 21 | rand('state', count); 22 | randn('state', 2*count); 23 | % 24 | % random +/- 1 signal 25 | x1 = zeros(N,1); 26 | x2 = zeros(N,1); 27 | q = randperm(N); 28 | x1(q(1:T)) = sign(randn(T,1)); 29 | x2(q([1:10,T+1:T+10])) = sign(randn(T,1)); % 50% similarity 30 | 31 | % projection matrix 32 | A1 = randn(base,N); 33 | A1 = A1./repmat(sqrt(sum(A1.^2,2)),[1,N]); 34 | A2 = randn(base,N); 35 | A2 = A2./repmat(sqrt(sum(A2.^2,2)),[1,N]); 36 | A{1} = A1; 37 | A{2} = A2; 38 | % noisy observations 39 | e1 = sigma*randn(base,1); 40 | e2 = sigma*randn(base,1); 41 | y1 = A1*x1 + e1; 42 | y2 = A2*x2 + e2; 43 | y{1} = y1; 44 | y{2} = y2; 45 | 46 | for i = 1:ns 47 | 48 | K = base+i*dN; 49 | a1 = randn(dN,N); 50 | a1 = a1/sqrt(sum(a1.^2)); 51 | % noisy observations 52 | e1 = sigma*randn(dN,1); 53 | y1 = a1*x1 + e1; 54 | y{1} = [y{1};y1]; 55 | A{1} = [A{1};a1]; 56 | % 57 | a2 = randn(dN,N); 58 | a2 = a2/sqrt(sum(a2.^2)); 59 | % noisy observations 60 | e2 = sigma*randn(dN,1); 61 | y2 = a2*x2 + e2; 62 | y{2} = [y{2};y2]; 63 | A{2} = [A{2};a2]; 64 | 65 | % solve task 1 66 | a = 1e2/0.1; b = 1; 67 | weights = mt_CS(A{1},y{1},a,b,1e-8); 68 | x1_BCS = weights; 69 | 70 | % solve task 2 71 | a = 1e2/0.1; b = 1; 72 | weights = mt_CS(A{2},y{2},a,b,1e-8); 73 | x2_BCS = weights; 74 | 75 | % solve 1&2 by MT-CS 76 | a = 1e2/0.1; b = 1; 77 | weights = mt_CS(A,y,a,b,1e-8); 78 | x1_MT = weights(:,1); 79 | x2_MT = weights(:,2); 80 | 81 | % reconstruction error 82 | E1_BCS(count,i) = norm(x1-x1_BCS)/norm(x1); 83 | E2_BCS(count,i) = norm(x2-x2_BCS)/norm(x2); 84 | E1_MT(count,i) = norm(x1-x1_MT)/norm(x1); 85 | E2_MT(count,i) = norm(x2-x2_MT)/norm(x2); 86 | 87 | end 88 | save multi_results_50.mat E1_BCS E2_BCS E1_MT E2_MT; 89 | end 90 | save multi_results_50.mat E1_BCS E2_BCS E1_MT E2_MT; 91 | disp('Done!'); 92 | beep; 93 | -------------------------------------------------------------------------------- /Other_Methods/MT_CS_demo/multi_runs_75.m: -------------------------------------------------------------------------------- 1 | %-------------------------------------------------------------- 2 | % This code generates Figure 3 (MT 75%) of the following paper: 3 | % "Multi-Task Compressive Sensing" (Preprint, 2007) 4 | % This example is modified from l1qc_example.m, an example 5 | % from l1magic. 6 | % Coded by: Shihao Ji, ECE, Duke University 7 | % last change: May. 15, 2007 8 | %-------------------------------------------------------------- 9 | clear all 10 | % 11 | total_count = 100; 12 | N = 512; % signal length 13 | T = 20; % number of spikes 14 | dN = 1; 15 | base = 40; % number of initial random measurements 16 | ns = 100; % number of additional random measurements 17 | sigma = 0.005; 18 | 19 | for count = 1:total_count 20 | count 21 | rand('state', count); 22 | randn('state', 2*count); 23 | % 24 | % random +/- 1 signal 25 | x1 = zeros(N,1); 26 | x2 = zeros(N,1); 27 | q = randperm(N); 28 | x1(q(1:T)) = sign(randn(T,1)); 29 | x2(q([1:T-5,T+1:T+5])) = sign(randn(T,1)); % 75% similarity 30 | 31 | % projection matrix 32 | A1 = randn(base,N); 33 | A1 = A1./repmat(sqrt(sum(A1.^2,2)),[1,N]); 34 | A2 = randn(base,N); 35 | A2 = A2./repmat(sqrt(sum(A2.^2,2)),[1,N]); 36 | A{1} = A1; 37 | A{2} = A2; 38 | % noisy observations 39 | e1 = sigma*randn(base,1); 40 | e2 = sigma*randn(base,1); 41 | y1 = A1*x1 + e1; 42 | y2 = A2*x2 + e2; 43 | y{1} = y1; 44 | y{2} = y2; 45 | 46 | for i = 1:ns 47 | 48 | K = base+i*dN; 49 | a1 = randn(dN,N); 50 | a1 = a1/sqrt(sum(a1.^2)); 51 | % noisy observations 52 | e1 = sigma*randn(dN,1); 53 | y1 = a1*x1 + e1; 54 | y{1} = [y{1};y1]; 55 | A{1} = [A{1};a1]; 56 | % 57 | a2 = randn(dN,N); 58 | a2 = a2/sqrt(sum(a2.^2)); 59 | % noisy observations 60 | e2 = sigma*randn(dN,1); 61 | y2 = a2*x2 + e2; 62 | y{2} = [y{2};y2]; 63 | A{2} = [A{2};a2]; 64 | 65 | % solve task 1 66 | a = 1e2/0.1; b = 1; 67 | weights = mt_CS(A{1},y{1},a,b,1e-8); 68 | x1_BCS = weights; 69 | 70 | % solve task 2 71 | a = 1e2/0.1; b = 1; 72 | weights = mt_CS(A{2},y{2},a,b,1e-8); 73 | x2_BCS = weights; 74 | 75 | % solve 1&2 by MT-CS 76 | a = 1e2/0.1; b = 1; 77 | weights = mt_CS(A,y,a,b,1e-8); 78 | x1_MT = weights(:,1); 79 | x2_MT = weights(:,2); 80 | 81 | % reconstruction error 82 | E1_BCS(count,i) = norm(x1-x1_BCS)/norm(x1); 83 | E2_BCS(count,i) = norm(x2-x2_BCS)/norm(x2); 84 | E1_MT(count,i) = norm(x1-x1_MT)/norm(x1); 85 | E2_MT(count,i) = norm(x2-x2_MT)/norm(x2); 86 | 87 | end 88 | save multi_results_75.mat E1_BCS E2_BCS E1_MT E2_MT; 89 | end 90 | save multi_results_75.mat E1_BCS E2_BCS E1_MT E2_MT; 91 | disp('Done!'); 92 | beep; 93 | -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/MFOCUSS.m: -------------------------------------------------------------------------------- 1 | function [X, gamma_ind, gamma_est, count] = MFOCUSS(Phi, Y, lambda, varargin) 2 | % MFOCUSS algorithm for the MMV model 3 | % 4 | % ============================== INPUTS ============================== 5 | % Phi : N X M dictionary matrix 6 | % 7 | % Y : N X L measurement matrix, i.e. Y = Phi * X + V. 8 | % 9 | % lambda : Regularization parameter. Generally it is close to the noise variance. 10 | % In the noiseless cases, simply setting lambda = 1e-10 leads to good performance. 11 | % In noisy cases, need the modified L-curve method to find optimal lambda. 12 | % See Ref [1] for details. 13 | % 14 | % 'p' : p-norm. p lies in [0,1]. Default value: p = 0.8 15 | % 16 | % 'PRUNE_GAMMA' : Threshold for prunning small gamma_i. 17 | % In noisy cases, you can set PRUNE_GAMMA = 1e-3 or 1e-4. 18 | % In strongly noisy cases (SNR<=5 dB), suggest to set PRUNE_GAMMA = 0.01; 19 | % Default value: MIN_GAMMA = 1e-4. 20 | % 21 | % 'MAX_ITERS' : Maximum number of iterations. 22 | % Default value: MAX_ITERS = 800 23 | % 24 | % 'EPSILON' : Threshold to stop the whole algorithm. 25 | % Default value: EPSILON = 1e-8 26 | % 27 | % 'PRINT' : Display flag. If PRINT = 1: show output; If PRINT = 0: supress output 28 | % Default value: PRINT = 0 29 | % 30 | % ============================== OUTPUTS ============================== 31 | % X : Estimated solution matrix(size: M X L) 32 | % gamma_ind : Indexes of nonzero gamma_i 33 | % gamma_est : Final values of all the gamma_i (including zeros). An M X 1 vector 34 | % count : number of iterations used 35 | % 36 | % ============== Examples of Commands =============== 37 | % [Example 1] 38 | % lambda = 1e-3; 39 | % [X,gamma_ind,gamma_est,count] ... 40 | % = MFOCUSS(Phi,Y,lambda,'p',0.8,'prune_gamma',1e-4,'max_iters',500,'epsilon',1e-8,'print',0); 41 | % 42 | % [Example 2] 43 | % lambda = 1e-5; 44 | % [X,gamma_ind,gamma_est,count] = MFOCUSS(Phi,Y, lambda); 45 | % 46 | % ============================== Reference ============================= 47 | % [1] Cotter, S.F.; Rao, B.D.; Kjersti Engan; Kreutz-Delgado, K.; 48 | % Sparse solutions to linear inverse problems with multiple measurement vectors 49 | % 50 | % ============================== Author ============================== 51 | % Zhilin Zhang (z4zhang@ucsd.edu) 52 | % Mainbody was written by David Wipf 53 | % 54 | % ============================== Version ============================== 55 | % 1.0 (05/12/2011) 56 | % 57 | % ============================== See Also ============================== 58 | % TSBL TMSBL ARSBL tMFOCUSS 59 | % 60 | 61 | % Dimension of the Problem 62 | [N M] = size(Phi); 63 | [N L] = size(Y); 64 | 65 | % Default Control Parameters 66 | PRUNE_GAMMA = 1e-4; % threshold for prunning small gamma_i 67 | p = 0.8; % p-norm 68 | EPSILON = 1e-8; % threshold for stopping iteration. 69 | MAX_ITERS = 800; % maximum iterations 70 | PRINT = 0; % not show progress information 71 | 72 | % get input argument values 73 | if(mod(length(varargin),2)==1) 74 | error('Optional parameters should always go by pairs\n'); 75 | else 76 | for i=1:2:(length(varargin)-1) 77 | switch lower(varargin{i}) 78 | case 'p' 79 | p = varargin{i+1}; 80 | case 'prune_gamma' 81 | PRUNE_GAMMA = varargin{i+1}; 82 | case 'epsilon' 83 | EPSILON = varargin{i+1}; 84 | case 'print' 85 | PRINT = varargin{i+1}; 86 | case 'max_iters' 87 | MAX_ITERS = varargin{i+1}; 88 | otherwise 89 | error(['Unrecognized parameter: ''' varargin{i} '''']); 90 | end 91 | end 92 | end 93 | 94 | if (PRINT) fprintf('\nRunning M-FOCUSS for the MMV Problem...\n'); end 95 | 96 | 97 | % Initializations 98 | gamma = ones(M,1); % initialization of gamma_i 99 | keep_list = [1:M]'; % record the index of nonzero gamma_i 100 | m = length(keep_list); % number of nonzero gamma_i 101 | mu = zeros(M,L); % initialization of the solution matrix 102 | count = 0; % record iterations 103 | 104 | 105 | % Learning loop 106 | while (1) 107 | 108 | % =========== Prune weights as their hyperparameters go to zero =========== 109 | if (min(gamma) < PRUNE_GAMMA ) 110 | index = find(gamma > PRUNE_GAMMA); 111 | gamma = gamma(index); 112 | Phi = Phi(:,index); % corresponding columns in Phi 113 | keep_list = keep_list(index); 114 | m = length(gamma); 115 | 116 | if (m == 0) break; end; 117 | end; 118 | 119 | 120 | % ====== Compute new weights ====== 121 | G = repmat(sqrt(gamma)',N,1); 122 | PhiG = Phi.*G; 123 | [U,S,V] = svd(PhiG,'econ'); 124 | 125 | [d1,d2] = size(S); 126 | if (d1 > 1) diag_S = diag(S); 127 | else diag_S = S(1); end; 128 | 129 | U_scaled = U(:,1:min(N,m)).*repmat((diag_S./(diag_S.^2 + sqrt(lambda) + 1e-16))',N,1); 130 | Xi = G'.*(V*U_scaled'); 131 | 132 | mu_old = mu; 133 | mu = Xi*Y; 134 | 135 | 136 | % *** Update hyperparameters *** 137 | gamma_old = gamma; 138 | mu2_bar = sum(abs(mu).^2,2); 139 | gamma = (mu2_bar/L).^(1-p/2); 140 | 141 | 142 | % ========= Check stopping conditions, etc. ========= 143 | count = count + 1; 144 | if (PRINT) disp(['iters: ',num2str(count),' num coeffs: ',num2str(m), ... 145 | ' gamma change: ',num2str(max(abs(gamma - gamma_old)))]); end; 146 | if (count >= MAX_ITERS) break; end; 147 | 148 | if (size(mu) == size(mu_old)) 149 | dmu = max(max(abs(mu_old - mu))); 150 | if (dmu < EPSILON) break; end; 151 | end; 152 | 153 | end; 154 | 155 | 156 | gamma_ind = sort(keep_list); 157 | gamma_est = zeros(M,1); 158 | gamma_est(keep_list,1) = gamma; 159 | 160 | % expand the final solution 161 | X = zeros(M,L); 162 | X(keep_list,:) = mu; 163 | 164 | if (PRINT) fprintf('\nM-FOCUSS finished !\n'); end 165 | return; -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/MSBL.m: -------------------------------------------------------------------------------- 1 | function [X,gamma_ind,gamma_est,count] = MSBL(Phi, Y, lambda, Learn_Lambda, varargin) 2 | % Sparse Bayesian Learning for Mulitple Measurement Vector (MMV) problems. 3 | % *** The version is suitable for noisy cases *** 4 | % It can also be used for single measurement vector problem without any modification. 5 | % 6 | % Command Format: 7 | % [X,gamma_ind,gamma_est,count] ... 8 | % = MSBL(Phi,Y,lambda,Learn_Lambda,'prune_gamma',1e-4,'max_iters',500,'epsilon',1e-8,'print',0); 9 | % [X,gamma_ind,gamma_est,count] = MSBL(Phi,Y, lambda, Learn_Lambda, 'prune_gamma', 1e-4); 10 | % [X,gamma_ind,gamma_est,count] = MSBL(Phi,Y, lambda, Learn_Lambda); 11 | % 12 | % ===== INPUTS ===== 13 | % Phi : N X M dictionary matrix 14 | % 15 | % Y : N X L measurement matrix 16 | % 17 | % lambda : Regularization parameter. Sometimes you can set it being the 18 | % noise variance value, which leads to sub-optimal 19 | % performance. The optimal value is generally slightly larger than the 20 | % noise variance vlaue. You need cross-validation methods or 21 | % other methods to find it. 22 | % 23 | % Learn_Lambda : If Learn_Lambda = 1, use the lambda as initial value and learn the optimal lambda 24 | % using its lambda learning rule. But note the 25 | % learning rule is not robust when SNR <= 15 dB. 26 | % If Learn_Lambda = 0, not use the lambda learning rule, but instead, use the 27 | % input lambda as the final value. 28 | % 29 | % 'PRUNE_GAMMA' : Threshold for prunning small hyperparameters gamma_i. 30 | % In noisy cases, you can set MIN_GAMMA = 1e-3 or 1e-4. 31 | % In strong noisy cases (e.g. SNR < 5 dB), set MIN_GAMMA = 1e-2 for better 32 | % performance. 33 | % [ Default value: MIN_GAMMA = 1e-3 ] 34 | % 35 | % 'MAX_ITERS' : Maximum number of iterations. 36 | % [ Default value: MAX_ITERS = 2000 ] 37 | % 38 | % 'EPSILON' : Threshold to stop the whole algorithm. 39 | % [ Default value: EPSILON = 1e-8 ] 40 | % 41 | % 'PRINT' : Display flag. If = 1: show output; If = 0: supress output 42 | % [ Default value: PRINT = 0 ] 43 | % 44 | % ===== OUTPUTS ===== 45 | % X : the estimated solution matrix, or called source matrix (size: M X L) 46 | % gamma_ind : indexes of nonzero gamma_i 47 | % gamma_est : final value of the M X 1 vector of hyperparameter values 48 | % count : number of iterations used 49 | % 50 | % 51 | % *** Reference *** 52 | % [1] David P. Wipf, Bhaskar D. Rao, An Empirical Bayesian Strategy for Solving 53 | % the Simultaneous Sparse Approximation Problem, IEEE Trans. Signal 54 | % Processing, Vol.55, No.7, 2007. 55 | % 56 | % *** Author *** 57 | % Zhilin Zhang (z4zhang@ucsd.edu) 58 | % (Modified based on David Wipf's original code such that the code is suitable for noisy cases) 59 | % 60 | % *** Version *** 61 | % 1.1 (02/12/2011) 62 | % 63 | % *** See Also *** 64 | % TSBL TMSBL 65 | % 66 | 67 | 68 | 69 | % Dimension of the Problem 70 | [N M] = size(Phi); 71 | [N L] = size(Y); 72 | 73 | % Default Control Parameters 74 | PRUNE_GAMMA = 1e-3; % threshold for prunning small hyperparameters gamma_i 75 | EPSILON = 1e-8; % threshold for stopping iteration. 76 | MAX_ITERS = 2000; % maximum iterations 77 | PRINT = 0; % don't show progress information 78 | 79 | 80 | if(mod(length(varargin),2)==1) 81 | error('Optional parameters should always go by pairs\n'); 82 | else 83 | for i=1:2:(length(varargin)-1) 84 | switch lower(varargin{i}) 85 | case 'prune_gamma' 86 | PRUNE_GAMMA = varargin{i+1}; 87 | case 'epsilon' 88 | EPSILON = varargin{i+1}; 89 | case 'print' 90 | PRINT = varargin{i+1}; 91 | case 'max_iters' 92 | MAX_ITERS = varargin{i+1}; 93 | otherwise 94 | error(['Unrecognized parameter: ''' varargin{i} '''']); 95 | end 96 | end 97 | end 98 | 99 | if (PRINT) fprintf('\nRunning MSBL ...\n'); end 100 | 101 | 102 | % Initializations 103 | gamma = ones(M,1); 104 | keep_list = [1:M]'; 105 | m = length(keep_list); 106 | mu = zeros(M,L); 107 | count = 0; % iteration count 108 | 109 | 110 | % *** Learning loop *** 111 | while (1) 112 | 113 | % *** Prune weights as their hyperparameters go to zero *** 114 | if (min(gamma) < PRUNE_GAMMA ) 115 | index = find(gamma > PRUNE_GAMMA); 116 | gamma = gamma(index); % use all the elements larger than MIN_GAMMA to form new 'gamma' 117 | Phi = Phi(:,index); % corresponding columns in Phi 118 | keep_list = keep_list(index); 119 | m = length(gamma); 120 | end; 121 | 122 | 123 | mu_old =mu; 124 | Gamma = diag(gamma); 125 | G = diag(sqrt(gamma)); 126 | 127 | % ****** estimate the solution matrix ***** 128 | [U,S,V] = svd(Phi*G,'econ'); 129 | 130 | [d1,d2] = size(S); 131 | if (d1 > 1) diag_S = diag(S); 132 | else diag_S = S(1); end; 133 | 134 | Xi = G * V * diag((diag_S./(diag_S.^2 + lambda + 1e-16))) * U'; 135 | mu = Xi * Y; 136 | 137 | % *** Update hyperparameters, i.e. Eq(18) in the reference *** 138 | gamma_old = gamma; 139 | mu2_bar = sum(abs(mu).^2,2)/L; 140 | 141 | Sigma_w_diag = real( gamma - (sum(Xi'.*(Phi*Gamma)))'); 142 | gamma = mu2_bar + Sigma_w_diag; 143 | 144 | % ***** the lambda learning rule ***** 145 | % You can use it to estimate the lambda when SNR >= 15 dB. But when SNR < 15 dB, 146 | % you'd better use other methods to estimate the lambda, since it is not robust 147 | % in strongly noisy cases. 148 | if Learn_Lambda == 1 149 | lambda = (norm(Y - Phi * mu,'fro')^2/L)/(N-m + sum(Sigma_w_diag./gamma_old)); 150 | end; 151 | 152 | 153 | % *** Check stopping conditions, etc. *** 154 | count = count + 1; 155 | if (PRINT) disp(['iters: ',num2str(count),' num coeffs: ',num2str(m), ... 156 | ' gamma change: ',num2str(max(abs(gamma - gamma_old)))]); end; 157 | if (count >= MAX_ITERS) break; end; 158 | 159 | if (size(mu) == size(mu_old)) 160 | dmu = max(max(abs(mu_old - mu))); 161 | if (dmu < EPSILON) break; end; 162 | end; 163 | 164 | end; 165 | 166 | 167 | % Expand hyperparameters 168 | gamma_ind = sort(keep_list); 169 | gamma_est = zeros(M,1); 170 | gamma_est(keep_list,1) = gamma; 171 | 172 | % expand the final solution 173 | X = zeros(M,L); 174 | X(keep_list,:) = mu; 175 | 176 | if (PRINT) fprintf('\nFinish running ...\n'); end 177 | return; 178 | 179 | 180 | 181 | -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/ReadMe.txt: -------------------------------------------------------------------------------- 1 | 2 | 1. Description: 3 | This package includes the codes of T-SBL/T-MSBL and simulations in the following paper 4 | 5 | Zhilin Zhang, Bhaskar D. Rao, Sparse Signal Recovery with 6 | Temporally Correlated Source Vectors Using Sparse Bayesian Learning, 7 | IEEE Journal of Selected Topics in Signal Processing, 8 | Special Issue on Adaptive Sparse Representation of 9 | Data and Applications in Signal and Image Processing, 2011 10 | 11 | Feel free to contact me for any questions (z4zhang@ucsd.edu) 12 | 13 | 14 | 2. Version: 2.3 (updated on July 30, 2011) 15 | 16 | 17 | 3. I strongly suggest you to spend 3 minutes to read the pdf file before use the codes: 18 | 19 | Z.Zhang, Master the Usage of TSBL and TMSBL in 3 Minutes 20 | 21 | 22 | 4. File Description: 23 | [ Core codes ] 24 | TSBL.m : code of T-SBL (version: 1.4) 25 | TMSBL.m : code of T-MSBL (version: 1.9) 26 | 27 | [ Auxiliary codes] 28 | perfSupp.m : code for measuring failure rates 29 | MSBL.m : code of M-SBL (used for performance comparison) 30 | MFOCUSS.m : code of M-FOCUSS (used for performance comparison) 31 | 32 | [ Demo codes] 33 | demo.m : demo for comparison of T-SBL,T-MSBL and M-SBL 34 | demo_fig3.m : demo for re-producing Fig.3 in the above IEEE paper 35 | demo_fig6_SNR10.m : demo for re-producing Fig.6 in the above IEEE paper 36 | demo_fig8.m : demo for re-producing Fig.8 in the above IEEE paper 37 | demo_time_varying.m : demo showing the use of T-MSBL for the time-varying sparsity model 38 | demo_identicalVector.m : demo showing the use of T-MSBL for the MMV model with identical source vectors 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/TMSBL.m: -------------------------------------------------------------------------------- 1 | function [X, gamma_ind, gamma_est, count, B_est] = TMSBL(Phi, Y, varargin) 2 | % Sparse Bayesian Learning for the MMV model exploiting temporal correlation of each source. 3 | % It is the simplified version of T-SBL. It can also be used in: 4 | % (1) the single measurement vector (SMV) model (i.e. Y and X are single vectors, not matrices) 5 | % (2) the MMV model when there is no temporal correlation. 6 | % 7 | % Command Format: 8 | % (1) For general noisy case (6-22dB): [X, gamma_ind, gamma_est, count, B_est] = TMSBL(Phi, Y) 9 | % (2) For strongly noisy case (<=6dB): [X, gamma_ind, gamma_est, count, B_est] = TMSBL(Phi, Y, 'noise','large') 10 | % For mild noisy case (6-22dB) : [X, gamma_ind, gamma_est, count, B_est] = TMSBL(Phi, Y, 'noise','mild') 11 | % For small noisy case (>22dB) : [X, gamma_ind, gamma_est, count, B_est] = TMSBL(Phi, Y, 'noise','small') 12 | % For noiseless case : [X, gamma_ind, gamma_est, count, B_est] = TMSBL(Phi, Y, 'noise','no') 13 | % [*** Just roughly guess SNR and then choose the above command for a good result (not necessarily optimal) ***] 14 | % (3) Choose your own values for the parameters; for examples: 15 | % [X,gamma_ind,gamma_est,count,B_est] ... 16 | % = TMSBL(Phi,Y, 'Prune_Gamma', 1e-5, 'lambda', 0.015, 'Learn_Lambda',1, ... 17 | % 'Enhance_Lambda',1, 'Matrix_Reg', 2*eye(5), 'MAX_ITERS', 500, ... 18 | % 'Fix_B', eye(L),'EPSILON', 1e-8, 'PRINT', 1 ); 19 | % [X,gamma_ind,gamma_est,count,B_est] = TMSBL(Phi,Y, 'Learn_Lambda', 1, 'prune_gamma', 1e-4); 20 | % 21 | % 22 | % ============================== INPUT ARGUMENTS ============================== 23 | % Phi : N X M dictionary matrix 24 | % Y : N X L available measurement matrix, i.e. Y = Phi * X + V. 25 | % 26 | % 'noise' : If 'noise' = 'no' : use default values of parameters for noiseless cases 27 | % If 'noise' = 'small' : use default values for the cases 28 | % when noise is small (e.g. SNR > 22dB) 29 | % If 'noise' = 'mild' : use default values for the cases 30 | % when noise is mild (e.g. 6 dB < SNR <= 22dB) 31 | % If 'noise' = 'large' : use default values for the cases 32 | % when noise is strong (e.g. SNR <= 6dB). 33 | % But if you have some knowledge about T-MSBL, 34 | % I suggest you to select your own values 35 | % for input arguments. 36 | % **** Note: if you use the input argument 'noise', then all input arguments are invalid, except to: **** 37 | % 'Fix_B', 'MAX_ITERS', 'EPSILON', 'PRINT' 38 | % **** This input argument choose a set of suitable parameter values for the four cases. **** 39 | % **** But the used parameter values may not be optimal. ***** 40 | % 41 | % 'PRUNE_GAMMA' : Threshold for prunning small hyperparameters gamma_i. 42 | % In noisy cases, you can set MIN_GAMMA = 1e-3 or 1e-4. 43 | % In strong noisy cases (e.g. SNR <= 6 dB), set MIN_GAMMA = 1e-2 for better 44 | % performance. 45 | % [ Default value: MIN_GAMMA = 1e-3 for samlle or medium scale data or 1e-2 for 46 | % large-scale data] 47 | % 48 | % 'lambda' : Initial value for the regularization parameter. Note that setting lambda being 49 | % noise variance only leads to sub-optimal performance. 50 | % The optimal value is generally 2-3 times the true noise variance. 51 | % However, in most cases you can use the lambda learning 52 | % rule to estimate it, if you set Learn_Lambda = 1 53 | % (1) In the noiseless cases, you can simply set lambda = 1e-15 or any other 54 | % very small values (e.g. 1e-10 is also okay), 55 | % and run the algorithm without using the lambda 56 | % learning rule (i.e. setting Learn_Lambda = 0). 57 | % (2) When SNR >= 7dB, you can use the lambda learning rule 58 | % to learn lambda. In this case, just roughly set an initial value for lambda, 59 | % and set Learn_Lambda = 1. 60 | % (3) When SNR <= 6 dB, the lambda learning rule may not be very good. So, you can use 61 | % cross-validation or other methods to estimate it. In this case, set Learn_Lambda = 0. 62 | % However, you can still roughly guess an initial value for 63 | % lambda and use the lambda learning rule. Sometimes 64 | % the performance is still better than other algorithms. 65 | % [ Default value: lambda = 1e-3 ] 66 | % 67 | % 'Learn_Lambda': (1) If Learn_Lambda = 1, use the lambda learning rule (thus the input lambda 68 | % is just as initial value). But note the learning rule is not very good 69 | % if SNR <= 6 dB (but still lead to better performance than some other algorithms) 70 | % (2) If Learn_Lambda = 0, do not use the lambda learning rule, but use the input 71 | % lambda as its final value. 72 | % [ Default value: Learn_Lambda = 1 ] 73 | % 74 | % 'Enhance_Lambda': (1) In large/mild noisy cases (<=22dB), set Enhance_Lambda = 1 for better performance 75 | % (2) In other cases, set Enhance_Lambda = 0 76 | % [ Default value: Enhance_Lambda = 1 ] 77 | % 78 | % 'Matrix_Reg' : Matrix used to add to the estimated matrix B in each iteration 79 | % (mainly used in noisy environment), e.g. c * I, where c is a positive scalar 80 | % and I is the identity matrix. 81 | % (1)When noise is small (eg. SNR > 22 dB), use zero matrix of the size L x L 82 | % (2)When noise is mild (eg. 7 dB - 22 dB), suggest to set Matrix_Reg = 2 * eye(L) 83 | % (3)When noise is large (eg.SNR < 6 dB), suggest to set Matrix_Reg = 4 * eye(L) 84 | % [ Default value: PLUS_MAT = 2*eye(L) ] 85 | % 86 | % 'Fix_B' : Do not adaptively estimate the matrix B, but instead use the given value 87 | % as the estimate of B. 88 | % [ Default: do not fix B (adaptive estimate B) ] 89 | % 90 | % 'MAX_ITERS' : Maximum number of iterations. 91 | % [ Default value: MAX_ITERS = 2000 ] 92 | % 93 | % 'EPSILON' : Threshold to stop the whole algorithm (based on the change of the soluiton matrix). 94 | % [ Default value: EPSILON = 1e-8 ] 95 | % 96 | % 'PRINT' : Display flag. 97 | % If 'PRINT'= 0: supress output; 98 | % If 'PRINT'= 1: print only parameter information 99 | % If 'PRINT'= 2: print the algorithm's progress information and parameter information 100 | % [ Default value: PRINT = 0 ] 101 | % 102 | % 103 | % ============================== OUTPUTS ============================== 104 | % X : The estimated solution matrix(size: M X L) 105 | % gamma_ind : Indexes of nonzero rows in the solution matrix 106 | % gamma_est : M X 1 vector of gamma_i values 107 | % count : Number of iterations used 108 | % B_est : Estimated matrix B 109 | % 110 | % 111 | % ============================== Reference ============================= 112 | % [1] Zhilin Zhang, Bhaskar D. Rao, Sparse Signal Recovery with 113 | % Temporally Correlated Source Vectors Using Sparse Bayesian Learning, 114 | % IEEE Journal of Selected Topics in Signal Processing, 115 | % Special Issue on Adaptive Sparse Representation of 116 | % Data and Applications in Signal and Image Processing, 2011 117 | % 118 | % 119 | % ============================== Author ============================== 120 | % Zhilin Zhang (z4zhang@ucsd.edu) 121 | % 122 | % ============================== Version ============================== 123 | % 1.1 (08/12/2010) 124 | % 1.2 (09/12/2010) 125 | % 1.3 (03/04/2011) 126 | % 1.4 (03/10/2011) 127 | % 1.5 (05/12/2011) revised the learning rule for lambda and B 128 | % 1.6 (05/20/2011) modified the lambda learning rule 129 | % 1.7 (05/21/2011) add 'noise' input argument to relax the burden of selecting parameters 130 | % 1.8 (05/23/2011) change the default value of 'MAX_ITERS' to 2000 131 | % 1.9 (07/30/2011) change the default value of 'lambda' and 'prune_gamma' 132 | % when status == 0 (noiseless case) 133 | % 1.10 (11/11/2011) modify default value of 'prune_gamma' for large-scale dataset 134 | % 1.11 (11/12/2011) modify default value for learning correlation 135 | % 136 | % ============================== See Also ============================== 137 | % TSBL ARSBL tMFOCUSS MSBL 138 | % 139 | 140 | 141 | 142 | % Dimension of the Problem 143 | [N M] = size(Phi); 144 | [N L] = size(Y); 145 | 146 | % Default Parameter Values for Any Cases 147 | EPSILON = 1e-8; % threshold for stopping iteration. 148 | MAX_ITERS = 2000; % maximum iterations 149 | PRINT = 1; % don't show progress information 150 | 151 | % Default Parameter Values (suitable for MILD noisy cases, e.g. 6-22dB) 152 | STATUS = -1; % use the default parameter values for mild noisy cases 153 | PRUNE_GAMMA = 1e-3; % threshold for prunning small hyperparameters gamma_i 154 | if N >= 500 | M>= 1000, PRUNE_GAMMA = 1e-2; end; 155 | lambda = 1e-3; % initial value for lambda 156 | LearnLambda = 1; % use the lambda learning rule to estimate lambda 157 | EnhanceLambda = 1; % enhance the lambda learning rule 158 | LearnB = 1; % adaptively estimate the covariance matrix B 159 | if L >= 10, LearnB = 0; B = eye(L); end; % set for long time-series 160 | MatrixReg = 2*eye(L); % added this matrix to regulate the estimated B in each iteration. 161 | 162 | 163 | 164 | if(mod(length(varargin),2)==1) 165 | error('Optional parameters should always go by pairs\n'); 166 | else 167 | for i=1:2:(length(varargin)-1) 168 | switch lower(varargin{i}) 169 | case 'noise' 170 | if strcmp(lower(varargin{i+1}),'no'), STATUS = 0; 171 | elseif strcmp(lower(varargin{i+1}),'small'), STATUS = 1; 172 | elseif strcmp(lower(varargin{i+1}),'mild'), STATUS = 2; 173 | elseif strcmp(lower(varargin{i+1}),'large'), STATUS = 3; 174 | else error(['Unrecognized Value for Input Argument ''Noise''']); 175 | end 176 | case 'prune_gamma' 177 | PRUNE_GAMMA = varargin{i+1}; 178 | case 'lambda' 179 | lambda = varargin{i+1}; 180 | case 'learn_lambda' 181 | LearnLambda = varargin{i+1}; 182 | if LearnLambda ~= 1 & LearnLambda ~= 0 183 | error(['Unrecognized Value for Input Argument ''Learn_Lambda''']); 184 | end 185 | case 'enhance_lambda' 186 | EnhanceLambda = varargin{i+1}; 187 | if EnhanceLambda ~= 1 & EnhanceLambda ~= 0 188 | error(['Unrecognized Value for Input Argument ''Enhance_Lambda''']); 189 | end 190 | case 'fix_b' 191 | B = varargin{i+1}; 192 | LearnB = 0; 193 | if size(B,1) ~= size(B,2) | size(B)~= L 194 | error(['Unrecognized Value for Input Argument ''Fix_B''']); 195 | end 196 | case 'matrix_reg' 197 | MatrixReg = varargin{i+1}; 198 | if size(MatrixReg,1) ~= size(MatrixReg,2) | size(MatrixReg,1) ~= L 199 | error(['Unrecognized Value for Input Argument ''Matrix_Reg''']); 200 | end 201 | case 'epsilon' 202 | EPSILON = varargin{i+1}; 203 | case 'print' 204 | PRINT = varargin{i+1}; 205 | case 'max_iters' 206 | MAX_ITERS = varargin{i+1}; 207 | otherwise 208 | error(['Unrecognized parameter: ''' varargin{i} '''']); 209 | end 210 | end 211 | end 212 | 213 | 214 | if STATUS == 0 215 | % default values for noiseless case 216 | PRUNE_GAMMA = 1e-4; % threshold to prune out small gamma_i 217 | if N >= 500 | M>= 1000, PRUNE_GAMMA = 1e-3; end; % set for large-scale dataset 218 | lambda = 1e-15; % fix lambda to this value, and, 219 | LearnLambda = 0; % do not use lambda learning rule, and, 220 | EnhanceLambda = 0; % of course, no need the enhancing strategy 221 | MatrixReg = 0; % do not regulate the estimate matrix B 222 | 223 | elseif STATUS == 1 224 | % default values for small noise cases, e.g. SNR > 22 dB 225 | PRUNE_GAMMA = 1e-3; % threshold to prune out small gamma_i 226 | if N >= 500 | M>= 1000, PRUNE_GAMMA = 1e-2; end; % set for large-scale dataset 227 | lambda = 1e-3; % initial value for lambda and, 228 | LearnLambda = 1; % use the lambda learning rule, and, 229 | EnhanceLambda = 0; % no need to enhance the lambda learning rule 230 | MatrixReg = 0; % do not regulate the estimate matrix B 231 | 232 | elseif STATUS == 2 233 | % Default values for mild noise cases, e.g. 6 dB < SNR <= 22 dB 234 | PRUNE_GAMMA = 1e-3; % threshold for prunning small hyperparameters gamma_i 235 | if N >= 500 | M>= 1000, PRUNE_GAMMA = 1e-2; end; % set for large-scale dataset 236 | lambda = 1e-3; % initial value for lambda, and, 237 | LearnLambda = 1; % use the lambda learning rule to estimate lambda 238 | EnhanceLambda = 1; % enhance the lambda learning rule for mild noisy cases 239 | MatrixReg = 2*eye(L); % added this matrix to regulate the estimated B in each iteration. 240 | 241 | elseif STATUS == 3 242 | % Default values for mild noise cases, e.g. SNR <= 6 dB 243 | PRUNE_GAMMA = 1e-2; % threshold for prunning small hyperparameters gamma_i 244 | lambda = 1e-2; % initial value for lambda, and, 245 | LearnLambda = 1; % use the lambda learning rule to estimate lambda 246 | EnhanceLambda = 1; % enhance the lambda learning rule for mild noisy cases 247 | MatrixReg = 4*eye(L); % added this matrix to regulate the estimated B in each iteration. 248 | 249 | end 250 | 251 | 252 | 253 | 254 | if PRINT == 2, 255 | fprintf('\n\nRunning T-MSBL...\n'); 256 | end 257 | 258 | if PRINT 259 | fprintf('\n====================================================\n'); 260 | fprintf(' Information about parameters...\n'); 261 | fprintf('====================================================\n'); 262 | if STATUS ~= -1, 263 | fprintf('You Select the Global Input Argument ''Noise''\n'); 264 | fprintf('So, some input arguments'' values set by you may be changed automatically\n'); 265 | fprintf('If you want to use your own input arguments'' values, do not use the ''Noise'' argument\n\n'); 266 | end 267 | fprintf('PRUNE_GAMMA : %e\n',PRUNE_GAMMA); 268 | fprintf('lambda : %e\n',lambda); 269 | fprintf('LearnLambda : %d\n',LearnLambda); 270 | fprintf('EnhanceLambda: %d\n',EnhanceLambda); 271 | fprintf('LearnB : %d\n',LearnB); 272 | if LearnB == 0, 273 | fprintf('Freeze B to\n'); disp(B); 274 | else 275 | fprintf('MatrixReg :\n');disp(MatrixReg); 276 | end 277 | fprintf('EPSILON : %e\n',EPSILON); 278 | fprintf('MAX_ITERS : %d\n\n',MAX_ITERS); 279 | end 280 | 281 | 282 | 283 | % Initializations 284 | gamma = ones(M,1); 285 | keep_list = [1:M]'; 286 | usedNum = length(keep_list); 287 | mu = zeros(M,L); 288 | count = 0; 289 | 290 | % Learning loop 291 | while (1) 292 | 293 | % =========== Prune weights as their hyperparameters go to zero =========== 294 | if (min(gamma) < PRUNE_GAMMA ) 295 | index = find(gamma > PRUNE_GAMMA); 296 | gamma = gamma(index); 297 | Phi = Phi(:,index); 298 | keep_list = keep_list(index); 299 | usedNum = length(gamma); 300 | end; 301 | 302 | % ================ Evaluate the mean (i.e. current solution) =============== 303 | mu_old = mu; 304 | Gamma = diag(gamma); 305 | G = diag(sqrt(gamma)); 306 | [U,S,V] = svd(Phi*G,'econ'); 307 | [d1,d2] = size(S); 308 | if (d1 > 1) diag_S = diag(S); 309 | else diag_S = S(1); end; 310 | Xi = G * V * diag((diag_S./(diag_S.^2 + lambda + 1e-16))) * U'; 311 | mu = Xi * Y; 312 | 313 | 314 | % ================= The Lambda Learning Rule ================= 315 | % Generally, it is good when SNR >=7 dB. But when SNR <= 6 dB, it is not good enough, 316 | % but still leads to better performance than many other algorithms. 317 | if LearnLambda == 1 318 | 319 | if EnhanceLambda == 1 320 | % enhance the lambda learning rule for robustness in mild/strong noisy cases 321 | PGP = diag(Phi*Gamma*Phi'); 322 | lambda = norm(Y - Phi * mu,'fro')^2/(N*L) + lambda * sum(PGP./(lambda+PGP))/N; 323 | else 324 | % the original learning rule in the paper 325 | PGP = Phi*Gamma*Phi'; 326 | lambda = norm(Y - Phi * mu,'fro')^2/(N*L) + lambda * trace(PGP*inv(lambda*eye(N)+PGP))/N; 327 | end 328 | end 329 | 330 | % ======== modify mu to compensate negative affect of temporal correlation ====== 331 | if LearnB == 1 % adaptively estimate B 332 | if usedNum <= N 333 | B = eye(L); % This strategy improves MSE of the solution 334 | else 335 | B = zeros(L,L); 336 | for i = 1 : usedNum 337 | B = B + mu(i,:)' * mu(i,:)/gamma(i); 338 | end 339 | B = B + MatrixReg; % this modification is used in low SNR cases 340 | B = B./norm(B,'fro'); 341 | end 342 | end; 343 | 344 | mub = mu * inv(B); 345 | mu2_bar = sum(mub.*mu,2)/L; 346 | 347 | 348 | % ===================== Update hyperparameters =================== 349 | Sigma_w_diag = real( gamma - (sum(Xi'.*(Phi*Gamma)))'); 350 | 351 | gamma_old = gamma; 352 | gamma = mu2_bar + Sigma_w_diag; 353 | 354 | 355 | % ================= Check stopping conditions, etc. ============== 356 | count = count + 1; 357 | if (PRINT==2) disp([' iters: ',num2str(count),' num coeffs: ',num2str(usedNum), ... 358 | ' gamma change: ',num2str(max(abs(gamma - gamma_old)))]); end; 359 | if (count >= MAX_ITERS) break; end; 360 | 361 | if (size(mu) == size(mu_old)) 362 | dmu = max(max(abs(mu_old - mu))); 363 | if (dmu < EPSILON) break; end; 364 | end; 365 | 366 | end; 367 | 368 | % Expand hyperparameters 369 | gamma_ind = sort(keep_list); 370 | gamma_est = zeros(M,1); 371 | gamma_est(keep_list,1) = gamma; 372 | 373 | % expand the final solution 374 | X = zeros(M,L); 375 | X(keep_list,:) = mu; 376 | 377 | B_est = B; 378 | 379 | if (PRINT==2) fprintf('\nT-MSBL finished !\n'); end 380 | return; 381 | 382 | 383 | -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/TSBL.m: -------------------------------------------------------------------------------- 1 | function [X, gamma_ind, gamma_est, count, B] = TSBL(Phi, Y, varargin) 2 | % Sparse Bayesian Learning for the MMV model considering temporal correlation among source vectors. 3 | % For fast implementation, see TMSBL.m 4 | % 5 | % Command Format: 6 | % (1) For low-SNR case (<=13dB): [X, gamma_ind, gamma_est, count, B] = TSBL(Phi, Y) 7 | % (2) For low-SNR case (<=13dB): [X, gamma_ind, gamma_est, count, B] = TSBL(Phi, Y, 'SNR','low') 8 | % For high-SNR case (>13dB) : [X, gamma_ind, gamma_est, count, B] = TSBL(Phi, Y, 'SNR','high') 9 | % For noiseless case : [X, gamma_ind, gamma_est, count, B] = TSBL(Phi, Y, 'SNR','inf') 10 | % [*** Just roughly guess SNR and then choose the above command for a good result (not necessarily optimal) ***] 11 | % (3) Choose your own values for the parameters; for examples: 12 | % [X,gamma_ind,gamma_est,count,B] ... 13 | % = TSBL(Phi,Y, 'Prune_Gamma', 1e-3, 'lambda', 0.015, 'Learn_Lambda',1, ... 14 | % 'Enhance_Lambda',1, 'MAX_ITERS', 500, 'EPSILON', 1e-8, 'PRINT', 1); 15 | % [X,gamma_ind,gamma_est,count,B] = TSBL(Phi,Y, 'prune_gamma', 1e-4); 16 | % 17 | % ============================== INPUTS ============================== 18 | % Phi : N X M dictionary matrix 19 | % 20 | % Y : N X L measurement matrix, i.e. Y = Phi * X + V. 21 | % 22 | % 'noise' : If 'SNR' = 'inf' : use default values of parameters for noiseless cases 23 | % If 'SNR' = 'high' : use default values for the cases 24 | % when noise is small (e.g. SNR > 13dB) 25 | % If 'SNR' = 'low' : use default values for the cases 26 | % when noise is strong (e.g. SNR <= 13dB). 27 | % But if you have some knowledge about T-SBL, 28 | % I suggest you to select your own values 29 | % for input arguments. 30 | % **** Note: if you use the input argument 'SNR', then all input arguments are invalid, except to: **** 31 | % 'MAX_ITERS', 'EPSILON', 'PRINT' 32 | % **** This input argument choose a set of suitable parameter values for the four cases. **** 33 | % **** But the used parameter values may not be optimal. ***** 34 | % 35 | % 'PRUNE_GAMMA' : Threshold for prunning small hyperparameters gamma_i. 36 | % In lightly noisy cases, you can set MIN_GAMMA = 1e-3 or 1e-4. 37 | % In strongly noisy cases, set MIN_GAMMA = 1e-2 for better performance. 38 | % [ Default value: MIN_GAMMA = 1e-2 ] 39 | % 40 | % 'lambda' : Initial value for the regularization parameter. Note that setting lambda being 41 | % noise variance only leads to sub-optimal performance. 42 | % For T-SBL, the optimal value is generally close to the noise variance. 43 | % However, in most cases you can use the lambda learning 44 | % rule to learn a near-optimal value, if you set Learn_Lambda = 1 45 | % (1) In the noiseless cases, you can simply set lambda = 1e-10 or any other 46 | % very small values (e.g. 1e-9, 1e-11 are also okay), 47 | % and run the algorithm without using the lambda 48 | % learning rule (i.e. setting Learn_Lambda = 0). 49 | % (2) When SNR >= 5dB, you can use the lambda learning rule 50 | % to learn lambda. In this case, just roughly input an initial value for lambda, 51 | % and set Learn_Lambda = 1. 52 | % (3) When SNR < 5dB, the lambda learning rule is not robust. So, you may need 53 | % to use cross-validation or other methods to estimate it. 54 | % In this case, set Learn_Lambda = 0, and try your best to get an 55 | % optimal value for lambda as the input. You can 56 | % roughly estimate the noise variance as the lambda value. 57 | % [ Default value: lambda = 1e-3 ] 58 | % 59 | % 'Learn_Lambda': (1) If Learn_Lambda = 1, use the lambda learning rule (thus the input lambda 60 | % is just as initial value). But note the learning rule is not very good 61 | % if SNR <= 5 dB (but still lead to better performance than some other algorithms) 62 | % (2) If Learn_Lambda = 0, do not use the lambda learning rule, but use the input 63 | % lambda as its final value. 64 | % [ Default value: Learn_Lambda = 1 ] 65 | % 66 | % 'Enhance_Lambda': (1) In low-SNR case (<=13dB), set Enhance_Lambda = 1 for better performance 67 | % (2) In other cases, set Enhance_Lambda = 0 68 | % [ Default value: Enhance_Lambda = 1 ] 69 | % 70 | % 'MAX_ITERS' : Maximum number of iterations. 71 | % [ Default value: MAX_ITERS = 1000 ] 72 | % 73 | % 'EPSILON' : Threshold to stop the whole algorithm. 74 | % [ Default value: EPSILON = 1e-8 ] 75 | % 76 | % 'PRINT' : Display flag. If = 1: show output; If = 0: supress output 77 | % [ Default value: PRINT = 0 ] 78 | % 79 | % ============================== OUTPUTS ============================== 80 | % X : The estimated solution matrix(size: M X L) 81 | % gamma_ind : Indexes of nonzero rows in the solution matrix 82 | % gamma_est : M X 1 vector of gamma_i values 83 | % B : Estimated matrix B 84 | % count : Number of iterations used 85 | % 86 | % ============= See Also ================================================ 87 | % TMSBL ARSBL MSBL tMFOCUSS 88 | % 89 | % ============= Reference =============================================== 90 | % [1] Zhilin Zhang, Bhaskar D. Rao, Sparse Signal Recovery with 91 | % Temporally Correlated Source Vectors Using Sparse Bayesian Learning, 92 | % IEEE Journal of Selected Topics in Signal Processing, 93 | % Special Issue on Adaptive Sparse Representation of 94 | % Data and Applications in Signal and Image Processing, 2011 95 | % 96 | % 97 | % ============= Author ============= 98 | % Zhilin Zhang (z4zhang@ucsd.edu) 99 | % 100 | % ============= Version ============= 101 | % 1.0 (07/31/2010) 102 | % 1.1 (09/12/2010) 103 | % 1.2 (03/05/2011) 104 | % 1.3 (05/20/2011) modify the lambda rule 105 | % 1.4 (05/21/2011) modify the whole 106 | 107 | 108 | 109 | 110 | % Dimension of the Problem 111 | [N M] = size(Phi); 112 | [N L] = size(Y); 113 | 114 | % Default Parameter Values for Any Cases 115 | EPSILON = 1e-8; % threshold for stopping iteration. 116 | MAX_ITERS = 2000; % maximum iterations 117 | PRINT = 0; % don't show progress information 118 | 119 | 120 | % Default Parameter Values (suitable for most noisy cases, SNR <= 13 dB) 121 | STATUS = -1; % use the default parameter values for mild noisy cases 122 | PRUNE_GAMMA = 1e-2; % threshold for prunning small hyperparameters gamma_i 123 | lambda = 1e-3; % initial value for lambda 124 | LearnLambda = 1; % use the lambda learning rule to estimate lambda 125 | EnhanceLambda = 1; % use the enhance strategy to the lambda learning rule 126 | 127 | 128 | if(mod(length(varargin),2)==1) 129 | error('Optional parameters should always go by pairs\n'); 130 | else 131 | for i=1:2:(length(varargin)-1) 132 | switch lower(varargin{i}) 133 | case 'snr' 134 | if strcmp(lower(varargin{i+1}),'inf'), STATUS = 0; 135 | elseif strcmp(lower(varargin{i+1}),'high'), STATUS = 1; 136 | elseif strcmp(lower(varargin{i+1}),'low'), STATUS = 2; 137 | else error(['Unrecognized Value for Input Argument ''SNR''']); 138 | end 139 | case 'prune_gamma' 140 | PRUNE_GAMMA = varargin{i+1}; 141 | case 'lambda' 142 | lambda = varargin{i+1}; 143 | case 'learn_lambda' 144 | LearnLambda = varargin{i+1}; 145 | if LearnLambda ~= 1 & LearnLambda ~= 0 146 | error(['Unrecognized Value for Input Argument ''Learn_Lambda''']); 147 | end 148 | case 'enhance_lambda' 149 | EnhanceLambda = varargin{i+1}; 150 | if EnhanceLambda ~= 1 & EnhanceLambda ~= 0 151 | error(['Unrecognized Value for Input Argument ''Enhance_Lambda''']); 152 | end 153 | case 'epsilon' 154 | EPSILON = varargin{i+1}; 155 | case 'print' 156 | PRINT = varargin{i+1}; 157 | case 'max_iters' 158 | MAX_ITERS = varargin{i+1}; 159 | otherwise 160 | error(['Unrecognized parameter: ''' varargin{i} '''']); 161 | end 162 | end 163 | end 164 | 165 | 166 | if STATUS == 0 167 | % default values for noiseless case 168 | PRUNE_GAMMA = 1e-4; % threshold to prune out small gamma_i 169 | lambda = 1e-10; % fix lambda to this value, and, 170 | LearnLambda = 0; % do not use lambda learning rule, and, 171 | EnhanceLambda = 0; % do not use the enhance strategy 172 | 173 | elseif STATUS == 1 174 | % default values for high-SNR cases, e.g. SNR > 13 dB 175 | PRUNE_GAMMA = 1e-3; % threshold to prune out small gamma_i 176 | lambda = 1e-3; % initial value for lambda and, 177 | LearnLambda = 1; % use the lambda learning rule, and, 178 | EnhanceLambda = 0; % do not use the enhance strategy 179 | 180 | elseif STATUS == 2 181 | % Default values for low-SNR cases, e.g. SNR <= 13 dB 182 | PRUNE_GAMMA = 1e-2; % threshold for prunning small hyperparameters gamma_i 183 | lambda = 1e-2; % initial value for lambda, and, 184 | LearnLambda = 1; % use the lambda learning rule to estimate lambda 185 | EnhanceLambda = 1; % use the enhance strategy to the lambda learning rule for low SNR cases 186 | 187 | end 188 | 189 | 190 | if PRINT 191 | fprintf('\n====================================================\n'); 192 | fprintf(' Running T-SBL...Information about parameters...\n'); 193 | fprintf('====================================================\n'); 194 | if STATUS ~= -1, 195 | fprintf('You Select the Global Input Argument ''Noise''. Other Arguments May be Invalid\n'); 196 | end 197 | fprintf('PRUNE_GAMMA : %e\n',PRUNE_GAMMA); 198 | fprintf('lambda : %e\n',lambda); 199 | fprintf('LearnLambda : %d\n',LearnLambda); 200 | fprintf('EnhanceLambda: %d\n',EnhanceLambda); 201 | fprintf('EPSILON : %e\n',EPSILON); 202 | fprintf('MAX_ITERS : %d\n\n',MAX_ITERS); 203 | end 204 | 205 | 206 | 207 | % Transfer the multiple measurements into single measurement by vectorization 208 | y = reshape(Y',N*L,1); 209 | D = kron(Phi,eye(L)); 210 | 211 | % Initialization 212 | Sigma0 = repmat(eye(L),[1 1 M]); 213 | gamma = ones(M,1); 214 | keep_list = [1:M]'; 215 | usedNum = length(keep_list); 216 | mu_x = zeros(M*L,1); 217 | count = 0; 218 | 219 | % Iteration 220 | while (1) 221 | count = count + 1; 222 | 223 | %=========== Prune weights as their hyperparameters go to zero============== 224 | 225 | if (min(gamma) < PRUNE_GAMMA) 226 | index = find(gamma > PRUNE_GAMMA); 227 | usedNum = length(index); 228 | 229 | % prune gamma and associated components in Sigma0, Phi 230 | gamma = gamma(index); 231 | Sigma0 = Sigma0(:,:,index); 232 | Phi = Phi(:,index); 233 | D = kron(Phi,eye(L)); 234 | 235 | % post-processing 236 | keep_list = keep_list(index); 237 | end 238 | 239 | %=================== Compute new weights ================= 240 | mu_old = mu_x; 241 | 242 | DBD = zeros(N*L); 243 | for i = 1 : usedNum 244 | DBD = DBD + D(:, (i-1)*L+1: i*L ) * Sigma0(:,:,i) * D(:, (i-1)*L+1: i*L )'; 245 | end 246 | H = D'*inv(DBD + lambda * eye(N*L)); 247 | Ht = H*y; HD = H * D; 248 | 249 | mu_x = zeros(usedNum*L,1); 250 | Sigma_x = repmat(zeros(L),[1 1 usedNum]); 251 | Cov_x = Sigma_x; 252 | B = zeros(L,L); 253 | for i = 1 : usedNum 254 | seg = [(i-1)*L+1 : i*L]; 255 | mu_x(seg) = Sigma0(:,:,i) * Ht(seg); % solution 256 | Sigma_x(:,:,i) = Sigma0(:,:,i) - Sigma0(:,:,i) * HD(seg,seg) * Sigma0(:,:,i); 257 | Cov_x(:,:,i) = Sigma_x(:,:,i) + mu_x(seg) * mu_x(seg)'; 258 | 259 | % Estimate covariance matrix 260 | B = B + Cov_x(:,:,i)/gamma(i); 261 | end 262 | 263 | if usedNum <= N, 264 | B = eye(L); invB = B; 265 | else 266 | B = B./norm(B,'fro'); 267 | invB = inv(B); 268 | end 269 | 270 | gamma_old = gamma; 271 | if LearnLambda == 1 % learn lambda (note that this rule is not robust at low SNR cases) 272 | 273 | for i = 1 : usedNum 274 | gamma(i) = sum(sum(invB .* Cov_x(:,:,i)))/L; % learn gamma_i 275 | Sigma0(:,:,i) = B * gamma(i); 276 | end 277 | 278 | if EnhanceLambda == 1 % modify the lambda rule for low/mild SNR cases 279 | PGP = diag(Phi*diag(gamma)*Phi'); 280 | lambda = norm(y - D * mu_x,2)^2/(N*L) + lambda * sum(PGP./(lambda+PGP))/N; 281 | else % use the original lambda rule for high SNR cases 282 | PGP = Phi*diag(gamma)*Phi'; 283 | lambda = norm(y - D * mu_x,2)^2/(N*L) + lambda * trace(PGP*inv(lambda*eye(N)+PGP))/N; 284 | end 285 | 286 | else % not learn lambda, but only learn gamma_i 287 | for i = 1 : usedNum 288 | gamma(i) = sum(sum(invB .* Cov_x(:,:,i)))/L; 289 | Sigma0(:,:,i) = B * gamma(i); 290 | end 291 | end 292 | 293 | 294 | % ================= Check stopping conditions, etc. ============== 295 | if (PRINT) disp([' iters: ',num2str(count),' num coeffs: ',num2str(usedNum), ... 296 | ' gamma change: ',num2str(max(abs(gamma - gamma_old)))]); end; 297 | if (count >= MAX_ITERS), if PRINT, fprintf('Reach max iterations. Stop\n\n'); end; break; end; 298 | 299 | if (size(mu_x) == size(mu_old)) 300 | dmu = max(max(abs(mu_old - mu_x))); 301 | if (dmu < EPSILON) break; end; 302 | end; 303 | 304 | 305 | end; 306 | 307 | 308 | % Expand hyperparameters 309 | gamma_ind = sort(keep_list); 310 | gamma_est = zeros(M,1); 311 | gamma_est(keep_list,1) = gamma; 312 | 313 | 314 | % Transfer to original weight matrix size (vec -> matrix) 315 | X = zeros(M,L); 316 | for i = 1 : usedNum 317 | X(keep_list(i),:) = mu_x((i-1)*L+1 : i*L)'; 318 | end 319 | 320 | if (PRINT) fprintf('\nT-SBL finished !\n'); end 321 | return; 322 | 323 | 324 | 325 | -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/demo.m: -------------------------------------------------------------------------------- 1 | % Goal: Compare T-MSBL, T-SBL, MSBL, MFOCUSS 2 | % In this demo, you will see T-MSBL and T-SBL outperform other two 3 | % algorithms. Particularly, please note that T-MSBL and T-SBL 4 | % adaptively learn the regularization parameter lambda, while MSBL 5 | % freezes lambda to the true noise variance (which is a near 6 | % optimal value). When T-MSBL and T-SBL freeze their lambda to the 7 | % optimal values, their performance can be further better. 8 | % 9 | % This demo also shows the lambda learning rule of MSBL is very 10 | % poor. It should not be used in low SNR cases. 11 | % 12 | % Author: Zhilin Zhang (z4zhang@ucsd.edu) 13 | % Date: May 20, 2011 14 | % Version: 1.1 15 | 16 | 17 | clear all; 18 | 19 | % Experiment Assignment 20 | run_TMSBL = 1; % Run T-MSBL 21 | run_TSBL = 1; % Run T-SBL 22 | run_MSBL = 1; % Run MSBL 23 | run_FOCUSS= 1; % Run M-FOCUSS 24 | iterNum = 50; % Trial number (i.e. number of repeating the experiment) 25 | % For statistical result, iterNum should not less than 50 26 | 27 | % Problem dimension 28 | N = 30; % Row number of the dictionary matrix 29 | M = N * 5; % Column number of the dictionary matrix 30 | L = 3; % Number of measurement vectors 31 | K = 7; % Number of nonzero rows (i.e. source number) in the solution matrix 32 | beta = ones(K,1)*0.9; % Temporal correlation of each nonzero row in the solution matrix. 33 | % In biomedical applications, such as EEG, MRI,etc, such temporal 34 | % correlation could be higher than 0.95 35 | 36 | %==================================================================== 37 | SNR = 10; % Note: When you change the SNR, you may need to accordingly change 38 | % the input arguments for each algorithm 39 | %==================================================================== 40 | 41 | for it = 1 : iterNum 42 | fprintf('\n\nTrial #%d:\n',it); 43 | 44 | % Generate dictionary matrix with columns draw uniformly from the surface of a unit hypersphere 45 | Phi = randn(N,M); 46 | Phi = Phi./(ones(N,1)*sqrt(sum(Phi.^2))); 47 | 48 | % Generate the K nonzero rows, each row being an AR(1) process. All the AR(1) 49 | % processes have different AR coefficients, which are randomly chosen from [0.7,1) 50 | nonzeroW(:,1) = randn(K,1); 51 | for i = 2 : L*100 52 | nonzeroW(:,i) = beta .* nonzeroW(:,i-1) + sqrt(1-beta.^2).*(ones(K,1).*randn(K,1)); 53 | end 54 | nonzeroW = nonzeroW(:,end-L+1:end); % Ensure the AR processes are stable 55 | 56 | % Normalize each row 57 | nonzeroW = nonzeroW./( sqrt(sum(nonzeroW.^2,2)) * ones(1,L) ); 58 | 59 | % Rescale each row such that the squared row-norm distributes in [1,scalefactor] 60 | scalefactor = 3; 61 | mag = rand(1,K); mag = mag - min(mag); 62 | mag = mag/(max(mag))*(scalefactor-1) + 1; 63 | nonzeroW = diag(sqrt(mag)) * nonzeroW; 64 | 65 | % Locations of nonzero rows are randomly chosen 66 | ind = randperm(M); 67 | indice = ind(1:K); 68 | Wgen = zeros(M,L); 69 | Wgen(indice,:) = nonzeroW; 70 | 71 | % Noiseless signal 72 | signal = Phi * Wgen; 73 | 74 | % Observation noise 75 | stdnoise = std(reshape(signal,N*L,1))*10^(-SNR/20); 76 | noise = randn(N,L) * stdnoise; 77 | 78 | % Noisy signal 79 | Y = signal + noise; 80 | 81 | 82 | %=================== T-MSBL (adaptively learn the lambda) ==================== 83 | if run_TMSBL == 1, 84 | tic; 85 | 86 | % Depends on the SNR, choosing suitable values for input arguments: 87 | % If no noise, Weight = TMSBL(Phi, Y, 'noise','no'); 88 | % If SNR >= 23 dB, Weight = TMSBL(Phi, Y, 'noise','small'); 89 | % If 6dB < SNR <= 22 dB, Weight = TMSBL(Phi, Y, 'noise','mild'); 90 | % If SNR <= 6 dB, Weight = TMSBL(Phi, Y, 'noise','large'); 91 | % Note that you need not to know exactly the SNR value. Just roughly estimate it. 92 | % For example, when the true SNR is 6 dB, either set 'noise'='mild' or 93 | % 'noise'='large' has the similar performance. 94 | % 95 | % Note: See the TMSBL code for details on input arguments ans set your 96 | % own values for specific problems. 97 | 98 | Weight1 = TMSBL(Phi, Y, 'noise','mild'); 99 | 100 | time1 = toc; 101 | TIME1(it) = time1; 102 | 103 | 104 | % failure rate: F1 = 1: perfect recovery; F1 = 0: worst recovery 105 | F1 = perfSupp(Weight1,indice,'firstlargest', K); 106 | fail_TMSBL(it) = (F1~=1); 107 | 108 | % Mean Square Error (MSE) 109 | mse_TMSBL(it) = (norm(Wgen - Weight1,'fro')/norm(Wgen,'fro'))^2; 110 | 111 | fprintf(' T-MSBL(learn lambda): time = %5.2f; Findex = %3.2f, Ave-MSE = %3.2f%%; Ave-Time = %4.3f\n',... 112 | time1,F1,mean(mse_TMSBL)*100,mean(TIME1)); 113 | end 114 | %==================================================================== 115 | 116 | 117 | 118 | %======================= T-SBL (adaptively learn the lambda)========== 119 | if run_TSBL == 1, 120 | % Similar to TMSBL, see the code for details about the input argument 121 | tic; 122 | Weight2 = TSBL(Phi, Y, 'SNR','low'); 123 | time2 = toc; 124 | TIME2(it) = time2; 125 | 126 | 127 | % failure rate: F2 = 1: perfect recovery; F2 = 0: worst recovery 128 | F2 = perfSupp(Weight2,indice,'firstlargest', K); 129 | fail_TSBL(it) = (F2~=1); 130 | 131 | % Mean Square Error (MSE) 132 | mse_TSBL(it) = (norm(Wgen - Weight2,'fro')/norm(Wgen,'fro'))^2; 133 | 134 | fprintf(' T-SBL(learn lambda): time = %5.2f; Findex = %3.2f, Ave-MSE = %3.2f%%; Ave-Time = %4.3f\n',... 135 | time2,F2,mean(mse_TSBL)*100,mean(TIME2)); 136 | end 137 | 138 | 139 | 140 | 141 | %================== MSBL (adaptively learn the lambda) ======================= 142 | lambda = 1e-2; % Initial value for the regularization parameter. 143 | Learn_Lambda = 1; % Using its lambda learning rule 144 | 145 | if run_MSBL == 1, 146 | tic; 147 | Weight3 = MSBL(Phi,Y, lambda, Learn_Lambda); 148 | time3 = toc; 149 | TIME3(it) = time3; 150 | 151 | 152 | % Failure rate 153 | F3 = perfSupp(Weight3,indice,'firstlargest', K); 154 | fail_MSBL(it) = (F3~=1); 155 | 156 | % MSE 157 | perf_MSBL(it) = (norm(Wgen - Weight3,'fro')/norm(Wgen,'fro'))^2; 158 | 159 | fprintf(' MSBL(learn lambda): time = %5.2f; Findex = %3.2f, Ave-MSE = %3.2f%%; Ave-Time = %4.3f\n',... 160 | time3,F3,mean(perf_MSBL)*100,mean(TIME3)); 161 | 162 | 163 | % now MSBL using the true noise variance as the lambda value and freeze it 164 | lambda0 = stdnoise^2; 165 | Learn_Lambda0 = 0; % do not use its lambda learning rule 166 | tic; 167 | Weight33 = MSBL(Phi,Y, lambda0, Learn_Lambda0); 168 | time33 = toc; 169 | TIME33(it) = time33; 170 | 171 | 172 | % Failure rate 173 | F33 = perfSupp(Weight33,indice,'firstlargest', K); 174 | fail_MSBL33(it) = (F33~=1); 175 | 176 | % MSE 177 | perf_MSBL33(it) = (norm(Wgen - Weight33,'fro')/norm(Wgen,'fro'))^2; 178 | 179 | fprintf('MSBL(lambda=noise var): time = %5.2f; Findex = %3.2f, Ave-MSE = %3.2f%%; Ave-Time = %4.3f\n',... 180 | time33,F33,mean(perf_MSBL33)*100,mean(TIME33)); 181 | 182 | 183 | end 184 | 185 | 186 | % ====================== MFOCUSS with near-optimal regularization ========= 187 | if run_FOCUSS == 1, 188 | tic; 189 | lambda_opt = stdnoise^2; % use the noise variance 190 | Weight4 = MFOCUSS(Phi, Y, lambda_opt); 191 | 192 | time4 = toc; 193 | TIME4(it) = time4; 194 | 195 | 196 | % Failure rate 197 | F4 = perfSupp(Weight4,indice,'firstlargest', K); 198 | fail_MFOCUSS(it) = (F4~=1); 199 | 200 | % MSE 201 | perf_MFOCUSS(it) = (norm(Wgen - Weight4,'fro')/norm(Wgen,'fro'))^2; 202 | 203 | fprintf('MFOCUSS(lambda=noise var): time = %5.2f; Findex = %3.2f, Ave-MSE = %3.2f%%; Ave-Time = %4.3f\n',... 204 | time4,F4,mean(perf_MFOCUSS)*100,mean(TIME4)); 205 | end 206 | 207 | end 208 | 209 | 210 | 211 | 212 | -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/demo_fig3.m: -------------------------------------------------------------------------------- 1 | % Goal: This demo produces the Fig.3(d) when source number is 14 2 | 3 | clear; 4 | 5 | % Experiment variables (will be varied to produce each sub-figures) 6 | % Here we produce Fig.3(d) when source nubmer is 14 7 | K = 14; % soure number 8 | beta = ones(K,1)*0.99; % temporal correlation of each source 9 | 10 | % problem dimension 11 | N = 25; % column number of the dictionary matrix 12 | M = 125; % row number of the dictionary matrix 13 | L = 4; % number of measurement vectors 14 | iterNum = 50; % In the paper we repeated 1000 times. But here we repeat 50 times to save time 15 | 16 | for it = 1:iterNum 17 | fprintf('\n\nTrial #%d:\n',it); 18 | 19 | % Dictionary matrix with columns draw uniformly from the surface of a unit hypersphere 20 | Phi = randn(N,M); 21 | Phi = Phi./(ones(N,1)*sqrt(sum(Phi.^2))); 22 | 23 | % Generate L source vectors 24 | nonzeroW(:,1) = randn(K,1); 25 | alpha = ones(K,1); 26 | for i = 2 : L*20 27 | nonzeroW(:,i) = beta .* nonzeroW(:,i-1) + sqrt(1-beta.^2).*(sqrt(alpha).*randn(K,1)); 28 | end 29 | % get stable AR signals with length L 30 | nonzeroW = nonzeroW(:,end-L+1:end); 31 | 32 | % normalize along row 33 | nonzeroW = nonzeroW./( sqrt(sum(nonzeroW.^2,2)) * ones(1,L) ); 34 | 35 | % rescale rows such that their norms uniformly distribute in [1/3, 1] 36 | AdjustNorm = 1/3+rand(K,1)*(1-1/3); 37 | nonzeroW = diag(AdjustNorm) * nonzeroW; 38 | 39 | % select active sources at random locations 40 | ind = randperm(M); 41 | indice = ind(1:K); 42 | Wgen = zeros(M,L); 43 | Wgen(indice,:) = nonzeroW; 44 | 45 | % noiseless signal 46 | Y = Phi * Wgen; 47 | 48 | 49 | %============================ 1.T-MSBL ========================== 50 | [Weight1, gamma_ind1, gamma_est1, count1] = TMSBL(Phi, Y, 'noise','no'); 51 | 52 | F1 = perfSupp(Weight1,indice,'firstlargest',K); 53 | FR_TMSBL(it) = (F1~=1); 54 | 55 | MSE1 = (norm(Weight1-Wgen,'fro')/norm(Wgen,'fro'))^2; 56 | MSE_TMSBL(it) = MSE1; 57 | 58 | fprintf(' T-MSBL: MSE = %3.2f%%; Fail_Rate = %4.3f%%; \n',mean(MSE_TMSBL)*100,mean(FR_TMSBL)*100); 59 | 60 | 61 | %============================ 2.T-SBL ========================== 62 | [Weight2, gamma_ind2, gamma_est2, count2, B2] = TSBL(Phi, Y, 'SNR','inf'); 63 | 64 | F2 = perfSupp(Weight2,indice,'firstlargest',K); 65 | FR_TSBL(it) = (F2~=1); 66 | 67 | MSE2 = (norm(Weight2-Wgen,'fro')/norm(Wgen,'fro'))^2; 68 | MSE_TSBL(it) = MSE2; 69 | 70 | fprintf(' T-SBL: MSE = %3.2f%%; Fail_Rate = %4.3f%%; \n',mean(MSE_TSBL)*100,mean(FR_TSBL)*100); 71 | 72 | 73 | 74 | %============================ 3.MSBL ========================== 75 | lambda1 = 1e-10; 76 | Learn_Lambda = 0; 77 | [Weight3,gamma3,gamma_used3,count3] = MSBL(Phi, Y, lambda1, Learn_Lambda); 78 | 79 | F3 = perfSupp(Weight3,indice,'firstlargest',K); 80 | FR_MSBL(it) = (F3~=1); 81 | 82 | MSE3 = (norm(Weight3-Wgen,'fro')/norm(Wgen,'fro'))^2; 83 | MSE_MSBL(it) = MSE3; 84 | 85 | fprintf(' M-SBL: MSE = %3.2f%%; Fail_Rate = %4.3f%%; \n',mean(MSE_MSBL)*100,mean(FR_MSBL)*100); 86 | 87 | 88 | 89 | %============================ 4.MFOCUSS =========================== 90 | [Weight4, gamma_ind4, gamma_est4, count4] = MFOCUSS(Phi, Y, lambda1); 91 | 92 | F4 = perfSupp(Weight4,indice,'firstlargest',K); 93 | FR_FOCUSS(it) = (F4~=1); 94 | 95 | MSE4 = (norm(Weight4-Wgen,'fro')/norm(Wgen,'fro'))^2; 96 | MSE_FOCUSS(it) = MSE4; 97 | 98 | fprintf(' FOCUSS: MSE = %3.2f%%; Fail_Rate = %4.3f%%; \n',mean(MSE_FOCUSS)*100,mean(FR_FOCUSS)*100); 99 | 100 | 101 | end -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/demo_fig6_SNR10.m: -------------------------------------------------------------------------------- 1 | % This demo produces the Fig.6 when SNR = 10dB and T-MSBL and MSBL use 2 | % their lambda lerning rules to learn lambda during learning procedures 3 | 4 | clear; 5 | 6 | SNR = 10; % SNR 7 | N = 25; % row number of the dictionary matrix 8 | M = 125; % column number of the dictionary matrix 9 | K = 7; % source number 10 | L = 4; % number of measurement vectors 11 | beta = ones(K,1)*(0.8); % temporal correlation of each source 12 | 13 | for it = 1:100 % repeat 100 times for this demo 14 | 15 | fprintf('\nTrial #%d: \n',it); 16 | 17 | % dictionary matrix with columns draw uniformly from the surface of a unit hypersphere 18 | Phi = randn(N,M); 19 | Phi = Phi./(ones(N,1)*sqrt(sum(Phi.^2))); 20 | 21 | % Generate L source vectors 22 | nonzeroW(:,1) = randn(K,1); 23 | alpha = ones(K,1); 24 | for i = 2 : L*100 25 | nonzeroW(:,i) = beta .* nonzeroW(:,i-1) + sqrt(1-beta.^2).*(sqrt(alpha).*randn(K,1)); 26 | end 27 | nonzeroW = nonzeroW(:,end-L+1:end); 28 | 29 | % normalize along row 30 | nonzeroW = nonzeroW./( sqrt(sum(nonzeroW.^2,2)) * ones(1,L) ); 31 | 32 | % select active sources at random locations 33 | ind = randperm(M); 34 | indice = ind(1:K); 35 | Wgen = zeros(M,L); 36 | Wgen(indice,:) = nonzeroW; 37 | 38 | 39 | % noiseless signal 40 | signal = Phi * Wgen; 41 | 42 | 43 | % observation noise 44 | stdnoise = std(reshape(signal,N*L,1))*10^(-SNR/20); 45 | noise = randn(N,L) * stdnoise; 46 | 47 | 48 | % noisy signal 49 | Y = signal + noise; 50 | 51 | 52 | 53 | %============================ T-MSBL ========================== 54 | [Weight3, gamma_ind3, gamma_est3, count3] = TMSBL(Phi, Y, 'noise','mild'); 55 | 56 | [F3,P3,R3,IND3] = perfSupp(Weight3,indice,'firstlargest',K); 57 | fail_TMSBL(it) = (F3~=1); 58 | mse_TMSBL(it) = (norm(Wgen - Weight3,'fro')/norm(Wgen,'fro'))^2; 59 | 60 | fprintf(' TMSBL: Mean MSE = %4.3f; Mean FR = %4.3f; \n',mean(mse_TMSBL), mean(fail_TMSBL)); 61 | 62 | 63 | %============================ MSBL ========================== 64 | lambda_ini = 1e-2; 65 | [Weight1,gamma1,gamma_used1,count1] = MSBL(Phi, Y, lambda_ini, 1); 66 | 67 | [F1,P1,R1,IND1] = perfSupp(Weight1,indice,'firstlargest',K); 68 | fail_MSBL(it) = (F1~=1); 69 | mse_MSBL(it) = (norm(Wgen - Weight1,'fro')/norm(Wgen,'fro'))^2; 70 | 71 | fprintf(' MSBL: Mean MSE = %4.3f; Mean FR = %4.3f; \n', mean(mse_MSBL), mean(fail_MSBL)); 72 | 73 | 74 | 75 | 76 | end -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/demo_fig8.m: -------------------------------------------------------------------------------- 1 | % This demo produces the Fig.8 in the paper. 2 | % Enjoy increasing the temporal correlation to see the performance of T-MSBL 3 | 4 | clear; 5 | 6 | double tmpCorr; 7 | 8 | % Set the temporal correlation value. You can add more "9"s to it (i.e. allow 9 | % the temporal correlation approximates 1 closer. But note that, 10 | % practically, when tmpCorr is larger than a value, the matrix computation 11 | % will have numerical problems. 12 | % In this demo, it is advised that |tmpCorr| <= 1e-13. 13 | tmpCorr = 0.999999999; 14 | 15 | 16 | % Problem dimension 17 | K = 12; % source number 18 | L = 3; % number of measurement vectors 19 | N = 40; % row number of the dictionary matrix 20 | M = 128; % column number of the dictionary matrix 21 | iterNum = 100; % number of repeating the experiment 22 | 23 | % generate Hadamard matrix of the size 128 x 128 24 | H = hadamard(M); 25 | 26 | 27 | for it = 1:iterNum 28 | fprintf('\n\nTrial #%d:\n',it); 29 | 30 | % generate the dictionary matrix, whose 40 rows are randomly chosen 31 | % from the rows of the Hadamard matrix 32 | rowLoc = randperm(M); 33 | loc = rowLoc(1:N); 34 | Phi = H(loc,:); 35 | 36 | % Generate L source vectors 37 | beta = ones(K,1)*tmpCorr; 38 | nonzeroW(:,1) = randn(K,1); 39 | alpha = ones(K,1); 40 | for i = 2 : L 41 | nonzeroW(:,i) = beta .* nonzeroW(:,i-1) + sqrt(1-beta.^2).*(sqrt(alpha).*randn(K,1)); 42 | end 43 | 44 | % rescale the rows 45 | nonzeroW = nonzeroW./( sqrt(sum(nonzeroW.^2,2)) * ones(1,L) ); 46 | AdjustNorm = 0.3+rand(K,1)*(1-0.3); 47 | nonzeroW = diag(AdjustNorm) * nonzeroW; 48 | 49 | 50 | % select active sources at random locations 51 | ind = randperm(M); 52 | indice = ind(1:K); 53 | Wgen = zeros(M,L); 54 | Wgen(indice,:) = nonzeroW; 55 | 56 | 57 | % noiseless signal 58 | Y = Phi * Wgen; 59 | 60 | 61 | 62 | %============================ 1.T-MSBL ========================== 63 | [Weight1, gamma_ind1, gamma_est1, count1] = TMSBL(Phi, Y, 'noise','no'); 64 | 65 | % Failure Rate 66 | F1 = perfSupp(Weight1,indice,'firstlargest',K); 67 | FR_TMSBL(it) = (F1~=1); 68 | 69 | % MSE 70 | MSE1 = (norm(Weight1-Wgen,'fro')/norm(Wgen,'fro'))^2; 71 | MSE_TMSBL(it) = MSE1; 72 | 73 | fprintf('T-MSBL(Using 3 Measurement Vectors): Fail_Rate = %4.3f%%; MSE = %3.2f%%; \n',mean(FR_TMSBL)*100,mean(MSE_TMSBL)*100); 74 | 75 | 76 | 77 | %============================ 3.MSBL for MMV ========================== 78 | lambda1 = 1e-15; 79 | Learn_Lambda = 0; 80 | [Weight3,gamma3,gamma_used3,count3] = MSBL(Phi, Y, lambda1, Learn_Lambda); 81 | 82 | % Failure Rate 83 | F3 = perfSupp(Weight3,indice,'firstlargest',K); 84 | FR_MSBL(it) = (F3~=1); 85 | 86 | % MSE 87 | MSE3 = (norm(Weight3-Wgen,'fro')/norm(Wgen,'fro'))^2; 88 | MSE_MSBL(it) = MSE3; 89 | 90 | fprintf(' MSBL(Using 3 Measurement Vectors): Fail_Rate = %4.3f%%; MSE = %3.2f%%; \n',mean(FR_MSBL)*100,mean(MSE_MSBL)*100); 91 | 92 | 93 | %============================ 4.MSBL for SMV ========================== 94 | [Weight4,gamma4,gamma_used4,count4] = MSBL(Phi, Y(:,1), lambda1, Learn_Lambda); 95 | 96 | % Failure Rate 97 | F4 = perfSupp(Weight4,indice,'firstlargest',K); 98 | FR_SBL(it) = (F4~=1); 99 | 100 | % MSE 101 | MSE4 = (norm(Weight4-Wgen(:,1),'fro')/norm(Wgen(:,1),'fro'))^2; 102 | MSE_SBL(it) = MSE4; 103 | 104 | fprintf(' MSBL(Using 1 Measurement Vector) : Fail_Rate = %4.3f%%; MSE = %3.2f%%; \n',mean(FR_SBL)*100,mean(MSE_SBL)*100); 105 | end 106 | 107 | -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/demo_identicalVector.m: -------------------------------------------------------------------------------- 1 | % Goal: compare T-MSBL, MSBL, MFOCUSS when source vectors are identical 2 | % Author: Zhilin Zhang (z4zhang@ucsd.edu) 3 | % Date: May 23, 2011 4 | % Version: 1.2 5 | 6 | 7 | clear all; 8 | 9 | % Experiment Assignment 10 | run_TMSBL = 1; % Run T-MSBL 11 | run_MSBL = 1; % Run MSBL 12 | run_FOCUSS= 1; % Run M-FOCUSS 13 | iterNum = 50; % Trial number (i.e. number of repeating the experiment) 14 | % For statistical result, iterNum should not less than 50 15 | 16 | % Problem dimension 17 | N = 40; % Row number of the dictionary matrix 18 | M = N * 5; % Column number of the dictionary matrix 19 | L = 4; % Number of measurement vectors 20 | K = 7; % Number of nonzero rows (i.e. source number) in the solution matrix 21 | beta = ones(K,1)*1; % The source vectors are identical 22 | 23 | %==================================================================== 24 | SNR = 10; % Note: When you change the SNR, you may need to accordingly change 25 | % the input arguments for each algorithm 26 | %==================================================================== 27 | 28 | for it = 1 : iterNum 29 | fprintf('\n\nTrial #%d:\n',it); 30 | 31 | % Generate dictionary matrix with columns draw uniformly from the surface of a unit hypersphere 32 | Phi = randn(N,M); 33 | Phi = Phi./(ones(N,1)*sqrt(sum(Phi.^2))); 34 | 35 | % Generate the K nonzero rows, each row being an AR(1) process. All the AR(1) 36 | % processes have different AR coefficients, which are randomly chosen from [0.7,1) 37 | nonzeroW(:,1) = randn(K,1); 38 | for i = 2 : L*100 39 | nonzeroW(:,i) = beta .* nonzeroW(:,i-1) + sqrt(1-beta.^2).*(ones(K,1).*randn(K,1)); 40 | end 41 | nonzeroW = nonzeroW(:,end-L+1:end); % Ensure the AR processes are stable 42 | 43 | % Normalize each row 44 | nonzeroW = nonzeroW./( sqrt(sum(nonzeroW.^2,2)) * ones(1,L) ); 45 | 46 | % Rescale each row such that the squared row-norm distributes in [1,scalefactor] 47 | scalefactor = 3; 48 | mag = rand(1,K); mag = mag - min(mag); 49 | mag = mag/(max(mag))*(scalefactor-1) + 1; 50 | nonzeroW = diag(sqrt(mag)) * nonzeroW; 51 | 52 | % Locations of nonzero rows are randomly chosen 53 | ind = randperm(M); 54 | indice = ind(1:K); 55 | Wgen = zeros(M,L); 56 | Wgen(indice,:) = nonzeroW; 57 | 58 | % Noiseless signal 59 | signal = Phi * Wgen; 60 | 61 | % Observation noise 62 | stdnoise = std(reshape(signal,N*L,1))*10^(-SNR/20); 63 | noise = randn(N,L) * stdnoise; 64 | 65 | % Noisy signal 66 | Y = signal + noise; 67 | 68 | 69 | %============================ T-MSBL ========================== 70 | if run_TMSBL == 1, 71 | tic; 72 | 73 | % Depends on the SNR, choosing suitable values for input arguments: 74 | % If no noise, Weight = TMSBL(Phi, Y, 'noise','no','fix_B',eye(L)); 75 | % If SNR >= 23 dB, Weight = TMSBL(Phi, Y, 'noise','small','fix_B',eye(L)); 76 | % If 6dB < SNR <= 22 dB, Weight = TMSBL(Phi, Y, 'noise','mild','fix_B',eye(L)); 77 | % If SNR <= 6 dB, Weight = TMSBL(Phi, Y, 'noise','large','fix_B',eye(L)); 78 | % Note: See the TMSBL code for details on input arguments ans set your 79 | % own values for specific problems. 80 | 81 | [Weight1, gamma_ind1, gamma_est1, count1] = TMSBL(Phi, Y, 'noise','mild','fix_B',eye(L)); 82 | 83 | time1 = toc; 84 | TIME1(it) = time1; 85 | 86 | 87 | % failure rate: F1 = 1: perfect recovery; F1 = 0: worst recovery 88 | F1 = perfSupp(Weight1,indice,'firstlargest', K); 89 | fail_TMSBL(it) = (F1~=1); 90 | 91 | % Mean Square Error (MSE) 92 | mse_TMSBL(it) = (norm(Wgen - Weight1,'fro')/norm(Wgen,'fro'))^2; 93 | 94 | fprintf(' T-MSBL: time = %5.2f; Findex = %3.2f, Ave-MSE = %3.2f%%; Ave-Fail_Rate = %4.3f%%; Ave-Time = %4.3f\n',... 95 | time1,F1,mean(mse_TMSBL)*100,mean(fail_TMSBL)*100,mean(TIME1)); 96 | end 97 | %============================================================ 98 | 99 | 100 | 101 | 102 | %============================ MSBL ========================== 103 | lambda = 1e-3; % Initial value for the regularization parameter. 104 | Learn_Lambda = 1; % Using its lambda learning rule 105 | 106 | if run_MSBL == 1, 107 | tic; 108 | [Weight3,gamma_est3,gamma_used3,count3] = MSBL(Phi,Y, lambda, Learn_Lambda); 109 | time3 = toc; 110 | TIME3(it) = time3; 111 | 112 | 113 | % Failure rate 114 | F3 = perfSupp(Weight3,indice,'firstlargest', K); 115 | fail_MSBL(it) = (F3~=1); 116 | 117 | % MSE 118 | perf_MSBL(it) = (norm(Wgen - Weight3,'fro')/norm(Wgen,'fro'))^2; 119 | 120 | fprintf(' MSBL: time = %5.2f; Findex = %3.2f, Ave-MSE = %3.2f%%; Ave-Fail_Rate = %4.3f%%; Ave-Time = %4.3f\n',... 121 | time3,F3,mean(perf_MSBL)*100,mean(fail_MSBL)*100,mean(TIME3)); 122 | end 123 | 124 | 125 | % ====================== MFOCUSS with near-optimal regularization ========= 126 | if run_FOCUSS == 1, 127 | tic; 128 | lambda_opt = stdnoise^2; % use the noise variance 129 | [Weight4, gamma_ind4, gamma_est4, count4] = MFOCUSS(Phi, Y, lambda_opt); 130 | 131 | time4 = toc; 132 | TIME4(it) = time4; 133 | 134 | 135 | % Failure rate 136 | F4 = perfSupp(Weight4,indice,'firstlargest', K); 137 | fail_MFOCUSS(it) = (F4~=1); 138 | 139 | % MSE 140 | perf_MFOCUSS(it) = (norm(Wgen - Weight4,'fro')/norm(Wgen,'fro'))^2; 141 | 142 | fprintf('MFOCUSS(optimal): time = %5.2f; Findex = %3.2f, Ave-MSE = %3.2f%%; Ave-Fail_Rate = %4.3f%%; Ave-Time = %4.3f\n',... 143 | time4,F4,mean(perf_MFOCUSS)*100,mean(fail_MFOCUSS)*100,mean(TIME4)); 144 | end 145 | 146 | end 147 | 148 | 149 | 150 | 151 | -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/demo_time_varying.m: -------------------------------------------------------------------------------- 1 | % Show how to use T-MSBL in the time-varying sparsity model. 2 | % To show the results by KF-CS and LS-CS, you need to download the codes 3 | % from: http://home.engineering.iastate.edu/~namrata/research/LSCS_KFCS_code.zip 4 | % 5 | % However, you can see the comparison in the following reference: 6 | % 7 | % Z. Zhang, B. D. Rao, Exploiting Correlation in Sparse Signal Recovery Problems: 8 | % Multiple Measurement Vectors, Block Sparsity, and Time-Varying Sparsity 9 | % ICML 2011 Workshop on Structured Sparsity: Learning and Inference 10 | % 11 | % Or similar comparison in my blogs: 12 | % 13 | % http://marchonscience.blogspot.com/2011/04/is-mmv-more-suitable-for-dynamic.html 14 | % http://marchonscience.blogspot.com/2011/04/is-mmv-more-suitable-for-dynamic_17.html 15 | % http://marchonscience.blogspot.com/2011/04/is-mmv-more-suitable-for-dynamic_18.html 16 | % 17 | % Feel free to send any questions to me: z4zhang@ucsd.edu 18 | % May 19, 2010 19 | 20 | clear; clc; close all; 21 | 22 | 23 | L = 50; % snapshot number 24 | M = 256; % column number of the dictionary matrix 25 | N = 60; % row number of the dictionary matrix 26 | K = 15; % number of the nonzero coefficients at the initial stage 27 | addNZ = 10; % added number of nonzero coefficients at each addition time 28 | segLen = 20; % duration of each added coefficient 29 | remNZ = 5; % removed number of nonzero coefficients at each deletion time 30 | 31 | iteration = 20; % independently repeat the experiments to see the averaged results 32 | 33 | for it = 1 : iteration 34 | fprintf('Trial #%d\n',it); 35 | 36 | % generate the dictionary matrix from Gaussian distribution 37 | Phi = randn(N,M); 38 | Phi = Phi./(ones(N,1)*sqrt(sum(Phi.^2))); 39 | 40 | % generate the K time series existing during the whole time 41 | beta = 1- rand(K,1)*0.3; % randomly choose temporal correlation of 42 | % time series of each coefficient 43 | longS(:,1) = randn(K,1); 44 | for i = 2 : L*20 45 | longS(:,i) = beta .* longS(:,i-1) + sqrt(1-beta.^2).*(randn(K,1)); 46 | end 47 | longS = longS(:,end-L+1:end); % stable time series 48 | longS = longS./( sqrt(sum(longS.^2,2)) * ones(1,L) ); 49 | 50 | % generate time series of coefficients 51 | D = addNZ * 3; % possible largest number of nonzero coefficients 52 | beta = 1- rand(D,1)*0.3; % randomly choose temporal correlation of 53 | % time series of each coefficient 54 | pool(:,1) = randn(D,1); 55 | for i = 2 : segLen*20 56 | pool(:,i) = beta .* pool(:,i-1) + sqrt(1-beta.^2).*(randn(D,1)); 57 | end 58 | pool = pool(:,end-segLen+1:end); % stable time series 59 | pool = pool./( sqrt(sum(pool.^2,2)) * ones(1,segLen) ); 60 | 61 | % initial stage of a time series should have increasing magnitude 62 | pool(:,[1,2,3]) = pool(:,[1,2,3]) .* (ones(D,1) * [0.2, 0.6, 1]); 63 | 64 | % when a time series disappears, the magnitude should be decreasing 65 | pool(:,[segLen-2,segLen-1,segLen]) = pool(:,[segLen-2,segLen-1,segLen]).*(ones(D,1)*[1,0.6,0.2]); 66 | 67 | % select K time series at L=1, 68 | ind = randperm(M); 69 | indice0 = ind(1:K); 70 | Wgen = zeros(M,L); 71 | Wgen(indice0,:) = longS; 72 | 73 | % select another addNZ time series at L = 16 74 | restLoc = setdiff([1:M],indice0); 75 | ind = randperm(length(restLoc)); 76 | indice1 = restLoc(ind(1:addNZ)); 77 | Wgen(indice1,[16:15+segLen]) = pool(1:addNZ,:); 78 | 79 | % delete existing time series at L=26 80 | candidateLoc = union(indice0,indice1); 81 | ind_rem = randperm(K+addNZ); 82 | indice_rem = candidateLoc(ind_rem(1:remNZ)); 83 | Wgen(indice_rem,[24:end]) = Wgen(indice_rem,[24:end]).*(ones(remNZ,1)*[0.8,0.4,zeros(1,25)]); 84 | 85 | % select another addNZ time series at L = 31 86 | restLoc2 = setdiff([1:M],union(indice0,indice1)); 87 | ind2 = randperm(length(restLoc2)); 88 | indice2 = restLoc2(ind2(1:addNZ)); 89 | Wgen(indice2,[31:30+segLen]) = pool(addNZ+1:addNZ*2,:); 90 | 91 | % % See the pictures of the generated signals 92 | % imagesc(Wgen); colorbar; 93 | 94 | % noiseless signal 95 | signal = Phi * Wgen; 96 | 97 | % observation noise 98 | stdnoise = 0.01; 99 | noise = randn(N,L) * stdnoise; 100 | 101 | % noisy signal 102 | Y = signal + noise; 103 | 104 | % compute total SNR 105 | SNR_rec(it) = 20*log10(norm(signal,'fro')/norm(noise,'fro')); 106 | 107 | 108 | % ===================Run T-MSBL step by 5 ================= 109 | tic; 110 | X_tsbl5 = []; 111 | for i=1:10 112 | 113 | % According to the SNR range, choose suitable input arguments 114 | % See the codes for details 115 | [X_tsbl1] = TMSBL(Phi, Y(:,[(i-1)*5+1:i*5]), 'noise','small'); 116 | 117 | 118 | X_tsbl5 = [X_tsbl5,X_tsbl1]; 119 | end 120 | TIME_tsbl5(it) = toc; 121 | 122 | for t = 1:L 123 | err_tsbl5(it,t) = norm( Wgen(:,t)-X_tsbl5(:,t) )^2; 124 | energy(it,t) = norm( Wgen(:,t) )^2; 125 | end 126 | 127 | 128 | % =================== Run T-MSBL step by 2 ================= 129 | tic; 130 | X_tsbl2 = []; 131 | for i=1:25 132 | % According to the SNR range, choose suitable input arguments 133 | % See the codes for details 134 | [X_tsbl1] = TMSBL(Phi, Y(:,[(i-1)*2+1:i*2]), 'noise','small'); 135 | 136 | X_tsbl2 = [X_tsbl2,X_tsbl1]; 137 | end 138 | TIME_tsbl2(it) = toc; 139 | 140 | for t = 1:L 141 | err_tsbl2(it,t) = norm( Wgen(:,t)-X_tsbl2(:,t) )^2; 142 | end 143 | 144 | 145 | 146 | % ===================Run MSBL steps by 5================= 147 | lambda = std(Y(:,1))*1e-2; 148 | Learn_Lambda = 1; 149 | 150 | tic; 151 | X_msbl5 = []; 152 | for i = 1 : 10 153 | [X_msbl1,gamma2,gamma_used2,count2] = MSBL(Phi, Y(:,[(i-1)*5+1:i*5]), lambda, Learn_Lambda); 154 | 155 | X_msbl5 = [X_msbl5,X_msbl1]; 156 | end 157 | 158 | TIME_msbl5(it) = toc; 159 | 160 | for t = 1:L 161 | err_msbl5(it,t) = norm( Wgen(:,t)-X_msbl5(:,t) )^2; 162 | end 163 | 164 | % ===================Run MSBL2 steps by 2================= 165 | lambda = std(Y(:,1))*1e-2; 166 | Learn_Lambda = 1; 167 | tic; 168 | X_msbl2 = []; 169 | for i = 1 : 25 170 | [X_msbl1,gamma2,gamma_used2,count2] = MSBL(Phi, Y(:,[(i-1)*2+1:i*2]), lambda, Learn_Lambda); 171 | X_msbl2 = [X_msbl2,X_msbl1]; 172 | end 173 | 174 | TIME_msbl2(it) = toc; 175 | 176 | for t = 1:L 177 | err_msbl2(it,t) = norm( Wgen(:,t)-X_msbl2(:,t) )^2; 178 | end 179 | 180 | 181 | % % ===================Run KF-CS =============== 182 | % tot = 50; n = N; m = M; lambdap = 0.1; 183 | % global n 184 | % global m 185 | % global tot 186 | % global lambdap 187 | % global tot 188 | % Pi0 = 0.8 * eye(M); 189 | % Q = 0.8 * eye(M); 190 | % R = stdnoise^2 * eye(N); 191 | % tic; 192 | % [X_kfcs,T_hat] = kfcs_full(Y,Pi0,Phi,Q,R); 193 | % TIME_kfcs(it) = toc; 194 | % 195 | % for t = 1:L 196 | % err_kfcs(it,t) = norm( Wgen(:,t)-X_kfcs(:,t) )^2; 197 | % end 198 | % 199 | % %========================== LS-CS ===================== 200 | % tic; 201 | % [x_upd_lscs,T_hat_lscs,x_upd_csres] = lscs_full(Y,Pi0,Phi,Q,R); 202 | % TIME_LSCS(it) = toc; 203 | % for t = 1:tot 204 | % err_lscs(it,t) = norm( Wgen(:,t)-x_upd_lscs(:,t) )^2; 205 | % end 206 | 207 | save data_demo_timevarying; 208 | end 209 | 210 | figure(1); 211 | semilogy(mean(err_tsbl2,1)./mean(energy,1),'r','linewidth',2.5); 212 | hold on;semilogy(mean(err_tsbl5,1)./mean(energy,1),'r-.','linewidth',2.5); 213 | hold on;semilogy(mean(err_msbl2,1)./mean(energy,1),'linewidth',2.5); 214 | hold on;semilogy(mean(err_msbl5,1)./mean(energy,1),'-.','linewidth',2.5); 215 | % hold on;semilogy(mean(err_kfcs,1)./mean(energy,1),'k','linewidth',2.5); 216 | % hold on;semilogy(mean(err_lscs,1)./mean(energy,1),'c','linewidth',2.5); 217 | legend('T-MSBL steps by 2','T-MSBL steps by 5','M-SBL steps by 2','M-SBL steps by 5'); 218 | ylabel('\bf\fontsize{20}Normalized MSE'); 219 | xlabel('\bf\fontsize{20}Time'); 220 | 221 | 222 | -------------------------------------------------------------------------------- /Other_Methods/TMSBL_code/perfSupp.m: -------------------------------------------------------------------------------- 1 | function [F, P, R, ind] = perfSupp(estSource, trueSourceIndex, varargin) 2 | % F-Measure of support recovery 3 | % 4 | % *** INPUT Parameters *** 5 | % estSource : estimated sources 6 | % trueSourceIndex : true support location index 7 | % varargin{1} == 'firstLargest' & varargin{2} == D 8 | % : select the support of D largest sources (in terms of 2-norm) 9 | % varargin{1} == 'largerThan' & varargin{2} == D 10 | % : select the support of elements larger than D 11 | % 12 | % *** OUTPUT Parameters *** 13 | % F : F-Measure of support recovery (=1: perfect): 14 | % F = (2*P*R)/(R+P) 15 | % P : Precision of support recovery: 16 | % P = (common set of estimated support and true support)/(set of estimated support); 17 | % R : Recall of support recovery: 18 | % R = (common set of estimated support and true support)/(set of true support); 19 | % ind : the support index used to measure performance 20 | % 21 | % Use: 22 | % F = perfSupp(estSource, trueSourceIndex, 'firstLargest', 10); 23 | % [F,P,R] = perfSupp(estSource, trueSourceIndex, 'largerThan', 1e-5); 24 | % 25 | % Author: Zhilin Zhang 26 | % Date : July, 2010 27 | % Version : 1.2 28 | 29 | 30 | 31 | if length(varargin) == 2 32 | switch lower(varargin{1}) 33 | case 'firstlargest' 34 | numThreshold = varargin{2}; 35 | estSource = sum(estSource.^2,2); 36 | [sortedSource,sortInd] = sort(estSource, 1, 'descend'); 37 | ind = sortInd(1:numThreshold); 38 | 39 | case 'largerthan' 40 | valThreshold = varargin{2}; 41 | estSource = sum(estSource.^2,2); 42 | ind = find(estSource >= valThreshold); 43 | 44 | end 45 | else 46 | error('Optional parameters are wrong!\n'); 47 | end 48 | 49 | commonSupp = intersect(ind,trueSourceIndex); 50 | 51 | if isempty(commonSupp), 52 | F = 0; P = 0; R = 0; 53 | else 54 | P = length(commonSupp)/length(ind); 55 | R = length(commonSupp)/length(trueSourceIndex); 56 | F = 2 * P * R/(P + R); 57 | end 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Distributed Compressive Sensing: A Deep Learning Approach 2 | This repository includes all necessary programs to implement the LSTM-CS method and generate Fig.4 of the following 3 | [paper](https://www.microsoft.com/en-us/research/wp-content/uploads/2017/02/LSTM_CS_TSP.pdf). To refer please use the following: 4 | 5 | ``` 6 | @ARTICLE{hp_LSTM_CS, 7 | author={Hamid Palangi and Rabab Ward and Li Deng}, 8 | journal={IEEE Transactions on Signal Processing}, 9 | title={Distributed Compressive Sensing: A Deep Learning Approach}, 10 | year={2016}, 11 | volume={64}, 12 | number={17}, 13 | pages={4504-4518}, 14 | month={Sept}, 15 | } 16 | ``` 17 | 18 | In the folder "Other_Methods", the codes for Bayesian Compressive Sensing from 19 | following references are used. Please refer to their website for copy 20 | right terms and conditions. 21 | ``` 22 | [1] Bayesian and Multitask Compressive Sensing: "http://www.ece.duke.edu/~lcarin/bcs_ver0.1.zip" 23 | [2] Sparse Signal Recovery with Temporally Correlated Source Vectors 24 | Using Sparse Bayesian Learning: "http://sccn.ucsd.edu/~zhang/TMSBL_code.zip" 25 | ``` 26 | To use, simply clone to your local machine and run the program in "Main.m". This will 27 | regenerate the Fig. 4 of the above paper. Please note that the codes are in MATLAB. 28 | 29 | The running time to regenerate Fig. 4 will be about 16 hours on one core i7 computer. This is because: 30 | (a) It runs on CPU not GPU. 31 | (b) It runs reconstruction algorithm 1,440 times (we have 144 different sparsity levels and 10 realization for each one). 32 | 33 | If you want just use the codes for your research and learn how they work, 34 | I recommend to create a faster debug setting for yourself by following modifications to "Main.m": 35 | (a) Set "ncell" in line 47 to 5. 36 | (b) Set "maxEpochLSTM" in line 50 to 5. 37 | (c) Comment line 178 and use "k_vec = 1:20:nsparse;" instead. 38 | 39 | Please note that debug setting will give you very bad results but it runs quite fast and helps you to understand the code. 40 | 41 | For this work, I have implemented Long Short-Term Memory module of the proposed model from scratch in MATLAB. 42 | I did this because I wanted to understand it better. 43 | For sure there are better implementations of LSTM out there. 44 | This is also a good opportunity to learn how exactly it is implemented if you are intereted. 45 | I have put another README file for LSTM. 46 | 47 | The description of the content of it is as follows: 48 | 49 | 1. Main.m: Calls all necessary sub-functions to train LSTM-CS and NWSOMP models, and evaluate their performance. 50 | 2. FP_LSTM.m: Forward pass of LSTM 51 | 3. ISOMP_LSTM.m: LSTM-CS solver 52 | 4. LSTM_Grad_Clip.m: Performs gradient clipping for LSTM-CS training 53 | 5. LSTM_Grads.m: Calculates LSTM-CS gradients. 54 | 6. LSTM_TrainCE.m: Call all subfunctions to train LSTM-CS. 55 | 7. mnist_fun.m: Sorts a sparse vector and keeps the k largest non-zero values. 56 | 8. NISOMPnew.m: NWSOMP solver. 57 | 9. OneLayerBackprop_new.m: For training NWSOMP solver. 58 | 10. OneLayerBackprop_new_Biased.m: For training NWSOMP solver when we have bias. 59 | 11. SOMP.m: SOMP Solver. 60 | 12. TgenerateISOMP_MNIST_MMV.m: Generates training data for NWSOMP. 61 | 13. TgenerateISOMP_MNIST_MMV_CE.m: Generates training data for LSTM-CS. 62 | 14. MNIST_Data: This folder includes the prepared MNIST data. If you want to prepare it yourself please 63 | download the MNIST data from "http://yann.lecun.com/exdb/mnist/" and run the "Prepare_MNIST_Data.m". 64 | 15. Other_Methods: This folder includes the mfiles for Bayesian methods. Please refer to the corresponding 65 | websites for mentioned above for copy right terms and conditions. 66 | 16. Results: The output curves are saved here. 67 | 17. Trained_Network: The trained model parameters are saved here. 68 | 69 | 70 | If you need further information or if you have any feedback, please contact Hamid Palangi "hpalangi@microsoft.com". 71 | 72 | Good luck! 73 | 74 | ## Copy Right: 75 | Permission is granted for anyone to copy, use, modify, or distribute 76 | this program and accompanying programs and documents for any purpose, 77 | provided this copyright notice is retained and prominently displayed, 78 | along with a note saying that the original programs are available from 79 | Hamid Palangi, "hpalangi@microsoft.com". 80 | The programs and documents are distributed without any warranty, 81 | express or implied. As the programs were written for research purposes only, 82 | they have not been tested to the degree that would be advisable in any important 83 | application. All use of these programs is entirely at the user's own risk. 84 | -------------------------------------------------------------------------------- /README_LSTM.txt: -------------------------------------------------------------------------------- 1 | Copy right: 2 | Permission is granted for anyone to copy, use, modify, or distribute 3 | this program and accompanying programs and documents for any purpose, 4 | provided this copyright notice is retained and prominently displayed, 5 | along with a note saying that the original programs are available from 6 | Hamid Palangi, "hamidp@ece.ubc.ca". 7 | The programs and documents are distributed without any warranty, 8 | express or implied. As the programs were written for research purposes only, 9 | they have not been tested to the degree that would be advisable in any important 10 | application. All use of these programs is entirely at the user's own risk. 11 | 12 | This README file describes how to use implemented Long Shor-Term Memory (LSTM) module. 13 | To refer please use the following: 14 | @ARTICLE{hp_LSTM_CS, 15 | author={Hamid Palangi and Rabab Ward and Li Deng}, 16 | journal={IEEE Transactions on Signal Processing}, 17 | title={Distributed Compressive Sensing: A Deep Learning Approach}, 18 | year={2016}, 19 | volume={64}, 20 | number={17}, 21 | pages={4504-4518}, 22 | month={Sept}, 23 | } 24 | 25 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 26 | I have implemented Long Short-Term Memory module of the proposed model from scratch in MATLAB. 27 | I did this because I wanted to understand it better. 28 | For sure there are better implementations of LSTM out there. 29 | This is also a good opportunity to learn how exactly it is implemented if you are intereted. 30 | 31 | Sample use: 32 | 33 | [Ws,ce_lstm,ce_lstm_dev] = LSTM_TrainCE('input train',In,'target train',T... 34 | ,'input validation',In_Dev,'target validation',T_Dev,'number of cells',ncell... 35 | ,'step size',StepSize,'epoch max',maxEpochLSTM,'number of mini-batches',nBatch,... 36 | 'trained network path',NetPath);%,'initial weight matrices',Ws_init); 37 | 38 | Inputs: 39 | In: is a NxLxnT matrix where "N" is the length of input vector, "L" is the length of time dependency and "nT" is the number of training examples. 40 | T: The same as "In" but includes output labels. 41 | In_Dev: The same as "In" for validation examples. 42 | T_Dev: The same as "T" for validation examples. 43 | NetPath: After each iteration trained LSTM parametes are saved here. 44 | Ws_init: If training interrupts we can load the last set of parameters in Ws_init and fine tune from there. 45 | 46 | Outputs: 47 | Ws: This is a struct including all LSTM parameters. 48 | Input weights: Ws.W1,Ws.W2,Ws.W3,Ws.W4 49 | Recurrent weights: Ws.Wrec1,Ws.Wrec2,Ws.Wrec3,Ws.Wrec4 50 | ce_lstm: value of cost function over epochs on training set. 51 | ce_lstm_dev: value of cost function over epochs on validation set. 52 | 53 | Description of rest of the M files: 54 | 55 | 1. FP_LSTM.m: Forward pass of LSTM 56 | 2. LSTM_Grad_Clip.m: Performs gradient clipping for LSTM training 57 | 3. LSTM_Grads.m: Calculates LSTM gradients. 58 | 4. LSTM_TrainCE.m: Call all subfunctions to train LSTM. 59 | 60 | If you need further information or if you have any feedback, please contact Hamid Palangi "hamidp@ece.ubc.ca". 61 | 62 | Good luck! -------------------------------------------------------------------------------- /SOMP.m: -------------------------------------------------------------------------------- 1 | function S_OMP = SOMP(phi_m,Y,res_min) 2 | 3 | 4 | %% Contact: Hamid Palangi, Email: hamidp@ece.ubc.ca 5 | % Simaltaneous Orthogonal Matching Pursuit 6 | m = size(phi_m,1); 7 | n = size(phi_m,2); 8 | L = size(Y,2); 9 | iter_max = L*m;done=0; 10 | j = 0; R = Y; S_OMP = zeros(n,L); C = 0; 11 | I = []; 12 | while done == 0 13 | j = j+1; 14 | C = phi_m'*R; 15 | for iC=1:size(C,1) 16 | Cnew(iC,1) = norm(C(iC,:)); 17 | end 18 | [val1,idx1] = max( Cnew ); 19 | if length( find( I == idx1 ) ) == 0 20 | I = [I;idx1]; 21 | phiT = phi_m(:,I); 22 | S_temp = lscov(phiT,Y); % Least squares 23 | R = Y-phiT*S_temp; 24 | normR = 0; 25 | for iR=1:size(R,2) 26 | normR = normR + norm(R(:,iR)); 27 | end 28 | if (j > iter_max) || (normR < res_min) 29 | done = 1; 30 | end 31 | else 32 | done = 1; 33 | end 34 | if done == 1 35 | for iSOMP=1:size(Y,2) 36 | S_OMP(:,iSOMP) = zeros(n,1); 37 | S_OMP(I,iSOMP) = S_temp(:,iSOMP); 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /TgenerateISOMP_MNIST_MMV.m: -------------------------------------------------------------------------------- 1 | function [rMM,sMM] = TgenerateISOMP_MNIST_MMV(phi_m,L,DataPath,nImage,Idim,nsparseTrain,Bdim,TransformProps) 2 | 3 | % This function generates training samples for MMV scenario 4 | %% Contact: Hamid Palangi, Email: hamidp@ece.ubc.ca 5 | 6 | % L: number of measurement vectors 7 | % nImage: number of images 8 | 9 | % Signal definition 10 | n = size(phi_m,2); % length of the signal 11 | m = size(phi_m,1); % number of measurements 12 | k = nsparseTrain; 13 | % The number of training samples is "M = nImage*k" 14 | % Bdim: block dimension 15 | noise = TransformProps.noise; 16 | load(DataPath); 17 | if strcmp(TransformProps.DataType,'Tr') 18 | Ims = ImsTr; 19 | clear ImsTr; 20 | elseif strcmp(TransformProps.DataType,'Val') 21 | Ims = ImsVal; 22 | clear ImsVal; 23 | elseif strcmp(TransformProps.DataType,'Test') 24 | Ims = ImsTest; 25 | clear ImsTest; 26 | end 27 | %% Generating nImage random sparse signals and measurements 28 | rM = zeros(m,L,k); 29 | sM = zeros(n,L,k); 30 | rMM = []; 31 | sMM = []; 32 | B2 = []; 33 | for iL=1:L 34 | Im1 = Ims{iL,1}; 35 | for i2=1:nImage % for the # of images per channel. 36 | Im=Im1(:,:,i2); 37 | Im=imresize(Im,[Idim,Idim]); 38 | B0 = im2col(Im,[Bdim Bdim],'distinct'); 39 | B1 = zeros( Bdim*Bdim , size(B0,2) ); 40 | for iB0=1:size(B0,2) % for the all number of blocks in image 41 | B1(:,iB0) = mnist_fun(B0(:,iB0),n,k); 42 | end 43 | B2 = [B2 B1]; 44 | end 45 | end 46 | nBlockPerImage = Idim*Idim / (Bdim*Bdim); 47 | nBlockPerChannel = nBlockPerImage * nImage; 48 | for i2B=1:nBlockPerChannel 49 | S0 = zeros(n,L); 50 | for iL=1:L 51 | S0(:,iL) = B2( : , i2B + (iL-1)*nBlockPerChannel ); % amplitudes of sparse signal 52 | end 53 | Y = phi_m*S0 + noise; % Measurements 54 | j = 1; rM(:,:,j) = Y; sM(:,:,j) = S0; 55 | cell_I = cell(L,1); 56 | % Outputs of following "while" are sM (targets) and rM (inputs) 57 | while j 1e-6 % To prevent division by zero. 86 | temp4(:,iL) = temp4(:,iL) / maxval; 87 | end 88 | end 89 | rMM(:,:,i4) = temp4; 90 | end 91 | nTrainSamples = size(sMM,3); 92 | for i4=1:nTrainSamples 93 | temp5 = sMM(:,:,i4); 94 | for iL=1:L 95 | [val1,idx1] = max( abs ( temp5(:,iL) ) ); 96 | temp6 = zeros(n,1); 97 | temp6(idx1,1) = 1; 98 | temp5(:,iL) = temp6; 99 | end 100 | sMM(:,:,i4) = temp5; 101 | end 102 | % Normalization ends****************************************** 103 | end -------------------------------------------------------------------------------- /mnist_fun.m: -------------------------------------------------------------------------------- 1 | function mnists = mnist_fun(vec_in,n,k) 2 | mnists = vec_in; 3 | [temp111,sel] = sort(abs(mnists)); 4 | mnists( sel(1:n-k) ) = 0; 5 | end --------------------------------------------------------------------------------