├── .gitignore ├── PartHadamardMtx.m ├── README.md ├── bpdn_jbb ├── BPDN_quadprog.m └── bpdn_test.m ├── l1magic-1.11 └── l1magic │ ├── Data │ ├── RandomStates.mat │ ├── boats.mat │ └── camera.mat │ ├── Measurements │ ├── A_f.m │ ├── A_fhp.m │ ├── At_f.m │ ├── At_fhp.m │ └── LineMask.m │ ├── Notes │ ├── Figs │ │ └── CVS │ │ │ ├── Entries │ │ │ ├── Repository │ │ │ └── Root │ ├── l1magic.bib │ └── l1magic_notes.tex │ ├── Optimization │ ├── cgsolve.m │ ├── l1dantzig_pd.m │ ├── l1decode_pd.m │ ├── l1eq_pd.m │ ├── l1qc_logbarrier.m │ ├── l1qc_newton.m │ ├── tvdantzig_logbarrier.m │ ├── tvdantzig_newton.m │ ├── tveq_logbarrier.m │ ├── tveq_newton.m │ ├── tvqc_logbarrier.m │ └── tvqc_newton.m │ ├── README │ ├── l1dantzig_example.m │ ├── l1decode_example.m │ ├── l1eq_example.m │ ├── l1qc_example.m │ ├── tvdantzig_example.m │ ├── tveq_example.m │ ├── tveq_phantom_example.m │ ├── tvqc_example.m │ └── tvqc_quantization_example.m ├── omp.m ├── omp_jbb ├── CS_OMP.m ├── CS_Reconstuction_KtoPercentage.m ├── CS_Reconstuction_MtoPercentage.m ├── CS_Reconstuction_Test.m └── MtoPercentage1000.mat ├── plygd.m ├── ricCalcu.m ├── testMatrix.m └── waveletplygd.m /.gitignore: -------------------------------------------------------------------------------- 1 | # on Mac 2 | .DS_Store 3 | l1magic-1.11/__MACOSX/ 4 | *.asv 5 | *.jpg 6 | 7 | # tex 8 | 9 | *.eps 10 | *.pdf 11 | *.aux 12 | *.bbl 13 | *.blg 14 | *.log 15 | 16 | -------------------------------------------------------------------------------- /PartHadamardMtx.m: -------------------------------------------------------------------------------- 1 | function [ Phi ] = PartHadamardMtx( M,N ) 2 | %PartHadamardMtx Summary of this function goes here 3 | % Generate part Hadamard matrix 4 | % M -- RowNumber 5 | % N -- ColumnNumber 6 | % Phi -- The part Hadamard matrix 7 | 8 | %% parameter initialization 9 | %Because the MATLAB function hadamard handles only the cases where n, n/12, 10 | %or n/20 is a power of 2 11 | L_t = max(M,N);%Maybe L_t does not meet requirement of function hadamard 12 | L_t1 = (12 - mod(L_t,12)) + L_t; 13 | L_t2 = (20 - mod(L_t,20)) + L_t; 14 | L_t3 = 2^ceil(log2(L_t)); 15 | L = min([L_t1,L_t2,L_t3]);%Get the minimum L 16 | %% Generate part Hadamard matrix 17 | Phi = []; 18 | Phi_t = hadamard(L); 19 | RowIndex = randperm(L); %生成随机的行数 20 | Phi_t_r = Phi_t(RowIndex(1:M),:); %随机取m行 21 | ColIndex = randperm(L); %生成随机列数 22 | Phi = Phi_t_r(:,ColIndex(1:N)); %打乱列 23 | end -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # csensing 2 | 3 | some matlab code for compressive sensing 4 | -------------------------------------------------------------------------------- /bpdn_jbb/BPDN_quadprog.m: -------------------------------------------------------------------------------- 1 | function [ x ] = BPDN_quadprog( y,A,tao ) 2 | %BPDN_quadprog(Basis Pursuit DeNoising with quadprog) Summary of this function goes here 3 | %Version: 1.0 written by jbb0523 @2016-07-22 4 | %Reference:Nowak R D, Wright S J. Gradient projection for sparse reconstruction: 5 | %Application to compressed sensing and other inverse problems[J]. 6 | %IEEE Journal of selected topics in signal processing, 2007, 1(4): 586-597. 7 | %(Available at: http://pages.cs.wisc.edu/~swright/papers/FigNW07a.pdf) 8 | [y_rows,y_columns] = size(y); 9 | if y_rows 3*pi/4)) 23 | yr = round(tan(thc(ll))*(-N/2+1:N/2-1))+N/2+1; 24 | for nn = 1:N-1 25 | M(yr(nn),nn+1) = 1; 26 | end 27 | else 28 | xc = round(cot(thc(ll))*(-N/2+1:N/2-1))+N/2+1; 29 | for nn = 1:N-1 30 | M(nn+1,xc(nn)) = 1; 31 | end 32 | end 33 | 34 | end 35 | 36 | 37 | % upper half plane mask (not including origin) 38 | Mh = zeros(N); 39 | Mh = M; 40 | Mh(N/2+2:N,:) = 0; % 取上半部分，下半部分赋值0 41 | Mh(N/2+1,N/2+1:N) = 0; % 去掉了一根线（应该是避免重复） 42 | 43 | 44 | M = ifftshift(M);%ifftshift 对换数据左右两边（我看结果是对换了上下和左右） 把星形变成了四个角一样的 45 | mi = find(M); %find(M) 找出M中非零元素的位置 46 | Mh = ifftshift(Mh); % 也是对换了上下和左右，于是留下了下半部分非零 47 | mhi = find(Mh); 48 | 49 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Notes/Figs/CVS/Entries: -------------------------------------------------------------------------------- 1 | /l1eqexample_minl2.eps/1.1/Mon Nov 21 08:22:29 2005// 2 | /l1eqexample_minl2.pdf/1.1/Mon Nov 21 08:22:29 2005// 3 | /l1eqexample_recovered.eps/1.1/Mon Nov 21 08:22:29 2005// 4 | /l1eqexample_recovered.pdf/1.1/Mon Nov 21 08:22:29 2005// 5 | /l1eqexample_signal.eps/1.1/Mon Nov 21 08:22:29 2005// 6 | /l1eqexample_signal.pdf/1.1/Mon Nov 21 08:22:29 2005// 7 | /phantom_backproj.eps/1.1/Mon Nov 21 23:31:13 2005// 8 | /phantom_backproj.pdf/1.1/Mon Nov 21 23:31:13 2005// 9 | /phantom_orig.eps/1.1/Mon Nov 21 23:31:13 2005// 10 | /phantom_orig.pdf/1.1/Mon Nov 21 23:31:13 2005// 11 | /phantom_sampling.eps/1.1/Mon Nov 21 23:31:13 2005// 12 | /phantom_sampling.pdf/1.1/Mon Nov 21 23:31:13 2005// 13 | /phantom_tv.eps/1.1/Mon Nov 21 23:31:13 2005// 14 | /phantom_tv.pdf/1.1/Mon Nov 21 23:31:13 2005// 15 | D 16 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Notes/Figs/CVS/Repository: -------------------------------------------------------------------------------- 1 | l1magic/Notes/Figs -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Notes/Figs/CVS/Root: -------------------------------------------------------------------------------- 1 | :ext:jrom@acm.caltech.edu:/home/jrom/CVS/ -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Notes/l1magic.bib: -------------------------------------------------------------------------------- 1 | 2 | 3 | @article{alizadeh03se, 4 | Author = {F. Alizadeh and D. Goldfarb}, 5 | Date-Added = {2005-07-15 15:21:25 -0700}, 6 | Date-Modified = {2005-07-15 15:23:10 -0700}, 7 | Journal = {Math. Program., Ser. B}, 8 | Keywords = {optimization, second-order cone programming}, 9 | Local-Url = {alizadeh03se.pdf}, 10 | Pages = {3--51}, 11 | Title = {Second-order cone programming}, 12 | Volume = {95}, 13 | Year = {2003}} 14 | 15 | 16 | @book{boyd04co, 17 | Author = {S. Boyd and L. Vandenberghe}, 18 | Date-Modified = {2005-02-01 19:48:02 -0800}, 19 | Local-Url = {boyd04co.pdf}, 20 | Publisher = {Cambridge University Press}, 21 | Title = {Convex Optimization}, 22 | Year = {2004}} 23 | 24 | 25 | @article{candes04ne, 26 | Author = {E. Cand\`es and T. Tao}, 27 | Date-Modified = {2005-02-01 18:39:06 -0800}, 28 | Journal = {submitted to IEEE Trans.\ Inform.\ Theory}, 29 | Local-Url = {candes04ne.pdf}, 30 | Month = {November}, 31 | Note = {Available on the ArXiV preprint server: {\tt math.CA/0410542}}, 32 | Title = {Near-optimal signal recovery from random projections and universal encoding strategies}, 33 | Year = {2004}} 34 | 35 | 36 | @article{candes04ro, 37 | Author = {E. Cand\`es and J. Romberg and T. Tao}, 38 | Date-Modified = {2005-02-01 18:38:11 -0800}, 39 | Journal = {Submitted to IEEE Trans.\ Inform.\ Theory}, 40 | Local-Url = {candes04ro.pdf}, 41 | Month = {June}, 42 | Note = {Available on theArXiV preprint server: {\tt math.GM/0409186}}, 43 | Title = {Robust uncertainty principles: {E}xact signal reconstruction from highly incomplete frequency information}, 44 | Year = {2004}} 45 | 46 | 47 | @misc{candes05da, 48 | Author = {E. Cand\`es and T. Tao}, 49 | Date-Added = {2005-06-09 15:50:04 -0700}, 50 | Date-Modified = {2005-06-09 15:51:39 -0700}, 51 | Howpublished = {Manuscript}, 52 | Keywords = {l1 minimization}, 53 | Local-Url = {candes05da.pdf}, 54 | Month = {May}, 55 | Title = {The {D}antzig selector: statistical estimation when $p$ is much smaller than $n$}, 56 | Year = {2005}} 57 | 58 | 59 | @article{candes05de, 60 | Author = {E. J. Cand\`es and T. Tao}, 61 | Date-Modified = {2005-11-10 18:35:41 -0800}, 62 | Journal = {To appear in IEEE Trans. Inform. Theory}, 63 | Keywords = {l1 minimization, channel coding, sparse approximation}, 64 | Local-Url = {candes04de.pdf}, 65 | Month = {December}, 66 | Title = {Decoding by Linear Programming}, 67 | Year = {2005}} 68 | 69 | 70 | @article{candes05qu, 71 | Author = {E. Cand\`es and J. Romberg}, 72 | Date-Modified = {2005-07-25 16:02:31 -0700}, 73 | Journal = {To appear in Foundations of Comput. Math.}, 74 | Local-Url = {candes05qu.pdf}, 75 | Title = {Quantitative robust uncertainty principles and optimally sparse decompositions}, 76 | Year = {2005}} 77 | 78 | 79 | @article{candes05st, 80 | Author = {E. Cand\`es and J. Romberg and T. Tao}, 81 | Date-Added = {2005-03-09 15:46:42 -0800}, 82 | Date-Modified = {2005-07-06 11:37:57 -0700}, 83 | Journal = {Submitted to Communications on Pure and Applied Mathematics}, 84 | Keywords = {basis pursuit, exact recovery, sparse approximation}, 85 | Local-Url = {StableRecovery_submittedJune18.pdf}, 86 | Month = {March}, 87 | Title = {Stable signal recovery from incomplete and inaccurate measurements}, 88 | Year = {2005}} 89 | 90 | @article{chan99no, 91 | Author = {T. Chan and G. Golub and P. Mulet}, 92 | Date-Modified = {2005-07-25 11:01:52 -0700}, 93 | Journal = {SIAM J. Sci. Comput.}, 94 | Keywords = {total variation, SOCP}, 95 | Local-Url = {chan99no.pdf}, 96 | Pages = {1964--1977}, 97 | Title = {A nonlinear primal-dual method for total variation-based image restoration}, 98 | Volume = {20}, 99 | Year = {1999}} 100 | 101 | 102 | @article{chen99at, 103 | Author = {S. S. Chen and D. L. Donoho and M. A. Saunders}, 104 | Date-Modified = {2005-02-01 19:31:45 -0800}, 105 | Journal = {SIAM J. Sci. Comput.}, 106 | Local-Url = {chen99at.pdf}, 107 | Pages = {33--61}, 108 | Title = {Atomic decomposition by basis pursuit}, 109 | Volume = {20}, 110 | Year = {1999}} 111 | 112 | 113 | @techreport{goldfarb04se, 114 | Author = {D. Goldfarb and W. Yin}, 115 | Date-Added = {2005-04-28 23:28:23 -0700}, 116 | Date-Modified = {2005-04-28 23:30:01 -0700}, 117 | Institution = {Columbia University}, 118 | Keywords = {total variation, socp}, 119 | Local-Url = {goldfarb04se.pdf}, 120 | Title = {Second-order cone programming methods for total variation-based image restoration}, 121 | Year = {2004}} 122 | 123 | 124 | @article{hintermueller05in, 125 | Author = {H. Hinterm\"uller and G. Stadler}, 126 | Date-Added = {2005-11-01 15:55:09 -0800}, 127 | Date-Modified = {2005-11-01 15:57:35 -0800}, 128 | Journal = {To appear in SIAM J. Sci. Comput.}, 129 | Keywords = {total variation}, 130 | Local-Url = {hintermueller05in.pdf}, 131 | Title = {An infeasible primal-dual algorithm for {TV}-based inf-convolution-type image restoration}, 132 | Year = {2005}} 133 | 134 | 135 | @article{lobo98ap, 136 | Author = {M. Lobo and L. Vanderberghe and S. Boyd and H. Lebret}, 137 | Date-Added = {2005-10-27 13:24:01 -0700}, 138 | Date-Modified = {2005-10-27 13:26:03 -0700}, 139 | Journal = {Linear Algebra and its Applications}, 140 | Local-Url = {lobo98ap.pdf}, 141 | Pages = {193--228}, 142 | Title = {Applications of second-order cone programming}, 143 | Volume = {284}, 144 | Year = {1998}} 145 | 146 | 147 | @book{nesterov94in, 148 | Address = {Philadelphia}, 149 | Author = {Y. E. Nesterov and A. S. Nemirovski}, 150 | Publisher = {SIAM Publications}, 151 | Title = {Interior Point Polynomial Methods in Convex Programming}, 152 | Year = {1994}} 153 | 154 | @book{nocedal99nu, 155 | Address = {New York}, 156 | Author = {J. Nocedal and S. J. Wright}, 157 | Publisher = {Springer}, 158 | Title = {Numerical Optimization}, 159 | Year = {1999}} 160 | 161 | 162 | @article{paige75so, 163 | Author = {C. C. Paige and M. Saunders}, 164 | Date-Added = {2005-11-09 15:39:31 -0800}, 165 | Date-Modified = {2005-11-09 15:41:44 -0800}, 166 | Journal = {SIAM J. Numer. Anal.}, 167 | Keywords = {symmlq, linear systems}, 168 | Local-Url = {paige75so.pdf}, 169 | Month = {September}, 170 | Number = {4}, 171 | Title = {Solution of sparse indefinite systems of linear equations}, 172 | Volume = {12}, 173 | Year = {1975}} 174 | 175 | 176 | @book{renegar01ma, 177 | Author = {J. Renegar}, 178 | Date-Added = {2005-11-10 13:38:49 -0800}, 179 | Date-Modified = {2005-11-10 13:41:09 -0800}, 180 | Keywords = {optimization, interior point methods}, 181 | Publisher = {SIAM}, 182 | Series = {MPS-SIAM Series on Optimization}, 183 | Title = {A mathematical view of interior-point methods in convex optimization}, 184 | Year = {2001}} 185 | 186 | 187 | @article{rudin92no, 188 | Author = {L. I. Rudin and S. Osher and E. Fatemi}, 189 | Journal = {Physica D}, 190 | Pages = {259--68}, 191 | Title = {Nonlinear total variation noise removal algorithm}, 192 | Volume = {60}, 193 | Year = {1992}} 194 | 195 | 196 | @misc{shewchuk94in, 197 | Author = {J. R. Shewchuk}, 198 | Date-Added = {2005-05-02 15:38:06 -0700}, 199 | Date-Modified = {2005-07-15 15:55:22 -0700}, 200 | Howpublished = {Manuscript}, 201 | Keywords = {conjugate gradients}, 202 | Local-Url = {shewchuk94in.pdf}, 203 | Month = {August}, 204 | Title = {An introduction to the conjugate gradient method without the agonizing pain}, 205 | Url = {www.cs.cmu.edu/~quake-papers/painless-conjugate-gradient.pdf}, 206 | Year = {1994}} 207 | 208 | 209 | @book{wright97pr, 210 | Author = {S. J. Wright}, 211 | Publisher = {SIAM Publications}, 212 | Title = {Primal-Dual Interior-Point Methods}, 213 | Year = {1997}} 214 | 215 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Notes/l1magic_notes.tex: -------------------------------------------------------------------------------- 1 | \documentclass{article} 2 | 3 | \usepackage{amsmath,amssymb,cite,graphicx,array} 4 | 5 | \newcommand{\bpm}{\left(\begin{matrix}} 6 | \newcommand{\epm}{\end{matrix}\right)} 7 | \newcommand{\grad}{\nabla} 8 | \newcommand{\dx}{\Delta x} 9 | \newcommand{\du}{\Delta u} 10 | \newcommand{\dt}{\Delta t} 11 | \newcommand{\dv}{\Delta v} 12 | \newcommand{\dz}{\Delta z} 13 | \newcommand{\dlam}{\Delta\lambda} 14 | \newcommand{\dnu}{\Delta\nu} 15 | \newcommand{\packname}{{\sc $\ell_1$-magic}\ } 16 | 17 | \newcommand{\R}{\mathbb{R}} 18 | \newcommand{\<}{\langle} 19 | \renewcommand{\>}{\rangle} 20 | 21 | \newcommand{\diag}{\operatorname{diag}} 22 | \newcommand{\vzero}{\mathbf{0}} 23 | \newcommand{\vone}{\mathbf{1}} 24 | 25 | % formatting 26 | \parindent = 0 pt 27 | \parskip = 8 pt 28 | \addtolength{\textwidth}{1in} 29 | \addtolength{\oddsidemargin}{-0.5in} 30 | \addtolength{\textheight}{1.6in} 31 | \addtolength{\topmargin}{-0.8in} 32 | 33 | %------------------------------------------------------------------------------- 34 | 35 | \title{\packname: Recovery of Sparse Signals\\ via Convex Programming} 36 | 37 | \author{Emmanuel Cand\`es and Justin Romberg, Caltech} 38 | 39 | \date{October 2005} 40 | 41 | \begin{document} 42 | 43 | \maketitle 44 | 45 | %------------------------------------------------------------------------------- 46 | \section{Seven problems} 47 | 48 | A recent series of papers 49 | \cite{candes04ro,candes04ne,candes05qu,candes05st,candes05da,candes05de} develops a theory of signal recovery from highly incomplete information. The central results state that a sparse vector $x_0\in\R^N$ can be recovered from a small number of linear measurements $b=Ax_0\in\R^K,~K\ll N$ (or $b=Ax_0+e$ when there is measurement noise) by solving a convex program. 50 | 51 | As a companion to these papers, this package includes MATLAB code that implements this recovery procedure in the seven contexts described below. The code is not meant to be cutting-edge, rather it is a proof-of-concept showing that these recovery procedures are computationally tractable, even for large scale problems where the number of data points is in the millions. 52 | 53 | The problems fall into two classes: those which can be recast as linear programs (LPs), and those which can be recast as second-order cone programs (SOCPs). The LPs are solved using a generic path-following primal-dual method. The SOCPs are solved with a generic log-barrier algorithm. The implementations follow Chapter 11 of 54 | \cite{boyd04co}. 55 | 56 | For maximum computational efficiency, the solvers for each of the seven problems are implemented separately. They all have the same basic structure, however, with the computational bottleneck being the calculation of the Newton step (this is discussed in detail below). The code can be used in either ``small scale'' mode, where the system is constructed explicitly and solved exactly, or in ``large scale'' mode, where an iterative matrix-free algorithm such as conjugate gradients (CG) is used to approximately solve the system. 57 | 58 | Our seven problems of interest are: 59 | \begin{itemize} 60 | % 61 | \item {\bf Min-$\ell_1$ with equality constraints.} The program 62 | \[ 63 | (P_1) \quad \min~\|x\|_1\quad\text{subject~to}\quad Ax=b, 64 | \] 65 | also known as {\em basis pursuit}, finds the vector with smallest {\em $\ell_1$ norm} 66 | \[ 67 | \|x\|_1 ~:=~ \sum_i |x_i| 68 | \] 69 | that explains the observations $b$. 70 | As the results in \cite{candes04ro,candes04ne} show, if a sufficiently sparse $x_0$ exists such that $Ax_0=b$, 71 | then $(P_1)$ will find it. When $x,A,b$ have real-valued entries, $(P_1)$ can be recast as an LP (this is discussed in detail in \cite{chen99at}). 72 | % 73 | \item {\bf Minimum $\ell_1$ error approximation.} Let $A$ be a $M\times N$ matrix with full rank. Given $y\in\R^M$, the program 74 | \[ 75 | (P_A) \quad \min_x~\|y-Ax\|_1 76 | \] 77 | finds the vector $x\in\R^N$ such that the {\em error} $y-Ax$ has minimum 78 | $\ell_1$ norm (i.e. we are asking that the difference between $Ax$ and $y$ be sparse). 79 | This program arises in the context of channel coding \cite{candes05de}. 80 | 81 | Suppose we have a channel code that produces a codeword $c=Ax$ for a message $x$. The message travels over the channel, and has an unknown number of its entries corrupted. The decoder observes $y = c + e$, where $e$ is the corruption. 82 | If $e$ is sparse enough, then the decoder can use $(P_A)$ to recover $x$ exactly. Again, $(P_A)$ can be recast as an LP. 83 | % 84 | \item {\bf Min-$\ell_1$ with quadratic constraints.} This program finds the 85 | vector with minimum $\ell_1$ norm that comes close to explaining the observations: 86 | \[ 87 | (P_2) \quad \min~\|x\|_1\quad\text{subject~to}\quad \|Ax-b\|_2\leq \epsilon, 88 | \] 89 | where $\epsilon$ is a user specified parameter. As shown in \cite{candes05st}, if a sufficiently sparse $x_0$ exists such that $b = Ax_0 + e$, for some small error term $\|e\|_2\leq\epsilon$, then the solution $x^\star_2$ to $(P_2)$ will be close to $x_0$. That is, $\|x^\star_2-x_0\|_2\leq C\cdot\epsilon$, where $C$ is a small constant. $(P_2)$ can be recast as a SOCP. 90 | % 91 | \item {\bf Min-$\ell_1$ with bounded residual correlation.} Also referred to as the {\em Dantzig Selector}, 92 | the program 93 | \[ 94 | (P_D) \quad \min~\|x\|_1\quad\text{subject~to}\quad \|A^*(Ax-b)\|_\infty\leq\gamma, 95 | \] 96 | where $\gamma$ is a user specified parameter, 97 | relaxes the equality constraints of $(P_1)$ in a different way. $(P_D)$ requires that the residual $Ax-b$ 98 | of a candidate vector $x$ not be too correlated with any of the columns of $A$ (the product $A^*(Ax-b)$ measures each of these correlations). If $b = Ax_0 + e$, where $e_i\sim \mathcal{N}(0,\sigma^2)$, then the solution $x^\star_D$ to $(P_D)$ has near-optimal minimax risk: 99 | \[ 100 | E\|x^\star_D-x_0\|^2_2 \leq C (\log N)\cdot\sum_i \min(x_0(i)^2, \sigma^2), 101 | \] 102 | (see \cite{candes05da} for details). For real-valued $x,A,b$, $(P_D)$ can again be recast as an LP; in the complex case, there is an equivalent SOCP. 103 | % 104 | \end{itemize} 105 | It is also true that when $x,A,b$ are complex, the programs $(P_1),(P_A),(P_D)$ can be written as SOCPs, but we will not pursue this here. 106 | 107 | If the underlying signal is a 2D image, an alternate recovery model is that the 108 | {\em gradient} is sparse \cite{rudin92no}. 109 | Let $x_{ij}$ denote the pixel in the $i$th row and $j$ column of an $n\times n$ image $x$, and 110 | define the operators 111 | \[ 112 | D_{h;ij}x = \begin{cases} x_{i+1,j}-x_{ij} & i < n\\ 113 | 0 & i = n \end{cases} 114 | \qquad 115 | D_{v;ij}x = \begin{cases} x_{i,j+1}-x_{ij} & j < n\\ 116 | 0 & j = n \end{cases}, 117 | \] 118 | and 119 | \begin{equation} 120 | \label{eq:Dij} 121 | D_{ij}x = \left(\begin{array}{c} D_{h;ij}x \\ D_{v;ij}x \end{array}\right). 122 | \end{equation} 123 | The $2$-vector $D_{ij}x$ can be interpreted as a kind of discrete gradient of the digital image $x$. 124 | The {\em total variation} of $x$ is simply the sum of the magnitudes of this discrete gradient at every point: 125 | \[ 126 | \operatorname{TV}(x) := \sum_{ij} \sqrt{(D_{h;ij}x)^2 + (D_{v;ij}x)^2} = 127 | \sum_{ij} \|D_{ij}x\|_2. 128 | \] 129 | 130 | With these definitions, we have three programs for image recovery, 131 | each of which can be recast as a SOCP: 132 | \begin{itemize} 133 | % 134 | \item {\bf Min-TV with equality constraints.} 135 | \[ 136 | (TV_1) \quad \min~\operatorname{TV}(x) \quad\text{subject~to}\quad Ax=b 137 | \] 138 | If there exists a piecewise constant $x_0$ with sufficiently few edges (i.e.\ $D_{ij}x_0$ is nonzero for only a small number of indices $ij$), then $(TV_1)$ will recover $x_0$ exactly --- see \cite{candes04ro}. 139 | % 140 | \item {\bf Min-TV with quadratic constraints.} 141 | \[ 142 | (TV_2) \quad \min~\operatorname{TV}(x) \quad\text{subject~to}\quad \|Ax-b\|_2\leq\epsilon 143 | \] 144 | Examples of recovering images from noisy observations using $(TV_2)$ were presented in \cite{candes05st}. Note that if $A$ is the identity matrix, $(TV_2)$ reduces to the standard Rudin-Osher-Fatemi image restoration problem \cite{rudin92no}. See also 145 | \cite{chan99no,goldfarb04se,hintermueller05in,lobo98ap} for SOCP solvers specifically designed for the total-variational functional. 146 | 147 | % 148 | \item {\bf Dantzig TV.} 149 | \[ 150 | (TV_D) \quad \min~\operatorname{TV}(x) \quad\text{subject~to}\quad \|A^*(Ax-b)\|_\infty\leq\gamma 151 | \] 152 | This program was used in one of the numerical experiments in \cite{candes05da}. 153 | % 154 | \end{itemize} 155 | 156 | In the next section, we describe how to solve linear and second-order cone programs using modern interior point methods. 157 | 158 | 159 | %------------------------------------------------------------------------------- 160 | \section{Interior point methods} 161 | 162 | Advances in interior point methods for convex optimization over the past 15 years, led by the seminal work \cite{nesterov94in}, have made large-scale solvers for the seven problems above feasible. Below we overview the generic LP and SOCP solvers used in the \packname package to solve these problems. 163 | 164 | %------------------------------------------------------------------------------- 165 | \subsection{A primal-dual algorithm for linear programming} 166 | \label{sec:primaldual} 167 | 168 | In Chapter 11 of \cite{boyd04co}, Boyd and Vandenberghe outline a relatively simple primal-dual 169 | algorithm for linear programming which we have followed very closely for the implementation of 170 | $(P_1)$,$(P_A)$, and $(P_D)$. For the sake of completeness, and to set up the notation, we briefly review their algorithm here. 171 | 172 | The standard-form linear program is 173 | \begin{align*} 174 | \min_{z}~ \ \quad\text{subject~to}\quad 175 | A_0 z & = b, \\[-2mm] 176 | f_i(z) &\leq 0, 177 | \end{align*} 178 | where the search vector $z\in\R^N$, $b\in\R^K$, $A_0$ is a $K\times N$ matrix, and each of the $f_i,~i=1,\ldots,m$ is a linear functional: 179 | \[ 180 | f_i(z) = \ + d_i, 181 | \] 182 | for some $c_i\in\R^N$, $d_i\in\R$. At the optimal point $z^\star$, there will exist dual vectors $\nu^\star\in\R^K,\lambda^\star\in\R^m,\lambda^\star\geq 0$ such that the Karush-Kuhn-Tucker conditions are satisfied: 183 | \begin{align*} 184 | (KKT)\quad\quad 185 | c_0 + A_0^T\nu^\star + \sum_i \lambda^\star_i c_i & = \mathbf{0}, \\ 186 | \lambda^\star_i f_i(z^\star) & = 0,~~i=1,\ldots,m, \\ 187 | A_0 z^\star & = b, \\ 188 | f_i(z^\star) & \leq 0, ~~i=1,\ldots,m.\\ 189 | \end{align*} 190 | In a nutshell, the primal dual algorithm finds the optimal $z^\star$ (along with optimal dual vectors $\nu^\star$ and $\lambda^\star$) by solving this system of nonlinear equations. The solution procedure is the classical Newton method: at an {\em interior point} $(z^k, \nu^k, \lambda^k)$ (by which we mean $f_i(z^k) < 0$, $\lambda^k > 0$), the system is linearized and solved. 191 | However, the step to new point $(z^{k+1}, \nu^{k+1}, \lambda^{k+1})$ must be modified so that we remain in the interior. 192 | 193 | In practice, we relax the {\em complementary slackness} condition $\lambda_i f_i = 0$ to 194 | \begin{equation} 195 | \label{eq:relaxedcs} 196 | \lambda^k_i f_i(z^k) = -1/\tau^k, 197 | \end{equation} 198 | where we judiciously increase the parameter $\tau^k$ as we progress through the Newton iterations. This biases the solution of the linearized equations towards the interior, allowing a smooth, well-defined ``central path'' from an interior point to the solution on the boundary (see 199 | \cite{nocedal99nu,wright97pr} for an extended discussion). 200 | 201 | The primal, dual, and central residuals quantify how close a point $(z,\nu,\lambda)$ is to satisfying $(KKT)$ with \eqref{eq:relaxedcs} in place of the slackness condition: 202 | \begin{eqnarray*} 203 | r_{\mathrm{dual}} & = & c_0 + A_0^T\nu + \sum_i \lambda_i c_i \\ 204 | r_{\mathrm{cent}} & = & -\Lambda f - (1/\tau)\mathbf{1} \\ 205 | r_{\mathrm{pri}} & = & A_0 z-b,\\ 206 | \end{eqnarray*} 207 | where $\Lambda$ is a diagonal matrix with $(\Lambda)_{ii} = \lambda_i$, and 208 | $f = \bpm f_1(z) & \ldots & f_m(z) \epm^T$. 209 | 210 | From a point $(z,\nu,\lambda)$, we want to find a step $(\dz,\dnu,\dlam)$ such that 211 | \begin{equation} 212 | \label{eq:res0} 213 | r_\tau(z+\dz,\nu+\dnu,\lambda+\dlam) = 0. 214 | \end{equation} 215 | Linearizing \eqref{eq:res0} with the Taylor expansion around $(z,\nu,\lambda)$, 216 | \[ 217 | r_\tau(z+\dz,\nu+\dnu,\lambda+\dlam) \approx 218 | r_\tau(z,\nu,\lambda) + J_{r_\tau}(z,\nu\lambda) 219 | \bpm \dz \\ \dnu \\ \dlam \epm, 220 | \] 221 | where $J_{r_\tau}(z,\nu\lambda)$ is the Jacobian of $r_\tau$, we have the system 222 | \[ 223 | \bpm \mathbf{0} & A_0^T & C^T \\ 224 | -\Lambda C & \mathbf{0} & -F \\ 225 | A_0 & \mathbf{0} & \mathbf{0} 226 | \epm 227 | \bpm \dz \\ \dv \\ \dlam \epm = 228 | - \bpm 229 | c_0 + A_0^T\nu + \sum_i \lambda_i c_i \\ 230 | -\Lambda f - (1/\tau)\mathbf{1} \\ 231 | A_0 z-b 232 | \epm, 233 | \] 234 | where $m\times N$ matrix $C$ has the $c^T_i$ as rows, and $F$ is diagonal with 235 | $(F)_{ii} = f_i(z)$. 236 | We can eliminate $\dlam$ using: 237 | \begin{equation} 238 | \label{eq:dlambda} 239 | \dlam = -\Lambda F^{-1}C\dz - \lambda - (1/\tau)f^{-1} 240 | \end{equation} 241 | leaving us with the core system 242 | \begin{equation} 243 | \label{eq:pdnewton} 244 | \bpm -C^TF^{-1}\Lambda C & A_0^T \\ A_0 & \mathbf{0} \epm \bpm \dz \\ \dnu \epm = 245 | \bpm -c_0 + (1/\tau)C^Tf^{-1} - A_0^T\nu 246 | \\ b-A_0 z \epm. 247 | \end{equation} 248 | 249 | With the $(\dz,\dnu,\dlam)$ we have a step direction. To choose the step length $0 0$ for all $i$. 252 | % 253 | \item The norm of the residuals has decreased sufficiently: 254 | \[ 255 | \|r_\tau(z+s\dz,\nu+s\dnu,\lambda+s\dlam)\|_2 \leq (1-\alpha s)\cdot\|r_\tau(z,\nu,\lambda) \|_2, 256 | \] 257 | where $\alpha$ is a user-sprecified parameter (in all of our implementations, we have set $\alpha=0.01$). 258 | \end{enumerate} 259 | Since the $f_i$ are linear functionals, item $1$ is easily addressed. 260 | We choose the maximum step size that just keeps us in the interior. Let 261 | \[ 262 | \mathcal{I}^+_f = \{i : \ > 0\},\quad 263 | \mathcal{I}^-_\lambda \{i : \dlam < 0\}, 264 | \] 265 | and set 266 | \[ 267 | s_{\mathrm{max}} = 0.99\cdot\min\{1,~ 268 | \{-f_i(z)/\,~i\in\mathcal{I}^+_f\},~ 269 | \{-\lambda_i/\dlam_i,~i\in\mathcal{I}^-_\lambda\}\}. 270 | \] 271 | Then starting with $s=s_{\mathrm{max}}$, we check if item $2$ above is satisfied; if not, we set $s^\prime = \beta\cdot s$ and try again. 272 | We have taken $\beta=1/2$ in all of our implementations. 273 | 274 | When $r_{\mathrm{dual}}$ and $r_{\mathrm{pri}}$ are small, the {\em surrogate duality gap} $\eta = -f^T\lambda$ is an approximation to how close a certain $(z,\nu,\lambda)$ is to being opitmal 275 | (i.e.\ $\-\\approx\eta$). The primal-dual algorithm repeats the Newton iterations described above until $\eta$ has decreased below a given tolerance. 276 | 277 | Almost all of the computational burden falls on solving \eqref{eq:pdnewton}. When the matrix $-C^TF^{-1}\Lambda C$ is easily invertible (as it is for $(P_1)$), or there are no equality constraints (as in $(P_A),(P_D)$), \eqref{eq:pdnewton} can be reduced to a symmetric positive definite set of equations. 278 | 279 | When $N$ and $K$ are large, forming the matrix and then solving the linear system of equations in \eqref{eq:pdnewton} is infeasible. However, if fast algorithms exist for applying $C,C^T$ and $A_0,A_0^T$, we can use a ``matrix free'' solver such as Conjugate Gradients. CG is iterative, requiring a few hundred applications of 280 | the constraint matrices (roughly speaking) to get an accurate solution. A CG solver (based on the very nice exposition in \cite{shewchuk94in}) is included with the \packname package. 281 | 282 | The implementations of $(P_1),(P_A),(P_D)$ are nearly identical, save for the calculation of the Newton step. In the Appendix, we derive the Newton step for each of these problems using notation mirroring that used in the actual MATLAB code. 283 | 284 | %------------------------------------------------------------------------------- 285 | \subsection{A log-barrier algorithm for SOCPs} 286 | \label{sec:logbarrier} 287 | 288 | Although primal-dual techniques exist for solving second-order cone programs 289 | (see \cite{alizadeh03se,lobo98ap}), their implementation is not quite as straightforward as in the LP case. Instead, we have implemented each of the SOCP recovery problems using a {\em log-barrier method}. The log-barrier method, for which we will again closely follow the generic (but effective) algorithm described in \cite[Chap. 11]{boyd04co}, is conceptually more straightforward than the primal-dual method described above, but at its core is again solving for a series of Newton steps. 290 | 291 | Each of $(P_2),(TV_1),(TV_2),(TV_D)$ can be written in the form 292 | \begin{align} 293 | \nonumber 294 | \min_z~\ \quad\text{subject~to}\quad A_0z & = b \\ 295 | \label{eq:socp} 296 | f_i(z) &\leq 0,~~i=1,\ldots,m 297 | \end{align} 298 | where each $f_i$ describes either a constraint which is linear 299 | \[ 300 | f_i = \ + d_i 301 | \] 302 | or second-order conic 303 | \[ 304 | f_i(z) = \frac{1}{2}\left(\|A_i z\|_2^2 - (\ + d_i)^2\right) 305 | \] 306 | (the $A_i$ are matrices, the $c_i$ are vectors, and the $d_i$ are scalars). 307 | 308 | 309 | The standard log-barrier method transforms \eqref{eq:socp} 310 | into a series of linearly constrained programs: 311 | \begin{equation} 312 | \label{eq:logbarrier} 313 | \min_z ~ \ + \frac{1}{\tau^k} \sum_{i} 314 | -\log(-f_i(z)) \quad\text{subject~to}\quad A_0 z=b, 315 | \end{equation} 316 | where $\tau^k > \tau^{k-1}$. The inequality constraints have been incorporated into the functional via a penalty function\footnote{The choice of $-\log(-x)$ for the barrier function is not arbitrary, it has a property (termed {\em self-concordance}) that is very important for quick convergence of \eqref{eq:logbarrier} to \eqref{eq:socp} both in theory and in practice (see the very nice exposition in \cite{renegar01ma}).} 317 | which is infinite when the constraint is violated (or even met exactly), and smooth elsewhere. As $\tau^k$ gets large, the solution $z^k$ to \eqref{eq:logbarrier} approaches the solution $z^\star$ to \eqref{eq:socp}: 318 | it can be shown that $\ - \ < m/\tau^k$, i.e.\ we are within $m/\tau^k$ of the optimal value after iteration $k$ ($m/\tau^k$ is called the {\em duality gap}). 319 | The idea here is that each of the smooth subproblems can be solved to fairly high accuracy with just a few iterations of Newton's method, especially 320 | since we can use the solution $z^k$ as a starting point for subproblem $k+1$. 321 | 322 | At log-barrier iteration $k$, Newton's method (which is again iterative) proceeds by forming a series of quadratic approximations to \eqref{eq:logbarrier}, and minimizing each by solving a system of equations (again, we might need to modify the step length to stay in the interior). The quadratic approximation of the functional 323 | \[ 324 | f_0(z) = \ + \frac{1}{\tau}\sum_i -\log(-f_i(z)) 325 | \] 326 | in \eqref{eq:logbarrier} around a point $z$ is given by 327 | \[ 328 | f_0(z+\dz) ~\approx~ z + \ + \frac{1}{2}\ ~:=~ q(z+\dz), 329 | \] 330 | where $g_z$ is the gradient 331 | \[ 332 | g_z = c_0 + \frac{1}{\tau}\sum_i \frac{1}{-f_i(z)}\grad f_i(z) 333 | \] 334 | and $H_z$ is the Hessian matrix 335 | \[ 336 | H_z ~=~ \frac{1}{\tau}\sum_i \frac{1}{f_i(z)^2} \grad f_i(z) (\grad f_i(z))^T ~+ ~ 337 | \frac{1}{\tau}\sum_i \frac{1}{-f_i(z)}\grad^2 f_i(z). 338 | \] 339 | Given that $z$ is feasible (that $A_0 z=b$, in particular), the $\dz$ that minimizes $q(z+\dz)$ subject to $A_0 z=b$ is the solution to the set of linear equations 340 | \begin{equation} 341 | \label{eq:lbnewton} 342 | \tau \bpm H_z & A_0^T \\ A_0 & \vzero \epm \bpm \dz \\ v \epm= -\tau g_z. 343 | \end{equation} 344 | (The vector $v$, which can be interpreted as the Lagrange multipliers for the quality constraints in the quadratic minimization problem, is not directly used.) 345 | 346 | In all of the recovery problems below with the exception of $(TV_1)$, there are no equality constraints ($A_0=\vzero$). In these cases, the system \eqref{eq:lbnewton} is symmetric positive definite, and thus can be solved using CG when the problem is ``large scale''. For the $(TV_1)$ problem, we use the SYMMLQ algorithm (which is similar to CG, and works on symmetric but indefinite systems, see\cite{paige75so}). 347 | 348 | With $\dz$ in hand, we have the Newton step direction. The step length $s\leq 1$ is chosen so that 349 | \begin{enumerate} 350 | % 351 | \item $f_i(z+s\dz) < 0$ for all $i=1,\ldots,m$, 352 | % 353 | \item The functional has decreased suffiently: 354 | \[ 355 | f_0(z+s\dz) < f_0(z) + \alpha s\dz \, 356 | \] 357 | where $\alpha$ is a user-specified parameter (each of the implementations below uses $\alpha=0.01$). This requirement basically states that the decrease must be within a certain percentage of that predicted by the linear model at $z$. 358 | % 359 | \end{enumerate} 360 | As before, we start with $s=1$, and decrease by multiples of $\beta$ until both these conditions are satisfied (all implementations use $\beta = 1/2$). 361 | 362 | 363 | The complete log-barrier implementation for each problem follows the outline: 364 | \begin{enumerate} 365 | \item Inputs: a feasible starting point $z^0$, a tolerance $\eta$, and parameters $\mu$ and an initial $\tau^1$. Set $k=1$. 366 | \item Solve \eqref{eq:logbarrier} via Newton's method (followed by the backtracking line search), using $z^{k-1}$ as an initial point. Call the solution $z^k$. 367 | \item If $m/\tau^k < \eta$, terminate and return $z^k$. 368 | \item Else, set $\tau^{k+1} = \mu\tau^k,~k=k+1$ and go to step 2. 369 | \end{enumerate} 370 | In fact, we can calculate in advance how many iterations the log-barrier algorithm will need: 371 | \[ 372 | \mathrm{barrier~iterations} = \left\lceil \frac{\log m - \log\eta -\log\tau^1}{\log\mu}\right\rceil. 373 | \] 374 | The final issue is the selection of $\tau^1$. Our implementation chooses $\tau^1$ conservatively; it is set so that the duality gap $m/\tau^1$ after the first iteration is equal to $\$. 375 | 376 | In Appendix, we explicitly derive the equations for the Newton step for each of $(P_2),(TV_1),(TV_2),(TV_D)$, again using notation that mirrors the variable names in the code. 377 | 378 | 379 | %------------------------------------------------------------------------------- 380 | \section{Examples} 381 | \label{sec:examples} 382 | 383 | To illustrate how to use the code, the \packname package includes m-files for solving specific instances of each of the above problems (these end in ``\texttt{\_example.m}'' in the main directory). 384 | 385 | \subsection{$\ell_1$ with equality constraints} 386 | 387 | We will begin by going through \texttt{l1eq\_example.m} in detail. This m-file creates a sparse signal, takes a limited number of measurements of that signal, and recovers the signal exactly by solving $(P_1)$. The first part of the procedure is for the most part self-explainatory: 388 | \begin{verbatim} 389 | % put key subdirectories in path if not already there 390 | path(path, './Optimization'); 391 | path(path, './Data'); 392 | 393 | % load random states for repeatable experiments 394 | load RandomStates 395 | rand('state', rand_state); 396 | randn('state', randn_state); 397 | 398 | % signal length 399 | N = 512; 400 | % number of spikes in the signal 401 | T = 20; 402 | % number of observations to make 403 | K = 120; 404 | 405 | % random +/- 1 signal 406 | x = zeros(N,1); 407 | q = randperm(N); 408 | x(q(1:T)) = sign(randn(T,1)); 409 | \end{verbatim} 410 | We add the 'Optimization' directory (where the interior point solvers reside) and the 'Data' directories to the path. The file \texttt{RandomStates.m} contains two variables: \texttt{rand\_state} and \texttt{randn\_state}, which we use to set the states of the random number generators on the next two lines (we want this to be a ``repeatable experiment''). The next few lines set up the problem: a length $512$ signal that contains $20$ spikes is created by choosing $20$ locations at random and then putting $\pm 1$ at these locations. The original signal is shown in Figure~\ref{fig:l1eqexample}(a). The next few lines: 411 | \begin{verbatim} 412 | % measurement matrix 413 | disp('Creating measurment matrix...'); 414 | A = randn(K,N); 415 | A = orth(A')'; 416 | disp('Done.'); 417 | 418 | % observations 419 | y = A*x; 420 | 421 | % initial guess = min energy 422 | x0 = A'*y; 423 | \end{verbatim} 424 | create a measurement ensemble by first creating a $K\times N$ matrix with iid Gaussian entries, and then orthogonalizing the rows. The measurements \texttt{y} are taken, 425 | and the ``minimum energy'' solution \texttt{x0} is calculated (\texttt{x0}, which is shown in Figure~\ref{fig:l1eqexample} is the vector in $\{x: Ax=y\}$ that is closest to the origin). Finally, we recover the signal with: 426 | \begin{verbatim} 427 | % solve the LP 428 | tic 429 | xp = l1eq_pd(x0, A, [], y, 1e-3); 430 | toc 431 | \end{verbatim} 432 | The function \texttt{l1eq\_pd.m} (found in the 'Optimization' subdirectory) implements the primal-dual algorithm presented in Section~\ref{sec:primaldual}; we are sending it our initial guess \texttt{x0} for the solution, the measurement matrix (the third argument, which is used to specify the transpose of the measurement matrix, is unnecessary here --- and hence left empty --- since we are providing \texttt{A} explicitly), the measurements, and the precision to which we want the problem solved (\texttt{l1eq\_pd} will terminate when the surrogate duality gap is below $10^{-3}$). Running the example file at the MATLAB prompt, we have the following output: 433 | {\tiny 434 | \begin{verbatim} 435 | >> l1eq_example 436 | Creating measurment matrix... 437 | Done. 438 | Iteration = 1, tau = 1.921e+02, Primal = 5.272e+01, PDGap = 5.329e+01, Dual res = 9.898e+00, Primal res = 1.466e-14 439 | H11p condition number = 1.122e-02 440 | Iteration = 2, tau = 3.311e+02, Primal = 4.383e+01, PDGap = 3.093e+01, Dual res = 5.009e+00, Primal res = 7.432e-15 441 | H11p condition number = 2.071e-02 442 | Iteration = 3, tau = 5.271e+02, Primal = 3.690e+01, PDGap = 1.943e+01, Dual res = 2.862e+00, Primal res = 1.820e-14 443 | H11p condition number = 2.574e-04 444 | Iteration = 4, tau = 7.488e+02, Primal = 3.272e+01, PDGap = 1.368e+01, Dual res = 1.902e+00, Primal res = 1.524e-14 445 | H11p condition number = 8.140e-05 446 | Iteration = 5, tau = 9.731e+02, Primal = 2.999e+01, PDGap = 1.052e+01, Dual res = 1.409e+00, Primal res = 1.380e-14 447 | H11p condition number = 5.671e-05 448 | Iteration = 6, tau = 1.965e+03, Primal = 2.509e+01, PDGap = 5.210e+00, Dual res = 6.020e-01, Primal res = 4.071e-14 449 | H11p condition number = 2.054e-05 450 | Iteration = 7, tau = 1.583e+04, Primal = 2.064e+01, PDGap = 6.467e-01, Dual res = 6.020e-03, Primal res = 3.126e-13 451 | H11p condition number = 1.333e-06 452 | Iteration = 8, tau = 1.450e+05, Primal = 2.007e+01, PDGap = 7.062e-02, Dual res = 6.020e-05, Primal res = 4.711e-13 453 | H11p condition number = 1.187e-07 454 | Iteration = 9, tau = 1.330e+06, Primal = 2.001e+01, PDGap = 7.697e-03, Dual res = 6.020e-07, Primal res = 2.907e-12 455 | H11p condition number = 3.130e-09 456 | Iteration = 10, tau = 1.220e+07, Primal = 2.000e+01, PDGap = 8.390e-04, Dual res = 6.020e-09, Primal res = 1.947e-11 457 | H11p condition number = 3.979e-11 458 | Elapsed time is 0.141270 seconds. 459 | \end{verbatim} 460 | } 461 | The recovered signal \texttt{xp} is shown in Figure~\ref{fig:l1eqexample}(c). The signal is recovered to fairly high accuracy: 462 | \begin{verbatim} 463 | >> norm(xp-x) 464 | 465 | ans = 466 | 467 | 8.9647e-05 468 | 469 | \end{verbatim} 470 | 471 | %%% 472 | \begin{figure} 473 | \centerline{ 474 | \begin{tabular}{ccc} 475 | \includegraphics[height=1.5in]{Figs/l1eqexample_signal.pdf} & 476 | \includegraphics[height=1.5in]{Figs/l1eqexample_minl2.pdf}& 477 | \includegraphics[height=1.5in]{Figs/l1eqexample_recovered.pdf} \\ 478 | (a) Original & (b) Minimum energy reconstruction & (c) Recovered 479 | \end{tabular} 480 | } 481 | \caption{\small\sl 1D recovery experiment for $\ell_1$ minimization with equality constraints. (a) Original length 512 signal \texttt{x} consisting of 20 spikes. (b) Minimum energy (linear) reconstruction \texttt{x0}. (c) Minimum $\ell_1$ reconstruction \texttt{xp}.} 482 | \label{fig:l1eqexample} 483 | \end{figure} 484 | %%% 485 | 486 | 487 | \subsection{Phantom reconstruction} 488 | 489 | A large scale example is given in \texttt{tveq\_phantom\_example.m}. This files recreates the phantom reconstruction experiment first published in \cite{candes04ro}. The $256\times 256$ Shepp-Logan phantom, shown in Figure~\ref{fig:phantom}(a), is measured at $K=5481$ locations in the 2D Fourier plane; the sampling pattern is shown in Figure~\ref{fig:phantom}(b). The image is then reconstructed exactly using $(TV_1)$. 490 | 491 | The star-shaped Fourier-domain sampling pattern is created with 492 | \begin{verbatim} 493 | % number of radial lines in the Fourier domain 494 | L = 22; 495 | 496 | % Fourier samples we are given 497 | [M,Mh,mh,mhi] = LineMask(L,n); 498 | OMEGA = mhi; 499 | \end{verbatim} 500 | The auxiliary function \texttt{LineMask.m} (found in the `Measurements' subdirectory) creates the star-shaped pattern consisting of 22 lines through the origin. The vector \texttt{OMEGA} 501 | contains the locations of the frequencies used in the sampling pattern. 502 | 503 | This example differs from the previous one in that the code operates in {\em large-scale} mode. The measurement matrix in this example is $5481\times 65536$, making the system \eqref{eq:lbnewton} far too large to solve (or even store) explicitly. (In fact, the measurment matrix itself would require almost 3 gigabytes of memory if stored in double precision.) Instead of creating the measurement matrix explicitly, we provide {\em function handles} that take a vector $x$, and return $Ax$. As discussed above, the Newton steps are solved for using an implicit algorithm. 504 | 505 | To create the implicit matrix, we use the function handles 506 | \begin{verbatim} 507 | A = @(z) A_fhp(z, OMEGA); 508 | At = @(z) At_fhp(z, OMEGA, n); 509 | \end{verbatim} 510 | The function \texttt{A\_fhp.m} takes a length $N$ vector (we treat $n\times n$ images as $N:=n^2$ vectors), and returns samples on the $K$ frequencies. (Actually, since the underlying image is real, \texttt{A\_fhp.m} return the real and imaginary parts of the 2D FFT on the upper half-plane of the domain shown in Figure~\ref{fig:phantom}(b).) 511 | 512 | To solve $(TV_1)$, we call 513 | \begin{verbatim} 514 | xp = tveq_logbarrier(xbp, A, At, y, 1e-1, 2, 1e-8, 600); 515 | \end{verbatim} 516 | The variable \texttt{xbp} is the initial guess (which is again the minimal energy reconstruction shown in Figure~\ref{fig:phantom}(c)), \texttt{y} are the measurements, and\texttt{1e-1} is the desired precision. The sixth input is the value of $\mu$ (the amount by which to increase $\tau^k$ at each iteration; see Section~\ref{sec:logbarrier}). The last two inputs are parameters for the large-scale solver used to find the Newton step. The solvers are iterative, with each iteration requiring one application of \texttt{A} and one application of \texttt{At}. The seventh and eighth arguments above state that we want the solver to iterate until the solution has precision $10^{-8}$ (that is, it finds a $z$ such that $\|Hz-g\|_2/\|g\|_2 \leq 10^{-8}$), or it has reached 600 iterations. 517 | 518 | The recovered phantom is shown in Figure~\ref{fig:phantom}(d). 519 | We have $\|X_{TV}-X\|_2/\|X\|_2 \approx 8\cdot 10^{-3}$. 520 | 521 | 522 | %%% 523 | \begin{figure} 524 | \centerline{ 525 | \begin{tabular}{cccc} 526 | \includegraphics[height=1.5in]{Figs/phantom_orig} & 527 | \includegraphics[height=1.5in]{Figs/phantom_sampling} & 528 | \includegraphics[height=1.5in]{Figs/phantom_backproj} & 529 | \includegraphics[height=1.5in]{Figs/phantom_tv} \\ 530 | {\small (a) Phantom} & {\small (b) Sampling pattern} & 531 | {\small (c) Min energy} & {\small (d) min-TV reconstruction} 532 | \end{tabular} 533 | } 534 | \caption{\small\sl Phantom recovery experiment.} 535 | \label{fig:phantom} 536 | \end{figure} 537 | %%% 538 | 539 | %------------------------------------------------------------------------------- 540 | \subsection{Optimization routines} 541 | 542 | We include a brief description of each of the main optimization routines (type 543 | \texttt{help } in MATLAB for details). Each of these m-files is found in the \texttt{Optimization} subdirectory. 544 | 545 | \begin{tabular}{|p{1.5 in}m{4 in}|} \hline 546 | % 547 | \texttt{cgsolve} & Solves $Ax=b$, where $A$ is symmetric positive definite, using the Conjugate Gradient method. \\[2mm]\hline 548 | % 549 | \texttt{l1dantzig\_pd} & Solves $(P_D)$ (the Dantzig selector) using a primal-dual algorithm. \\[2mm]\hline 550 | % 551 | \texttt{l1decode\_pd} & Solves the norm approximation problem $(P_A)$ (for decoding via linear programming) using a primal-dual algorithm. \\[2mm]\hline 552 | % 553 | \texttt{l1eq\_pd} & Solves the standard Basis Pursuit problem $(P_1)$ using a primal-dual algorithm. \\[2mm]\hline 554 | % 555 | \texttt{l1qc\_logbarrier} & Barrier (``outer'') iterations for solving quadratically constrained $\ell_1$ minimization $(P_2)$. \\[2mm]\hline 556 | % 557 | \texttt {l1qc\_newton} & Newton (``inner'') iterations for solving quadratically constrained $\ell_1$ minimization $(P_2)$. \\[2mm]\hline 558 | % 559 | \texttt{tvdantzig\_logbarrier} & Barrier iterations for solving the TV Dantzig selector $(TV_D)$. \\[2mm]\hline 560 | % 561 | \texttt{tvdantzig\_newton} & Newton iterations for $(TV_D)$. \\[2mm]\hline 562 | % 563 | \texttt{tveq\_logbarrier} & Barrier iterations for equality constrained TV minimizaiton $(TV_1)$. \\[2mm]\hline 564 | % 565 | \texttt{tveq\_newton} & Newton iterations for $(TV_1)$. \\[2mm]\hline 566 | % 567 | \texttt{tvqc\_logbarrier} & Barrier iterations for quadratically constrained TV minimization $(TV_2)$. \\[2mm]\hline 568 | % 569 | \texttt{tvqc\_newton} & Newton iterations for $(TV_2)$. \\[2mm]\hline 570 | % 571 | 572 | \end{tabular} 573 | 574 | %------------------------------------------------------------------------------- 575 | \section{Error messages} 576 | 577 | Here we briefly discuss each of the error messages that the $\ell_1$-{\sc magic} may produce. 578 | 579 | \begin{itemize} 580 | % 581 | \item \texttt{Matrix ill-conditioned. Returning previous iterate.} 582 | This error can occur when the code is running in {\em small-scale} mode; that is, the matrix $A$ is provided explicitly. The error message is produced when the condition number of the 583 | linear system we need to solve to find the step direction (i.e.\ \eqref{eq:pdnewton} for the linear programs, and \eqref{eq:lbnewton} for the SOCPs) has an estimated condition number of less than $10^{-14}$. 584 | 585 | This error most commonly occurs during the last iterations of the primal-dual or log-barrier algorithms. While it means that the solution is not within the tolerance specified (by the primal-dual gap), in practice it is usually pretty close. 586 | % 587 | \item \texttt{Cannot solve system. Returning previous iterate.} 588 | This error is the large-scale analog to the above. The error message is produced when the residual produced by the conjugate gradients algorithm was above $1/2$; essentially this means that CG has not solved the system in any meaningful way. Again, this error typically occurs in the late stages of the optimization algorithm, and is a symptom of the system being ill-conditioned. 589 | % 590 | \item \texttt{Stuck backtracking, returning last iterate.} 591 | This error occurs when the algorithm, after computing the step direction, cannot find a step size small enough that decreases the objective. It is generally occurs in large-scale mode, and is a symptom of CG not solving for the step direction to sufficient precision (if the system is solved perfectly, a small enough step size will always be found). Again, this will typically occur in the late stages of the optimization algorithm. 592 | % 593 | \item \texttt{Starting point infeasible; using x0 = At*inv(AAt)*y.} 594 | Each of the optimization programs expects an initial guess which is {\em feasible} (obeys the constraints). If the \texttt{x0} provided is not, this message is produced, and the algorithm proceeds using the least-squares starting point $x_0 = A^T(AA^T)^{-1}b$. 595 | % 596 | \end{itemize} 597 | 598 | %------------------------------------------------------------------------------- 599 | 600 | \newpage 601 | \section*{Appendix} 602 | \appendix 603 | 604 | %------------------------------------------------------------------------------- 605 | \section{$\ell_1$ minimization with equality constraints} 606 | 607 | When $x$, $A$ and $b$ are real, then $(P_1)$ can be recast as the linear program 608 | \begin{align*} 609 | \min_{x,u}~\sum_i u_i \quad\text{subject~to}\quad 610 | x_i - u_i & \leq 0 \\[-4mm] 611 | -x_i - u_i & \leq 0, \\ 612 | Ax & = b 613 | \end{align*} 614 | which can be solved using the standard primal-dual algorithm outlined in Section~\ref{sec:primaldual} 615 | (again, see \cite[Chap.11]{boyd04co} for a full discussion). 616 | Set 617 | \begin{eqnarray*} 618 | f_{u_1;i} & := & x_i - u_i \\ 619 | f_{u_2;i} & := & -x_i - u_i, 620 | \end{eqnarray*} 621 | with $\lambda_{u_1;i},\lambda_{u_2;i}$ the corresponding dual variables, 622 | and let $f_{u_1}$ be the vector $(f_{u_1;1}~~\ldots~~f_{u_1;N})^T$ (and likewise for $f_{u_2},\lambda_{u_1},\lambda_{u_2}$). 623 | Note that 624 | \[ 625 | \grad f_{u_1;i} = \bpm \delta_i \\ -\delta_i \epm, 626 | \quad 627 | \grad f_{u_2;i} = \bpm -\delta_i \\ -\delta_i \epm, 628 | \quad 629 | \grad^2 f_{u_1;i} = 0, 630 | \quad 631 | \grad^2 f_{u_2;i} = 0, 632 | \] 633 | where $\delta_i$ is the standard basis vector for component $i$. 634 | Thus at a point $(x,u; v,\lambda_{u_1},\lambda_{u_2})$, the central and dual 635 | residuals are 636 | \begin{eqnarray*} 637 | r_{\mathrm{cent}} & = & \bpm -\Lambda_{u_1} f_{u_1} \\ -\Lambda_{u_2} f_{u_2} \epm 638 | - (1/\tau)\mathbf{1}, \\ 639 | r_{\mathrm{dual}} & = & \bpm \lambda_{u_1}-\lambda_{u_2} + A^Tv \\ 640 | \mathbf{1} - \lambda_{u_1}-\lambda_{u_2} \epm, 641 | \end{eqnarray*} 642 | and the Newton step 643 | \eqref{eq:pdnewton} is given by: 644 | \[ 645 | \bpm \Sigma_1 & \Sigma_2 & A^T \\ \Sigma_2 & \Sigma_1 & 0 \\ A & 0 & 0 \epm 646 | \bpm \dx \\ \du \\ \dv \epm = 647 | \bpm w_1 \\ w_2 \\ w_3 \epm := 648 | \bpm (-1/\tau)\cdot(-f^{-1}_{u_1} + f^{-1}_{u_2}) - A^Tv \\ 649 | -\mathbf{1} - (1/\tau)\cdot(f^{-1}_{u_1} + f^{-1}_{u_2}) \\ 650 | b - Ax \epm, 651 | \] 652 | with 653 | \[ 654 | \Sigma_1 = \Lambda_{u_1} F^{-1}_{u_1} - \Lambda_{u_2} F^{-1}_{u_2}, \quad 655 | \Sigma_2 = \Lambda_{u_1} F^{-1}_{u_1} + \Lambda_{u_2} F^{-1}_{u_2}, 656 | \] 657 | (The $F_\bullet$, for example, are diagonal matrices with $(F_\bullet)_{ii} = f_{\bullet;i}$, and $f^{-1}_{\bullet;i} = 1/f_{\bullet;i}$.) 658 | Setting 659 | \[ 660 | \Sigma_x = \Sigma_1 - \Sigma_2^2\Sigma_1^{-1}, 661 | \] 662 | we can eliminate 663 | \begin{eqnarray*} 664 | \dx & = & \Sigma_x^{-1}(w_1 - \Sigma_2\Sigma_1^{-1}w_2 - A^T\dv)\\ 665 | \du & = & \Sigma_1^{-1}(w_2 - \Sigma_2\dx), 666 | \end{eqnarray*} 667 | and solve 668 | \[ 669 | -A\Sigma_x^{-1}A^T\dv = w_3 - A(\Sigma_x^{-1}w_1 - \Sigma_x^{-1}\Sigma_2\Sigma_1^{-1}w_2). 670 | \] 671 | This is a $K\times K$ positive definite system of equations, and can be solved using conjugate gradients. 672 | 673 | Given $\dx,\du,\dv$, we calculate the change in the inequality dual variables as in \eqref{eq:dlambda}: 674 | \begin{eqnarray*} 675 | \dlam_{u_1} & = & \Lambda_{u_1} F^{-1}_{u_1}(-\dx + \du) - \lambda_{u_1} - (1/\tau)f^{-1}_{u_1} \\ 676 | \dlam_{u_2} & = & \Lambda_{u_2} F^{-1}_{u_2}(\dx+\du) - \lambda_{u_2} - (1/\tau)f^{-1}_{u_2}. 677 | \end{eqnarray*} 678 | 679 | %The line search proceeds exactly as described in Section~\ref{sec:primaldual}. 680 | 681 | %------------------------------------------------------------------------------- 682 | \section{$\ell_1$ norm approximation} 683 | \label{sec:l1approx} 684 | 685 | The $\ell_1$ norm approximation problem $(P_A)$ can also be recast as a linear program: 686 | \begin{align*} 687 | \min_{x,u}~\sum_{m=1}^M u_m \quad\text{subject~to}\quad 688 | Ax - u -y & \leq 0 \\[-4mm] 689 | -Ax - u + y & \leq 0, 690 | \end{align*} 691 | (recall that unlike the other 6 problems, here the $M\times N$ matrix $A$ has more rows than columns). 692 | For the primal-dual algorithm, we define 693 | \[ 694 | f_{u_1} = A x - u - y,\quad 695 | f_{u_2} = -A x - u + y. 696 | \] 697 | Given a vector of weights $\sigma\in\R^M$, 698 | \[ 699 | \sum_m \sigma_m\grad f_{u_1;m} = 700 | \bpm A^T\sigma \\ -\sigma \epm, \quad 701 | \sum_m \sigma_m\grad f_{u_2;m} = 702 | \bpm -A^T\sigma \\ -\sigma \epm, 703 | \] 704 | \[ 705 | \sum_m \sigma_m\grad f_{u_1;m}\grad f_{u_1;m}^T = 706 | \bpm A^T\Sigma A & -A^T\Sigma \\ -\Sigma A & \Sigma \epm,\quad 707 | \sum_m \sigma_m\grad f_{u_2;m}\grad f_{u_2;m}^T = 708 | \bpm A^T\Sigma A & A^T\Sigma \\ \Sigma A & \Sigma \epm. 709 | \] 710 | At a point $(x,u;\lambda_{u_1},\lambda_{u_2})$, the dual residual is 711 | \[ 712 | r_{\mathrm{dual}} = 713 | \bpm A^T(\lambda_{u_1}-\lambda_{u_2}) \\ -\lambda_{u_1}-\lambda_{u_2} \epm, 714 | \] 715 | and the Newton step is the solution to 716 | \[ 717 | \bpm A^T\Sigma_{11}A & A^T\Sigma_{12} \\ \Sigma_{12}A & \Sigma_{11}\epm 718 | \bpm \dx \\ \du \epm = 719 | \bpm -(1/\tau)\cdot A^T(-f^{-1}_{u_1} + f^{-1}_{u_2}) \\ 720 | -\mathbf{1} - (1/\tau)\cdot(f^{-1}_{u_1} + f^{-1}_{u_2}) \epm := 721 | \bpm w_1 \\ w_2 \epm 722 | \] 723 | where 724 | \begin{eqnarray*} 725 | \Sigma_{11} & = & -\Lambda_{u_1} F^{-1}_{u_1} - \Lambda_{u_2} F^{-1}_{u_2} \\ 726 | \Sigma_{12} & = & \Lambda_{u_1} F^{-1}_{u_1} - \Lambda_{u_2} F^{-1}_{u_2}. 727 | \end{eqnarray*} 728 | Setting 729 | \[ 730 | \Sigma_x = \Sigma_{11} - \Sigma^2_{12}\Sigma^{-1}_{11}, 731 | \] 732 | we can eliminate $\du = \Sigma_{11}^{-1}(w_2 - \Sigma_{22}A\dx)$, and solve 733 | \[ 734 | A^T\Sigma_x A\dx = w_1 - A^T\Sigma_{22}\Sigma^{-1}_{11}w_2 735 | \] 736 | for $\dx$. Again, $A^T\Sigma_x A$ is a $N\times N$ symmetric positive definite matrix (it is straightforward to verify that each element on the diagonal of $\Sigma_x$ will be strictly positive), 737 | and so the Conjugate Gradients algorithm can be used for large-scale problems. 738 | 739 | Given $\dx,\du$, the step directions for the inequality dual variables are given by 740 | \begin{eqnarray*} 741 | \dlam_{u_1} & = & 742 | -\Lambda_{u_1} F^{-1}_{u_1} (A\dx-\du) - \lambda_{u_1} - (1/\tau)f^{-1}_{u_1} \\ 743 | \dlam_{u_2} & = & 744 | \Lambda_{u_2} F^{-1}_{u_2}(A\dx+\du) - \lambda_{u_2} - (1/\tau)f^{-1}_{u_2}. 745 | \end{eqnarray*} 746 | 747 | %------------------------------------------------------------------------------- 748 | \section{$\ell_1$ Dantzig selection} 749 | \label{sec:l1dantzig} 750 | 751 | An equivalent linear program to $(P_D)$ in the real case is given by: 752 | \begin{align*} 753 | \min_{x,u}~\sum_i u_i \quad\text{subject~to}\quad 754 | x - u & \leq 0, \\[-4mm] 755 | -x - u & \leq 0, \\ 756 | A^T r - \epsilon & \leq 0, \\ 757 | -A^T r - \epsilon & \leq 0, 758 | \end{align*} 759 | where $r = Ax-b$. Taking 760 | \[ 761 | f_{u_1} = x - u,\quad 762 | f_{u_2} = -x -u,\quad 763 | f_{\epsilon_1} = A^T r - \epsilon,\quad 764 | f_{\epsilon_2} = -A^T r - \epsilon, 765 | \] 766 | the residuals at a point 767 | $(x,u;\lambda_{u_1},\lambda_{u_2},\lambda_{\epsilon_1},\lambda_{\epsilon_2})$, 768 | the dual residual is 769 | \[ 770 | r_{\mathrm{dual}} = 771 | \bpm \lambda_{u_1}-\lambda_{u_2} + A^TA(\lambda_{\epsilon_1} - \lambda_{\epsilon_2}) 772 | \\ \mathbf{1} - \lambda_{u_1}-\lambda_{u_2}\epm, 773 | \] 774 | and the Newton step is the solution to 775 | \[ 776 | \bpm A^TA\Sigma_{a}A^TA + \Sigma_{11} & \Sigma_{12} \\ 777 | \Sigma_{12} & \Sigma_{11}\epm 778 | \bpm \dx \\ \du \epm = 779 | \bpm -(1/\tau)\cdot (A^TA(-f^{-1}_{\epsilon_1} + f^{-1}_{\epsilon_2})) - 780 | f^{-1}_{u_1} + f^{-1}_{u_2} \\ 781 | -\mathbf{1} - (1/\tau)\cdot(f^{-1}_{u_1} + f^{-1}_{u_2}) \epm := 782 | \bpm w_1 \\ w_2 \epm 783 | \] 784 | where 785 | \begin{eqnarray*} 786 | \Sigma_{11} & = & -\Lambda_{u_1}F^{-1}_{u_1} - \Lambda_{u_2}F^{-1}_{u_2} \\ 787 | \Sigma_{12} & = & \Lambda_{u_1}F^{-1}_{u_1} - \Lambda_{u_2}F^{-1}_{u_2} \\ 788 | \Sigma_a & = & -\Lambda_{\epsilon_1}F^{-1}_{\epsilon_1} - \Lambda_{\epsilon_2}F^{-1}_{\epsilon_2}. 789 | \end{eqnarray*} 790 | Again setting 791 | \[ 792 | \Sigma_x = \Sigma_{11} - \Sigma^2_{12}\Sigma^{-1}_{11}, 793 | \] 794 | we can eliminate 795 | \[ 796 | \du = \Sigma^{-1}_{11}(w_2 - \Sigma_{12}\dx), 797 | \] 798 | and solve 799 | \[ 800 | (A^TA\Sigma_a A^TA + \Sigma_x)\dx = w_1 - \Sigma_{12}\Sigma^{-1}_{11}w_2 801 | \] 802 | for $\dx$. As before, the system is symmetric positive definite, and the CG algorithm can be used to solve it. 803 | 804 | Given $\dx,\du$, the step directions for the inequality dual variables are given by 805 | \begin{eqnarray*} 806 | \dlam_{u_1} & = & -\Lambda_{u_1} F^{-1}_{u_1}(\dx-\du) - \lambda_{u_1} - (1/\tau)f^{-1}_{u_1} \\ 807 | \dlam_{u_2} & = & -\Lambda_{u_2} F^{-1}_{u_2}(-\dx-\du) - \lambda_{u_2} - (1/\tau)f^{-1}_{u_2} \\ 808 | \dlam_{\epsilon_1} & = & -\Lambda_{\epsilon_1} F^{-1}_{\epsilon_1}(A^TA\dx) - 809 | \lambda_{\epsilon_1} - (1/\tau)f^{-1}_{\epsilon_1} \\ 810 | \dlam_{\epsilon_2} & = & -\Lambda_{\epsilon_2} F^{-1}_{\epsilon_2}(-A^TA\dx) - 811 | \lambda_{\epsilon_2} - (1/\tau)f^{-1}_{\epsilon_2}. 812 | \end{eqnarray*} 813 | 814 | 815 | 816 | %------------------------------------------------------------------------------- 817 | \section{$\ell_1$ minimization with quadratic constraints} 818 | \label{sec:l1qc} 819 | 820 | The quadractically constrained $\ell_1$ minimization problem $(P_2)$ can be recast as the second-order cone program 821 | \begin{align*} 822 | \min_{x,u}~\sum_i u_i \quad\text{subject~to}\quad 823 | x - u & \leq 0, \\[-4mm] 824 | -x - u & \leq 0, \\ 825 | \frac{1}{2}\left(\|Ax-b\|^2_2 - \epsilon^2\right) & \leq 0. 826 | \end{align*} 827 | Taking 828 | \[ 829 | f_{u_1} = x - u,\quad 830 | f_{u_2} = -x -u,\quad 831 | f_\epsilon = \frac{1}{2}\left(\|Ax-b\|^2_2 - \epsilon^2\right), 832 | \] 833 | we can write the Newton step (as in \eqref{eq:lbnewton}) at a point $(x,u)$ for a given $\tau$ as 834 | \[ 835 | \bpm \Sigma_{11} - f_\epsilon^{-1}A^TA + f_\epsilon^{-2}A^Trr^TA & \Sigma_{12} \\ 836 | \Sigma_{12} & \Sigma_{11} \epm 837 | \bpm \dx \\ \du \epm = 838 | \bpm 839 | f_{u_1}^{-1} - f_{u_2}^{-1} + f_\epsilon^{-1}A^Tr \\ 840 | -\tau\mathbf{1} - f_{u_1}^{-1} - f_{u_2}^{-1} 841 | \epm := 842 | \bpm w_1 \\ w_2 \epm 843 | \] 844 | where $r = Ax-b$, and 845 | \begin{eqnarray*} 846 | \Sigma_{11} & = & F_{u_1}^{-2} + F_{u_2}^{-2} \\ 847 | \Sigma_{12} & = & -F_{u_1}^{-2} + F_{u_2}^{-2}. 848 | \end{eqnarray*} 849 | As before, we set 850 | \[ 851 | \Sigma_x = \Sigma_{11} - \Sigma_{12}^2\Sigma_{11}^{-1} 852 | \] 853 | and eliminate $\du$ 854 | \[ 855 | \du = \Sigma_{11}^{-1}(w_2 - \Sigma_{12}\dx), 856 | \] 857 | leaving us with the reduced system 858 | \[ 859 | (\Sigma_x - f_\epsilon^{-1}A^TA + f_\epsilon^{-2}A^Trr^TA)\dx = 860 | w_1 - \Sigma_{12}\Sigma_{11}^{-1}w_2 861 | \] 862 | which is symmetric positive definite and can be solved using CG. 863 | 864 | %------------------------------------------------------------------------------- 865 | \section{Total variation minimization with equality constraints} 866 | \label{sec:tveq} 867 | 868 | The equality constrained TV minimization problem 869 | \[ 870 | \min_{x}~\operatorname{TV}(x)\quad\text{subject~to}\quad Ax=b, 871 | \] 872 | can be rewritten as the SOCP 873 | \begin{align*} 874 | \min_{t,x}~\sum_{ij} t_{ij} \quad\text{s.t.}\quad \|D_{ij}x\|_2 & \leq t_{ij} \\[-2mm] 875 | Ax &= b. 876 | \end{align*} 877 | Defining the inequality functions 878 | \begin{equation} 879 | \label{eq:ftij} 880 | f_{t_{ij}} = \frac{1}{2}\left(\|D_{ij}\|_2^2 - t^2_{ij}\right)\quad i,j=1,\ldots,n 881 | \end{equation} 882 | we have 883 | \[ 884 | \grad f_{t_{ij}} = 885 | \left( \begin{array}{c} D^T_{ij}D_{ij} x \\ -t_{ij}\delta_{ij} \end{array}\right) 886 | \] 887 | \[ 888 | \grad f_{t_{ij}} \grad f_{t_{ij}}^T = 889 | \bpm D^T_{ij}D_{ij} xx^TD^T_{ij}D_{ij} & 890 | -t_{ij}D^T_{ij}D_{ij}x\delta_{ij}^T \\ 891 | -t_{ij}\delta_{ij}x^T D^T_{ij}D_{ij} & 892 | t_{ij}^2\delta_{ij}\delta_{ij}^T \\ 893 | \epm,\quad 894 | \grad^2 f_{t_{ij}} = 895 | \bpm D_{ij}^*D_{ij} & \vzero \\ 896 | \vzero & -\delta_{ij}\delta_{ij}^T 897 | \epm, 898 | \] 899 | where $\delta_{ij}$ is the Kronecker vector that is $1$ in entry $ij$ and zero elsewhere. 900 | For future reference: 901 | \[ 902 | \sum_{ij}\sigma_{ij} \grad f_{t_{ij}} = 903 | \bpm D_h^T\Sigma D_h x + D_v^T\Sigma D_v x \\ -\sigma t\epm, 904 | \] 905 | \[ 906 | \sum_{ij} \sigma_{ij} \grad f_{t_{ij}} \grad f_{t_{ij}}^T = 907 | \left(\begin{array}{cc} 908 | B\Sigma B^T & -BT\Sigma \\ -\Sigma TB^T & \Sigma T^2 909 | \end{array}\right), 910 | \] 911 | \[ 912 | \sum_{ij} \sigma_{ij} \grad^2 f_{t_{ij}} = 913 | \left(\begin{array}{cc} 914 | D_h^T \Sigma D_h + D_v^T \Sigma D_v & \vzero \\ \vzero & -\Sigma 915 | \end{array}\right) 916 | \] 917 | where $\Sigma = \diag(\{\sigma_{ij}\})$, $T = \diag(t)$, $D_h$ has the $D_{h;ij}$ as rows (and likewise for $D_v$), and $B$ is a matrix that depends on $x$: 918 | \[ 919 | B = D_h^T\Sigma_{\partial h} + D_v^T\Sigma_{\partial v}. 920 | \] 921 | with $\Sigma_{\partial h} = \diag(D_h x),~ \Sigma_{\partial v} = \diag(D_v x)$. 922 | 923 | 924 | The Newton system \eqref{eq:lbnewton} for the log-barrier algorithm is then 925 | \[ 926 | \bpm H_{11} & B\Sigma_{12} & A^T \\ 927 | \Sigma_{12}B^T & \Sigma_{22} & \vzero \\ 928 | A & \vzero & \vzero \epm 929 | \bpm \dx \\ \dt \\ \dv \epm ~=~ 930 | \bpm D_h^T F_t^{-1} D_h x + D_v^T F_t^{-1} D_vx \\ 931 | -\tau\mathbf{1} - F_{t}^{-1}t \\ 932 | \vzero \epm ~:= ~ 933 | \bpm w_1 \\ w_2 \\ \vzero \epm, 934 | \] 935 | where 936 | \[ 937 | H_{11} = D_h^T(-F_t^{-1})D_h ~+~ D_v^T(-F_t^{-1})D_v ~+~ B F_t^{-2}B^T. 938 | \] 939 | Eliminating $\dt$ 940 | \begin{eqnarray*} 941 | \dt & = & \Sigma_{22}^{-1}(w_2 - \Sigma_{12}B^T\dx) \\ 942 | & = & \Sigma_{22}^{-1}(w_2 - \Sigma_{12}\Sigma_{\partial h}D_h\dx - \Sigma_{12}\Sigma_{\partial v}D_v\dx), 943 | \end{eqnarray*} 944 | the reduced $(N+K)\times (N+K)$ system is 945 | \begin{equation} 946 | \label{eq:tveqsys} 947 | \bpm H'_{11} & A^T \\ A & \vzero \epm 948 | \bpm \dx \\ \dv \epm = 949 | \bpm w'_1 \\ \vzero \epm 950 | \end{equation} 951 | with 952 | \begin{align*} 953 | H'_{11} = ~& H_{11} - B\Sigma^2_{12}\Sigma_{22}^{-1}B^T \\ 954 | = ~& D_h^T(\Sigma_b\Sigma^2_{\partial h}-F_t^{-1})D_h ~+~ 955 | D_v^T(\Sigma_b\Sigma^2_{\partial v}-F_t^{-1})D_v ~+~\\ 956 | ~& D_h^T(\Sigma_b\Sigma_{\partial h}\Sigma_{\partial v})D_v ~+~ 957 | D_v^T(\Sigma_b\Sigma_{\partial h}\Sigma_{\partial v})D_h\\ 958 | w'_1 = ~& w_1 - B\Sigma_{12}\Sigma_{22}^{-1}w_2 \\ 959 | = ~& w_1 - (D_h^T\Sigma_{\partial h} + D_v^T\Sigma_{\partial v} )\Sigma_{12}\Sigma_{22}^{-1}w_2 \\ 960 | \Sigma_b = ~& F_t^{-2} - \Sigma_{22}^{-1}\Sigma_{12}^2. 961 | \end{align*} 962 | 963 | The system of equations \eqref{eq:tveqsys} is symmetric, but not positive definite. 964 | Note that $D_h$ and $D_v$ are (very) sparse matrices, and hence can be stored and applied very efficiently. This allows us to again solve the system above using an iterative method such as SYMMLQ \cite{paige75so}. 965 | 966 | 967 | 968 | %------------------------------------------------------------------------------- 969 | \section{Total variation minimization with quadratic constraints} 970 | \label{sec:tvqc} 971 | 972 | We can rewrite $(TV_2)$ as the SOCP 973 | \begin{align*} 974 | \min_{x,t}~\sum_{ij} t_{ij} 975 | \quad\text{subject~to}\quad 976 | & \|D_{ij} x\|_2\leq t_{ij}, ~~ i,j=1,\ldots,n \\[-4mm] 977 | & \|Ax - b\|_2 \leq \epsilon 978 | \end{align*} 979 | where $D_{ij}$ is as in \eqref{eq:Dij}. 980 | % 981 | Taking $f_{t_{ij}}$ as in \eqref{eq:ftij} and 982 | \[ 983 | f_\epsilon = \frac{1}{2}\left( \|Ax-b\|^2_2 - \epsilon^2\right), 984 | \] 985 | with 986 | \[ 987 | \grad f_\epsilon = \bpm A^Tr \\ \vzero \epm,\quad 988 | \grad f_\epsilon \grad f_\epsilon^T = 989 | \bpm A^Trr^TA & \vzero \\ \vzero & \vzero \epm, \quad 990 | \grad^2 f_\epsilon = \bpm A^*A & \vzero \\ 991 | \vzero & \vzero \epm 992 | \] 993 | where $r = Ax-b$. 994 | 995 | 996 | Also, 997 | \[ 998 | \grad^2 f_{t_{ij}} = \left(\begin{array}{cc} D_{ij}^*D_{ij} & \vzero \\ 999 | \vzero & -\delta_{ij}\delta_{ij}^T \end{array}\right) 1000 | \quad 1001 | \grad^2 f_\epsilon = \left(\begin{array}{cc} A^*A & \vzero \\ 1002 | \vzero & \vzero \end{array}\right). 1003 | \] 1004 | 1005 | The Newton system is similar to that in equality constraints case: 1006 | \[ 1007 | \bpm H_{11} & B\Sigma_{12} \\ 1008 | \Sigma_{12}B^T & \Sigma_{22} \epm 1009 | \bpm \dx \\ \dt \epm = 1010 | \bpm D_h^T F_t^{-1}D_h x + D_v^TF_t^{-1}D_v x + f_\epsilon^{-1}A^Tr \\ 1011 | -\tau\mathbf{1} - tf^{-1}_t \epm := 1012 | \bpm w_1 \\ w_2 \epm. 1013 | \] 1014 | where $(tf^{-1}_t)_{ij} = t_{ij}/f_{t_{ij}}$, and 1015 | \begin{align*} 1016 | H_{11} = ~& D_h^T(-F_t^{-1})D_h ~+~ D_v^T(-F_t^{-1})D_v ~+~ B F_t^{-2}B^T ~- \\ 1017 | & f_\epsilon^{-1} A^TA ~+~ f_\epsilon^{-2} A^Trr^TA, \\ 1018 | \Sigma_{12} = ~& -TF_t^{-2}, \\ 1019 | \Sigma_{22} = ~& F_t^{-1} + F_t^{-2}T^2, 1020 | \end{align*} 1021 | Again eliminating $\dt$ 1022 | \[ 1023 | \dt = \Sigma_{22}^{-1}(w_2 - \Sigma_{12}\Sigma_{\partial h}D_h\dx - \Sigma_{12}\Sigma_{\partial v}D_v\dx), 1024 | \] 1025 | the key system is 1026 | \[ 1027 | H'_{11}\dx = 1028 | w_1 - (D_h^T\Sigma_{\partial h} + D_v^T\Sigma_{\partial v} )\Sigma_{12}\Sigma_{22}^{-1}w_2 1029 | \] 1030 | where 1031 | \begin{align*} 1032 | H'_{11} = ~& H_{11} - B\Sigma^2_{12}\Sigma_{22}^{-1}B^T \\ 1033 | = ~& D_h^T(\Sigma_b\Sigma^2_{\partial h}-F_t^{-1})D_h ~+~ 1034 | D_v^T(\Sigma_b\Sigma^2_{\partial v}-F_t^{-1})D_v ~+~\\ 1035 | ~& D_h^T(\Sigma_b\Sigma_{\partial h}\Sigma_{\partial v})D_v ~+~ 1036 | D_v^T(\Sigma_b\Sigma_{\partial h}\Sigma_{\partial v})D_h ~-~ \\ 1037 | ~& f_\epsilon^{-1} A^TA ~+~ f_\epsilon^{-2} A^Trr^TA, \\ 1038 | \Sigma_b = ~& F_t^{-2} - \Sigma_{12}^2\Sigma_{22}^{-1}. 1039 | \end{align*} 1040 | 1041 | The system above is symmetric positive definite, and can be solved with CG. 1042 | 1043 | 1044 | %------------------------------------------------------------------------------- 1045 | \section{Total variation minimization with bounded residual correlation} 1046 | \label{sec:tvdantzig} 1047 | 1048 | The $TV$ Dantzig problem has an equivalent SOCP as well: 1049 | \begin{align*} 1050 | \min_{x,t}~\sum_{ij} t_{ij} 1051 | \quad\text{subject to}\quad\quad 1052 | \|D_{ij} x\|_2 & \leq t_{ij}, ~~ i,j=1,\ldots,n \\[-4mm] 1053 | A^T(Ax-b) -\epsilon & \leq 0 \\ 1054 | -A^T(Ax-b) -\epsilon & \leq 0. 1055 | \end{align*} 1056 | % 1057 | The inequality constraint functions are 1058 | \begin{eqnarray*} 1059 | f_{t_{ij}} & = & \frac{1}{2}\left( \|D_{ij}x\|^2_2 - t_{ij}^2\right) \quad i,j=1,\ldots,n\\ 1060 | f_{\epsilon_1} & = & A^T(Ax-b)-\epsilon, \\ 1061 | f_{\epsilon_2} & = & -A^T(Ax-b)-\epsilon, 1062 | \end{eqnarray*} 1063 | with 1064 | \[ 1065 | \sum_{ij}\sigma_{ij} \grad f_{\epsilon_1;ij} = 1066 | \bpm A^TA\sigma \\ \vzero \epm, \quad 1067 | \sum_{ij}\sigma_{ij} \grad f_{\epsilon_2;ij} = 1068 | \bpm -A^TA\sigma \\ \vzero \epm, 1069 | \] 1070 | and 1071 | \[ 1072 | \sum_{ij} \sigma_{ij} \grad f_{\epsilon_1;ij} \grad f_{\epsilon_1;ij}^T = 1073 | \sum_{ij} \sigma_{ij} \grad f_{\epsilon_2;ij} \grad f_{\epsilon_2;ij}^T = 1074 | \bpm A^TA\Sigma A^TA & \vzero \\ \vzero & \vzero \epm. 1075 | \] 1076 | Thus the log barrier Newton system is nearly the same as in the quadratically constrained case: 1077 | \[ 1078 | \bpm H_{11} & B\Sigma_{12} \\ 1079 | \Sigma_{12}B^T & \Sigma_{22} \epm 1080 | \bpm \dx \\ \dt \epm = 1081 | \bpm D_h^T F_t^{-1}D_h x + D_v^TF_t^{-1}D_v x + 1082 | A^TA(f_{\epsilon_1}^{-1}-f_{\epsilon_2}^{-1}) \\ 1083 | -\tau\mathbf{1} - tf^{-1}_t \epm := 1084 | \bpm w_1 \\ w_2 \epm. 1085 | \] 1086 | where 1087 | \begin{align*} 1088 | H_{11} = ~& D_h^T(-F_t^{-1})D_h ~+~ D_v^T(-F_t^{-1})D_v ~+~ B F_t^{-2}B^T ~+~ A^TA\Sigma_a A^TA, \\ 1089 | \Sigma_{12} = ~& -TF_t^{-2}, \\ 1090 | \Sigma_{22} = ~& F_t^{-1} + F_t^{-2}T^2, \\ 1091 | \Sigma_a = ~& F_{\epsilon_1}^{-2} + F_{\epsilon_2}^{-2}. 1092 | \end{align*} 1093 | Eliminating $\dt$ as before 1094 | \[ 1095 | \dt ~=~ \Sigma_{22}^{-1}(w_2 - \Sigma_{12}\Sigma_{\partial h}D_h\dx - 1096 | \Sigma_{12}\Sigma_{\partial v}D_v\dx), 1097 | \] 1098 | the key system is 1099 | \[ 1100 | H'_{11}\dx = 1101 | w_1 - (D_h^T\Sigma_{\partial h} + D_v^T\Sigma_{\partial v} )\Sigma_{12}\Sigma_{22}^{-1}w_2 1102 | \] 1103 | where 1104 | \begin{align*} 1105 | H'_{11} = ~& D_h^T(\Sigma_b\Sigma^2_{\partial h}-F_t^{-1})D_h ~+~ 1106 | D_v^T(\Sigma_b\Sigma^2_{\partial v}-F_t^{-1})D_v ~+~\\ 1107 | ~& D_h^T(\Sigma_b\Sigma_{\partial h}\Sigma_{\partial v})D_v ~+~ 1108 | D_v^T(\Sigma_b\Sigma_{\partial h}\Sigma_{\partial v})D_h ~+~ 1109 | A^TA\Sigma_a A^TA, \\ 1110 | \Sigma_b = ~& F_t^{-2} - \Sigma_{12}^2\Sigma_{22}^{-1}. 1111 | \end{align*} 1112 | 1113 | 1114 | 1115 | 1116 | 1117 | 1118 | %------------------------------------------------------------------------------- 1119 | \vspace{10mm} 1120 | 1121 | \bibliographystyle{plain} 1122 | \bibliography{l1magic} 1123 | 1124 | \end{document} 1125 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/cgsolve.m: -------------------------------------------------------------------------------- 1 | % cgsolve.m 2 | % 3 | % 大概是正定系统的求解 4 | % 5 | % Solve a symmetric positive definite system Ax = b via conjugate gradients. 6 | % 7 | % Usage: [x, res, iter] = cgsolve(A, b, tol, maxiter, verbose) 8 | % 9 | % A - Either an NxN matrix, or a function handle. 10 | % 11 | % b - N vector 12 | % 13 | % tol - Desired precision. Algorithm terminates when 14 | % norm(Ax-b)/norm(b) < tol . 15 | % 16 | % maxiter - Maximum number of iterations. 17 | % 18 | % verbose - If 0, do not print out progress messages. 19 | % If and integer greater than 0, print out progress every 'verbose' iters. 20 | % 21 | % Written by: Justin Romberg, Caltech 22 | % Email: jrom@acm.caltech.edu 23 | % Created: October 2005 24 | % 25 | 26 | function [x, res, iter] = cgsolve(A, b, tol, maxiter, verbose) 27 | 28 | if (nargin < 5), verbose = 1; end 29 | 30 | implicit = isa(A,'function_handle'); 31 | 32 | x = zeros(length(b),1); 33 | r = b; 34 | d = r; 35 | delta = r'*r; 36 | delta0 = b'*b; 37 | numiter = 0; 38 | bestx = x; 39 | bestres = sqrt(delta/delta0); 40 | while ((numiter < maxiter) && (delta > tol^2*delta0)) 41 | 42 | % q = A*d 43 | if (implicit), q = A(d); else q = A*d; end 44 | 45 | alpha = delta/(d'*q); 46 | x = x + alpha*d; 47 | 48 | if (mod(numiter+1,50) == 0) 49 | % r = b - Aux*x 50 | if (implicit), r = b - A(x); else r = b - A*x; end 51 | else 52 | r = r - alpha*q; 53 | end 54 | 55 | deltaold = delta; 56 | delta = r'*r; 57 | beta = delta/deltaold; 58 | d = r + beta*d; 59 | numiter = numiter + 1; 60 | if (sqrt(delta/delta0) < bestres) 61 | bestx = x; 62 | bestres = sqrt(delta/delta0); 63 | end 64 | 65 | if ((verbose) && (mod(numiter,verbose)==0)) 66 | disp(sprintf('cg: Iter = %d, Best residual = %8.3e, Current residual = %8.3e', ... 67 | numiter, bestres, sqrt(delta/delta0))); 68 | end 69 | 70 | end 71 | 72 | if (verbose) 73 | disp(sprintf('cg: Iterations = %d, best residual = %14.8e', numiter, bestres)); 74 | end 75 | x = bestx; 76 | res = bestres; 77 | iter = numiter; 78 | 79 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/l1dantzig_pd.m: -------------------------------------------------------------------------------- 1 | % l1dantzig_pd.m 2 | % 3 | % 对应P_D l1 dantzig的pd求解法 4 | % 5 | % Solves 6 | % min_x ||x||_1 subject to ||A'(Ax-b)||_\infty <= epsilon 7 | % 8 | % Recast as linear program 9 | % min_{x,u} sum(u) s.t. x - u <= 0 10 | % -x - u <= 0 11 | % A'(Ax-b) - epsilon <= 0 12 | % -A'(Ax-b) - epsilon <= 0 13 | % and use primal-dual interior point method. 14 | % 15 | % Usage: xp = l1dantzig_pd(x0, A, At, b, epsilon, pdtol, pdmaxiter, cgtol, cgmaxiter) 16 | % 17 | % x0 - Nx1 vector, initial point. 18 | % 19 | % A - Either a handle to a function that takes a N vector and returns a K 20 | % vector , or a KxN matrix. If A is a function handle, the algorithm 21 | % operates in "largescale" mode, solving the Newton systems via the 22 | % Conjugate Gradients algorithm. 23 | % 24 | % At - Handle to a function that takes a K vector and returns an N vector. 25 | % If A is a KxN matrix, At is ignored. 26 | % 27 | % b - Kx1 vector of observations. 28 | % 29 | % epsilon - scalar or Nx1 vector of correlation constraints 30 | % 31 | % pdtol - Tolerance for primal-dual algorithm (algorithm terminates if 32 | % the duality gap is less than pdtol). 33 | % Default = 1e-3. 34 | % 35 | % pdmaxiter - Maximum number of primal-dual iterations. 36 | % Default = 50. 37 | % 38 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 39 | % Default = 1e-8. 40 | % 41 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 42 | % if A is a matrix. 43 | % Default = 200. 44 | % 45 | % Written by: Justin Romberg, Caltech 46 | % Email: jrom@acm.caltech.edu 47 | % Created: October 2005 48 | % 49 | 50 | function xp = l1dantzig_pd(x0, A, At, b, epsilon, pdtol, pdmaxiter, cgtol, cgmaxiter) 51 | 52 | largescale = isa(A,'function_handle'); 53 | 54 | if (nargin < 6), pdtol = 1e-3; end 55 | if (nargin < 7), pdmaxiter = 50; end 56 | if (nargin < 8), cgtol = 1e-8; end 57 | if (nargin < 9), cgmaxiter = 200; end 58 | 59 | N = length(x0); 60 | 61 | alpha = 0.01; 62 | beta = 0.5; 63 | mu = 10; 64 | 65 | gradf0 = [zeros(N,1); ones(N,1)]; 66 | 67 | 68 | % starting point --- make sure that it is feasible 69 | if (largescale) 70 | if (max( abs(At(A(x0) - b)) - epsilon ) > 0) 71 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 72 | AAt = @(z) A(At(z)); 73 | [w, cgres] = cgsolve(AAt, b, cgtol, cgmaxiter, 0); 74 | if (cgres > 1/2) 75 | disp('A*At is ill-conditioned: cannot find starting point'); 76 | xp = x0; 77 | return; 78 | end 79 | x0 = At(w); 80 | end 81 | else 82 | if (max(abs(A'*(A*x0 - b)) - epsilon ) > 0) 83 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 84 | opts.POSDEF = true; opts.SYM = true; 85 | [w, hcond] = linsolve(A*A', b, opts); 86 | if (hcond < 1e-14) 87 | disp('A*At is ill-conditioned: cannot find starting point'); 88 | xp = x0; 89 | return; 90 | end 91 | x0 = A'*w; 92 | end 93 | end 94 | x = x0; 95 | u = (0.95)*abs(x0) + (0.10)*max(abs(x0)); 96 | 97 | % set up for the first iteration 98 | if (largescale) 99 | Atr = At(A(x) - b); 100 | else 101 | Atr = A'*(A*x - b); 102 | end 103 | fu1 = x - u; 104 | fu2 = -x - u; 105 | fe1 = Atr - epsilon; 106 | fe2 = -Atr - epsilon; 107 | lamu1 = -(1./fu1); 108 | lamu2 = -(1./fu2); 109 | lame1 = -(1./fe1); 110 | lame2 = -(1./fe2); 111 | if (largescale) 112 | AtAv = At(A(lame1-lame2)); 113 | else 114 | AtAv = A'*(A*(lame1-lame2)); 115 | end 116 | 117 | % sdg = surrogate duality gap 118 | sdg = -[fu1; fu2; fe1; fe2]'*[lamu1; lamu2; lame1; lame2]; 119 | tau = mu*(4*N)/sdg; 120 | 121 | % residuals 122 | rdual = gradf0 + [lamu1-lamu2 + AtAv; -lamu1-lamu2]; 123 | rcent = -[lamu1.*fu1; lamu2.*fu2; lame1.*fe1; lame2.*fe2] - (1/tau); 124 | resnorm = norm([rdual; rcent]); 125 | 126 | % iterations 127 | pditer = 0; 128 | done = (sdg < pdtol) | (pditer >= pdmaxiter); 129 | while (~done) 130 | 131 | % solve for step direction 132 | w2 = - 1 - (1/tau)*(1./fu1 + 1./fu2); 133 | 134 | sig11 = -lamu1./fu1 - lamu2./fu2; 135 | sig12 = lamu1./fu1 - lamu2./fu2; 136 | siga = -(lame1./fe1 + lame2./fe2); 137 | sigx = sig11 - sig12.^2./sig11; 138 | 139 | if (largescale) 140 | w1 = -(1/tau)*( At(A(1./fe2-1./fe1)) + 1./fu2 - 1./fu1 ); 141 | w1p = w1 - (sig12./sig11).*w2; 142 | hpfun = @(z) At(A(siga.*At(A(z)))) + sigx.*z; 143 | [dx, cgres, cgiter] = cgsolve(hpfun, w1p, cgtol, cgmaxiter, 0); 144 | if (cgres > 1/2) 145 | disp('Cannot solve system. Returning previous iterate. (See Section 4 of notes for more information.)'); 146 | xp = x; 147 | return 148 | end 149 | AtAdx = At(A(dx)); 150 | else 151 | w1 = -(1/tau)*( A'*(A*(1./fe2-1./fe1)) + 1./fu2 - 1./fu1 ); 152 | w1p = w1 - (sig12./sig11).*w2; 153 | Hp = A'*(A*sparse(diag(siga))*A')*A + diag(sigx); 154 | opts.POSDEF = true; opts.SYM = true; 155 | [dx, hcond] = linsolve(Hp, w1p,opts); 156 | if (hcond < 1e-14) 157 | disp('Matrix ill-conditioned. Returning previous iterate. (See Section 4 of notes for more information.)'); 158 | xp = x; 159 | return 160 | end 161 | AtAdx = A'*(A*dx); 162 | end 163 | du = w2./sig11 - (sig12./sig11).*dx; 164 | 165 | dlamu1 = -(lamu1./fu1).*(dx-du) - lamu1 - (1/tau)*1./fu1; 166 | dlamu2 = -(lamu2./fu2).*(-dx-du) - lamu2 - (1/tau)*1./fu2; 167 | dlame1 = -(lame1./fe1).*(AtAdx) - lame1 - (1/tau)*1./fe1; 168 | dlame2 = -(lame2./fe2).*(-AtAdx) - lame2 - (1/tau)*1./fe2; 169 | if (largescale) 170 | AtAdv = At(A(dlame1-dlame2)); 171 | else 172 | AtAdv = A'*(A*(dlame1-dlame2)); 173 | end 174 | 175 | 176 | % find minimal step size that keeps ineq functions < 0, dual vars > 0 177 | iu1 = find(dlamu1 < 0); iu2 = find(dlamu2 < 0); 178 | ie1 = find(dlame1 < 0); ie2 = find(dlame2 < 0); 179 | ifu1 = find((dx-du) > 0); ifu2 = find((-dx-du) > 0); 180 | ife1 = find(AtAdx > 0); ife2 = find(-AtAdx > 0); 181 | smax = min(1,min([... 182 | -lamu1(iu1)./dlamu1(iu1); -lamu2(iu2)./dlamu2(iu2); ... 183 | -lame1(ie1)./dlame1(ie1); -lame2(ie2)./dlame2(ie2); ... 184 | -fu1(ifu1)./(dx(ifu1)-du(ifu1)); -fu2(ifu2)./(-dx(ifu2)-du(ifu2)); ... 185 | -fe1(ife1)./AtAdx(ife1); -fe2(ife2)./(-AtAdx(ife2)) ])); 186 | s = 0.99*smax; 187 | 188 | % backtracking line search 189 | suffdec = 0; 190 | backiter = 0; 191 | while (~suffdec) 192 | xp = x + s*dx; up = u + s*du; 193 | Atrp = Atr + s*AtAdx; AtAvp = AtAv + s*AtAdv; 194 | fu1p = fu1 + s*(dx-du); fu2p = fu2 + s*(-dx-du); 195 | fe1p = fe1 + s*AtAdx; fe2p = fe2 + s*(-AtAdx); 196 | lamu1p = lamu1 + s*dlamu1; lamu2p = lamu2 + s*dlamu2; 197 | lame1p = lame1 + s*dlame1; lame2p = lame2 + s*dlame2; 198 | rdp = gradf0 + [lamu1p-lamu2p + AtAvp; -lamu1p-lamu2p]; 199 | rcp = -[lamu1p.*fu1p; lamu2p.*fu2p; lame1p.*fe1p; lame2p.*fe2p] - (1/tau); 200 | suffdec = (norm([rdp; rcp]) <= (1-alpha*s)*resnorm); 201 | s = beta*s; 202 | backiter = backiter+1; 203 | if (backiter > 32) 204 | disp('Stuck backtracking, returning last iterate. (See Section 4 of notes for more information.)') 205 | xp = x; 206 | return 207 | end 208 | end 209 | 210 | % setup for next iteration 211 | x = xp; u = up; 212 | Atr = Atrp; AtAv = AtAvp; 213 | fu1 = fu1p; fu2 = fu2p; 214 | fe1 = fe1p; fe2 = fe2p; 215 | lamu1 = lamu1p; lamu2 = lamu2p; 216 | lame1 = lame1p; lame2 = lame2p; 217 | 218 | sdg = -[fu1; fu2; fe1; fe2]'*[lamu1; lamu2; lame1; lame2]; 219 | tau = mu*(4*N)/sdg; 220 | 221 | rdual = rdp; 222 | rcent = -[lamu1.*fu1; lamu2.*fu2; lame1.*fe1; lame2.*fe2] - (1/tau); 223 | resnorm = norm([rdual; rcent]); 224 | 225 | pditer = pditer+1; 226 | done = (sdg < pdtol) | (pditer >= pdmaxiter); 227 | 228 | disp(sprintf('Iteration = %d, tau = %8.3e, Primal = %8.3e, PDGap = %8.3e, Dual res = %8.3e',... 229 | pditer, tau, sum(u), sdg, norm(rdual))); 230 | if (largescale) 231 | disp(sprintf(' CG Res = %8.3e, CG Iter = %d', cgres, cgiter)); 232 | else 233 | disp(sprintf(' H11p condition number = %8.3e', hcond)); 234 | end 235 | 236 | end 237 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/l1decode_pd.m: -------------------------------------------------------------------------------- 1 | % l1decode_pd.m 2 | % 3 | % 对应P_2 解码的pd求解法 4 | % 5 | % Decoding via linear programming. 6 | % Solve 7 | % min_x ||b-Ax||_1 . 8 | % 9 | % Recast as the linear program 10 | % min_{x,u} sum(u) s.t. -Ax - u + y <= 0 11 | % Ax - u - y <= 0 12 | % and solve using primal-dual interior point method. 13 | % 14 | % Usage: xp = l1decode_pd(x0, A, At, y, pdtol, pdmaxiter, cgtol, cgmaxiter) 15 | % 16 | % x0 - Nx1 vector, initial point. 17 | % 18 | % A - Either a handle to a function that takes a N vector and returns a M 19 | % vector, or a MxN matrix. If A is a function handle, the algorithm 20 | % operates in "largescale" mode, solving the Newton systems via the 21 | % Conjugate Gradients algorithm. 22 | % 23 | % At - Handle to a function that takes an M vector and returns an N vector. 24 | % If A is a matrix, At is ignored. 25 | % 26 | % y - Mx1 observed code (M > N). 27 | % 28 | % pdtol - Tolerance for primal-dual algorithm (algorithm terminates if 29 | % the duality gap is less than pdtol). 30 | % Default = 1e-3. 31 | % 32 | % pdmaxiter - Maximum number of primal-dual iterations. 33 | % Default = 50. 34 | % 35 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 36 | % Default = 1e-8. 37 | % 38 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 39 | % if A is a matrix. 40 | % Default = 200. 41 | % 42 | % Written by: Justin Romberg, Caltech 43 | % Email: jrom@acm.caltech.edu 44 | % Created: October 2005 45 | % 46 | 47 | function xp = l1decode_pd(x0, A, At, y, pdtol, pdmaxiter, cgtol, cgmaxiter) 48 | 49 | largescale = isa(A,'function_handle'); 50 | 51 | if (nargin < 5), pdtol = 1e-3; end 52 | if (nargin < 6), pdmaxiter = 50; end 53 | if (nargin < 7), cgtol = 1e-8; end 54 | if (nargin < 8), cgmaxiter = 200; end 55 | 56 | N = length(x0); 57 | M = length(y); 58 | 59 | alpha = 0.01; 60 | beta = 0.5; 61 | mu = 10; 62 | 63 | gradf0 = [zeros(N,1); ones(M,1)]; 64 | 65 | x = x0; 66 | if (largescale), Ax = A(x); else Ax = A*x; end 67 | u = (0.95)*abs(y-Ax) + (0.10)*max(abs(y-Ax)); 68 | 69 | fu1 = Ax - y - u; 70 | fu2 = -Ax + y - u; 71 | 72 | lamu1 = -1./fu1; 73 | lamu2 = -1./fu2; 74 | 75 | if (largescale), Atv = At(lamu1-lamu2); else Atv = A'*(lamu1-lamu2); end 76 | 77 | sdg = -(fu1'*lamu1 + fu2'*lamu2); 78 | tau = mu*2*M/sdg; 79 | 80 | rcent = [-lamu1.*fu1; -lamu2.*fu2] - (1/tau); 81 | rdual = gradf0 + [Atv; -lamu1-lamu2]; 82 | resnorm = norm([rdual; rcent]); 83 | 84 | pditer = 0; 85 | done = (sdg < pdtol)| (pditer >= pdmaxiter); 86 | while (~done) 87 | 88 | pditer = pditer + 1; 89 | 90 | w2 = -1 - 1/tau*(1./fu1 + 1./fu2); 91 | 92 | sig1 = -lamu1./fu1 - lamu2./fu2; 93 | sig2 = lamu1./fu1 - lamu2./fu2; 94 | sigx = sig1 - sig2.^2./sig1; 95 | 96 | if (largescale) 97 | w1 = -1/tau*(At(-1./fu1 + 1./fu2)); 98 | w1p = w1 - At((sig2./sig1).*w2); 99 | h11pfun = @(z) At(sigx.*A(z)); 100 | [dx, cgres, cgiter] = cgsolve(h11pfun, w1p, cgtol, cgmaxiter, 0); 101 | if (cgres > 1/2) 102 | disp('Cannot solve system. Returning previous iterate. (See Section 4 of notes for more information.)'); 103 | xp = x; 104 | return 105 | end 106 | Adx = A(dx); 107 | else 108 | w1 = -1/tau*(A'*(-1./fu1 + 1./fu2)); 109 | w1p = w1 - A'*((sig2./sig1).*w2); 110 | H11p = A'*(sparse(diag(sigx))*A); 111 | opts.POSDEF = true; opts.SYM = true; 112 | [dx, hcond] = linsolve(H11p, w1p,opts); 113 | if (hcond < 1e-14) 114 | disp('Matrix ill-conditioned. Returning previous iterate. (See Section 4 of notes for more information.)'); 115 | xp = x; 116 | return 117 | end 118 | Adx = A*dx; 119 | end 120 | 121 | du = (w2 - sig2.*Adx)./sig1; 122 | 123 | dlamu1 = -(lamu1./fu1).*(Adx-du) - lamu1 - (1/tau)*1./fu1; 124 | dlamu2 = (lamu2./fu2).*(Adx + du) -lamu2 - (1/tau)*1./fu2; 125 | if (largescale), Atdv = At(dlamu1-dlamu2); else Atdv = A'*(dlamu1-dlamu2); end 126 | 127 | % make sure that the step is feasible: keeps lamu1,lamu2 > 0, fu1,fu2 < 0 128 | indl = find(dlamu1 < 0); indu = find(dlamu2 < 0); 129 | s = min([1; -lamu1(indl)./dlamu1(indl); -lamu2(indu)./dlamu2(indu)]); 130 | indl = find((Adx-du) > 0); indu = find((-Adx-du) > 0); 131 | s = (0.99)*min([s; -fu1(indl)./(Adx(indl)-du(indl)); -fu2(indu)./(-Adx(indu)-du(indu))]); 132 | 133 | % backtrack 134 | suffdec = 0; 135 | backiter = 0; 136 | while(~suffdec) 137 | xp = x + s*dx; up = u + s*du; 138 | Axp = Ax + s*Adx; Atvp = Atv + s*Atdv; 139 | lamu1p = lamu1 + s*dlamu1; lamu2p = lamu2 + s*dlamu2; 140 | fu1p = Axp - y - up; fu2p = -Axp + y - up; 141 | rdp = gradf0 + [Atvp; -lamu1p-lamu2p]; 142 | rcp = [-lamu1p.*fu1p; -lamu2p.*fu2p] - (1/tau); 143 | suffdec = (norm([rdp; rcp]) <= (1-alpha*s)*resnorm); 144 | s = beta*s; 145 | backiter = backiter + 1; 146 | if (backiter > 32) 147 | disp('Stuck backtracking, returning last iterate. (See Section 4 of notes for more information.)') 148 | xp = x; 149 | return 150 | end 151 | end 152 | 153 | % next iteration 154 | x = xp; u = up; 155 | Ax = Axp; Atv = Atvp; 156 | lamu1 = lamu1p; lamu2 = lamu2p; 157 | fu1 = fu1p; fu2 = fu2p; 158 | 159 | % surrogate duality gap 160 | sdg = -(fu1'*lamu1 + fu2'*lamu2); 161 | tau = mu*2*M/sdg; 162 | rcent = [-lamu1.*fu1; -lamu2.*fu2] - (1/tau); 163 | rdual = rdp; 164 | resnorm = norm([rdual; rcent]); 165 | 166 | done = (sdg < pdtol) | (pditer >= pdmaxiter); 167 | 168 | disp(sprintf('Iteration = %d, tau = %8.3e, Primal = %8.3e, PDGap = %8.3e, Dual res = %8.3e',... 169 | pditer, tau, sum(u), sdg, norm(rdual))); 170 | if (largescale) 171 | disp(sprintf(' CG Res = %8.3e, CG Iter = %d', cgres, cgiter)); 172 | else 173 | disp(sprintf(' H11p condition number = %8.3e', hcond)); 174 | end 175 | 176 | end 177 | 178 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/l1eq_pd.m: -------------------------------------------------------------------------------- 1 | % l1eq_pd.m 2 | % 3 | % 对应P_1 等式约束的pd求解 4 | % 5 | % Solve 6 | % min_x ||x||_1 s.t. Ax = b 7 | % 8 | % Recast as linear program 9 | % min_{x,u} sum(u) s.t. -u <= x <= u, Ax=b 10 | % and use primal-dual interior point method 11 | % 12 | % Usage: xp = l1eq_pd(x0, A, At, b, pdtol, pdmaxiter, cgtol, cgmaxiter) 13 | % 14 | % x0 - Nx1 vector, initial point. 15 | % 16 | % A - Either a handle to a function that takes a N vector and returns a K 17 | % vector , or a KxN matrix. If A is a function handle, the algorithm 18 | % operates in "largescale" mode, solving the Newton systems via the 19 | % Conjugate Gradients algorithm. 20 | % 21 | % At - Handle to a function that takes a K vector and returns an N vector. 22 | % If A is a KxN matrix, At is ignored. 23 | % 24 | % b - Kx1 vector of observations. 25 | % 26 | % pdtol - Tolerance for primal-dual algorithm (algorithm terminates if 27 | % the duality gap is less than pdtol). 28 | % Default = 1e-3. 29 | % 30 | % pdmaxiter - Maximum number of primal-dual iterations. 31 | % Default = 50. 32 | % 33 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 34 | % Default = 1e-8. 35 | % 36 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 37 | % if A is a matrix. 38 | % Default = 200. 39 | % 40 | % Written by: Justin Romberg, Caltech 41 | % Email: jrom@acm.caltech.edu 42 | % Created: October 2005 43 | % 44 | 45 | function xp = l1eq_pd(x0, A, At, b, pdtol, pdmaxiter, cgtol, cgmaxiter) 46 | 47 | largescale = isa(A,'function_handle'); 48 | 49 | if (nargin < 5), pdtol = 1e-3; end 50 | if (nargin < 6), pdmaxiter = 50; end 51 | if (nargin < 7), cgtol = 1e-8; end 52 | if (nargin < 8), cgmaxiter = 200; end 53 | 54 | N = length(x0); 55 | 56 | alpha = 0.01; 57 | beta = 0.5; 58 | mu = 10; 59 | 60 | gradf0 = [zeros(N,1); ones(N,1)]; 61 | 62 | % starting point --- make sure that it is feasible 63 | if (largescale) 64 | if (norm(A(x0)-b)/norm(b) > cgtol) 65 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 66 | AAt = @(z) A(At(z)); 67 | [w, cgres, cgiter] = cgsolve(AAt, b, cgtol, cgmaxiter, 0); 68 | if (cgres > 1/2) 69 | disp('A*At is ill-conditioned: cannot find starting point'); 70 | xp = x0; 71 | return; 72 | end 73 | x0 = At(w); 74 | end 75 | else 76 | if (norm(A*x0-b)/norm(b) > cgtol) 77 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 78 | opts.POSDEF = true; opts.SYM = true; 79 | [w, hcond] = linsolve(A*A', b, opts); 80 | if (hcond < 1e-14) 81 | disp('A*At is ill-conditioned: cannot find starting point'); 82 | xp = x0; 83 | return; 84 | end 85 | x0 = A'*w; 86 | end 87 | end 88 | x = x0; 89 | u = (0.95)*abs(x0) + (0.10)*max(abs(x0)); 90 | 91 | % set up for the first iteration 92 | fu1 = x - u; 93 | fu2 = -x - u; 94 | lamu1 = -1./fu1; 95 | lamu2 = -1./fu2; 96 | if (largescale) 97 | v = -A(lamu1-lamu2); 98 | Atv = At(v); 99 | rpri = A(x) - b; 100 | else 101 | v = -A*(lamu1-lamu2); 102 | Atv = A'*v; 103 | rpri = A*x - b; 104 | end 105 | 106 | sdg = -(fu1'*lamu1 + fu2'*lamu2); 107 | tau = mu*2*N/sdg; 108 | 109 | rcent = [-lamu1.*fu1; -lamu2.*fu2] - (1/tau); 110 | rdual = gradf0 + [lamu1-lamu2; -lamu1-lamu2] + [Atv; zeros(N,1)]; 111 | resnorm = norm([rdual; rcent; rpri]); 112 | 113 | pditer = 0; 114 | done = (sdg < pdtol) | (pditer >= pdmaxiter); 115 | while (~done) 116 | 117 | pditer = pditer + 1; 118 | 119 | w1 = -1/tau*(-1./fu1 + 1./fu2) - Atv; 120 | w2 = -1 - 1/tau*(1./fu1 + 1./fu2); 121 | w3 = -rpri; 122 | 123 | sig1 = -lamu1./fu1 - lamu2./fu2; 124 | sig2 = lamu1./fu1 - lamu2./fu2; 125 | sigx = sig1 - sig2.^2./sig1; 126 | 127 | if (largescale) 128 | w1p = w3 - A(w1./sigx - w2.*sig2./(sigx.*sig1)); 129 | h11pfun = @(z) -A(1./sigx.*At(z)); 130 | [dv, cgres, cgiter] = cgsolve(h11pfun, w1p, cgtol, cgmaxiter, 0); 131 | if (cgres > 1/2) 132 | disp('Cannot solve system. Returning previous iterate. (See Section 4 of notes for more information.)'); 133 | xp = x; 134 | return 135 | end 136 | dx = (w1 - w2.*sig2./sig1 - At(dv))./sigx; 137 | Adx = A(dx); 138 | Atdv = At(dv); 139 | else 140 | w1p = -(w3 - A*(w1./sigx - w2.*sig2./(sigx.*sig1))); 141 | H11p = A*(sparse(diag(1./sigx))*A'); 142 | opts.POSDEF = true; opts.SYM = true; 143 | [dv,hcond] = linsolve(H11p, w1p, opts); 144 | if (hcond < 1e-14) 145 | disp('Matrix ill-conditioned. Returning previous iterate. (See Section 4 of notes for more information.)'); 146 | xp = x; 147 | return 148 | end 149 | dx = (w1 - w2.*sig2./sig1 - A'*dv)./sigx; 150 | Adx = A*dx; 151 | Atdv = A'*dv; 152 | end 153 | 154 | du = (w2 - sig2.*dx)./sig1; 155 | 156 | dlamu1 = (lamu1./fu1).*(-dx+du) - lamu1 - (1/tau)*1./fu1; 157 | dlamu2 = (lamu2./fu2).*(dx+du) - lamu2 - 1/tau*1./fu2; 158 | 159 | % make sure that the step is feasible: keeps lamu1,lamu2 > 0, fu1,fu2 < 0 160 | indp = find(dlamu1 < 0); indn = find(dlamu2 < 0); 161 | s = min([1; -lamu1(indp)./dlamu1(indp); -lamu2(indn)./dlamu2(indn)]); 162 | indp = find((dx-du) > 0); indn = find((-dx-du) > 0); 163 | s = (0.99)*min([s; -fu1(indp)./(dx(indp)-du(indp)); -fu2(indn)./(-dx(indn)-du(indn))]); 164 | 165 | % backtracking line search 166 | suffdec = 0; 167 | backiter = 0; 168 | while (~suffdec) 169 | xp = x + s*dx; up = u + s*du; 170 | vp = v + s*dv; Atvp = Atv + s*Atdv; 171 | lamu1p = lamu1 + s*dlamu1; lamu2p = lamu2 + s*dlamu2; 172 | fu1p = xp - up; fu2p = -xp - up; 173 | rdp = gradf0 + [lamu1p-lamu2p; -lamu1p-lamu2p] + [Atvp; zeros(N,1)]; 174 | rcp = [-lamu1p.*fu1p; -lamu2p.*fu2p] - (1/tau); 175 | rpp = rpri + s*Adx; 176 | suffdec = (norm([rdp; rcp; rpp]) <= (1-alpha*s)*resnorm); 177 | s = beta*s; 178 | backiter = backiter + 1; 179 | if (backiter > 32) 180 | disp('Stuck backtracking, returning last iterate. (See Section 4 of notes for more information.)') 181 | xp = x; 182 | return 183 | end 184 | end 185 | 186 | 187 | % next iteration 188 | x = xp; u = up; 189 | v = vp; Atv = Atvp; 190 | lamu1 = lamu1p; lamu2 = lamu2p; 191 | fu1 = fu1p; fu2 = fu2p; 192 | 193 | % surrogate duality gap 194 | sdg = -(fu1'*lamu1 + fu2'*lamu2); 195 | tau = mu*2*N/sdg; 196 | rpri = rpp; 197 | rcent = [-lamu1.*fu1; -lamu2.*fu2] - (1/tau); 198 | rdual = gradf0 + [lamu1-lamu2; -lamu1-lamu2] + [Atv; zeros(N,1)]; 199 | resnorm = norm([rdual; rcent; rpri]); 200 | 201 | done = (sdg < pdtol) | (pditer >= pdmaxiter); 202 | 203 | disp(sprintf('Iteration = %d, tau = %8.3e, Primal = %8.3e, PDGap = %8.3e, Dual res = %8.3e, Primal res = %8.3e',... 204 | pditer, tau, sum(u), sdg, norm(rdual), norm(rpri))); 205 | if (largescale) 206 | disp(sprintf(' CG Res = %8.3e, CG Iter = %d', cgres, cgiter)); 207 | else 208 | disp(sprintf(' H11p condition number = %8.3e', hcond)); 209 | end 210 | 211 | end 212 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/l1qc_logbarrier.m: -------------------------------------------------------------------------------- 1 | % l1qc_logbarrier.m 2 | % 3 | % 对应P_D 噪声情况下不等式约束的logbarrier求解方法 4 | % 5 | % Solve quadratically constrained l1 minimization: 6 | % min ||x||_1 s.t. ||Ax - b||_2 <= \epsilon 7 | % 8 | % Reformulate as the second-order cone program 9 | % min_{x,u} sum(u) s.t. x - u <= 0, 10 | % -x - u <= 0, 11 | % 1/2(||Ax-b||^2 - \epsilon^2) <= 0 12 | % and use a log barrier algorithm. 13 | % 14 | % Usage: xp = l1qc_logbarrier(x0, A, At, b, epsilon, lbtol, mu, cgtol, cgmaxiter) 15 | % 16 | % x0 - Nx1 vector, initial point. 17 | % 18 | % A - Either a handle to a function that takes a N vector and returns a K 19 | % vector , or a KxN matrix. If A is a function handle, the algorithm 20 | % operates in "largescale" mode, solving the Newton systems via the 21 | % Conjugate Gradients algorithm. 22 | % 23 | % At - Handle to a function that takes a K vector and returns an N vector. 24 | % If A is a KxN matrix, At is ignored. 25 | % 26 | % b - Kx1 vector of observations. 27 | % 28 | % epsilon - scalar, constraint relaxation parameter 29 | % 30 | % lbtol - The log barrier algorithm terminates when the duality gap <= lbtol. 31 | % Also, the number of log barrier iterations is completely 32 | % determined by lbtol. 33 | % Default = 1e-3. 34 | % 35 | % mu - Factor by which to increase the barrier constant at each iteration. 36 | % Default = 10. 37 | % 38 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 39 | % Default = 1e-8. 40 | % 41 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 42 | % if A is a matrix. 43 | % Default = 200. 44 | % 45 | % Written by: Justin Romberg, Caltech 46 | % Email: jrom@acm.caltech.edu 47 | % Created: October 2005 48 | % 49 | 50 | function xp = l1qc_logbarrier(x0, A, At, b, epsilon, lbtol, mu, cgtol, cgmaxiter) 51 | 52 | largescale = isa(A,'function_handle'); 53 | 54 | if (nargin < 6), lbtol = 1e-3; end 55 | if (nargin < 7), mu = 10; end 56 | if (nargin < 8), cgtol = 1e-8; end 57 | if (nargin < 9), cgmaxiter = 200; end 58 | 59 | newtontol = lbtol; 60 | newtonmaxiter = 50; 61 | 62 | N = length(x0); 63 | 64 | % starting point --- make sure that it is feasible 65 | if (largescale) 66 | if (norm(A(x0)-b) > epsilon) 67 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 68 | AAt = @(z) A(At(z)); 69 | [w, cgres] = cgsolve(AAt, b, cgtol, cgmaxiter, 0); 70 | if (cgres > 1/2) 71 | disp('A*At is ill-conditioned: cannot find starting point'); 72 | xp = x0; 73 | return; 74 | end 75 | x0 = At(w); 76 | end 77 | else 78 | if (norm(A*x0-b) > epsilon) 79 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 80 | opts.POSDEF = true; opts.SYM = true; 81 | [w, hcond] = linsolve(A*A', b, opts); 82 | if (hcond < 1e-14) 83 | disp('A*At is ill-conditioned: cannot find starting point'); 84 | xp = x0; 85 | return; 86 | end 87 | x0 = A'*w; 88 | end 89 | end 90 | x = x0; 91 | u = (0.95)*abs(x0) + (0.10)*max(abs(x0)); 92 | 93 | disp(sprintf('Original l1 norm = %.3f, original functional = %.3f', sum(abs(x0)), sum(u))); 94 | 95 | % choose initial value of tau so that the duality gap after the first 96 | % step will be about the origial norm 97 | tau = max((2*N+1)/sum(abs(x0)), 1); 98 | 99 | lbiter = ceil((log(2*N+1)-log(lbtol)-log(tau))/log(mu)); 100 | disp(sprintf('Number of log barrier iterations = %d\n', lbiter)); 101 | 102 | totaliter = 0; 103 | 104 | for ii = 1:lbiter 105 | 106 | [xp, up, ntiter] = l1qc_newton(x, u, A, At, b, epsilon, tau, newtontol, newtonmaxiter, cgtol, cgmaxiter); 107 | totaliter = totaliter + ntiter; 108 | 109 | disp(sprintf('\nLog barrier iter = %d, l1 = %.3f, functional = %8.3f, tau = %8.3e, total newton iter = %d\n', ... 110 | ii, sum(abs(xp)), sum(up), tau, totaliter)); 111 | 112 | x = xp; 113 | u = up; 114 | 115 | tau = mu*tau; 116 | 117 | end 118 | 119 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/l1qc_newton.m: -------------------------------------------------------------------------------- 1 | % l1qc_newton.m 2 | % 3 | % 对应P_D 噪声情况下不等式约束的newton求解方法 4 | % 5 | % Newton algorithm for log-barrier subproblems for l1 minimization 6 | % with quadratic constraints. 7 | % 8 | % Usage: 9 | % [xp,up,niter] = l1qc_newton(x0, u0, A, At, b, epsilon, tau, 10 | % newtontol, newtonmaxiter, cgtol, cgmaxiter) 11 | % 12 | % x0,u0 - starting points 13 | % 14 | % A - Either a handle to a function that takes a N vector and returns a K 15 | % vector , or a KxN matrix. If A is a function handle, the algorithm 16 | % operates in "largescale" mode, solving the Newton systems via the 17 | % Conjugate Gradients algorithm. 18 | % 19 | % At - Handle to a function that takes a K vector and returns an N vector. 20 | % If A is a KxN matrix, At is ignored. 21 | % 22 | % b - Kx1 vector of observations. 23 | % 24 | % epsilon - scalar, constraint relaxation parameter 25 | % 26 | % tau - Log barrier parameter. 27 | % 28 | % newtontol - Terminate when the Newton decrement is <= newtontol. 29 | % Default = 1e-3. 30 | % 31 | % newtonmaxiter - Maximum number of iterations. 32 | % Default = 50. 33 | % 34 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 35 | % Default = 1e-8. 36 | % 37 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 38 | % if A is a matrix. 39 | % Default = 200. 40 | % 41 | % Written by: Justin Romberg, Caltech 42 | % Email: jrom@acm.caltech.edu 43 | % Created: October 2005 44 | % 45 | 46 | 47 | function [xp, up, niter] = l1qc_newton(x0, u0, A, At, b, epsilon, tau, newtontol, newtonmaxiter, cgtol, cgmaxiter) 48 | 49 | % check if the matrix A is implicit or explicit 50 | largescale = isa(A,'function_handle'); 51 | 52 | % line search parameters 53 | alpha = 0.01; 54 | beta = 0.5; 55 | 56 | if (~largescale), AtA = A'*A; end 57 | 58 | % initial point 59 | x = x0; 60 | u = u0; 61 | if (largescale), r = A(x) - b; else r = A*x - b; end 62 | fu1 = x - u; 63 | fu2 = -x - u; 64 | fe = 1/2*(r'*r - epsilon^2); 65 | f = sum(u) - (1/tau)*(sum(log(-fu1)) + sum(log(-fu2)) + log(-fe)); 66 | 67 | niter = 0; 68 | done = 0; 69 | while (~done) 70 | 71 | if (largescale), atr = At(r); else atr = A'*r; end 72 | 73 | ntgz = 1./fu1 - 1./fu2 + 1/fe*atr; 74 | ntgu = -tau - 1./fu1 - 1./fu2; 75 | gradf = -(1/tau)*[ntgz; ntgu]; 76 | 77 | sig11 = 1./fu1.^2 + 1./fu2.^2; 78 | sig12 = -1./fu1.^2 + 1./fu2.^2; 79 | sigx = sig11 - sig12.^2./sig11; 80 | 81 | w1p = ntgz - sig12./sig11.*ntgu; 82 | if (largescale) 83 | h11pfun = @(z) sigx.*z - (1/fe)*At(A(z)) + 1/fe^2*(atr'*z)*atr; 84 | [dx, cgres, cgiter] = cgsolve(h11pfun, w1p, cgtol, cgmaxiter, 0); 85 | if (cgres > 1/2) 86 | disp('Cannot solve system. Returning previous iterate. (See Section 4 of notes for more information.)'); 87 | xp = x; up = u; 88 | return 89 | end 90 | Adx = A(dx); 91 | else 92 | H11p = diag(sigx) - (1/fe)*AtA + (1/fe)^2*atr*atr'; 93 | opts.POSDEF = true; opts.SYM = true; 94 | [dx,hcond] = linsolve(H11p, w1p, opts); 95 | if (hcond < 1e-14) 96 | disp('Matrix ill-conditioned. Returning previous iterate. (See Section 4 of notes for more information.)'); 97 | xp = x; up = u; 98 | return 99 | end 100 | Adx = A*dx; 101 | end 102 | du = (1./sig11).*ntgu - (sig12./sig11).*dx; 103 | 104 | % minimum step size that stays in the interior 105 | ifu1 = find((dx-du) > 0); ifu2 = find((-dx-du) > 0); 106 | aqe = Adx'*Adx; bqe = 2*r'*Adx; cqe = r'*r - epsilon^2; 107 | smax = min(1,min([... 108 | -fu1(ifu1)./(dx(ifu1)-du(ifu1)); -fu2(ifu2)./(-dx(ifu2)-du(ifu2)); ... 109 | (-bqe+sqrt(bqe^2-4*aqe*cqe))/(2*aqe) 110 | ])); 111 | s = (0.99)*smax; 112 | 113 | % backtracking line search 114 | suffdec = 0; 115 | backiter = 0; 116 | while (~suffdec) 117 | xp = x + s*dx; up = u + s*du; rp = r + s*Adx; 118 | fu1p = xp - up; fu2p = -xp - up; fep = 1/2*(rp'*rp - epsilon^2); 119 | fp = sum(up) - (1/tau)*(sum(log(-fu1p)) + sum(log(-fu2p)) + log(-fep)); 120 | flin = f + alpha*s*(gradf'*[dx; du]); 121 | suffdec = (fp <= flin); 122 | s = beta*s; 123 | backiter = backiter + 1; 124 | if (backiter > 32) 125 | disp('Stuck on backtracking line search, returning previous iterate. (See Section 4 of notes for more information.)'); 126 | xp = x; up = u; 127 | return 128 | end 129 | end 130 | 131 | % set up for next iteration 132 | x = xp; u = up; r = rp; 133 | fu1 = fu1p; fu2 = fu2p; fe = fep; f = fp; 134 | 135 | lambda2 = -(gradf'*[dx; du]); 136 | stepsize = s*norm([dx; du]); 137 | niter = niter + 1; 138 | done = (lambda2/2 < newtontol) | (niter >= newtonmaxiter); 139 | 140 | disp(sprintf('Newton iter = %d, Functional = %8.3f, Newton decrement = %8.3f, Stepsize = %8.3e', ... 141 | niter, f, lambda2/2, stepsize)); 142 | if (largescale) 143 | disp(sprintf(' CG Res = %8.3e, CG Iter = %d', cgres, cgiter)); 144 | else 145 | disp(sprintf(' H11p condition number = %8.3e', hcond)); 146 | end 147 | 148 | end 149 | 150 | 151 | 152 | 153 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/tvdantzig_logbarrier.m: -------------------------------------------------------------------------------- 1 | % tvdantzig_logbarrier.m 2 | % 3 | % 对应P_D 噪声情况下不等式约束的logbarrier求解方法 4 | % 5 | % Solve the total variation Dantzig program 6 | % 7 | % min_x TV(x) subject to ||A'(Ax-b)||_\infty <= epsilon 8 | % 9 | % Recast as the SOCP 10 | % min sum(t) s.t. ||D_{ij}x||_2 <= t, i,j=1,...,n 11 | % <= epsilon i,j=1,...,n 12 | % and use a log barrier algorithm. 13 | % 14 | % Usage: xp = tvdantzig_logbarrier(x0, A, At, b, epsilon, lbtol, mu, cgtol, cgmaxiter) 15 | % 16 | % x0 - Nx1 vector, initial point. 17 | % 18 | % A - Either a handle to a function that takes a N vector and returns a K 19 | % vector , or a KxN matrix. If A is a function handle, the algorithm 20 | % operates in "largescale" mode, solving the Newton systems via the 21 | % Conjugate Gradients algorithm. 22 | % 23 | % At - Handle to a function that takes a K vector and returns an N vector. 24 | % If A is a KxN matrix, At is ignored. 25 | % 26 | % b - Kx1 vector of observations. 27 | % 28 | % epsilon - scalar, constraint relaxation parameter 29 | % 30 | % lbtol - The log barrier algorithm terminates when the duality gap <= lbtol. 31 | % Also, the number of log barrier iterations is completely 32 | % determined by lbtol. 33 | % Default = 1e-3. % 可忍受的误差，在此误差下停止迭代 34 | % 35 | % mu - Factor by which to increase the barrier constant at each iteration. 36 | % Default = 10. 迭代的步长 37 | % 38 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 39 | % Default = 1e-8. 可忍受的误差 如果是矩阵则忽略此参数 40 | % 41 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 42 | % if A is a matrix. 43 | % Default = 200. 最大迭代次数 44 | % 45 | % Written by: Justin Romberg, Caltech 46 | % Email: jrom@acm.caltech.edu 47 | % Created: October 2005 48 | % 49 | 50 | function xp = tvdantzig_logbarrier(x0, A, At, b, epsilon, lbtol, mu, cgtol, cgmaxiter) 51 | 52 | largescale = isa(A,'function_handle'); 53 | 54 | if (nargin < 6), lbtol = 1e-3; end 55 | if (nargin < 7), mu = 10; end 56 | if (nargin < 8), cgtol = 1e-8; end 57 | if (nargin < 9), cgmaxiter = 200; end 58 | 59 | newtontol = lbtol; 60 | newtonmaxiter = 50; 61 | 62 | N = length(x0); 63 | n = round(sqrt(N)); 64 | 65 | % create (sparse) differencing matrices for TV 66 | Dv = spdiags([reshape([-ones(n-1,n); zeros(1,n)],N,1) ... 67 | reshape([zeros(1,n); ones(n-1,n)],N,1)], [0 1], N, N); 68 | Dh = spdiags([reshape([-ones(n,n-1) zeros(n,1)],N,1) ... 69 | reshape([zeros(n,1) ones(n,n-1)],N,1)], [0 n], N, N); 70 | 71 | if (largescale) 72 | if (norm(A(x0)-b) > epsilon) 73 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 74 | AAt = @(z) A(At(z)); 75 | [w, cgres] = cgsolve(AAt, b, cgtol, cgmaxiter, 0); 76 | if (cgres > 1/2) 77 | disp('A*At is ill-conditioned: cannot find starting point'); 78 | xp = x0; 79 | return; 80 | end 81 | x0 = At(w); 82 | end 83 | else 84 | if (norm(A*x0-b) > epsilon) 85 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 86 | opts.POSDEF = true; opts.SYM = true; 87 | [w, hcond] = linsolve(A*A', b, opts); 88 | if (hcond < 1e-14) 89 | disp('A*At is ill-conditioned: cannot find starting point'); 90 | xp = x0; 91 | return; 92 | end 93 | x0 = A'*w; 94 | end 95 | end 96 | x = x0; 97 | Dhx = Dh*x; Dvx = Dv*x; 98 | t = 1.05*sqrt(Dhx.^2 + Dvx.^2) + .01*max(sqrt(Dhx.^2 + Dvx.^2)); 99 | 100 | % choose initial value of tau so that the duality gap after the first 101 | % step will be about the origial TV 102 | tau = 3*N/sum(sqrt(Dhx.^2+Dvx.^2)); 103 | 104 | lbiter = ceil((log(3*N)-log(lbtol)-log(tau))/log(mu)); 105 | disp(sprintf('Number of log barrier iterations = %d\n', lbiter)); 106 | totaliter = 0; 107 | for ii = 1:lbiter 108 | 109 | [xp, tp, ntiter] = tvdantzig_newton(x, t, A, At, b, epsilon, tau, newtontol, newtonmaxiter, cgtol, cgmaxiter); 110 | totaliter = totaliter + ntiter; 111 | tvxp = sum(sqrt((Dh*xp).^2 + (Dv*xp).^2)); 112 | 113 | disp(sprintf('\nLog barrier iter = %d, TV = %.3f, functional = %8.3f, tau = %8.3e, total newton iter = %d\n', ... 114 | ii, tvxp, sum(tp), tau, totaliter)); 115 | 116 | x = xp; 117 | t = tp; 118 | 119 | tau = mu*tau; 120 | 121 | end 122 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/tvdantzig_newton.m: -------------------------------------------------------------------------------- 1 | % tvdantzig_newton.m 2 | % 3 | % 对应P_D 噪声情况下不等式约束的newton求解方法 4 | % 5 | % Newton iterations for TV Dantzig log-barrier subproblem. 6 | % 7 | % Usage : [xp, tp, niter] = tvdantzig_newton(x0, t0, A, At, b, epsilon, tau, 8 | % newtontol, newtonmaxiter, cgtol, cgmaxiter) 9 | % 10 | % x0,t0 - Nx1 vectors, initial points. 11 | % 12 | % A - Either a handle to a function that takes a N vector and returns a K 13 | % vector , or a KxN matrix. If A is a function handle, the algorithm 14 | % operates in "largescale" mode, solving the Newton systems via the 15 | % Conjugate Gradients algorithm. 16 | % 17 | % At - Handle to a function that takes a K vector and returns an N vector. 18 | % If A is a KxN matrix, At is ignored. 19 | % 20 | % b - Kx1 vector of observations. 21 | % 22 | % epsilon - scalar, constraint relaxation parameter 23 | % 24 | % tau - Log barrier parameter. 25 | % 26 | % newtontol - Terminate when the Newton decrement is <= newtontol. 27 | % 28 | % newtonmaxiter - Maximum number of iterations. 29 | % 30 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 31 | % 32 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 33 | % if A is a matrix. 34 | % 35 | % Written by: Justin Romberg, Caltech 36 | % Email: jrom@acm.caltech.edu 37 | % Created: October 2005 38 | % 39 | 40 | 41 | function [xp, tp, niter] = tvdantzig_newton(x0, t0, A, At, b, epsilon, tau, newtontol, newtonmaxiter, cgtol, cgmaxiter) 42 | 43 | largescale = isa(A,'function_handle'); 44 | 45 | alpha = 0.01; 46 | beta = 0.5; 47 | 48 | N = length(x0); 49 | n = round(sqrt(N)); 50 | 51 | % create (sparse) differencing matrices for TV 52 | Dv = spdiags([reshape([-ones(n-1,n); zeros(1,n)],N,1) ... 53 | reshape([zeros(1,n); ones(n-1,n)],N,1)], [0 1], N, N); 54 | Dh = spdiags([reshape([-ones(n,n-1) zeros(n,1)],N,1) ... 55 | reshape([zeros(n,1) ones(n,n-1)],N,1)], [0 n], N, N); 56 | 57 | % initial point 58 | x = x0; 59 | t = t0; 60 | if (largescale) 61 | r = A(x) - b; 62 | Atr = At(r); 63 | else 64 | AtA = A'*A; 65 | r = A*x - b; 66 | Atr = A'*r; 67 | end 68 | Dhx = Dh*x; Dvx = Dv*x; 69 | ft = 1/2*(Dhx.^2 + Dvx.^2 - t.^2); 70 | fe1 = Atr - epsilon; 71 | fe2 = -Atr - epsilon; 72 | f = sum(t) - (1/tau)*(sum(log(-ft)) + sum(log(-fe1)) + sum(log(-fe2))); 73 | 74 | niter = 0; 75 | done = 0; 76 | while (~done) 77 | 78 | if (largescale) 79 | ntgx = Dh'*((1./ft).*Dhx) + Dv'*((1./ft).*Dvx) + At(A(1./fe1-1./fe2)); 80 | else 81 | ntgx = Dh'*((1./ft).*Dhx) + Dv'*((1./ft).*Dvx) + AtA*(1./fe1-1./fe2); 82 | end 83 | ntgt = -tau - t./ft; 84 | gradf = -(1/tau)*[ntgx; ntgt]; 85 | 86 | sig22 = 1./ft + (t.^2)./(ft.^2); 87 | sig12 = -t./ft.^2; 88 | sigb = 1./ft.^2 - (sig12.^2)./sig22; 89 | siga = 1./fe1.^2 + 1./fe2.^2; 90 | 91 | w11 = ntgx - Dh'*(Dhx.*(sig12./sig22).*ntgt) - Dv'*(Dvx.*(sig12./sig22).*ntgt); 92 | if (largescale) 93 | h11pfun = @(w) H11p(w, A, At, Dh, Dv, Dhx, Dvx, sigb, ft, siga); 94 | [dx, cgres, cgiter] = cgsolve(h11pfun, w11, cgtol, cgmaxiter, 0); 95 | if (cgres > 1/2) 96 | disp('Cannot solve system. Returning previous iterate. (See Section 4 of notes for more information.)'); 97 | xp = x; tp = t; 98 | return 99 | end 100 | Adx = A(dx); 101 | AtAdx = At(Adx); 102 | else 103 | H11p = Dh'*sparse(diag(-1./ft + sigb.*Dhx.^2))*Dh + ... 104 | Dv'*sparse(diag(-1./ft + sigb.*Dvx.^2))*Dv + ... 105 | Dh'*sparse(diag(sigb.*Dhx.*Dvx))*Dv + ... 106 | Dv'*sparse(diag(sigb.*Dhx.*Dvx))*Dh + ... 107 | AtA*sparse(diag(siga))*AtA; 108 | opts.POSDEF = true; opts.SYM = true; 109 | [dx,hcond] = linsolve(H11p, w11, opts); 110 | if (hcond < 1e-14) 111 | disp('Matrix ill-conditioned. Returning previous iterate. (See Section 4 of notes for more information.)'); 112 | xp = x; tp = t; 113 | return 114 | end 115 | Adx = A*dx; 116 | AtAdx = A'*Adx; 117 | end 118 | Dhdx = Dh*dx; Dvdx = Dv*dx; 119 | dt = (1./sig22).*(ntgt - sig12.*(Dhx.*Dhdx + Dvx.*Dvdx)); 120 | 121 | % minimum step size that stays in the interior 122 | ife1 = find(AtAdx > 0); ife2 = find(-AtAdx > 0); 123 | aqt = Dhdx.^2 + Dvdx.^2 - dt.^2; 124 | bqt = 2*(Dhdx.*Dhx + Dvdx.*Dvx - t.*dt); 125 | cqt = Dhx.^2 + Dvx.^2 - t.^2; 126 | tsols = [(-bqt+sqrt(bqt.^2-4*aqt.*cqt))./(2*aqt); ... 127 | (-bqt-sqrt(bqt.^2-4*aqt.*cqt))./(2*aqt) ]; 128 | indt = find([(bqt.^2 > 4*aqt.*cqt); (bqt.^2 > 4*aqt.*cqt)] & (tsols > 0)); 129 | smax = min(1, min([-fe1(ife1)./AtAdx(ife1); -fe2(ife2)./(-AtAdx(ife2)); tsols(indt)])); 130 | s = (0.99)*smax; 131 | 132 | % backtracking line search 133 | suffdec = 0; 134 | backiter = 0; 135 | while (~suffdec) 136 | xp = x + s*dx; tp = t + s*dt; 137 | rp = r + s*Adx; Atrp = Atr + s*AtAdx; 138 | Dhxp = Dhx + s*Dhdx; Dvxp = Dvx + s*Dvdx; 139 | ftp = 1/2*(Dhxp.^2 + Dvxp.^2 - tp.^2); 140 | fe1p = Atrp - epsilon; 141 | fe2p = -Atrp - epsilon; 142 | fp = sum(tp) - (1/tau)*(sum(log(-ftp)) + sum(log(-fe1p)) + sum(log(-fe2p))); 143 | flin = f + alpha*s*(gradf'*[dx; dt]); 144 | suffdec = (fp <= flin); 145 | s = beta*s; 146 | backiter = backiter + 1; 147 | if (backiter > 32) 148 | disp('Stuck backtracking, returning previous iterate. (See Section 4 of notes for more information.)'); 149 | xp = x; tp = t; 150 | return 151 | end 152 | end 153 | 154 | % set up for next iteration 155 | x = xp; t = tp; 156 | r = rp; Atr = Atrp; 157 | Dvx = Dvxp; Dhx = Dhxp; 158 | ft = ftp; fe1 = fe1p; fe2 = fe2p; f = fp; 159 | 160 | lambda2 = -(gradf'*[dx; dt]); 161 | stepsize = s*norm([dx; dt]); 162 | niter = niter + 1; 163 | done = (lambda2/2 < newtontol) | (niter >= newtonmaxiter); 164 | 165 | disp(sprintf('Newton iter = %d, Functional = %8.3f, Newton decrement = %8.3f, Stepsize = %8.3e', ... 166 | niter, f, lambda2/2, stepsize)); 167 | if (largescale) 168 | disp(sprintf(' CG Res = %8.3e, CG Iter = %d', cgres, cgiter)); 169 | else 170 | disp(sprintf(' H11p condition number = %8.3e', hcond)); 171 | end 172 | 173 | end 174 | 175 | 176 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 177 | % H11p auxiliary function 178 | function y = H11p(v, A, At, Dh, Dv, Dhx, Dvx, sigb, ft, siga) 179 | 180 | Dhv = Dh*v; 181 | Dvv = Dv*v; 182 | 183 | y = Dh'*((-1./ft + sigb.*Dhx.^2).*Dhv + sigb.*Dhx.*Dvx.*Dvv) + ... 184 | Dv'*((-1./ft + sigb.*Dvx.^2).*Dvv + sigb.*Dhx.*Dvx.*Dhv) + ... 185 | At(A(siga.*At(A(v)))); 186 | 187 | 188 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/tveq_logbarrier.m: -------------------------------------------------------------------------------- 1 | % tveq_logbarrier.m 2 | % 3 | % 对应tv P_1 等式约束的logbarrier求解方法 4 | % 5 | % Solve equality constrained TV minimization 6 | % min TV(x) s.t. Ax=b. 7 | % 8 | % Recast as the SOCP 9 | % min sum(t) s.t. ||D_{ij}x||_2 <= t, i,j=1,...,n 10 | % Ax=b 11 | % and use a log barrier algorithm. 12 | % 13 | % Usage: xp = tveq_logbarrier(x0, A, At, b, lbtol, mu, slqtol, slqmaxiter) 14 | % 15 | % x0 - Nx1 vector, initial point. 16 | % 17 | % A - Either a handle to a function that takes a N vector and returns a K 18 | % vector , or a KxN matrix. If A is a function handle, the algorithm 19 | % operates in "largescale" mode, solving the Newton systems via the 20 | % Conjugate Gradients algorithm. 21 | % 22 | % At - Handle to a function that takes a K vector and returns an N vector. 23 | % If A is a KxN matrix, At is ignored. 24 | % 25 | % b - Kx1 vector of observations. 26 | % 27 | % lbtol - The log barrier algorithm terminates when the duality gap <= lbtol. 28 | % Also, the number of log barrier iterations is completely 29 | % determined by lbtol. 30 | % Default = 1e-3. % 可忍受的误差，在此误差下停止迭代 31 | % 32 | % mu - Factor by which to increase the barrier constant at each iteration. 33 | % Default = 10. 迭代的步长 34 | % 35 | % slqtol - Tolerance for SYMMLQ; ignored if A is a matrix. 36 | % Default = 1e-8. 可忍受的误差 如果是矩阵则忽略此参数，因为矩阵不是largescal 37 | % 38 | % slqmaxiter - Maximum number of iterations for SYMMLQ; ignored 39 | % if A is a matrix. 40 | % Default = 200. 最大迭代次数 41 | % 42 | % Written by: Justin Romberg, Caltech 43 | % Email: jrom@acm.caltech.edu 44 | % Created: October 2005 45 | % 46 | 47 | function xp = tveq_logbarrier(x0, A, At, b, lbtol, mu, slqtol, slqmaxiter) 48 | 49 | % 判断是不是large scale模式，如果是largescale为1.这里如果A用的是fuction handle，那么判定为启用large scale模式 50 | largescale = isa(A,'function_handle'); 51 | 52 | % 默认参数的判断 53 | if (nargin < 5), lbtol = 1e-3; end % logbarrier停止迭代的误差 54 | if (nargin < 6), mu = 10; end % 步长默认10 55 | if (nargin < 7), slqtol = 1e-8; end % 可以忍受的误差 56 | if (nargin < 8), slqmaxiter = 200; end % 最大迭代次数 57 | 58 | newtontol = lbtol; % 停止迭代的误差 59 | newtonmaxiter = 50; % 最大迭代次数 60 | 61 | N = length(x0); % 传入的是一维向量，这里N=nxn 62 | n = round(sqrt(N)); % 通过N计算出n 63 | 64 | % create (sparse) differencing matrices for TV 65 | % spdiags 取出非零元素 66 | % [-ones(n-1,n); zeros(1,n)]是n*n的矩阵，n-1*n为-1，最后一行为0 67 | % reshape以后按列取，变成n-1个-1，然后一个0重复...的一维向量 68 | % [reshape([-ones(n-1,n); zeros(1,n)],N,1) ... 69 | % reshape([zeros(1,n); ones(n-1,n)],N,1)]就成了两行向量 70 | % 经过spdiags函数以后，Dv就成了对角线是-1从(1,1)-(63,63)，对角线左上的斜线是1的稀疏矩阵(1,2)(63,64) 71 | % Dh是对角线为-1 距离为16的地方(1,17)开始是1的稀疏矩阵 72 | 73 | Dv = spdiags([reshape([-ones(n-1,n); zeros(1,n)],N,1) ... 74 | reshape([zeros(1,n); ones(n-1,n)],N,1)], [0 1], N, N); 75 | Dh = spdiags([reshape([-ones(n,n-1) zeros(n,1)],N,1) ... 76 | reshape([zeros(n,1) ones(n,n-1)],N,1)], [0 n], N, N); 77 | 78 | % starting point --- make sure that it is feasible 79 | if (largescale) 80 | if (norm(A(x0)-b)/norm(b) > slqtol) % 对传入的x0投影，减去观测值得到残差，求残差和观测值的2范数比例 大于可忍受的 81 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 82 | AAt = @(z) A(At(z)); 83 | [w,cgres] = cgsolve(AAt, b, slqtol, slqmaxiter, 0); % 这里用到了函数cgsolve 84 | if (cgres > 1/2) % 检查是否满足求解条件 85 | disp('A*At is ill-conditioned: cannot find starting point'); 86 | xp = x0; 87 | return; 88 | end 89 | x0 = At(w); % 求解后的w反投影为x0 90 | end 91 | else %　小规模下是否满足求解条件 92 | if (norm(A*x0-b)/norm(b) > slqtol) 93 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 94 | opts.POSDEF = true; opts.SYM = true; 95 | [w, hcond] = linsolve(A*A', b, opts); 96 | if (hcond < 1e-14) 97 | disp('A*At is ill-conditioned: cannot find starting point'); 98 | xp = x0; 99 | return; 100 | end 101 | x0 = A'*w; % 对解出的w反投影为x0 102 | end 103 | end 104 | x = x0; % 为x赋初始值 105 | Dhx = Dh*x; Dvx = Dv*x; %　求gradient　horizontal and vertical 106 | t = (0.95)*sqrt(Dhx.^2 + Dvx.^2) + (0.1)*max(sqrt(Dhx.^2 + Dvx.^2)); % 加权方法算出来的gradient值 107 | 108 | % choose initial value of tau so that the duality(二元) gap after the first 109 | % step will be about the origial TV 110 | tau = N/sum(sqrt(Dhx.^2+Dvx.^2)); % 这个tau不知道是什么 111 | 112 | lbiter = ceil((log(N)-log(lbtol)-log(tau))/log(mu)); %计算logbarrier的迭代次数 113 | disp(sprintf('Number of log barrier iterations = %d\n', lbiter)); 114 | totaliter = 0; 115 | for ii = 1:lbiter % 进行logbarrier迭代 116 | 117 | [xp, tp, ntiter] = tveq_newton(x, t, A, At, b, tau, newtontol, newtonmaxiter, slqtol, slqmaxiter); % 进行迭代 返回值中有迭代次数 118 | totaliter = totaliter + ntiter; %总迭代次数由牛顿迭代的次数累加 119 | 120 | tvxp = sum(sqrt((Dh*xp).^2 + (Dv*xp).^2)); % 总的gradient值 121 | disp(sprintf('\nLog barrier iter = %d, TV = %.3f, functional = %8.3f, tau = %8.3e, total newton iter = %d\n', ... 122 | ii, tvxp, sum(tp), tau, totaliter)); % 这里的functional不知道是什么 123 | 124 | x = xp; 125 | t = tp; 126 | 127 | tau = mu*tau; %　新的tau是步长mu*tau 128 | 129 | end 130 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/tveq_newton.m: -------------------------------------------------------------------------------- 1 | % tveq_newton.m 2 | % 3 | % 对应tv P_1 等式约束的newton求解方法 4 | % 5 | % Newton algorithm for log-barrier subproblems for TV minimization 6 | % with equality constraints. 7 | % 8 | % Usage: 9 | % [xp,tp,niter] = tveq_newton(x0, t0, A, At, b, tau, 10 | % newtontol, newtonmaxiter, slqtol, slqmaxiter) 11 | % 12 | % x0,t0 - starting points 13 | % 14 | % A - Either a handle to a function that takes a N vector and returns a K 15 | % vector , or a KxN matrix. If A is a function handle, the algorithm 16 | % operates in "largescale" mode, solving the Newton systems via the 17 | % Conjugate Gradients algorithm. 18 | % 19 | % At - Handle to a function that takes a K vector and returns an N vector. 20 | % If A is a KxN matrix, At is ignored. 21 | % 22 | % b - Kx1 vector of observations. 23 | % 24 | % tau - Log barrier parameter. 25 | % 26 | % newtontol - Terminate when the Newton decrement is <= newtontol. 27 | % 28 | % newtonmaxiter - Maximum number of iterations. 29 | % 30 | % slqtol - Tolerance for SYMMLQ; ignored if A is a matrix. 31 | % 32 | % slqmaxiter - Maximum number of iterations for SYMMLQ; ignored 33 | % if A is a matrix. 34 | % 35 | % Written by: Justin Romberg, Caltech 36 | % Email: jrom@acm.caltech.edu 37 | % Created: October 2005 38 | % 39 | 40 | function [xp, tp, niter] = tveq_newton(x0, t0, A, At, b, tau, newtontol, newtonmaxiter, slqtol, slqmaxiter) 41 | 42 | largescale = isa(A,'function_handle'); 43 | 44 | alpha = 0.01; 45 | beta = 0.5; 46 | 47 | N = length(x0); 48 | n = round(sqrt(N)); 49 | K = length(b); 50 | 51 | % create (sparse) differencing matrices for TV 52 | Dv = spdiags([reshape([-ones(n-1,n); zeros(1,n)],N,1) ... 53 | reshape([zeros(1,n); ones(n-1,n)],N,1)], [0 1], N, N); 54 | Dh = spdiags([reshape([-ones(n,n-1) zeros(n,1)],N,1) ... 55 | reshape([zeros(n,1) ones(n,n-1)],N,1)], [0 n], N, N); 56 | 57 | % auxillary(辅助的) matrices for preconditioning(预处理) 58 | Mdv = spdiags([reshape([ones(n-1,n); zeros(1,n)],N,1) ... 59 | reshape([zeros(1,n); ones(n-1,n)],N,1)], [0 1], N, N); 60 | Mdh = spdiags([reshape([ones(n,n-1) zeros(n,1)],N,1) ... 61 | reshape([zeros(n,1) ones(n,n-1)],N,1)], [0 n], N, N); 62 | Mmd = reshape([ones(n-1,n-1) zeros(n-1,1); zeros(1,n)],N,1); 63 | 64 | 65 | % initial point 66 | x = x0; 67 | t = t0; 68 | Dhx = Dh*x; Dvx = Dv*x; 69 | ft = 1/2*(Dhx.^2 + Dvx.^2 - t.^2); 70 | f = sum(t) - (1/tau)*(sum(log(-ft))); 71 | 72 | niter = 0; 73 | done = 0; 74 | while (~done) 75 | 76 | ntgx = Dh'*((1./ft).*Dhx) + Dv'*((1./ft).*Dvx); 77 | ntgt = -tau - t./ft; 78 | gradf = -(1/tau)*[ntgx; ntgt]; 79 | 80 | sig22 = 1./ft + (t.^2)./(ft.^2); 81 | sig12 = -t./ft.^2; 82 | sigb = 1./ft.^2 - (sig12.^2)./sig22; 83 | 84 | w1p = ntgx - Dh'*(Dhx.*(sig12./sig22).*ntgt) - Dv'*(Dvx.*(sig12./sig22).*ntgt); 85 | wp = [w1p; zeros(K,1)]; 86 | if (largescale) 87 | % diagonal of H11p 88 | dg11p = Mdh'*(-1./ft + sigb.*Dhx.^2) + Mdv'*(-1./ft + sigb.*Dvx.^2) + 2*Mmd.*sigb.*Dhx.*Dvx; 89 | afac = max(dg11p); 90 | hpfun = @(z) Hpeval(z, A, At, Dh, Dv, Dhx, Dvx, sigb, ft, afac); 91 | [dxv,slqflag,slqres,slqiter] = symmlq(hpfun, wp, slqtol, slqmaxiter); 92 | if (slqres > 1/2) 93 | disp('Cannot solve system. Returning previous iterate. (See Section 4 of notes for more information.)'); 94 | xp = x; 95 | return 96 | end 97 | else 98 | H11p = Dh'*sparse(diag(-1./ft + sigb.*Dhx.^2))*Dh + ... 99 | Dv'*sparse(diag(-1./ft + sigb.*Dvx.^2))*Dv + ... 100 | Dh'*sparse(diag(sigb.*Dhx.*Dvx))*Dv + ... 101 | Dv'*sparse(diag(sigb.*Dhx.*Dvx))*Dh; 102 | afac = max(diag(H11p)); 103 | Hp = full([H11p afac*A'; afac*A zeros(K)]); 104 | %keyboard 105 | opts.SYM = true; 106 | [dxv, hcond] = linsolve(Hp, wp, opts); 107 | if (hcond < 1e-14) 108 | disp('Matrix ill-conditioned. Returning previous iterate. (See Section 4 of notes for more information.)'); 109 | xp = x; tp = t; 110 | return 111 | end 112 | end 113 | dx = dxv(1:N); 114 | Dhdx = Dh*dx; Dvdx = Dv*dx; 115 | dt = (1./sig22).*(ntgt - sig12.*(Dhx.*Dhdx + Dvx.*Dvdx)); 116 | 117 | % minimum step size that stays in the interior 118 | aqt = Dhdx.^2 + Dvdx.^2 - dt.^2; 119 | bqt = 2*(Dhdx.*Dhx + Dvdx.*Dvx - t.*dt); 120 | cqt = Dhx.^2 + Dvx.^2 - t.^2; 121 | tsols = [(-bqt+sqrt(bqt.^2-4*aqt.*cqt))./(2*aqt); ... 122 | (-bqt-sqrt(bqt.^2-4*aqt.*cqt))./(2*aqt) ]; 123 | indt = find([(bqt.^2 > 4*aqt.*cqt); (bqt.^2 > 4*aqt.*cqt)] & (tsols > 0)); 124 | smax = min(1, min(tsols(indt))); 125 | s = (0.99)*smax; 126 | 127 | % line search 128 | suffdec = 0; 129 | backiter = 0; 130 | while (~suffdec) 131 | xp = x + s*dx; tp = t + s*dt; 132 | Dhxp = Dhx + s*Dhdx; Dvxp = Dvx + s*Dvdx; 133 | ftp = 1/2*(Dhxp.^2 + Dvxp.^2 - tp.^2); 134 | fp = sum(tp) - (1/tau)*(sum(log(-ftp))); 135 | flin = f + alpha*s*(gradf'*[dx; dt]); 136 | suffdec = (fp <= flin); 137 | s = beta*s; 138 | backiter = backiter + 1; 139 | if (backiter > 32) 140 | disp('Stuck backtracking, returning last iterate. (See Section 4 of notes for more information.)'); 141 | xp = x; tp = t; 142 | return 143 | end 144 | end 145 | 146 | % set up for next iteration 147 | x = xp; t = tp; 148 | Dvx = Dvxp; Dhx = Dhxp; 149 | ft = ftp; f = fp; 150 | 151 | lambda2 = -(gradf'*[dx; dt]); 152 | stepsize = s*norm([dx; dt]); 153 | niter = niter + 1; 154 | done = (lambda2/2 < newtontol) | (niter >= newtonmaxiter); 155 | 156 | disp(sprintf('Newton iter = %d, Functional = %8.3f, Newton decrement = %8.3f, Stepsize = %8.3e', ... 157 | niter, f, lambda2/2, stepsize)); 158 | if (largescale) 159 | disp(sprintf(' SYMMLQ Res = %8.3e, SYMMLQ Iter = %d', slqres, slqiter)); 160 | else 161 | disp(sprintf(' H11p condition number = %8.3e', hcond)); 162 | end 163 | 164 | end 165 | 166 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 167 | % Implicit application of Hessian 168 | function y = Hpeval(z, A, At, Dh, Dv, Dhx, Dvx, sigb, ft, afac) 169 | 170 | N = length(ft); 171 | K = length(z)-N; 172 | w = z(1:N); 173 | v = z(N+1:N+K); 174 | 175 | Dhw = Dh*w; 176 | Dvw = Dv*w; 177 | 178 | y1 = Dh'*((-1./ft + sigb.*Dhx.^2).*Dhw + sigb.*Dhx.*Dvx.*Dvw) + ... 179 | Dv'*((-1./ft + sigb.*Dvx.^2).*Dvw + sigb.*Dhx.*Dvx.*Dhw) + afac*At(v); 180 | y2 = afac*A(w); 181 | 182 | y = [y1; y2]; 183 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/tvqc_logbarrier.m: -------------------------------------------------------------------------------- 1 | % tvqc_logbarrier.m 2 | % 3 | % 对应tv P_D 噪声情况下不等式约束的logbarrier求解方法 4 | % 5 | % Solve quadractically constrained TV minimization 6 | % min TV(x) s.t. ||Ax-b||_2 <= epsilon. 7 | % 8 | % Recast as the SOCP 9 | % min sum(t) s.t. ||D_{ij}x||_2 <= t, i,j=1,...,n 10 | % ||Ax - b||_2 <= epsilon 11 | % and use a log barrier algorithm. 12 | % 13 | % Usage: xp = tvqc_logbarrier(x0, A, At, b, epsilon, lbtol, mu, cgtol, cgmaxiter) 14 | % 15 | % x0 - Nx1 vector, initial point. 16 | % 17 | % A - Either a handle to a function that takes a N vector and returns a K 18 | % vector , or a KxN matrix. If A is a function handle, the algorithm 19 | % operates in "largescale" mode, solving the Newton systems via the 20 | % Conjugate Gradients algorithm. 21 | % 22 | % At - Handle to a function that takes a K vector and returns an N vector. 23 | % If A is a KxN matrix, At is ignored. 24 | % 25 | % b - Kx1 vector of observations. 26 | % 27 | % epsilon - scalar, constraint relaxation parameter 28 | % 29 | % lbtol - The log barrier algorithm terminates when the duality gap <= lbtol. 30 | % Also, the number of log barrier iterations is completely 31 | % determined by lbtol. 32 | % Default = 1e-3. 33 | % 34 | % mu - Factor by which to increase the barrier constant at each iteration. 35 | % Default = 10. 36 | % 37 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 38 | % Default = 1e-8. 39 | % 40 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 41 | % if A is a matrix. 42 | % Default = 200. 43 | % 44 | % Written by: Justin Romberg, Caltech 45 | % Email: jrom@acm.caltech.edu 46 | % Created: October 2005 47 | % 48 | 49 | 50 | function [xp, tp] = tvqc_logbarrier(x0, A, At, b, epsilon, lbtol, mu, cgtol, cgmaxiter) 51 | 52 | largescale = isa(A,'function_handle'); 53 | 54 | if (nargin < 6), lbtol = 1e-3; end 55 | if (nargin < 7), mu = 10; end 56 | if (nargin < 8), cgtol = 1e-8; end 57 | if (nargin < 9), cgmaxiter = 200; end 58 | 59 | newtontol = lbtol; 60 | newtonmaxiter = 50; 61 | 62 | N = length(x0); 63 | n = round(sqrt(N)); 64 | 65 | % create (sparse) differencing matrices for TV 66 | Dv = spdiags([reshape([-ones(n-1,n); zeros(1,n)],N,1) ... 67 | reshape([zeros(1,n); ones(n-1,n)],N,1)], [0 1], N, N); 68 | Dh = spdiags([reshape([-ones(n,n-1) zeros(n,1)],N,1) ... 69 | reshape([zeros(n,1) ones(n,n-1)],N,1)], [0 n], N, N); 70 | 71 | % starting point --- make sure that it is feasible 72 | if (largescale) 73 | if (norm(A(x0)-b) > epsilon) 74 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 75 | AAt = @(z) A(At(z)); 76 | [w, cgres] = cgsolve(AAt, b, cgtol, cgmaxiter, 0); 77 | if (cgres > 1/2) 78 | disp('A*At is ill-conditioned: cannot find starting point'); 79 | xp = x0; 80 | return; 81 | end 82 | x0 = At(w); 83 | end 84 | else 85 | if (norm(A*x0-b) > epsilon) 86 | disp('Starting point infeasible; using x0 = At*inv(AAt)*y.'); 87 | opts.POSDEF = true; opts.SYM = true; 88 | [w, hcond] = linsolve(A*A', b, opts); 89 | if (hcond < 1e-14) 90 | disp('A*At is ill-conditioned: cannot find starting point'); 91 | xp = x0; 92 | return; 93 | end 94 | x0 = A'*w; 95 | end 96 | end 97 | x = x0; 98 | Dhx = Dh*x; Dvx = Dv*x; 99 | t = (0.95)*sqrt(Dhx.^2 + Dvx.^2) + (0.1)*max(sqrt(Dhx.^2 + Dvx.^2)); 100 | 101 | % choose initial value of tau so that the duality gap after the first 102 | % step will be about the origial TV 103 | tau = (N+1)/sum(sqrt(Dhx.^2+Dvx.^2)); 104 | 105 | lbiter = ceil((log((N+1))-log(lbtol)-log(tau))/log(mu)); 106 | disp(sprintf('Number of log barrier iterations = %d\n', lbiter)); 107 | totaliter = 0; 108 | for ii = 1:lbiter 109 | 110 | [xp, tp, ntiter] = tvqc_newton(x, t, A, At, b, epsilon, tau, newtontol, newtonmaxiter, cgtol, cgmaxiter); 111 | totaliter = totaliter + ntiter; 112 | 113 | tvxp = sum(sqrt((Dh*xp).^2 + (Dv*xp).^2)); 114 | disp(sprintf('\nLog barrier iter = %d, TV = %.3f, functional = %8.3f, tau = %8.3e, total newton iter = %d\n', ... 115 | ii, tvxp, sum(tp), tau, totaliter)); 116 | 117 | x = xp; 118 | t = tp; 119 | 120 | tau = mu*tau; 121 | 122 | end 123 | 124 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/Optimization/tvqc_newton.m: -------------------------------------------------------------------------------- 1 | % tvqc_newton.m 2 | % 3 | % 对应tv P_D 噪声情况下不等式约束的newton求解方法 4 | % 5 | % Newton algorithm for log-barrier subproblems for TV minimization 6 | % with quadratic constraints. 7 | % 8 | % Usage: 9 | % [xp,tp,niter] = tvqc_newton(x0, t0, A, At, b, epsilon, tau, 10 | % newtontol, newtonmaxiter, cgtol, cgmaxiter) 11 | % 12 | % x0,t0 - starting points 13 | % 14 | % A - Either a handle to a function that takes a N vector and returns a K 15 | % vector , or a KxN matrix. If A is a function handle, the algorithm 16 | % operates in "largescale" mode, solving the Newton systems via the 17 | % Conjugate Gradients algorithm. 18 | % 19 | % At - Handle to a function that takes a K vector and returns an N vector. 20 | % If A is a KxN matrix, At is ignored. 21 | % 22 | % b - Kx1 vector of observations. 23 | % 24 | % epsilon - scalar, constraint relaxation parameter 25 | % 26 | % tau - Log barrier parameter. 27 | % 28 | % newtontol - Terminate when the Newton decrement is <= newtontol. 29 | % 30 | % newtonmaxiter - Maximum number of iterations. 31 | % 32 | % cgtol - Tolerance for Conjugate Gradients; ignored if A is a matrix. 33 | % 34 | % cgmaxiter - Maximum number of iterations for Conjugate Gradients; ignored 35 | % if A is a matrix. 36 | % 37 | % Written by: Justin Romberg, Caltech 38 | % Email: jrom@acm.caltech.edu 39 | % Created: October 2005 40 | % 41 | 42 | function [xp, tp, niter] = tvqc_newton(x0, t0, A, At, b, epsilon, tau, newtontol, newtonmaxiter, cgtol, cgmaxiter) 43 | 44 | largescale = isa(A,'function_handle'); 45 | 46 | alpha = 0.01; 47 | beta = 0.5; 48 | 49 | N = length(x0); 50 | n = round(sqrt(N)); 51 | 52 | % create (sparse) differencing matrices for TV 53 | Dv = spdiags([reshape([-ones(n-1,n); zeros(1,n)],N,1) ... 54 | reshape([zeros(1,n); ones(n-1,n)],N,1)], [0 1], N, N); 55 | Dh = spdiags([reshape([-ones(n,n-1) zeros(n,1)],N,1) ... 56 | reshape([zeros(n,1) ones(n,n-1)],N,1)], [0 n], N, N); 57 | 58 | if (~largescale), AtA = A'*A; end; 59 | 60 | % initial point 61 | x = x0; 62 | t = t0; 63 | if (largescale), r = A(x) - b; else r = A*x - b; end 64 | Dhx = Dh*x; Dvx = Dv*x; 65 | ft = 1/2*(Dhx.^2 + Dvx.^2 - t.^2); 66 | fe = 1/2*(r'*r - epsilon^2); 67 | f = sum(t) - (1/tau)*(sum(log(-ft)) + log(-fe)); 68 | 69 | niter = 0; 70 | done = 0; 71 | while (~done) 72 | 73 | if (largescale), Atr = At(r); else Atr = A'*r; end 74 | ntgx = Dh'*((1./ft).*Dhx) + Dv'*((1./ft).*Dvx) + 1/fe*Atr; 75 | ntgt = -tau - t./ft; 76 | gradf = -(1/tau)*[ntgx; ntgt]; 77 | 78 | sig22 = 1./ft + (t.^2)./(ft.^2); 79 | sig12 = -t./ft.^2; 80 | sigb = 1./ft.^2 - (sig12.^2)./sig22; 81 | 82 | w1p = ntgx - Dh'*(Dhx.*(sig12./sig22).*ntgt) - Dv'*(Dvx.*(sig12./sig22).*ntgt); 83 | if (largescale) 84 | h11pfun = @(z) H11p(z, A, At, Dh, Dv, Dhx, Dvx, sigb, ft, fe, Atr); 85 | [dx, cgres, cgiter] = cgsolve(h11pfun, w1p, cgtol, cgmaxiter, 0); 86 | if (cgres > 1/2) 87 | disp('Cannot solve system. Returning previous iterate. (See Section 4 of notes for more information.)'); 88 | xp = x; tp = t; 89 | return 90 | end 91 | Adx = A(dx); 92 | else 93 | H11p = Dh'*sparse(diag(-1./ft + sigb.*Dhx.^2))*Dh + ... 94 | Dv'*sparse(diag(-1./ft + sigb.*Dvx.^2))*Dv + ... 95 | Dh'*sparse(diag(sigb.*Dhx.*Dvx))*Dv + ... 96 | Dv'*sparse(diag(sigb.*Dhx.*Dvx))*Dh - ... 97 | (1/fe)*AtA + (1/fe^2)*Atr*Atr'; 98 | opts.POSDEF = true; opts.SYM = true; 99 | [dx,hcond] = linsolve(H11p, w1p, opts); 100 | if (hcond < 1e-14) 101 | disp('Matrix ill-conditioned. Returning previous iterate. (See Section 4 of notes for more information.)'); 102 | xp = x; tp = t; 103 | return 104 | end 105 | Adx = A*dx; 106 | end 107 | Dhdx = Dh*dx; Dvdx = Dv*dx; 108 | dt = (1./sig22).*(ntgt - sig12.*(Dhx.*Dhdx + Dvx.*Dvdx)); 109 | 110 | % minimum step size that stays in the interior 111 | aqt = Dhdx.^2 + Dvdx.^2 - dt.^2; 112 | bqt = 2*(Dhdx.*Dhx + Dvdx.*Dvx - t.*dt); 113 | cqt = Dhx.^2 + Dvx.^2 - t.^2; 114 | tsols = [(-bqt+sqrt(bqt.^2-4*aqt.*cqt))./(2*aqt); ... 115 | (-bqt-sqrt(bqt.^2-4*aqt.*cqt))./(2*aqt) ]; 116 | indt = find([(bqt.^2 > 4*aqt.*cqt); (bqt.^2 > 4*aqt.*cqt)] & (tsols > 0)); 117 | aqe = Adx'*Adx; bqe = 2*r'*Adx; cqe = r'*r - epsilon^2; 118 | smax = min(1,min([... 119 | tsols(indt); ... 120 | (-bqe+sqrt(bqe^2-4*aqe*cqe))/(2*aqe) 121 | ])); 122 | s = (0.99)*smax; 123 | 124 | % backtracking line search 125 | suffdec = 0; 126 | backiter = 0; 127 | while (~suffdec) 128 | xp = x + s*dx; tp = t + s*dt; 129 | rp = r + s*Adx; Dhxp = Dhx + s*Dhdx; Dvxp = Dvx + s*Dvdx; 130 | ftp = 1/2*(Dhxp.^2 + Dvxp.^2 - tp.^2); 131 | fep = 1/2*(rp'*rp - epsilon^2); 132 | fp = sum(tp) - (1/tau)*(sum(log(-ftp)) + log(-fep)); 133 | flin = f + alpha*s*(gradf'*[dx; dt]); 134 | suffdec = (fp <= flin); 135 | s = beta*s; 136 | backiter = backiter + 1; 137 | if (backiter > 32) 138 | disp('Stuck on backtracking line search, returning previous iterate. (See Section 4 of notes for more information.)'); 139 | xp = x; tp = t; 140 | return 141 | end 142 | end 143 | 144 | % set up for next iteration 145 | x = xp; t = tp; 146 | r = rp; Dvx = Dvxp; Dhx = Dhxp; 147 | ft = ftp; fe = fep; f = fp; 148 | 149 | lambda2 = -(gradf'*[dx; dt]); 150 | stepsize = s*norm([dx; dt]); 151 | niter = niter + 1; 152 | done = (lambda2/2 < newtontol) | (niter >= newtonmaxiter); 153 | 154 | disp(sprintf('Newton iter = %d, Functional = %8.3f, Newton decrement = %8.3f, Stepsize = %8.3e', ... 155 | niter, f, lambda2/2, stepsize)); 156 | if (largescale) 157 | disp(sprintf(' CG Res = %8.3e, CG Iter = %d', cgres, cgiter)); 158 | else 159 | disp(sprintf(' H11p condition number = %8.3e', hcond)); 160 | end 161 | 162 | end 163 | 164 | 165 | 166 | 167 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 168 | % H11p auxiliary function 169 | function y = H11p(v, A, At, Dh, Dv, Dhx, Dvx, sigb, ft, fe, atr) 170 | 171 | Dhv = Dh*v; 172 | Dvv = Dv*v; 173 | 174 | y = Dh'*((-1./ft + sigb.*Dhx.^2).*Dhv + sigb.*Dhx.*Dvx.*Dvv) + ... 175 | Dv'*((-1./ft + sigb.*Dvx.^2).*Dvv + sigb.*Dhx.*Dvx.*Dhv) - ... 176 | 1/fe*At(A(v)) + 1/fe^2*(atr'*v)*atr; 177 | 178 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 179 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/README: -------------------------------------------------------------------------------- 1 | 2 | 3 | l1 magic 4 | -------- 5 | 6 | This package contains code for solving seven optimization problems. A detailed explanation is given in the file l1magic.pdf. 7 | 8 | The main directory contains MATLAB m-files which contain simple examples for each of the recovery problems. They illustrate how the code should be used (it is fairly straightforward). The prefixes on the example files are as follows: 9 | "l1eq" = L1 minimization with equality constraints, 10 | "l1qc" = L1 minimization with quadratic (L2 norm) constraints, 11 | "l1decode" = L1 norm approximation (for channel decoding), 12 | "11dantzig" = L1 minimization with minimal residual correlation (the Dantzig selector). 13 | "tveq" = TV minimization with equality constraints, 14 | "tvqc" = TV minimization with quadratic constrains, 15 | "tvdantzig" = TV minimization with minimal residual correlation. 16 | 17 | The 'Optimization' directory contains the m-files for the actual interior-point optimization algorithms. 18 | 19 | The 'Measurements' directory contains implementations of some relevant large-scale measurement matrices that are used in the examples. 20 | 21 | The 'Data' directory contains test images for the examples. 22 | 23 | The code will only run on MATLAB 7.0 and higher. The incompatibility with previous versions comes from the way function handles are used. 24 | 25 | If you have questions or comments, please contact 26 | Justin Romberg 27 | Caltech, Applied and Computational Mathematics, jrom@acm.caltech.edu 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/l1dantzig_example.m: -------------------------------------------------------------------------------- 1 | % l1dantzig_example.m 2 | % 3 | % Test out l1dantzig code (l1 minimization with bounded residual correlation). 4 | % 5 | % Written by: Justin Romberg, Caltech 6 | % Email: jrom@acm.caltech.edu 7 | % Created: October 2005 8 | % 9 | 10 | % put optimization code in path if not already there 11 | path(path, './Optimization'); 12 | 13 | % signal length 14 | N = 512; 15 | 16 | % number of spikes to put down 17 | T = 20; 18 | 19 | % number of observations to make 20 | K = 120; 21 | 22 | % random +/- 1 signal 23 | x = zeros(N,1); 24 | q = randperm(N); 25 | x(q(1:T)) = sign(randn(T,1)); 26 | 27 | % measurement matrix = random projection 28 | disp('Creating measurment matrix...'); 29 | A = randn(K,N); 30 | A = orth(A')'; 31 | disp('Done.'); 32 | 33 | % noisy observations 34 | sigma = 0.005; 35 | e = sigma*randn(K,1); 36 | y = A*x + e; 37 | 38 | % initial guess = min energy 39 | x0 = A'*y; 40 | 41 | % Dantzig selection 42 | epsilon = 3e-3; 43 | tic 44 | xp = l1dantzig_pd(x0, A, [], y, epsilon, 5e-2); 45 | toc 46 | 47 | 48 | % large scale 49 | % Afun = @(z) A*z; 50 | % Atfun = @(z) A'*z; 51 | % tic 52 | % xp = l1dantzig_pd(x0, Afun, Atfun, y, epsilon, 5e-2, 50, 1e-8, 500); 53 | % toc 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/l1decode_example.m: -------------------------------------------------------------------------------- 1 | % l1decode_example.m 2 | % 3 | % Test out l1decode code. 4 | % 5 | % Written by: Justin Romberg, Caltech 6 | % Email: jrom@acm.caltech.edu 7 | % Created: October 2005 8 | % 9 | 10 | path(path, './Optimization'); 11 | 12 | % source length 13 | N = 256; 14 | 15 | % codeword length 16 | M = 4*N; 17 | 18 | % number of perturbations 19 | T = round(.2*M); 20 | 21 | % coding matrix 22 | G = randn(M,N); 23 | 24 | % source word 25 | x = randn(N,1); 26 | 27 | % code word 28 | c = G*x; 29 | 30 | % channel: perturb T randomly chosen entries 31 | q = randperm(M); 32 | y = c; 33 | y(q(1:T)) = randn(T,1); 34 | 35 | % recover 36 | x0 = inv(G'*G)*G'*y; 37 | xp = l1decode_pd(x0, G, [], y, 1e-4, 30); 38 | 39 | % large scale 40 | % gfun = @(z) G*z; 41 | % gtfun = @(z) G'*z; 42 | % xp = l1decode_pd(x0, gfun, gtfun, y, 1e-3, 25, 1e-8, 200); 43 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/l1eq_example.m: -------------------------------------------------------------------------------- 1 | % l1eq_example.m 2 | % 3 | % Test out l1eq code (l1 minimization with equality constraints). 4 | % 5 | % Written by: Justin Romberg, Caltech 6 | % Email: jrom@acm.caltech.edu 7 | % Created: October 2005 8 | % 9 | 10 | % put key subdirectories in path if not already there 11 | path(path, './Optimization'); 12 | path(path, './Data'); 13 | 14 | % To reproduce the example in the documentation, uncomment the 15 | % two lines below 16 | %load RandomStates 17 | %rand('state', rand_state); 18 | %randn('state', randn_state); 19 | 20 | % signal length 21 | N = 512; 22 | % number of spikes in the signal 23 | T = 20; 24 | % number of observations to make 25 | K = 120; 26 | 27 | % random +/- 1 signal 28 | x = zeros(N,1); 29 | q = randperm(N); 30 | x(q(1:T)) = sign(randn(T,1)); 31 | 32 | % measurement matrix 33 | disp('Creating measurment matrix...'); 34 | A = randn(K,N); 35 | A = orth(A')'; 36 | disp('Done.'); 37 | 38 | % observations 39 | y = A*x; 40 | 41 | % initial guess = min energy 42 | x0 = A'*y; 43 | 44 | % solve the LP 45 | tic 46 | xp = l1eq_pd(x0, A, [], y, 1e-3); 47 | toc 48 | 49 | % large scale 50 | % Afun = @(z) A*z; 51 | % Atfun = @(z) A'*z; 52 | % tic 53 | % xp = l1eq_pd(x0, Afun, Atfun, y, 1e-3, 30, 1e-8, 200); 54 | % toc 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/l1qc_example.m: -------------------------------------------------------------------------------- 1 | % l1qc_example.m 2 | % 3 | % Test out l1qc code (l1 minimization with quadratic constraint). 4 | % 5 | % Written by: Justin Romberg, Caltech 6 | % Email: jrom@acm.caltech.edu 7 | % Created: October 2005 8 | % 9 | 10 | % put optimization code in path if not already there 11 | path(path, './Optimization'); 12 | 13 | % signal length 14 | N = 512; 15 | 16 | % number of spikes to put down 17 | T = 20; 18 | 19 | % number of observations to make 20 | K = 120; 21 | 22 | % random +/- 1 signal 23 | x = zeros(N,1); 24 | q = randperm(N); 25 | x(q(1:T)) = sign(randn(T,1)); 26 | 27 | % measurement matrix 28 | disp('Creating measurment matrix...'); 29 | A = randn(K,N); 30 | A = orth(A')'; 31 | disp('Done.'); 32 | 33 | % noisy observations 34 | sigma = 0.005; 35 | e = sigma*randn(K,1); 36 | y = A*x + e; 37 | 38 | % initial guess = min energy 39 | x0 = A'*y; 40 | 41 | % take epsilon a little bigger than sigma*sqrt(K) 42 | epsilon = sigma*sqrt(K)*sqrt(1 + 2*sqrt(2)/sqrt(K)); 43 | 44 | tic 45 | xp = l1qc_logbarrier(x0, A, [], y, epsilon, 1e-3); 46 | toc 47 | 48 | % large scale 49 | % Afun = @(z) A*z; 50 | % Atfun = @(z) A'*z; 51 | % tic 52 | % xp = l1qc_logbarrier(x0, Afun, Atfun, y, epsilon, 1e-3, 50, 1e-8, 500); 53 | % toc 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/tvdantzig_example.m: -------------------------------------------------------------------------------- 1 | % tvdantzig_example.m 2 | % 3 | % Test out tvdantzig code (TV minimization with bounded residual correlation). 4 | % 5 | % Written by: Justin Romberg, Caltech 6 | % Email: jrom@acm.caltech.edu 7 | % Created: October 2005 8 | % 9 | 10 | % use implicit, matrix-free algorithms ? 11 | largescale = 0; 12 | 13 | path(path, './Optimization'); 14 | path(path, './Measurements'); 15 | path(path, './Data'); 16 | 17 | 18 | % test image = 32x32 piece of cameraman's arm 19 | load camera 20 | I = camera(81:112,37:68); 21 | n = 32; 22 | N = n*n; 23 | I = I/norm(I(:)); 24 | I = I - mean(I(:)); 25 | x = reshape(I,N,1); 26 | 27 | 28 | % num obs 29 | K = 300; 30 | 31 | % permutation P and observation set OMEGA 32 | P = randperm(N)'; 33 | q = randperm(N/2-1)+1; 34 | OMEGA = q(1:K/2)'; 35 | 36 | 37 | 38 | % measurement matrix 39 | if (largescale) 40 | A = @(z) A_f(z, OMEGA, P); 41 | At = @(z) At_f(z, N, OMEGA, P); 42 | % obsevations 43 | b = A(x); 44 | % initial point 45 | x0 = At(b); 46 | else 47 | FT = 1/sqrt(N)*fft(eye(N)); 48 | A = sqrt(2)*[real(FT(OMEGA,:)); imag(FT(OMEGA,:))]; 49 | A = [1/sqrt(N)*ones(1,N); A]; 50 | At = []; 51 | % observations 52 | b = A*x; 53 | % initial point 54 | x0 = A'*b; 55 | end 56 | 57 | 58 | epsilon = 5e-3; 59 | 60 | 61 | 62 | tvI = sum(sum(sqrt([diff(I,1,2) zeros(n,1)].^2 + [diff(I,1,1); zeros(1,n)].^2 ))); 63 | disp(sprintf('Original TV = %.3f', tvI)); 64 | 65 | time0 = clock; 66 | xp = tvdantzig_logbarrier(x0, A, At, b, epsilon, 1e-3, 5, 1e-8, 1500); 67 | Ip = reshape(xp, n, n); 68 | disp(sprintf('Total elapsed time = %f secs\n', etime(clock,time0))); 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/tveq_example.m: -------------------------------------------------------------------------------- 1 | % tveq_example.m 2 | % 3 | % Test out tveq code (TV minimization with equality constraints). 4 | % 5 | % Written by: Justin Romberg, Caltech 6 | % Email: jrom@acm.caltech.edu 7 | % Created: October 2005 8 | % 9 | 10 | % use implicit, matrix-free algorithms ? 11 | largescale = 0; 12 | 13 | path(path, './Optimization'); 14 | path(path, './Measurements'); 15 | path(path, './Data'); 16 | 17 | 18 | % test image = 32x32 piece of cameraman's arm 19 | load camera 20 | I = camera(81:112,37:68); 21 | n = 32; 22 | N = n*n; 23 | I = I/norm(I(:)); 24 | I = I - mean(I(:)); 25 | x = reshape(I,N,1); 26 | 27 | 28 | % num obs 29 | K = 300; 30 | 31 | % permutation P and observation set OMEGA 32 | P = randperm(N)'; 33 | q = randperm(N/2-1)+1; 34 | OMEGA = q(1:K/2)'; 35 | 36 | 37 | 38 | % measurement matrix 39 | if (largescale) 40 | A = @(z) A_f(z, OMEGA, P); 41 | At = @(z) At_f(z, N, OMEGA, P); 42 | % obsevations 43 | b = A(x); 44 | % initial point 45 | x0 = At(b); 46 | else 47 | FT = 1/sqrt(N)*fft(eye(N)); 48 | A = sqrt(2)*[real(FT(OMEGA,:)); imag(FT(OMEGA,:))]; 49 | A = [1/sqrt(N)*ones(1,N); A]; 50 | At = []; 51 | % observations 52 | b = A*x; 53 | % initial point 54 | x0 = A'*b; 55 | end 56 | 57 | tvI = sum(sum(sqrt([diff(I,1,2) zeros(n,1)].^2 + [diff(I,1,1); zeros(1,n)].^2 ))); 58 | disp(sprintf('Original TV = %.3f', tvI)); 59 | 60 | time0 = clock; 61 | xp = tveq_logbarrier(x0, A, At, b, 1e-3, 5, 1e-8, 200); 62 | Ip = reshape(xp, n, n); 63 | disp(sprintf('Total elapsed time = %f secs\n', etime(clock,time0))); 64 | 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/tveq_phantom_example.m: -------------------------------------------------------------------------------- 1 | % tveq_phantom_example.m 2 | % 3 | % Phantom reconstruction from samples on 22 radial lines in the Fourier 4 | % plane. 5 | % 6 | % Written by: Justin Romberg, Caltech 7 | % Email: jrom@acm.caltech.edu 8 | % Created: October 2005 9 | % 10 | 11 | 12 | path(path, './Optimization'); 13 | path(path, './Measurements'); 14 | path(path, './Data'); 15 | 16 | 17 | % Phantom 18 | n = 256; 19 | N = n*n; 20 | X = phantom(n); % phantom的图像 大小n*n 21 | x = X(:); % 把phantom展开成一维向量（n*n）*1 22 | 23 | % number of radial lines in the Fourier domain 24 | L = 22; 25 | 26 | % Fourier samples we are given 27 | [M,Mh,mi,mhi] = LineMask(L,n); 28 | OMEGA = mhi; % mhi其实只有一半 29 | A = @(z) A_fhp(z, OMEGA); % 函数已输入OMEGA 调用A(x) 相当于调用A_fhp(x,OMEGA) 30 | At = @(z) At_fhp(z, OMEGA, n); % 函数已输入OMEGA和n 调用At(y) 相当于调用A_fhp(y,OMEGA,n) 31 | 32 | % measurements 33 | y = A(x); % 测量值 OMEGA个img和OMEGA个real以及一个均值，是mask以后的频域 34 | 35 | % min l2 reconstruction (backprojection) 36 | xbp = At(y); % minimal energy reconstruction initial guess 初始猜测的最小能量恢复 是用mask以后的频域恢复的图像向量 37 | Xbp = reshape(xbp,n,n); % 最小能量恢复的图 reshape把向量组织成图 38 | 39 | 40 | 41 | % recovery 42 | tic 43 | tvI = sum(sum(sqrt([diff(X,1,2) zeros(n,1)].^2 + [diff(X,1,1); zeros(1,n)].^2 ))); % 最开始的TV值 44 | % TV的值是每个像素sqrt(横向的‘相邻元素的差值’^2+纵向‘相邻元素的差值’^2)，总nxn个像素的gradient总和 45 | disp(sprintf('Original TV = %8.3f', tvI)); 46 | xp = tveq_logbarrier(xbp, A, At, y, 1e-1, 2, 1e-8, 600); % 进行优化 47 | Xtv = reshape(xp, n, n); % 范数优化输出的图像 48 | toc 49 | 50 | figure; 51 | subplot(1,2,1); 52 | title('minimal energy reconstruction'); 53 | imshow(Xbp); 54 | subplot(1,2,2); 55 | title('l1-magic reconstruction'); 56 | imshow(Xtv); 57 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/tvqc_example.m: -------------------------------------------------------------------------------- 1 | % tvqc_example.m 2 | % 3 | % Test out tvqc code (TV minimization with quadratic constraint). 4 | % 5 | % Written by: Justin Romberg, Caltech 6 | % Email: jrom@acm.caltech.edu 7 | % Created: October 2005 8 | % 9 | 10 | % use implicit, matrix-free algorithms ? 11 | largescale = 0; 12 | 13 | path(path, './Optimization'); 14 | path(path, './Measurements'); 15 | path(path, './Data'); 16 | 17 | 18 | % test image = 32x32 piece of cameraman's arm 19 | load camera 20 | I = camera(81:112,37:68); 21 | n = 32; 22 | N = n*n; 23 | I = I/norm(I(:)); 24 | I = I - mean(I(:)); 25 | x = reshape(I,N,1); 26 | 27 | 28 | % num obs 29 | K = 300; 30 | 31 | % permutation P and observation set OMEGA 32 | P = randperm(N)'; 33 | q = randperm(N/2-1)+1; 34 | OMEGA = q(1:K/2)'; 35 | 36 | 37 | 38 | % measurement matrix 39 | if (largescale) 40 | A = @(z) A_f(z, OMEGA, P); 41 | At = @(z) At_f(z, N, OMEGA, P); 42 | % obsevations 43 | b = A(x); 44 | % initial point 45 | x0 = At(b); 46 | else 47 | FT = 1/sqrt(N)*fft(eye(N)); 48 | A = sqrt(2)*[real(FT(OMEGA,:)); imag(FT(OMEGA,:))]; 49 | A = [1/sqrt(N)*ones(1,N); A]; 50 | At = []; 51 | % observations 52 | b = A*x; 53 | % initial point 54 | x0 = A'*b; 55 | end 56 | 57 | 58 | epsilon = 5e-3; 59 | 60 | 61 | tvI = sum(sum(sqrt([diff(I,1,2) zeros(n,1)].^2 + [diff(I,1,1); zeros(1,n)].^2 ))); 62 | disp(sprintf('Original TV = %.3f', tvI)); 63 | 64 | time0 = clock; 65 | xp = tvqc_logbarrier(x0, A, At, b, epsilon, 1e-3, 5, 1e-8, 200); 66 | Ip = reshape(xp, n, n); 67 | disp(sprintf('Total elapsed time = %f secs\n', etime(clock,time0))); 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /l1magic-1.11/l1magic/tvqc_quantization_example.m: -------------------------------------------------------------------------------- 1 | % tvqc_quantization_example.m 2 | % 3 | % Takes random measurements of the 'boats' image, quantizes, and 4 | % reconstructs using quadractically constrained TV. 5 | % 6 | % Written by: Justin Romberg, Caltech 7 | % Email: jrom@acm.caltech.edu 8 | % Created: October 2005 9 | % 10 | 11 | path(path, './Optimization'); 12 | path(path, './Measurements'); 13 | path(path, './Data'); 14 | 15 | % boats 16 | n = 256; 17 | N = n*n; 18 | load boats 19 | img = boats; 20 | I = img/norm(img(:)); 21 | I = I - mean(I(:)); 22 | x = reshape(I,N,1); 23 | 24 | % num obs 25 | K = 25000; 26 | 27 | % permutation P and observation set OMEGA 28 | P = randperm(N)'; 29 | q = randperm(N/2-1)+1; 30 | OMEGA = q(1:K/2)'; 31 | 32 | % measurement implicit matrices 33 | A = @(z) A_f(z, OMEGA, P); 34 | At = @(z) At_f(z, N, OMEGA, P); 35 | 36 | % take measurements 37 | Ax = A(x); 38 | % quantization codebook 39 | mxax = max(abs(Ax)); 40 | bins = 10; 41 | C = 2*mxax*(linspace(1/(2*bins),1-1/(2*bins),bins)-1/2); 42 | % quantize 43 | b = zeros(K,1); 44 | symbols = zeros(K,1); 45 | for kk = 1:K 46 | [mn,mni] = min(abs(Ax(kk)-C)); 47 | b(kk) = C(mni); 48 | symbols(kk) = mni; 49 | end 50 | % error (just for future reference) 51 | e = b - Ax; 52 | 53 | % the error should be roughly this size 54 | epsilon = (2*mxax/bins)/sqrt(12)*sqrt(K); 55 | 56 | 57 | % initial point = min l2 58 | x0 = At(b); 59 | 60 | 61 | tvI = sum(sum(sqrt([diff(I,1,2) zeros(n,1)].^2 + [diff(I,1,1); zeros(1,n)].^2 ))); 62 | disp(sprintf('original TV = %8.3f', tvI)); 63 | 64 | time0 = clock; 65 | xp = tvqc_logbarrier(x0, A, At, b, epsilon, 1e-1, 5); 66 | Ip = reshape(xp, n, n); 67 | disp(sprintf('Total elapsed time = %f secs\n', etime(clock,time0))); 68 | -------------------------------------------------------------------------------- /omp.m: -------------------------------------------------------------------------------- 1 | 2 | % 1-D信号压缩传感的实现(正交匹配追踪法Orthogonal Matching Pursuit) 3 | % 测量数M>=K*log(N/K),K是稀疏度,N信号长度,可以近乎完全重构 4 | % 编程人--香港大学电子工程系 沙威 Email: wsha@eee.hku.hk 5 | % 编程时间：2008年11月18日 6 | % 文档下载: http://www.eee.hku.hk/~wsha/Freecode/freecode.htm 7 | % 参考文献：Joel A. Tropp and Anna C. Gilbert 8 | % Signal Recovery From Random Measurements Via Orthogonal Matching 9 | % Pursuit，IEEE TRANSACTIONS ON INFORMATION THEORY, VOL. 53, NO. 12, 10 | % DECEMBER 2007. 11 | 12 | clc;clear 13 | 14 | %% 1. 时域测试信号生成 15 | K=7; % 稀疏度(做FFT可以看出来) 16 | N=256; % 信号长度 17 | M=64; % 测量数(M>=K*log(N/K),至少40,但有出错的概率) 18 | f1=50; % 信号频率1 19 | f2=100; % 信号频率2 20 | f3=200; % 信号频率3 21 | f4=400; % 信号频率4 22 | fs=800; % 采样频率 23 | ts=1/fs; % 采样间隔 24 | Ts=1:N; % 采样序列 25 | noise=rand(N)-0.5;%zeros(N); 26 | x=0.3*cos(2*pi*f1*Ts*ts)+0.6*cos(2*pi*f2*Ts*ts)+0.2*cos(2*pi*f3*Ts*ts)+0.9*cos(2*pi*f4*Ts*ts)+noise(Ts); % 完整信号,由4个信号叠加而来 27 | 28 | %% 2. 时域信号压缩传感 29 | Phi=randn(M,N); % 测量矩阵(高斯分布白噪声)64*256的扁矩阵，Phi也就是文中说的D矩阵 30 | s=Phi*x.'; % 获得线性测量 ，s相当于文中的y矩阵 31 | 32 | %% 3. 正交匹配追踪法重构信号(本质上是L_1范数最优化问题) 33 | %匹配追踪：找到一个其标记看上去与收集到的数据相关的小波；在数据中去除这个标记的所有印迹；不断重复直到我们能用小波标记“解释”收集到的所有数据。 34 | 35 | m=K*2; % 算法迭代次数(m>=K)，设x是K-sparse的 36 | Psi=fft(eye(N,N))/sqrt(N); % 傅里叶正变换矩阵 37 | T=Phi*Psi'; % 恢复矩阵(测量矩阵*正交反变换矩阵) 38 | 39 | hat_y=zeros(1,N); % 待重构的谱域(变换域)向量 40 | Aug_t=[]; % 增量矩阵(初始值为空矩阵) 41 | r_n=s; % 残差值 42 | 43 | for times=1:m; % 迭代次数(有噪声的情况下,该迭代次数为K) 44 | for col=1:N; % 恢复矩阵的所有列向量 45 | product(col)=abs(T(:,col)'*r_n); % 恢复矩阵的列向量和残差的投影系数(内积值) 46 | end 47 | [val,pos]=max(product); % 最大投影系数对应的位置，即找到一个其标记看上去与收集到的数据相关的小波 48 | Aug_t=[Aug_t,T(:,pos)]; % 矩阵扩充 ***** 最相关的基m*x ***** 49 | 50 | T(:,pos)=zeros(M,1); % 选中的列置零（实质上应该去掉，为了简单我把它置零），在数据中去除这个标记的所有印迹 51 | aug_y=(Aug_t'*Aug_t)^(-1)*Aug_t'*s; % 最小二乘,使残差最小 ***** Aug_t(基)*aug_y(系数)=s的最小二乘解aug_y(系数) ****** 52 | r_n=s-Aug_t*aug_y; % 残差 ***** aug_y就是使s-Aug_t*aug_y最小的解 ***** 53 | pos_array(times)=pos; % 纪录最大投影系数的位置 54 | end 55 | hat_y(pos_array)=aug_y; % 重构的谱域向量 56 | hat_x=real(Psi'*hat_y.'); % 做逆傅里叶变换重构得到时域信号 57 | 58 | %% 4. 恢复信号和原始信号对比 59 | figure(1); 60 | hold on; 61 | %plot(hat_x,'k.-') % 重建信号 62 | plot(abs(fft(hat_x)),'k.-') 63 | %plot(x,'r') % 原始信号 64 | plot(abs(fft(x)),'r') 65 | legend('Recovery','Original') 66 | norm(hat_x.'-x)/norm(x) % 重构误差 67 | 68 | -------------------------------------------------------------------------------- /omp_jbb/CS_OMP.m: -------------------------------------------------------------------------------- 1 | function [ theta ] = CS_OMP( y,A,t ) 2 | %CS_OMP Summary of this function goes here 3 | %Version: 1.0 written by jbb0523 @2015-04-18 4 | % Detailed explanation goes here 5 | % y = Phi * x 6 | % x = Psi * theta 7 | % y = Phi*Psi * theta 8 | % 令 A = Phi*Psi, 则y=A*theta 9 | % 现在已知y和A，求theta 10 | [y_rows,y_columns] = size(y); 11 | if y_rows