├── .gitignore ├── CONTRIBUTION_GUIDE.md ├── CONTRIBUTORS.md ├── COPYRIGHT.txt ├── Contents.m ├── INSTALL.md ├── LICENSE.txt ├── README.md ├── cstep.m ├── cvsrch.m ├── docs ├── README.md ├── examples_doc.m ├── gradientcheck_doc.m ├── helptoc.xml ├── html │ ├── examples_doc.html │ ├── examples_doc_eq00543772986039519777.png │ ├── examples_doc_eq02586487823804834451.png │ ├── examples_doc_eq02619125812656254059.png │ ├── examples_doc_eq02739270504201626537.png │ ├── examples_doc_eq02978325375087395648.png │ ├── examples_doc_eq03421469680577370400.png │ ├── examples_doc_eq03672095713503266041.png │ ├── examples_doc_eq05147331747641807187.png │ ├── examples_doc_eq05495199406804600545.png │ ├── examples_doc_eq05508344529756732484.png │ ├── examples_doc_eq06327244962903926757.png │ ├── examples_doc_eq07741076879175499892.png │ ├── examples_doc_eq11316155984473799093.png │ ├── examples_doc_eq11475201221685588019.png │ ├── examples_doc_eq11638118408029626293.png │ ├── examples_doc_eq12267211811494638899.png │ ├── examples_doc_eq12534387414789875675.png │ ├── examples_doc_eq13029888333516263543.png │ ├── examples_doc_eq15636846968047188835.png │ ├── examples_doc_eq17937321231062034804.png │ ├── gradientcheck_doc.html │ ├── gradientcheck_doc_eq00827914115064801059.png │ ├── gradientcheck_doc_eq05671228016298599287.png │ ├── gradientcheck_doc_eq08062630503172331818.png │ ├── gradientcheck_doc_eq12428413953531653171.png │ ├── gradientcheck_doc_eq13924783238156832738.png │ ├── gradientcheck_doc_eq15869567321110343067.png │ ├── gradientcheck_doc_eq16537821098686504121.png │ ├── images │ │ ├── banner-background.jpg │ │ └── logo.gif │ ├── lbfgs_doc.html │ ├── lbfgs_doc_eq00345914025344442474.png │ ├── lbfgs_doc_eq02499409918622348759.png │ ├── lbfgs_doc_eq02757236760158446565.png │ ├── lbfgs_doc_eq05671228016298599287.png │ ├── lbfgs_doc_eq07165327077797634723.png │ ├── lbfgs_doc_eq17452142193152507074.png │ ├── methods.html │ ├── ncg_doc.html │ ├── ncg_doc_eq00202142981986870057.png │ ├── ncg_doc_eq00232388002207087152.png │ ├── ncg_doc_eq00345914025344442474.png │ ├── ncg_doc_eq00906644648645161877.png │ ├── ncg_doc_eq02535271716144487441.png │ ├── ncg_doc_eq05857437662927211826.png │ ├── ncg_doc_eq07165327077797634723.png │ ├── ncg_doc_eq08047493418710975323.png │ ├── ncg_doc_eq11960540003268942809.png │ ├── ncg_doc_eq12955286862837271730.png │ ├── ncg_doc_eq16009458126034669820.png │ ├── ncg_doc_eq16519649717996589748.png │ ├── ncg_doc_eq17452142193152507074.png │ ├── overview_doc.html │ ├── overview_doc_eq00884557479967417095.png │ ├── overview_doc_eq07500982203449410655.png │ ├── overview_doc_eq08097522164358817053.png │ ├── overview_doc_eq13721435717236182146.png │ ├── overview_doc_eq13754963546198947976.png │ ├── overview_doc_eq16485556778937288589.png │ ├── parameters.html │ ├── parameters_input_doc.html │ ├── parameters_output_doc.html │ ├── poblano.css │ ├── tn_doc.html │ ├── tn_doc_eq00345914025344442474.png │ ├── tn_doc_eq03158747792916826732.png │ ├── tn_doc_eq06834187619862737782.png │ ├── tn_doc_eq07165327077797634723.png │ ├── tn_doc_eq08610859800392802260.png │ ├── tn_doc_eq11373214381793991308.png │ ├── tn_doc_eq13505831761284237672.png │ ├── tn_doc_eq13549337764080820674.png │ ├── tn_doc_eq15653174921336884406.png │ ├── tn_doc_eq17016985893546760461.png │ └── tn_doc_eq17452142193152507074.png ├── index.html ├── lbfgs_doc.m ├── mxdom2simplehtml_poblano.xsl ├── ncg_doc.m ├── overview_doc.m ├── parameters_input_doc.m ├── parameters_output_doc.m └── tn_doc.m ├── example1.m ├── example2.m ├── example2_extract.m ├── example2_init.m ├── gradientcheck.m ├── hessvec_fd.m ├── info.xml ├── install_poblano.m ├── lbfgs.m ├── ncg.m ├── poblano_linesearch.m ├── poblano_out.m ├── poblano_params.m ├── tests ├── README.md ├── matlab.mat ├── poblano_coverage.m ├── test_cvsrch.m ├── test_gradientcheck.m ├── test_hessvec_fd.m ├── test_lbfgs.m ├── test_ncg.m ├── test_poblano_linsearch.m ├── test_poblano_out.m ├── test_tn.m ├── unittest_sum_sin.m └── unittest_sum_squares.m └── tn.m /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | -------------------------------------------------------------------------------- /CONTRIBUTION_GUIDE.md: -------------------------------------------------------------------------------- 1 | # Poblano Toolbox Contribution Guide 2 | 3 | ## Checklist 4 | 5 | - [ ] **Issue** Before the pull request, submit an issue for the change, providing as much detailed information as possible. For bug reports, please provide enough information to reproduce the problem. 6 | 7 | - [ ] **Fork** Create a branch or fork of the code and make your changes. 8 | 9 | - [ ] **Help Comments** Create or update comments for the m-files, following the style of the existing files. Be sure to explain all code options. 10 | 11 | - [ ] **Documentation** For any major new functionality, please follow the following steps. 12 | - [ ] Add MATLAB script documentation in the `doc` directory with the name `XXX_doc.m` 13 | - [ ] Use the MATLAB `publish` command on `XXX_doc.m` and the `mxdom2simplehtml_poblano.xsl` as the stylesheet to create a new file in `doc\html` 14 | - [ ] Add a pointer to this documentation file in `doc\html\helptoc.xml` 15 | - [ ] Add pointers in any related higher-level files, e.g., a new method for be referenced in the `html/methods.html` file 16 | - [ ] Add link to HTML documentation from help comments in function 17 | 18 | - [ ] **Tests** Create or update tests in the `tests` directory, especially for bug fixes or strongly encouraged for new code. 19 | 20 | - [ ] **Contents** If new functions were added at top level, go to `maintenance` and run `update_topcontents` to update the Contents.m file at the top level. 21 | 22 | - [ ] **Release Notes** Include release notes in this checklist in the pull request associated with this contribution. Include any significant bug fixes or additions. 23 | 24 | - [ ] **Contributors List** Update `CONTRIBUTORS.md` with your name and a brief description of the contributions. 25 | 26 | - [ ] **Pass All Tests** Confirm that all tests (including existing tests) pass in `tests` directory. 27 | 28 | - [ ] **Create Pull Request** At any point, create a work-in-progress pull request, referencing the issue number and with this checklist and `WIP` in the header. 29 | 30 | - [ ] **Finalize Pull Request** Once all of the items of this checklist in the pull request have been completed, remove `WIP` from the title of the merge requests and add the Comment, "Completed, ready to submit", and check this item as completed. 31 | 32 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | # Primary POC 2 | * Daniel M. Dunlavy, dmdunla@sandia.gov 3 | 4 | # Contributors 5 | * Evrim Acar - Original author 6 | * Daniel M. Dunlavy - Original author, tests 7 | * Tamara G. Kolda - Original author 8 | * Dianne P. O'Leary - ```cstep.m```, ```cvsrch.m``` 9 | 10 | # Other Contributions 11 | * This product includes software developed by the University of 12 | Chicago, as Operator of Argonne National Laboratory. (```cstep.m```, 13 | ```cvsrch.m```) 14 | 15 | 16 | -------------------------------------------------------------------------------- /COPYRIGHT.txt: -------------------------------------------------------------------------------- 1 | Poblano Toolbox for MATLAB 2 | 3 | Copyright 4 | 5 | Copyright 2009 National Technology & Engineering Solutions of Sandia, 6 | LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 7 | U.S. Government retains certain rights in this software. 8 | 9 | NOTICE: 10 | 11 | For five (5) years from 06/20/2018 the United States Government is 12 | granted for itself and others acting on its behalf a paid-up, 13 | nonexclusive, irrevocable worldwide license in this data to reproduce, 14 | prepare derivative works, and perform publicly and display publicly, 15 | by or on behalf of the Government. There is provision for the possible 16 | extension of the term of this license. Subsequent to that period or 17 | any extension granted, the United States Government is granted for 18 | itself and others acting on its behalf a paid-up, nonexclusive, 19 | irrevocable worldwide license in this data to reproduce, prepare 20 | derivative works, distribute copies to the public, perform publicly 21 | and display publicly, and to permit others to do so. The specific term 22 | of the license can be identified by inquiry made to National 23 | Technology and Engineering Solutions of Sandia, LLC or DOE. 24 | 25 | NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT 26 | OF ENERGY, NOR NATIONAL TECHNOLOGY AND ENGINEERING SOLUTIONS OF 27 | SANDIA, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, EXPRESS 28 | OR IMPLIED, OR ASSUMES ANY LEGAL RESPONSIBILITY FOR THE ACCURACY, 29 | COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, APPARATUS, PRODUCT, OR 30 | PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE 31 | PRIVATELY OWNED RIGHTS. 32 | 33 | Any licensee of this software has the obligation and responsibility to 34 | abide by the applicable export control laws, regulations, and general 35 | prohibitions relating to the export of technical data. Failure to 36 | obtain an export control license or other authority from the 37 | Government may result in criminal liability under U.S. laws. 38 | 39 | 40 | -------------------------------------------------------------------------------- /Contents.m: -------------------------------------------------------------------------------- 1 | % Poblano Toolbox (Sandia National Labs) 2 | % Version 1.2 26-APR-2019 3 | % 4 | % Poblano Toolbox for large-scale nonlinear optimization. 5 | % 6 | % cstep - More-Thuente line search step from MINPACK. 7 | % cvsrch - More-Thuente line search from MINPACK. 8 | % example1 - Example function for Poblano Toolbox routines. 9 | % example2 - Matrix factorization. 10 | % example2_extract - Matrix factorization example: extraction. 11 | % example2_init - Matrix factorization example: initialization. 12 | % gradientcheck - Finite difference verification of analytic gradients. 13 | % hessvec_fd - Hessian vector product finite difference approximation. 14 | % install_poblano - Script to install Poblano path. 15 | % lbfgs - Limited-memory BFGS minimization (vector-based). 16 | % ncg - Nonlinear conjugate gradient minimization. 17 | % poblano_linesearch - Line search methods in the Poblano Toolbox. 18 | % poblano_out - Standard output parameters for the Poblano Toolbox. 19 | % poblano_params - Standard input parameters for the Poblano Toolbox. 20 | % tn - Truncated Newton minimization. 21 | % 22 | %Poblano Toolbox for MATLAB 23 | % 24 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 25 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 26 | %U.S. Government retains certain rights in this software. 27 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | ## Installing Poblano Toolbox for MATLAB 2 | 3 | You can obtain a copy of the Poblano Toolbox for MATLAB as follows: 4 | * Clone or download the `master` branch of the git repository at https://github.com/sandialabs/poblano_toolbox 5 | * Download a released version at https://github.com/sandialabs/poblano_toolbox/releases 6 | 7 | Once you have a version of the Poblano Toolbox for MATLAB: 8 | * Start MATLAB. 9 | * Within MATLAB, change to the `poblano_toolbox` directory from above and execute the following commands: 10 | 11 | addpath(pwd) %<-- Add the Poblano toolbox to the MATLAB path 12 | savepath %<-- Save for future MATLAB sessions 13 | or enter the following command: 14 | 15 | install_poblano 16 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2009 National Technology & Engineering Solutions of Sandia, 2 | LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 3 | U.S. Government retains certain rights in this software. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are 7 | met: 8 | 9 | 1. Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in the 14 | documentation and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Poblano Toolbox for MATLAB 2 | 3 | ``` 4 | Copyright 2009 National Technology & Engineering Solutions of Sandia, 5 | LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 6 | U.S. Government retains certain rights in this software. 7 | ``` 8 | 9 | Poblano is a Matlab toolbox of large-scale algorithms for 10 | unconstrained nonlinear optimization problems. The algorithms in 11 | Poblano require only first-order derivative information (e.g., 12 | gradients for scalar-valued objective functions), and therefore can 13 | scale to very large problems. The driving application for Poblano 14 | development has been tensor decompositions in data analysis 15 | applications (bibliometric analysis, social network analysis, 16 | chemometrics, etc.). 17 | 18 | Poblano optimizers find local minimizers of scalar-valued objective 19 | functions taking vector inputs. The gradient (i.e., first derivative) 20 | of the objective function is required for all Poblano optimizers. The 21 | optimizers converge to a stationary point where the gradient is 22 | approximately zero. A line search satisfying the strong Wolfe 23 | conditions is used to guarantee global convergence of the Poblano 24 | optimizers. The optimization methods in Poblano include several 25 | nonlinear conjugate gradient methods (Fletcher-Reeves, Polak-Ribiere, 26 | Hestenes-Stiefel), a limited-memory quasi-Newton method using BFGS 27 | updates to approximate second-order derivative information, and a 28 | truncated Newton method using finite differences to approximate 29 | second-order derivative information. 30 | -------------------------------------------------------------------------------- /cstep.m: -------------------------------------------------------------------------------- 1 | function [stx,fx,dx,sty,fy,dy,stp,fp,dp,brackt,info] ... 2 | = cstep(stx,fx,dx,sty,fy,dy,stp,fp,dp,brackt,stpmin,stpmax) 3 | %CSTEP More-Thuente line search step from MINPACK. 4 | % 5 | % Translation of minpack subroutine cstep 6 | % Dianne O'Leary July 1991 7 | % ********** 8 | % 9 | % Subroutine cstep 10 | % 11 | % The purpose of cstep is to compute a safeguarded step for 12 | % a linesearch and to update an interval of uncertainty for 13 | % a minimizer of the function. 14 | % 15 | % The parameter stx contains the step with the least function 16 | % value. The parameter stp contains the current step. It is 17 | % assumed that the derivative at stx is negative in the 18 | % direction of the step. If brackt is set true then a 19 | % minimizer has been bracketed in an interval of uncertainty 20 | % with endpoints stx and sty. 21 | % 22 | % The subroutine statement is 23 | % 24 | % subroutine cstep(stx,fx,dx,sty,fy,dy,stp,fp,dp,brackt, 25 | % stpmin,stpmax,info) 26 | % 27 | % where 28 | % 29 | % stx, fx, and dx are variables which specify the step, 30 | % the function, and the derivative at the best step obtained 31 | % so far. The derivative must be negative in the direction 32 | % of the step, that is, dx and stp-stx must have opposite 33 | % signs. On output these parameters are updated appropriately. 34 | % 35 | % sty, fy, and dy are variables which specify the step, 36 | % the function, and the derivative at the other endpoint of 37 | % the interval of uncertainty. On output these parameters are 38 | % updated appropriately. 39 | % 40 | % stp, fp, and dp are variables which specify the step, 41 | % the function, and the derivative at the current step. 42 | % If brackt is set true then on input stp must be 43 | % between stx and sty. On output stp is set to the new step. 44 | % 45 | % brackt is a logical variable which specifies if a minimizer 46 | % has been bracketed. If the minimizer has not been bracketed 47 | % then on input brackt must be set false. If the minimizer 48 | % is bracketed then on output brackt is set true. 49 | % 50 | % stpmin and stpmax are input variables which specify lower 51 | % and upper bounds for the step. 52 | % 53 | % info is an integer output variable set as follows: 54 | % If info = 1,2,3,4,5, then the step has been computed 55 | % according to one of the five cases below. Otherwise 56 | % info = 0, and this indicates improper input parameters. 57 | % 58 | % Subprograms called 59 | % 60 | % FORTRAN-supplied ... abs,max,min,sqrt 61 | % ... dble 62 | % 63 | % Argonne National Laboratory. MINPACK Project. June 1983 64 | % Jorge J. More', David J. Thuente 65 | % 66 | % ********** 67 | 68 | 69 | % Redistribution and use in source and binary forms, with or 70 | % without modification, are permitted provided that the 71 | % following conditions are met: 72 | % 73 | % 1. Redistributions of source code must retain the above 74 | % copyright notice, this list of conditions and the following 75 | % disclaimer. 76 | % 77 | % 2. Redistributions in binary form must reproduce the above 78 | % copyright notice, this list of conditions and the following 79 | % disclaimer in the documentation and/or other materials 80 | % provided with the distribution. 81 | % 82 | % 3. The end-user documentation included with the 83 | % redistribution, if any, must include the following 84 | % acknowledgment: 85 | % 86 | % "This product includes software developed by the 87 | % University of Chicago, as Operator of Argonne National 88 | % Laboratory. 89 | % 90 | % Alternately, this acknowledgment may appear in the software 91 | % itself, if and wherever such third-party acknowledgments 92 | % normally appear. 93 | % 94 | % 4. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS" 95 | % WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT HOLDER, THE 96 | % UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND 97 | % THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR 98 | % IMPLIED, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES 99 | % OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE 100 | % OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY 101 | % OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR 102 | % USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF 103 | % THE SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) 104 | % DO NOT WARRANT THAT THE SOFTWARE WILL FUNCTION 105 | % UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL 106 | % BE CORRECTED. 107 | % 108 | % 5. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT 109 | % HOLDER, THE UNITED STATES, THE UNITED STATES DEPARTMENT OF 110 | % ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT, 111 | % INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF 112 | % ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED TO LOSS OF 113 | % PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER 114 | % SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT 115 | % (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE, 116 | % EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE 117 | % POSSIBILITY OF SUCH LOSS OR DAMAGES. 118 | % 119 | 120 | p66 = 0.66; 121 | info = 0; 122 | % 123 | % Check the input parameters for errors. 124 | % 125 | if ((brackt & (stp <= min(stx,sty) | ... 126 | stp >= max(stx,sty))) | ... 127 | dx*(stp-stx) >= 0.0 | stpmax < stpmin) 128 | return 129 | end 130 | % 131 | % Determine if the derivatives have opposite sign. 132 | % 133 | sgnd = dp*(dx/abs(dx)); 134 | % 135 | % First case. A higher function value. 136 | % The minimum is bracketed. If the cubic step is closer 137 | % to stx than the quadratic step, the cubic step is taken, 138 | % else the average of the cubic and quadratic steps is taken. 139 | % 140 | if (fp > fx) 141 | info = 1; 142 | bound = 1; 143 | theta = 3*(fx - fp)/(stp - stx) + dx + dp; 144 | s = norm([theta,dx,dp],inf); 145 | gamma = s*sqrt((theta/s)^2 - (dx/s)*(dp/s)); 146 | if (stp < stx) 147 | gamma = -gamma; 148 | end 149 | p = (gamma - dx) + theta; 150 | q = ((gamma - dx) + gamma) + dp; 151 | r = p/q; 152 | stpc = stx + r*(stp - stx); 153 | stpq = stx + ((dx/((fx-fp)/(stp-stx)+dx))/2)*(stp - stx); 154 | if (abs(stpc-stx) < abs(stpq-stx)) 155 | stpf = stpc; 156 | else 157 | stpf = stpc + (stpq - stpc)/2; 158 | end 159 | brackt = 1; 160 | % 161 | % Second case. A lower function value and derivatives of 162 | % opposite sign. The minimum is bracketed. If the cubic 163 | % step is closer to stx than the quadratic (secant) step, 164 | % the cubic step is taken, else the quadratic step is taken. 165 | % 166 | elseif (sgnd < 0.0) 167 | info = 2; 168 | bound = 0; 169 | theta = 3*(fx - fp)/(stp - stx) + dx + dp; 170 | s = norm([theta,dx,dp],inf); 171 | gamma = s*sqrt((theta/s)^2 - (dx/s)*(dp/s)); 172 | if (stp > stx) 173 | gamma = -gamma; 174 | end 175 | p = (gamma - dp) + theta; 176 | q = ((gamma - dp) + gamma) + dx; 177 | r = p/q; 178 | stpc = stp + r*(stx - stp); 179 | stpq = stp + (dp/(dp-dx))*(stx - stp); 180 | if (abs(stpc-stp) > abs(stpq-stp)) 181 | stpf = stpc; 182 | else 183 | stpf = stpq; 184 | end 185 | brackt = 1; 186 | % 187 | % Third case. A lower function value, derivatives of the 188 | % same sign, and the magnitude of the derivative decreases. 189 | % The cubic step is only used if the cubic tends to infinity 190 | % in the direction of the step or if the minimum of the cubic 191 | % is beyond stp. Otherwise the cubic step is defined to be 192 | % either stpmin or stpmax. The quadratic (secant) step is also 193 | % computed and if the minimum is bracketed then the the step 194 | % closest to stx is taken, else the step farthest away is taken. 195 | % 196 | elseif (abs(dp) < abs(dx)) 197 | info = 3; 198 | bound = 1; 199 | theta = 3*(fx - fp)/(stp - stx) + dx + dp; 200 | s = norm([theta,dx,dp],inf); 201 | % 202 | % The case gamma = 0 only arises if the cubic does not tend 203 | % to infinity in the direction of the step. 204 | % 205 | gamma = s*sqrt(max(0.,(theta/s)^2 - (dx/s)*(dp/s))); 206 | if (stp > stx) 207 | gamma = -gamma; 208 | end 209 | p = (gamma - dp) + theta; 210 | q = (gamma + (dx - dp)) + gamma; 211 | r = p/q; 212 | if (r < 0.0 & gamma ~= 0.0) 213 | stpc = stp + r*(stx - stp); 214 | elseif (stp > stx) 215 | stpc = stpmax; 216 | else 217 | stpc = stpmin; 218 | end 219 | stpq = stp + (dp/(dp-dx))*(stx - stp); 220 | if (brackt) 221 | if (abs(stp-stpc) < abs(stp-stpq)) 222 | stpf = stpc; 223 | else 224 | stpf = stpq; 225 | end 226 | else 227 | if (abs(stp-stpc) > abs(stp-stpq)) 228 | stpf = stpc; 229 | else 230 | stpf = stpq; 231 | end 232 | end 233 | % 234 | % Fourth case. A lower function value, derivatives of the 235 | % same sign, and the magnitude of the derivative does 236 | % not decrease. If the minimum is not bracketed, the step 237 | % is either stpmin or stpmax, else the cubic step is taken. 238 | % 239 | else 240 | info = 4; 241 | bound = 0; 242 | if (brackt) 243 | theta = 3*(fp - fy)/(sty - stp) + dy + dp; 244 | s = norm([theta,dy,dp],inf); 245 | gamma = s*sqrt((theta/s)^2 - (dy/s)*(dp/s)); 246 | if (stp > sty) 247 | gamma = -gamma; 248 | end 249 | p = (gamma - dp) + theta; 250 | q = ((gamma - dp) + gamma) + dy; 251 | r = p/q; 252 | stpc = stp + r*(sty - stp); 253 | stpf = stpc; 254 | elseif (stp > stx) 255 | stpf = stpmax; 256 | else 257 | stpf = stpmin; 258 | end 259 | end 260 | % 261 | % Update the interval of uncertainty. This update does not 262 | % depend on the new step or the case analysis above. 263 | % 264 | if (fp > fx) 265 | sty = stp; 266 | fy = fp; 267 | dy = dp; 268 | else 269 | if (sgnd < 0.0) 270 | sty = stx; 271 | fy = fx; 272 | dy = dx; 273 | end 274 | stx = stp; 275 | fx = fp; 276 | dx = dp; 277 | end 278 | % 279 | % Compute the new step and safeguard it. 280 | % 281 | stpf = min(stpmax,stpf); 282 | stpf = max(stpmin,stpf); 283 | stp = stpf; 284 | if (brackt & bound) 285 | if (sty > stx) 286 | stp = min(stx+p66*(sty-stx),stp); 287 | else 288 | stp = max(stx+p66*(sty-stx),stp); 289 | end 290 | end 291 | return 292 | % 293 | % Last card of subroutine cstep. 294 | % 295 | 296 | 297 | 298 | 299 | -------------------------------------------------------------------------------- /cvsrch.m: -------------------------------------------------------------------------------- 1 | function [x,f,g,stp,info,nfev] = cvsrch(fcn,x,f,g,stp,s,params) 2 | %CVSRCH More-Thuente line search from MINPACK. 3 | % 4 | % Re-factor of API for use in the Poblano Toolbox, 5 | % Daniel M. Dunlavy, August 2008, March 2009 6 | % 7 | % Translation of minpack subroutine cvsrch 8 | % Dianne O'Leary July 1991 9 | % ********** 10 | % 11 | % Subroutine cvsrch 12 | % 13 | % The purpose of cvsrch is to find a step which satisfies 14 | % a sufficient decrease condition and a curvature condition. 15 | % The user must provide a subroutine which calculates the 16 | % function and the gradient. 17 | % 18 | % At each stage the subroutine updates an interval of 19 | % uncertainty with endpoints stx and sty. The interval of 20 | % uncertainty is initially chosen so that it contains a 21 | % minimizer of the modified function 22 | % 23 | % f(x+stp*s) - f(x) - ftol*stp*(gradf(x)'s). 24 | % 25 | % If a step is obtained for which the modified function 26 | % has a nonpositive function value and nonnegative derivative, 27 | % then the interval of uncertainty is chosen so that it 28 | % contains a minimizer of f(x+stp*s). 29 | % 30 | % The algorithm is designed to find a step which satisfies 31 | % the sufficient decrease condition 32 | % 33 | % f(x+stp*s) <= f(x) + ftol*stp*(gradf(x)'s), 34 | % 35 | % and the curvature condition 36 | % 37 | % abs(gradf(x+stp*s)'s)) <= gtol*abs(gradf(x)'s). 38 | % 39 | % If ftol is less than gtol and if, for example, the function 40 | % is bounded below, then there is always a step which satisfies 41 | % both conditions. If no step can be found which satisfies both 42 | % conditions, then the algorithm usually stops when rounding 43 | % errors prevent further progress. In this case stp only 44 | % satisfies the sufficient decrease condition. 45 | % 46 | % The subroutine statement is 47 | % 48 | % subroutine cvsrch(fcn,n,x,f,g,s,stp,ftol,gtol,xtol, 49 | % stpmin,stpmax,maxfev,info,nfev,wa) 50 | % where 51 | % 52 | % fcn is the name of the user-supplied subroutine which 53 | % calculates the function and the gradient. fcn must 54 | % be declared in an external statement in the user 55 | % calling program, and should be written as follows. 56 | % 57 | % subroutine fcn(n,x,f,g) 58 | % integer n 59 | % f 60 | % x(n),g(n) 61 | % ---------- 62 | % Calculate the function at x and 63 | % return this value in the variable f. 64 | % Calculate the gradient at x and 65 | % return this vector in g. 66 | % ---------- 67 | % return 68 | % end 69 | % 70 | % n is a positive integer input variable set to the number 71 | % of variables. 72 | % 73 | % x is an array of length n. On input it must contain the 74 | % base point for the line search. On output it contains 75 | % x + stp*s. 76 | % 77 | % f is a variable. On input it must contain the value of f 78 | % at x. On output it contains the value of f at x + stp*s. 79 | % 80 | % g is an array of length n. On input it must contain the 81 | % gradient of f at x. On output it contains the gradient 82 | % of f at x + stp*s. 83 | % 84 | % s is an input array of length n which specifies the 85 | % search direction. 86 | % 87 | % stp is a nonnegative variable. On input stp contains an 88 | % initial estimate of a satisfactory step. On output 89 | % stp contains the final estimate. 90 | % 91 | % ftol and gtol are nonnegative input variables. Termination 92 | % occurs when the sufficient decrease condition and the 93 | % directional derivative condition are satisfied. 94 | % 95 | % xtol is a nonnegative input variable. Termination occurs 96 | % when the relative width of the interval of uncertainty 97 | % is at most xtol. 98 | % 99 | % stpmin and stpmax are nonnegative input variables which 100 | % specify lower and upper bounds for the step. 101 | % 102 | % maxfev is a positive integer input variable. Termination 103 | % occurs when the number of calls to fcn is at least 104 | % maxfev by the end of an iteration. 105 | % 106 | % info is an integer output variable set as follows: 107 | % 108 | % info = 0 Improper input parameters. 109 | % 110 | % info = 1 The sufficient decrease condition and the 111 | % directional derivative condition hold. 112 | % 113 | % info = 2 Relative width of the interval of uncertainty 114 | % is at most xtol. 115 | % 116 | % info = 3 Number of calls to fcn has reached maxfev. 117 | % 118 | % info = 4 The step is at the lower bound stpmin. 119 | % 120 | % info = 5 The step is at the upper bound stpmax. 121 | % 122 | % info = 6 Rounding errors prevent further progress. 123 | % There may not be a step which satisfies the 124 | % sufficient decrease and curvature conditions. 125 | % Tolerances may be too small. 126 | % 127 | % nfev is an integer output variable set to the number of 128 | % calls to fcn. 129 | % 130 | % wa is a work array of length n. 131 | % 132 | % Subprograms called 133 | % 134 | % user-supplied......fcn 135 | % 136 | % MINPACK-supplied...cstep 137 | % 138 | % FORTRAN-supplied...abs,max,min 139 | % 140 | % Argonne National Laboratory. MINPACK Project. June 1983 141 | % Jorge J. More', David J. Thuente 142 | % 143 | % ********** 144 | 145 | % Redistribution and use in source and binary forms, with or 146 | % without modification, are permitted provided that the 147 | % following conditions are met: 148 | % 149 | % 1. Redistributions of source code must retain the above 150 | % copyright notice, this list of conditions and the following 151 | % disclaimer. 152 | % 153 | % 2. Redistributions in binary form must reproduce the above 154 | % copyright notice, this list of conditions and the following 155 | % disclaimer in the documentation and/or other materials 156 | % provided with the distribution. 157 | % 158 | % 3. The end-user documentation included with the 159 | % redistribution, if any, must include the following 160 | % acknowledgment: 161 | % 162 | % "This product includes software developed by the 163 | % University of Chicago, as Operator of Argonne National 164 | % Laboratory. 165 | % 166 | % Alternately, this acknowledgment may appear in the software 167 | % itself, if and wherever such third-party acknowledgments 168 | % normally appear. 169 | % 170 | % 4. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS" 171 | % WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT HOLDER, THE 172 | % UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND 173 | % THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR 174 | % IMPLIED, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES 175 | % OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE 176 | % OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY 177 | % OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR 178 | % USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF 179 | % THE SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) 180 | % DO NOT WARRANT THAT THE SOFTWARE WILL FUNCTION 181 | % UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL 182 | % BE CORRECTED. 183 | % 184 | % 5. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT 185 | % HOLDER, THE UNITED STATES, THE UNITED STATES DEPARTMENT OF 186 | % ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT, 187 | % INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF 188 | % ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED TO LOSS OF 189 | % PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER 190 | % SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT 191 | % (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE, 192 | % EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE 193 | % POSSIBILITY OF SUCH LOSS OR DAMAGES. 194 | % 195 | 196 | % Additions from D. Dunlavy 197 | n = length(x); 198 | 199 | % Initialize params 200 | xtol = params.LineSearch_xtol; 201 | ftol = params.LineSearch_ftol; 202 | gtol = params.LineSearch_gtol; 203 | stpmin = params.LineSearch_stpmin; 204 | stpmax = params.LineSearch_stpmax; 205 | maxfev = params.LineSearch_maxfev; 206 | 207 | % move up in case of early termination (before this was initialized 208 | nfev = 0; 209 | 210 | % Start of D. O'Leary translation 211 | p5 = .5; 212 | p66 = .66; 213 | xtrapf = 4; 214 | info = 0; 215 | infoc = 1; 216 | % 217 | % Check the input parameters for errors. 218 | % 219 | if (n <= 0 | stp <= 0.0 | ftol < 0.0 | ... 220 | gtol < 0.0 | xtol < 0.0 | stpmin < 0.0 ... 221 | | stpmax < stpmin | maxfev <= 0) 222 | return 223 | end 224 | % 225 | % Compute the initial gradient in the search direction 226 | % and check that s is a descent direction. 227 | % 228 | dginit = g'*s; 229 | if (dginit >= 0.0) 230 | return 231 | end 232 | % 233 | % Initialize local variables. 234 | % 235 | brackt = 0; 236 | stage1 = 1; 237 | % moved up to initialize before any potential return 238 | % nfev = 0; 239 | finit = f; 240 | dgtest = ftol*dginit; 241 | width = stpmax - stpmin; 242 | width1 = 2*width; 243 | wa = x; 244 | % 245 | % The variables stx, fx, dgx contain the values of the step, 246 | % function, and directional derivative at the best step. 247 | % The variables sty, fy, dgy contain the value of the step, 248 | % function, and derivative at the other endpoint of 249 | % the interval of uncertainty. 250 | % The variables stp, f, dg contain the values of the step, 251 | % function, and derivative at the current step. 252 | % 253 | stx = 0.0; 254 | fx = finit; 255 | dgx = dginit; 256 | sty = 0.0; 257 | fy = finit; 258 | dgy = dginit; 259 | % 260 | % Start of iteration. 261 | % 262 | while (1) 263 | % 264 | % Set the minimum and maximum steps to correspond 265 | % to the present interval of uncertainty. 266 | % 267 | if (brackt) 268 | stmin = min(stx,sty); 269 | stmax = max(stx,sty); 270 | else 271 | stmin = stx; 272 | stmax = stp + xtrapf*(stp - stx); 273 | end 274 | % 275 | % Force the step to be within the bounds stpmax and stpmin. 276 | % 277 | stp = max(stp,stpmin); 278 | stp = min(stp,stpmax); 279 | % 280 | % If an unusual termination is to occur then let 281 | % stp be the lowest point obtained so far. 282 | % 283 | if ((brackt & (stp <= stmin | stp >= stmax)) ... 284 | | nfev >= maxfev-1 | infoc == 0 ... 285 | | (brackt & stmax-stmin <= xtol*stmax)) 286 | stp = stx; 287 | end 288 | % 289 | % Evaluate the function and gradient at stp 290 | % and compute the directional derivative. 291 | % 292 | x = wa + stp * s; 293 | [f,g] = feval(fcn,x); 294 | nfev = nfev + 1; 295 | dg = g' * s; 296 | ftest1 = finit + stp*dgtest; 297 | % 298 | % Test for convergence. 299 | % 300 | if ((brackt & (stp <= stmin | stp >= stmax)) | infoc == 0) 301 | info = 6; 302 | end 303 | if (stp == stpmax & f <= ftest1 & dg <= dgtest) 304 | info = 5; 305 | end 306 | if (stp == stpmin & (f > ftest1 | dg >= dgtest)) 307 | info = 4; 308 | end 309 | if (nfev >= maxfev) 310 | info = 3; 311 | end 312 | if (brackt & stmax-stmin <= xtol*stmax) 313 | info = 2; 314 | end 315 | if (f <= ftest1 & abs(dg) <= gtol*(-dginit)) 316 | info = 1; 317 | end 318 | % 319 | % Check for termination. 320 | % 321 | if (info ~= 0) 322 | return 323 | end 324 | % 325 | % In the first stage we seek a step for which the modified 326 | % function has a nonpositive value and nonnegative derivative. 327 | % 328 | if (stage1 & f <= ftest1 & dg >= min(ftol,gtol)*dginit) 329 | stage1 = 0; 330 | end 331 | % 332 | % A modified function is used to predict the step only if 333 | % we have not obtained a step for which the modified 334 | % function has a nonpositive function value and nonnegative 335 | % derivative, and if a lower function value has been 336 | % obtained but the decrease is not sufficient. 337 | % 338 | if (stage1 & f <= fx & f > ftest1) 339 | % 340 | % Define the modified function and derivative values. 341 | % 342 | fm = f - stp*dgtest; 343 | fxm = fx - stx*dgtest; 344 | fym = fy - sty*dgtest; 345 | dgm = dg - dgtest; 346 | dgxm = dgx - dgtest; 347 | dgym = dgy - dgtest; 348 | % 349 | % Call cstep to update the interval of uncertainty 350 | % and to compute the new step. 351 | % 352 | [stx,fxm,dgxm,sty,fym,dgym,stp,fm,dgm,brackt,infoc] ... 353 | = cstep(stx,fxm,dgxm,sty,fym,dgym,stp,fm,dgm, ... 354 | brackt,stmin,stmax); 355 | % 356 | % Reset the function and gradient values for f. 357 | % 358 | fx = fxm + stx*dgtest; 359 | fy = fym + sty*dgtest; 360 | dgx = dgxm + dgtest; 361 | dgy = dgym + dgtest; 362 | else 363 | % 364 | % Call cstep to update the interval of uncertainty 365 | % and to compute the new step. 366 | % 367 | [stx,fx,dgx,sty,fy,dgy,stp,f,dg,brackt,infoc] ... 368 | = cstep(stx,fx,dgx,sty,fy,dgy,stp,f,dg, ... 369 | brackt,stmin,stmax); 370 | end 371 | % 372 | % Force a sufficient decrease in the size of the 373 | % interval of uncertainty. 374 | % 375 | if (brackt) 376 | if (abs(sty-stx) >= p66*width1) 377 | stp = stx + p5*(sty - stx); 378 | end 379 | width1 = width; 380 | width = abs(sty-stx); 381 | end 382 | % 383 | % End of iteration. 384 | % 385 | end 386 | % 387 | % Last card of subroutine cvsrch. 388 | % 389 | 390 | 391 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Poblano Toolbox for MATLAB 2 | 3 | ## Documentation 4 | 5 | To generate the documentation in HTML that will appear in the MATLAB 6 | Dcoumentation, you can run the following MATALB command from within 7 | this directory: 8 | 9 | ``` 10 | opts.stylesheet = 'mxdom2simplehtml_poblano.xsl'; D = dir('*_doc.m'); for i = 1:length(D), publish(D(i).name, opts); end 11 | ``` 12 | 13 | All of the HTML files will be placed in the ```html``` subdirectory. 14 | 15 | The file ```html/helptoc.xml``` specifies the order of the 16 | documentation as it will appear in the MATLAB Documentation. -------------------------------------------------------------------------------- /docs/examples_doc.m: -------------------------------------------------------------------------------- 1 | %% Poblano Toolbox for MATLAB: Examples 2 | % This section describes the examples provided with Poblano. These examples 3 | % can be used as templates for other function/gradient M-files. 4 | %% 5 | % 6 | %
7 | %% Solving a Problem Using Poblano 8 | % To solve a problem using Poblano, the following steps are performed: 9 | % 10 | % * *Step 1: Create M-file for objective and gradient.* An M-file which 11 | % takes a vector as input and provides a scalar function value and gradient 12 | % vector (the same size as the input) must be provided to the Poblano 13 | % optimizers. 14 | % * *Step 2: Call the optimizer.* One of the Poblano optimizers is called, 15 | % taking an anonymous function handle to the function to be minimized, a 16 | % starting point, and an optional set of optimizer parameters as inputs. 17 | % 18 | % Poblano provides two example function/gradient M-files: 19 | % 20 | % * |example1|: simple multivariate function 21 | % * |example2|: more complicated function of a matrix variable 22 | %% 23 | % 24 | %
25 | %% Example 1: Multivariate Sum 26 | % 27 | % The following example is a simple multivariate function, $f_1 : R^N \to R$, 28 | % that can be minimized using Poblano: 29 | %% 30 | % 31 | % $$f_1(x) = \sum_{n=1}^N \sin(ax_i)$$ 32 | %% 33 | % 34 | % $$\left(\nabla f_1(x)\right)_i = a \cos(ax_i)\; , \quad i=1,\dots,N$ 35 | %% 36 | % 37 | % where $a$ is a scalar parameter. 38 | %% 39 | % 40 | % Listed below are the contents of the |example1.m| M-file 41 | % distributed with the Poblano code. This is an example of a self-contained 42 | % function requiring a vector of independent variables and an 43 | % _optional_ scalar input parameter for $a$. 44 | % 45 | % function [f,g]=example1(x,a) 46 | % if nargin < 2, a = 1; end 47 | % f = sum(sin(a*x)); 48 | % g = a*cos(a*x); 49 | %% 50 | % 51 | % The following presents a call to |ncg| optimizer using the default 52 | % parameters (see documentation for more 53 | % details) along with the information displayed and output return by 54 | % Poblano. By default, at each iteration Poblano displays the number of 55 | % function evaluations (|FuncEvals|), the function value at the current 56 | % iterate (|F(X)|), and the Euclidean norm of the scaled gradient at the 57 | % current iterate (|||||G(X)||||||/N|, where |N| is the size of |X|) for 58 | % each iteration. The output of Poblano optimizers is a Matlab structure 59 | % containing the solution. See the documentation for more details. 61 | out = ncg(@(x) example1(x,3), pi/4) 62 | %% 63 | % The problem dimension, $N$, of |example1| is determined by the size of the 64 | % initial guess. For example, to solve a problem with $N=10^6$, simply use 65 | % an initial guess vector of size $N=10^6$ when calling the Poblano optimizer: 66 | randn('state',0); 67 | x0 = randn(1e6,1); 68 | out = ncg(@(x) example1(x,3), x0, 'Display', 'final') 69 | %% Example 2: Matrix Decomposition 70 | % 71 | % The following example is a more complicated function involving matrix 72 | % variables. As the Poblano methods require scalar functions with vector 73 | % inputs, variable matrices must be reshaped into vectors first. The 74 | % problem in this example is to find a rank-$k$ decomposition, $UV^T$, of 75 | % a $m \times n$ matrix, $A$, which minimizes the Frobenius norm of the fit 76 | %% 77 | % 78 | % $$f_2 = \frac12\| A - UV^T \|_F^2$$ 79 | %% 80 | % 81 | % $$\nabla_U f_2 = -(A-UV^T)V$$ 82 | %% 83 | % 84 | % $$\nabla_V f_2 = -(A-UV^T)^T U$$ 85 | %% 86 | % 87 | % where $A \in R^{m \times n}$ is a matrix with rank $\geq k$, $U 88 | % \in R^{m \times k}$, and $V \in R^{n \times k}$. This problem can be 89 | % solved using Poblano by providing an M-file that computes the function 90 | % and gradient shown above but that takes $U$ and $V$ as input in 91 | % vectorized form. 92 | % 93 | % This problem can be solved using Poblano by providing an M-?le that 94 | % computes the function and gradient shown above but that takes U and V as 95 | % input in vectorized form. 96 | %% 97 | % 98 | % Listed below are the contents of the |example2.m| M-file 99 | % distributed with the Poblano code. Note that the input |Data| is 100 | % required and is a structure containing the matrix to be decomposed, $A$, 101 | % and the desired rank, $k$. This example also illustrates how the 102 | % vectorized form of the factor matrices, $U$ and $V$, are converted to 103 | % matrix form for the function and gradient computations. 104 | % 105 | % function [f,g]=example2(x,Data) 106 | % % Data setup 107 | % [m,n] = size(Data.A); 108 | % k = Data.rank; 109 | % U = reshape(x(1:m*k),m,k); 110 | % V = reshape(x(m*k+1:m*k+n*k),n,k); 111 | % 112 | % % Function value (residual) 113 | % AmUVt = Data.A-U*V'; 114 | % f = 0.5*norm(AmUVt,'fro')^2; 115 | % 116 | % % First derivatives computed in matrix form 117 | % g = zeros((m+n)*k,1); 118 | % g(1:m*k) = -reshape(AmUVt*V,m*k,1); 119 | % g(m*k+1:end) = -reshape(AmUVt'*U,n*k,1); 120 | %% 121 | % 122 | % Included with Poblano are two helper functions and which can be used to 123 | % generate problems instances along with starting points 124 | % (|example2_init.m|) and extract the factors $U$ and $V$ from a solution 125 | % vector (|example2_extract.m|). We show an example of their use below. 126 | randn('state',0); 127 | m = 4; n = 3; k = 2; 128 | [x0,Data] = example2_init(m,n,k) 129 | out = ncg(@(x) example2(x,Data), x0, 'RelFuncTol', 1e-16, 'StopTol', 1e-8, ... 130 | 'MaxFuncEvals',1000,'Display','final') 131 | %% 132 | % 133 | % Extracting the factors from the solution, we see that we have found a 134 | % solution, since the the Euclidean norm of the difference between the 135 | % matrix and the approximate solution is equal to the the $k+1$ singular 136 | % value of $A$ [1, Theorem 2.5.3]. 137 | [U,V] = example2_extract(4,3,2,out.X); 138 | norm_diff = norm(Data.A-U*V') 139 | sv = svd(Data.A); 140 | sv_k_plus_1 = sv(k+1) 141 | %% 142 | % 143 | %
144 | %% References 145 | % 146 | % [1] Golub, G. H. and Loan, C. F. V. (1996). _Matrix Computations_. 147 | % Johns Hopkins University Press. 148 | % 149 | -------------------------------------------------------------------------------- /docs/gradientcheck_doc.m: -------------------------------------------------------------------------------- 1 | %% Poblano Toolbox for MATLAB: Checking Gradient Calculations 2 | % Analytic gradients can be checked using finite difference approximations. 3 | % The Poblano function |gradientcheck| computes the gradient approximations 4 | % and compares the results to the analytic gradient using a user-supplied 5 | % objective function/gradient M-file. The user can choose one of several 6 | % difference formulas as well as the difference step used in the 7 | % computations. 8 | %% 9 | % 10 | %
11 | %% Difference Formulas 12 | % The difference formulas for approximating the gradients in Poblano are 13 | % listed below. For more details on the different formulas, see [1]. 14 | % 15 | % _Forward Differences_ 16 | % 17 | % $$\frac{\partial f}{\partial x_i}(x) \approx \frac{f(x+he_i)-f(x)}{h}$$ 18 | % 19 | % _Backward Differences_ 20 | % 21 | % $$\frac{\partial f}{\partial x_i}(x) \approx \frac{f(x)-f(x-he_i)}{h}$$ 22 | % 23 | % _Centered Differences_ 24 | % 25 | % $$\frac{\partial f}{\partial x_i}(x) \approx \frac{f(x+he_i)-f(x-he_i)}{2h}$$ 26 | % 27 | % Note that $e_i$ is a vector the same size as $x$ with a 1 in element $i$ 28 | % and zeros elsewhere. The type of finite differences to use is specified 29 | % using the |DifferenceType| input parameter, and the value of $h$ is 30 | % specified using the |DifferenceStep| input parameter. 31 | % 32 | % For a detailed discussion on the impact of the choice of $h$ on the 33 | % quality of the approximation, see [2]. 34 | % 35 | %% 36 | % 37 | %
38 | %% Gradient Check Input Parameters 39 | % 40 | % The input parameters available for the |gradientcheck| function 41 | % are presented below. 42 | % 43 | % DifferenceType Difference formula to use {'forward'} 44 | % 'forward' g_i = (f(x+he_i)-f(x)) / h 45 | % 'backward' g_i = (f(x)-f(x-he_i)) / h 46 | % 'centered' g_i = (f(x+he_i)-f(x-he_i)) / (2h) 47 | % 48 | % DifferenceStep Value of h in difference formulae {1e-8} 49 | %% 50 | % 51 | %
52 | %% Gradient Check Output Parameters 53 | % 54 | % The fields in the structure of output parameters generated by the 55 | % |gradientcheck| function are presented below. 56 | % 57 | % G Analytic gradient 58 | % 59 | % GFD FD approximation of gradient 60 | % 61 | % MaxDiff Maximum difference between G and GFD 62 | % 63 | % MaxDiffInd Index of maximum difference between G and GFD 64 | % 65 | % NormGradientDiffs 2-norm of G - GFD 66 | % 67 | % GradientDiffs G - GFD 68 | % 69 | % Params Parameters used to compute FD approximations 70 | %% 71 | % 72 | %
73 | %% Examples 74 | %% 75 | % We use |example1| (described in detail in the 76 | % section) to illustrate 77 | % how to use the |gradientcheck| function to check user-supplied 78 | % gradients. The user provides a function handle to the M-file containing 79 | % their function and gradient computations, a point at which to check the 80 | % gradients, and the type of difference formula to use. Below is an example 81 | % of running the gradient check using each of the difference formulas. 82 | %% 83 | outFD = gradientcheck(@(x) example1(x,3), pi./[4 5 6]','DifferenceType','forward') 84 | outBD = gradientcheck(@(x) example1(x,3), pi./[4 5 6]','DifferenceType','backward') 85 | outCD = gradientcheck(@(x) example1(x,3), pi./[4 5 6]','DifferenceType','centered') 86 | %% 87 | % 88 | % Note the different gradients produced using the various differencing 89 | % formulas: 90 | format long 91 | [outFD.G outFD.GFD outBD.GFD outCD.GFD] 92 | format short 93 | %% References 94 | % 95 | % [1] Nocedal, J. and Wright S. J. (1999). 96 | % _Numerical Optimization_. Springer. 97 | % 98 | % [2] Overton, M. L. (2001). 99 | % _Numerical Computing with IEEE Floating Point Arithmetic_. 100 | % Society for Industrial and Applied Mathematics. 101 | -------------------------------------------------------------------------------- /docs/helptoc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Poblano Toolbox 6 | 7 | Overview 8 | 9 | 10 | Parameters 11 | Input 12 | Output 13 | 14 | 15 | Methods 16 | Nonlinear Conjugate Gradient (ncg) 17 | Limited-Memory BFGS (lbfgs) 18 | Truncated Newton (tn) 19 | 20 | 21 | Examples 22 | 23 | 24 | Checking Gradient Calculations 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /docs/html/examples_doc_eq00543772986039519777.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq00543772986039519777.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq02586487823804834451.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq02586487823804834451.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq02619125812656254059.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq02619125812656254059.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq02739270504201626537.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq02739270504201626537.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq02978325375087395648.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq02978325375087395648.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq03421469680577370400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq03421469680577370400.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq03672095713503266041.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq03672095713503266041.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq05147331747641807187.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq05147331747641807187.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq05495199406804600545.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq05495199406804600545.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq05508344529756732484.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq05508344529756732484.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq06327244962903926757.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq06327244962903926757.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq07741076879175499892.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq07741076879175499892.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq11316155984473799093.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq11316155984473799093.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq11475201221685588019.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq11475201221685588019.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq11638118408029626293.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq11638118408029626293.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq12267211811494638899.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq12267211811494638899.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq12534387414789875675.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq12534387414789875675.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq13029888333516263543.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq13029888333516263543.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq15636846968047188835.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq15636846968047188835.png -------------------------------------------------------------------------------- /docs/html/examples_doc_eq17937321231062034804.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/examples_doc_eq17937321231062034804.png -------------------------------------------------------------------------------- /docs/html/gradientcheck_doc.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | Poblano Toolbox for MATLAB: Checking Gradient Calculations
Sandia National Laboratories

Poblano Toolbox for MATLAB: Checking Gradient Calculations

Analytic gradients can be checked using finite difference approximations. The Poblano function gradientcheck computes the gradient approximations and compares the results to the analytic gradient using a user-supplied objective function/gradient M-file. The user can choose one of several difference formulas as well as the difference step used in the computations.

Contents


Difference Formulas

The difference formulas for approximating the gradients in Poblano are listed below. For more details on the different formulas, see [1].

Forward Differences

$$\frac{\partial f}{\partial x_i}(x) \approx \frac{f(x+he_i)-f(x)}{h}$$

Backward Differences

$$\frac{\partial f}{\partial x_i}(x) \approx \frac{f(x)-f(x-he_i)}{h}$$

Centered Differences

$$\frac{\partial f}{\partial x_i}(x) \approx \frac{f(x+he_i)-f(x-he_i)}{2h}$$

Note that $e_i$ is a vector the same size as $x$ with a 1 in element $i$ and zeros elsewhere. The type of finite differences to use is specified using the DifferenceType input parameter, and the value of $h$ is specified using the DifferenceStep input parameter.

For a detailed discussion on the impact of the choice of $h$ on the quality of the approximation, see [2].


Gradient Check Input Parameters

The input parameters available for the gradientcheck function are presented below.

DifferenceType     Difference formula to use {'forward'}
 72 |      'forward'     g_i = (f(x+he_i)-f(x)) / h
 73 |      'backward'    g_i = (f(x)-f(x-he_i)) / h
 74 |      'centered'    g_i = (f(x+he_i)-f(x-he_i)) / (2h)
DifferenceStep     Value of h in difference formulae {1e-8}


Gradient Check Output Parameters

The fields in the structure of output parameters generated by the gradientcheck function are presented below.

G                  Analytic gradient
GFD                FD approximation of gradient
MaxDiff            Maximum difference between G and GFD
MaxDiffInd         Index of maximum difference between G and GFD
NormGradientDiffs  2-norm of G - GFD
GradientDiffs      G - GFD
Params             Parameters used to compute FD approximations


Examples

We use example1 (described in detail in the Poblano Examples section) to illustrate how to use the gradientcheck function to check user-supplied gradients. The user provides a function handle to the M-file containing their function and gradient computations, a point at which to check the gradients, and the type of difference formula to use. Below is an example of running the gradient check using each of the difference formulas.

outFD = gradientcheck(@(x) example1(x,3), pi./[4 5 6]','DifferenceType','forward')
 75 | outBD = gradientcheck(@(x) example1(x,3), pi./[4 5 6]','DifferenceType','backward')
 76 | outCD = gradientcheck(@(x) example1(x,3), pi./[4 5 6]','DifferenceType','centered')
 77 | 
 78 | outFD = 
 79 | 
 80 |   struct with fields:
 81 | 
 82 |                     G: [3×1 double]
 83 |                   GFD: [3×1 double]
 84 |               MaxDiff: 6.4662e-08
 85 |            MaxDiffInd: 1
 86 |     NormGradientDiffs: 8.4203e-08
 87 |         GradientDiffs: [3×1 double]
 88 |                Params: [1×1 struct]
 89 | 
 90 | 
 91 | outBD = 
 92 | 
 93 |   struct with fields:
 94 | 
 95 |                     G: [3×1 double]
 96 |                   GFD: [3×1 double]
 97 |               MaxDiff: -4.4409e-08
 98 |            MaxDiffInd: 3
 99 |     NormGradientDiffs: 5.2404e-08
100 |         GradientDiffs: [3×1 double]
101 |                Params: [1×1 struct]
102 | 
103 | 
104 | outCD = 
105 | 
106 |   struct with fields:
107 | 
108 |                     G: [3×1 double]
109 |                   GFD: [3×1 double]
110 |               MaxDiff: 2.0253e-08
111 |            MaxDiffInd: 1
112 |     NormGradientDiffs: 2.1927e-08
113 |         GradientDiffs: [3×1 double]
114 |                Params: [1×1 struct]
115 | 
116 | 

Note the different gradients produced using the various differencing formulas:

format long
117 | [outFD.G outFD.GFD outBD.GFD outCD.GFD]
118 | format short
119 | 
120 | ans =
121 | 
122 |   Columns 1 through 3
123 | 
124 |   -2.121320343559642  -2.121320408221550  -2.121320319403708
125 |   -0.927050983124842  -0.927051013732694  -0.927050969323773
126 |    0.000000000000000  -0.000000044408921   0.000000044408921
127 | 
128 |   Column 4
129 | 
130 |   -2.121320363812629
131 |   -0.927050991528233
132 |                    0
133 | 
134 | 

References

[1] Nocedal, J. and Wright S. J. (1999). Numerical Optimization. Springer.

[2] Overton, M. L. (2001). Numerical Computing with IEEE Floating Point Arithmetic. Society for Industrial and Applied Mathematics.

-------------------------------------------------------------------------------- /docs/html/gradientcheck_doc_eq00827914115064801059.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/gradientcheck_doc_eq00827914115064801059.png -------------------------------------------------------------------------------- /docs/html/gradientcheck_doc_eq05671228016298599287.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/gradientcheck_doc_eq05671228016298599287.png -------------------------------------------------------------------------------- /docs/html/gradientcheck_doc_eq08062630503172331818.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/gradientcheck_doc_eq08062630503172331818.png -------------------------------------------------------------------------------- /docs/html/gradientcheck_doc_eq12428413953531653171.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/gradientcheck_doc_eq12428413953531653171.png -------------------------------------------------------------------------------- /docs/html/gradientcheck_doc_eq13924783238156832738.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/gradientcheck_doc_eq13924783238156832738.png -------------------------------------------------------------------------------- /docs/html/gradientcheck_doc_eq15869567321110343067.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/gradientcheck_doc_eq15869567321110343067.png -------------------------------------------------------------------------------- /docs/html/gradientcheck_doc_eq16537821098686504121.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/gradientcheck_doc_eq16537821098686504121.png -------------------------------------------------------------------------------- /docs/html/images/banner-background.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/images/banner-background.jpg -------------------------------------------------------------------------------- /docs/html/images/logo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/images/logo.gif -------------------------------------------------------------------------------- /docs/html/lbfgs_doc_eq00345914025344442474.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/lbfgs_doc_eq00345914025344442474.png -------------------------------------------------------------------------------- /docs/html/lbfgs_doc_eq02499409918622348759.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/lbfgs_doc_eq02499409918622348759.png -------------------------------------------------------------------------------- /docs/html/lbfgs_doc_eq02757236760158446565.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/lbfgs_doc_eq02757236760158446565.png -------------------------------------------------------------------------------- /docs/html/lbfgs_doc_eq05671228016298599287.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/lbfgs_doc_eq05671228016298599287.png -------------------------------------------------------------------------------- /docs/html/lbfgs_doc_eq07165327077797634723.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/lbfgs_doc_eq07165327077797634723.png -------------------------------------------------------------------------------- /docs/html/lbfgs_doc_eq17452142193152507074.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/lbfgs_doc_eq17452142193152507074.png -------------------------------------------------------------------------------- /docs/html/methods.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Poblano Toolbox for MATLAB: Methods 6 | 7 | 8 | 9 | 10 |
11 | Sandia National Laboratories 12 |
13 | 14 |
15 |

Poblano Toolbox for MATLAB: Methods

16 | 17 | 22 | 23 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq00202142981986870057.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq00202142981986870057.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq00232388002207087152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq00232388002207087152.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq00345914025344442474.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq00345914025344442474.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq00906644648645161877.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq00906644648645161877.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq02535271716144487441.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq02535271716144487441.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq05857437662927211826.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq05857437662927211826.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq07165327077797634723.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq07165327077797634723.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq08047493418710975323.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq08047493418710975323.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq11960540003268942809.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq11960540003268942809.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq12955286862837271730.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq12955286862837271730.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq16009458126034669820.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq16009458126034669820.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq16519649717996589748.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq16519649717996589748.png -------------------------------------------------------------------------------- /docs/html/ncg_doc_eq17452142193152507074.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/ncg_doc_eq17452142193152507074.png -------------------------------------------------------------------------------- /docs/html/overview_doc_eq00884557479967417095.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/overview_doc_eq00884557479967417095.png -------------------------------------------------------------------------------- /docs/html/overview_doc_eq07500982203449410655.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/overview_doc_eq07500982203449410655.png -------------------------------------------------------------------------------- /docs/html/overview_doc_eq08097522164358817053.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/overview_doc_eq08097522164358817053.png -------------------------------------------------------------------------------- /docs/html/overview_doc_eq13721435717236182146.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/overview_doc_eq13721435717236182146.png -------------------------------------------------------------------------------- /docs/html/overview_doc_eq13754963546198947976.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/overview_doc_eq13754963546198947976.png -------------------------------------------------------------------------------- /docs/html/overview_doc_eq16485556778937288589.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/overview_doc_eq16485556778937288589.png -------------------------------------------------------------------------------- /docs/html/parameters.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Poblano Toolbox for MATLAB: Parameters 6 | 7 | 8 | 9 | 10 |
11 | Sandia National Laboratories 12 |
13 | 14 |
15 |

Poblano Toolbox for MATLAB: Parameters

16 | 17 |
    18 |
  • Input Parameters - 19 | Information regarding how to set the inputs to the optimization methods.
  • 20 |
  • Output Parameters - 21 | Information regarding what the optimization methods return.
  • 22 |
23 | 24 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /docs/html/parameters_output_doc.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | Poblano Toolbox for MATLAB: Optimization Output Parameters
Sandia National Laboratories

Poblano Toolbox for MATLAB: Optimization Output Parameters

Each of the optimization methods in Poblano outputs a single structure containing fields for the approximate solution, function and gradient values at the solution, and various information about the optimization run (e.g., number of function evaluations, etc.). The Poblano function poblano_out is used by the optimization methods to set the output parameters.

Contents


Output Parameters

Each of the optimization methods in Poblano outputs a single structure containing fields described below.

X                Final iterate
F                Function value at X
G                Gradient at X
Params           Input parameters used for the minimization method (as
 72 |                  parsed Matlab inputParser object)
FuncEvals        Number of function evaluations performed
Iters            Number of iterations performed (see individual minimization
 73 |                  routines for details on what each iteration consists of
ExitFlag         Termination flag, with one of the following values
 74 |                  0 : scaled gradient norm < StopTol input parameter)
 75 |                  1 : maximum number of iterations exceeded
 76 |                  2 : maximum number of function values exceeded
 77 |                  3 : relative change in function value < RelFuncTol input parameter
 78 |                  4 : NaNs found in F, G, or ||G||
ExitDescription  Text description of the termination flag


Optional Trace Output Parameters

Additional output parameters returned by the Poblano optimization methods are presented below. The histories (i.e., traces) of different variables and parameters at each iteration are returned as output parameters if the corresponding input parameters are set to true (see the Optimization Input Parameters documentation for more details on the input parameters).

TraceX           History of X (iterates)
TraceFunc        History of the function values of the iterates
TraceRelFunc     History of the relative difference between the function
 79 |                  values at the current and previous iterates
TraceGrad        History of the gradients of the iterates
TraceGradNorm    History of the norm of the gradients of the iterates
TraceFuncEvals   History of the number of function evaluations performed
 80 |                  at each iteration


Example Output

The following example shows the output produced when the default input parameters are used.

out = ncg(@(x) example1(x,3), pi/4)
 81 | 
 Iter  FuncEvals       F(X)          ||G(X)||/N        
 82 | ------ --------- ---------------- ----------------
 83 |      0         1       0.70710678       2.12132034
 84 |      1        14      -0.99998885       0.01416497
 85 |      2        16      -1.00000000       0.00000147
 86 | 
 87 | out = 
 88 | 
 89 |   struct with fields:
 90 | 
 91 |              Params: [1×1 inputParser]
 92 |            ExitFlag: 0
 93 |     ExitDescription: 'Successful termination based on StopTol'
 94 |                   X: 70.6858
 95 |                   F: -1.0000
 96 |                   G: -1.4734e-06
 97 |           FuncEvals: 16
 98 |               Iters: 2
 99 | 
100 | 

The following example presents an example where a method terminates before convergence (due to a limit on the number of iterations allowed).

out = ncg(@(x) example1(x,3), pi/4,'MaxIters',1)
101 | 
 Iter  FuncEvals       F(X)          ||G(X)||/N        
102 | ------ --------- ---------------- ----------------
103 |      0         1       0.70710678       2.12132034
104 |      1        14      -0.99998885       0.01416497
105 | 
106 | out = 
107 | 
108 |   struct with fields:
109 | 
110 |              Params: [1×1 inputParser]
111 |            ExitFlag: 1
112 |     ExitDescription: 'Maximum number of iterations exceeded'
113 |                   X: 70.6843
114 |                   F: -1.0000
115 |                   G: -0.0142
116 |           FuncEvals: 14
117 |               Iters: 1
118 | 
119 | 

The following shows the ability to save traces of the different information for each iteration.

out = ncg(@(x) example1(x,3), [1 2 3]','TraceX',true,'TraceFunc', true, ...
120 |     'TraceRelFunc',true,'TraceGrad',true,'TraceGradNorm',true,'TraceFuncEvals',true)
121 | 
 Iter  FuncEvals       F(X)          ||G(X)||/N        
122 | ------ --------- ---------------- ----------------
123 |      0         1       0.27382300       1.65292785
124 |      1         5      -2.65134210       0.79522946
125 |      2        11      -2.93709563       0.35196906
126 |      3        14      -2.99999975       0.00070154
127 |      4        16      -3.00000000       0.00000000
128 | 
129 | out = 
130 | 
131 |   struct with fields:
132 | 
133 |              Params: [1×1 inputParser]
134 |            ExitFlag: 0
135 |     ExitDescription: 'Successful termination based on StopTol'
136 |                   X: [3×1 double]
137 |                   F: -3
138 |                   G: [3×1 double]
139 |           FuncEvals: 16
140 |               Iters: 4
141 |              TraceX: [3×5 double]
142 |           TraceFunc: [0.2738 -2.6513 -2.9371 -3.0000 -3]
143 |        TraceRelFunc: [10.6827 0.1078 0.0214 8.2026e-08]
144 |           TraceGrad: [3×5 double]
145 |       TraceGradNorm: [4.9588 2.3857 1.0559 0.0021 2.5726e-09]
146 |      TraceFuncEvals: [1 4 6 3 2]
147 | 
148 | 

We can examine the final solution and its gradient (which list only their sizes when viewing the output structure):

X = out.X
149 | G = out.G
150 | 
151 | X =
152 | 
153 |     3.6652
154 |    -0.5236
155 |     5.7596
156 | 
157 | 
158 | G =
159 | 
160 |    1.0e-08 *
161 | 
162 |     0.0072
163 |    -0.2052
164 |    -0.1550
165 | 
166 | 

We can also see the values of X (current iterate) and its gradient G for each iteration (including iteration 0, which just computes the function and gradient values of the initial point):

out.TraceX
167 | out.TraceGrad
168 | 
169 | ans =
170 | 
171 |     1.0000    3.7424    3.7441    3.6652    3.6652
172 |     2.0000   -0.6598   -0.6106   -0.5238   -0.5236
173 |     3.0000    5.5239    5.7756    5.7594    5.7596
174 | 
175 | 
176 | ans =
177 | 
178 |    -2.9700    0.6886    0.7031    0.0001    0.0000
179 |     2.8805   -1.1918   -0.7745   -0.0017   -0.0000
180 |    -2.7334   -1.9486    0.1437   -0.0013   -0.0000
181 | 
182 | 
-------------------------------------------------------------------------------- /docs/html/poblano.css: -------------------------------------------------------------------------------- 1 | html,body,div,span,applet,object,iframe,h1,h2,h3,h4,h5,h6,p,blockquote,pre,a,abbr,acronym,address,big,cite,code,del,dfn,em,font,img,ins,kbd,q,s,samp,small,strike,strong,sub,sup,tt,var,b,u,i,center,dl,dt,dd,ol,ul,li,fieldset,form,label,legend,table,caption,tbody,tfoot,thead,tr,th,td{margin:0;padding:0;border:0;outline:0;font-size:100%;vertical-align:baseline;background:transparent}body{line-height:1}ol,ul{list-style:none}blockquote,q{quotes:none}blockquote:before,blockquote:after,q:before,q:after{content:'';content:none}:focus{outine:0}ins{text-decoration:none}del{text-decoration:line-through}table{border-collapse:collapse;border-spacing:0} 2 | 3 | html { min-height:100%; margin-bottom:1px; } 4 | html body { height:100%; margin:0px; font-family:Arial, Helvetica, sans-serif; font-size:10px; color:#000; line-height:140%; background:#fff none; overflow-y:scroll; } 5 | html body td { vertical-align:top; text-align:left; } 6 | 7 | h1 { padding:0px; margin:0px 0px 25px; font-family:Arial, Helvetica, sans-serif; font-size:1.5em; color:#d55000; line-height:100%; font-weight:normal; } 8 | h2 { padding:0px; margin:0px 0px 8px; font-family:Arial, Helvetica, sans-serif; font-size:1.2em; color:#000; font-weight:bold; line-height:140%; border-bottom:1px solid #d6d4d4; display:block; } 9 | h3 { padding:0px; margin:0px 0px 5px; font-family:Arial, Helvetica, sans-serif; font-size:1.1em; color:#000; font-weight:bold; line-height:140%; } 10 | 11 | a { color:#005fce; text-decoration:none; } 12 | a:hover { color:#005fce; text-decoration:underline; } 13 | a:visited { color:#004aa0; text-decoration:none; } 14 | 15 | a.name { color:default; text-decoration:none; } 16 | a.name:hover { color:default; text-decoration:none; } 17 | a.name:visited { color:default; text-decoration:none; } 18 | 19 | 20 | p { padding:0px; margin:0px 0px 20px; } 21 | img { padding:0px; margin:0px 0px 20px; border:none; } 22 | p img, pre img, tt img, li img, h1 img, h2 img { margin-bottom:0px; } 23 | 24 | ul { padding:0px; margin:0px 0px 20px 23px; list-style:square; } 25 | ul li { padding:0px; margin:0px 0px 7px 0px; } 26 | ul li ul { padding:5px 0px 0px; margin:0px 0px 7px 23px; } 27 | ul li ol li { list-style:decimal; } 28 | ol { padding:0px; margin:0px 0px 20px 0px; list-style:decimal; } 29 | ol li { padding:0px; margin:0px 0px 7px 23px; list-style-type:decimal; } 30 | ol li ol { padding:5px 0px 0px; margin:0px 0px 7px 0px; } 31 | ol li ol li { list-style-type:lower-alpha; } 32 | ol li ul { padding-top:7px; } 33 | ol li ul li { list-style:square; } 34 | 35 | .content { font-size:1.2em; line-height:140%; padding: 20px; } 36 | .banner{ background-color:#15243c; text-align:center;} 37 | 38 | pre, code { font-size:12px; } 39 | tt { font-size: 1.2em; } 40 | pre { margin:0px 0px 20px; } 41 | pre.codeinput { padding:10px; border:1px solid #d3d3d3; background:#f7f7f7; } 42 | pre.codeoutput { padding:10px 11px; margin:0px 0px 20px; color:#4c4c4c; } 43 | pre.error { color:red; } 44 | 45 | @media print { pre.codeinput, pre.codeoutput { word-wrap:break-word; width:100%; } } 46 | 47 | span.keyword { color:#0000FF } 48 | span.comment { color:#228B22 } 49 | span.string { color:#A020F0 } 50 | span.untermstring { color:#B20000 } 51 | span.syscmd { color:#B28C00 } 52 | 53 | .footer { width:auto; padding:10px 0px; margin:25px 0px 0px; border-top:1px dotted #878787; font-size:0.8em; line-height:140%; font-style:italic; color:#878787; text-align:left; float:none; } 54 | .footer p { margin:0px; } 55 | .footer a { color:#878787; } 56 | .footer a:hover { color:#878787; text-decoration:underline; } 57 | .footer a:visited { color:#878787; } 58 | 59 | table th { padding:7px 5px; text-align:left; vertical-align:middle; border: 1px solid #d6d4d4; font-weight:bold; } 60 | table td { padding:7px 5px; text-align:left; vertical-align:top; border:1px solid #d6d4d4; } 61 | 62 | .portrait { 63 | background-image: url('../img/portrait.jpg'); 64 | width: 200px; 65 | height: 200px; 66 | margin: 0 auto; 67 | border-radius: 50%; 68 | background-size: cover; 69 | -webkit-background-size: cover; 70 | -moz-background-size: cover; 71 | } 72 | 73 | .background { 74 | background-color:#004889; 75 | background-image: url(images/banner-background.jpg); 76 | background-repeat:no-repeat; 77 | } -------------------------------------------------------------------------------- /docs/html/tn_doc_eq00345914025344442474.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq00345914025344442474.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq03158747792916826732.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq03158747792916826732.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq06834187619862737782.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq06834187619862737782.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq07165327077797634723.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq07165327077797634723.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq08610859800392802260.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq08610859800392802260.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq11373214381793991308.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq11373214381793991308.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq13505831761284237672.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq13505831761284237672.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq13549337764080820674.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq13549337764080820674.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq15653174921336884406.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq15653174921336884406.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq17016985893546760461.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq17016985893546760461.png -------------------------------------------------------------------------------- /docs/html/tn_doc_eq17452142193152507074.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/docs/html/tn_doc_eq17452142193152507074.png -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Poblano Toolbox for MATLAB 6 | 7 | 8 | 9 | 10 | 11 | 12 |
13 | Sandia National Laboratories 14 |
15 | 16 |
17 |

Poblano Toolbox for MATLAB

18 |

The Poblano Toolbox for MATLAB enables large-scale optimization of 19 | scalar-valued functions using first-order derivative information only.

20 | 21 |

Functionality

22 | 23 |

The Poblano Toolbox provides the following capabilities for 24 | large-scale optimization. This documention is provided via the 25 | MATLAB help interface under "Supplemental Software".

26 | 27 |
    28 | 29 |
  • Overview - An overview of 30 | the entire toolbox.
  • 31 |
  • Parameters - Input and 32 | output parameters for the optimziation methods are described 33 | in detail here.
  • 34 |
  • Methods - The optimization 35 | methods available in Poblano are described here. Current 36 | methods include Nonlinear Conjugate Gradient, Limited Memory 37 | BFGS, and Truncated Newton.
  • 38 |
  • Examples - Several 39 | examples are provided to illustrate how Poblano methods can be 40 | used in practice.
  • 41 |
  • Checking Gradient 42 | Calculations - To validate gradient functions, a function 43 | is provided to check the analytic gradients provided by the 44 | user with finite difference approximations. 45 | 46 |
47 | 48 |

Download

49 | 50 |

To download the Poblano Toolbox for MATLAB, go to 51 | http://github.com/sandialabs/poblano_toolbox.

52 | 53 |

Legal Stuff

54 | 58 | 59 |

How to Contribute

60 | 61 |

This is an open-source project hosted on GITLAB at 62 | http://github.com/sandialabs/poblano_toolbox. 63 | Visit this website to submit bug reports and suggestions for 64 | improvement. 65 |

66 | 76 | 77 |

Contact

78 |

Please email dmdunla@sandia.gov 79 | with any questions about the toolbox that cannot be resolved via issue reporting. 80 | Stories of its usefulness are especially welcome. 81 | We will try to respond to every email may not always be successful due to the volume of emails. 82 | 83 |

86 | 87 | 88 | -------------------------------------------------------------------------------- /docs/lbfgs_doc.m: -------------------------------------------------------------------------------- 1 | %% Poblano Toolbox for MATLAB: Limited-Memory BFGS Optimization 2 | % Limited-memory quasi-Newton methods [2] are a class of methods that 3 | % compute and/or maintain simple, compact approximations of the Hessian 4 | % matrices of second derivatives, which are used determining search 5 | % directions. Poblano includes the limited-memory BFGS (L-BFGS) method, a 6 | % variant of these methods whose Hessian approximations are based on the 7 | % BFGS method (see [1] for more details). 8 | % 9 | % The Poblano function for the L-BFGS method is called |lbfgs|. 10 | %% 11 | % 12 | %
13 | %% Introduction 14 | % The general steps of L-BFGS methods are given below in high-level 15 | % pseudo-code [2]: 16 | %% 17 | % 18 | % 19 | % $$ 20 | % \begin{tabular}{ll} 21 | % \emph{1.} & Input: $x_0$, a starting point; $M > 0$, an integer\\ 22 | % \emph{2.} & Evaluate $f_0 = f(x_0)$, $g_0 = \nabla f(x_0)$\\ 23 | % \emph{3.} & Set $p_0 = -g_0$, $\gamma_0 = 1$, $i = 0$\\ 24 | % \emph{4.} & \textbf{while} $\|g_i\| > 0$\\ 25 | % \emph{5.} & \qquad Choose an initial Hessian approximation: $H_i^0 = \gamma_i I$\\ 26 | % \emph{6.} & \qquad Compute a step direction $p_i = -r$ using {\tt TwoLoopRecursion} method\\ 27 | % \emph{7.} & \qquad Compute a step length $\alpha_i$ and set $x_{i+1} = x_i + \alpha_i p_i$\\ 28 | % \emph{8.} & \qquad Set $g_i = \nabla f(x_{i+1})$\\ 29 | % \emph{9.} & \qquad \textbf{if} $i > M$\\ 30 | % \emph{10.} & \qquad \qquad Discard vectors $\left\{s_{i-m}, y_{i-m}\right\}$ from storage\\ 31 | % \emph{11.} & \qquad \textbf{end if}\\ 32 | % \emph{12.} & \qquad \qquad Set and store $s_i = x_{i+1}-x_i$ and $y_i = g_{i+1} - g_i$\\ 33 | % \emph{13.} & \qquad Set $i = i + 1$\\ 34 | % \emph{14.} & \textbf{end while}\\ 35 | % \emph{15.} & Output: $x_i \approx x^*$\\ 36 | % \end{tabular} 37 | % $$ 38 | % 39 | %% 40 | % *Computing the step direction* 41 | % 42 | % In Step 6 in the above method, the computation of the step direction is 43 | % performed using the following method (assume we are at iteration $i$) [2]: 44 | %% 45 | % 46 | % $$ 47 | % \begin{tabular}{ll} 48 | % \multicolumn{2}{l}{\tt TwoLoopRecursion}\\ 49 | % \emph{1.} & $q = g_i$\\ 50 | % \emph{2.} & \textbf{for} $k = i-1, i-2, \dots, i-m$\\ 51 | % \emph{3.} & \qquad $a_k = (s_k^Tq)/(y_k^Ts_k)$\\ 52 | % \emph{4.} & \qquad $q = q - a_k y_k$\\ 53 | % \emph{5.} & \textbf{end for}\\ 54 | % \emph{6.} & $r = H_i^0 q$\\ 55 | % \emph{7.} & \textbf{for} $k = i-m, i-m+1, \dots, i-1$\\ 56 | % \emph{8.} & \qquad $b = (y_k^Tr) / (y_k^Ts_k)$\\ 57 | % \emph{9.} & \qquad $r = r + (a_k - b)s_k$\\ 58 | % \emph{10.} & \textbf{end for}\\ 59 | % \emph{11.} & Output: $r = H_ig_i$\\ 60 | % \end{tabular} 61 | % $$ 62 | %% 63 | % 64 | %
65 | %% Method Specific Input Parameters 66 | % 67 | % The input parameters specific to the |lbfgs| method are presented below. 68 | % See the 69 | % documentation for more details on the Poblano parameters shared across 70 | % all methods. 71 | % 72 | % M Limited memory parameter {5} 73 | %% 74 | % 75 | %
76 | %% Default Input Parameters 77 | % The default input parameters are returned with the following call to 78 | % |lbfgs|: 79 | params = lbfgs('defaults') 80 | %% 81 | % 82 | % See the 83 | % documentationfor more details on the Poblano parameters shared across 84 | % all methods. 85 | %% 86 | % 87 | %
88 | %% Examples 89 | % Below are the results of using the |lbfgs| method in Poblano to solve 90 | % example problems solved using the |ncg| method in the documentation. Note that the 92 | % different methods lead to slightly different solutions and a different 93 | % number of function evaluations. 94 | %% 95 | % *Example 1* (from ) 96 | % 97 | % In this example, we have $x \in R^{10}$ and $a = 3$, and use a random 98 | % starting point. 99 | randn('state',0); 100 | x0 = randn(10,1) 101 | out = lbfgs(@(x) example1(x,3), x0) 102 | %% 103 | % *Example 2* (from ) 104 | % 105 | % In this example, we compute a rank-4 approximation to a $4 \times 4$ 106 | % Pascal matrix (generated using the Matlab function |pascal(4)|). The 107 | % starting point is random vector. Note that in the interest of space, 108 | % Poblano is set to display only the final iteration is this example. 109 | m = 4; n = 4; k = 4; 110 | Data.rank = k; 111 | Data.A = pascal(m); 112 | randn('state',0); 113 | x0 = randn((m+n)*k,1); 114 | out = lbfgs(@(x) example2(x,Data), x0, 'Display', 'final') 115 | %% 116 | % 117 | % As for the |ncg| method, the fact that 118 | % |out.ExitFlag| > 0 indicates that the method did not 119 | % converge to the specified tolerance (i.e., using the default |StopTol| 120 | % input parameter value of |1e-5|). Since the maximum number of function 121 | % evaluations was exceeded, we can increasing the number of maximum numbers 122 | % of function evaluations and iterations allowed, and the optimizer 123 | % converges to a solution within the specified tolerance. 124 | out = lbfgs(@(x) example2(x,Data), x0, 'MaxIters',1000, ... 125 | 'MaxFuncEvals',10000,'Display','final') 126 | %% 127 | % 128 | % Verifying the solution, we see that we find a matrix decomposition which 129 | % fits the matrix with very small relative error (given the stopping 130 | % tolerance of |1e-5| used by the optimizer). 131 | [U,V] = example2_extract(m,n,k,out.X); 132 | norm(Data.A-U*V')/norm(Data.A) 133 | %% 134 | % 135 | % For Example 2, we see that |lbfgs| requires slightly fewer 136 | % function evaluations than |ncg| to solve the problem (143 versus 137 | % 175). Performance of the different methods in Poblano is dependent on 138 | % both the method and the particular parameters chosen. Thus, it is 139 | % recommended that several test runs on smaller problems are performed 140 | % initially using the different methods to help decide which method and set 141 | % of parameters works best for a particular class of problems. 142 | %% 143 | % 144 | %
145 | %% References 146 | % 147 | % [1] Dennis, Jr., J. E. and Schnabel, R. B. (1996). _Numerical Methods 148 | % for Unconstrained Optimization and Nonlinear Equations_. SIAM. 149 | % 150 | % [2] Nocedal, J. and Wright S. J. (1999). 151 | % _Numerical Optimization_. Springer. 152 | 153 | -------------------------------------------------------------------------------- /docs/mxdom2simplehtml_poblano.xsl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 9 | 10 | ]> 11 | 16 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | This HTML was auto-generated from MATLAB code. 37 | To make changes, update the MATLAB code and republish this document. 38 | 39 | 40 | <xsl:value-of select="$title"/> 41 | 42 | 43 | MATLAB 44 | 45 | 46 | 47 | 48 | 49 | 50 | .m 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 |
62 | 63 | 64 | 65 | 66 | 67 | 68 |

69 | introduction 70 | 71 | 72 | 73 | /introduction 74 |
75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | h1 92 | h2 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 |
115 | 116 | 117 | 118 | 119 | 120 |
121 | 122 | 123 | 186 | 187 | 188 | 189 | 190 |
191 | Sandia National Laboratories 192 |
193 |
194 | 195 | 196 | 197 | 201 | 202 | 203 | 204 | 205 | 206 |

Contents

207 |
    208 | 209 | 210 |
  • #
  • 211 |
    212 |
    213 |
214 |
215 | 216 | 217 | 218 | 219 |

220 |
221 | 222 |
223 |
224 | 225 |
226 |
227 | 228 |
  • 229 |
    230 | 231 | 232 | 233 |
    234 |
    235 | 236 |
    237 |
    238 |
    239 |
    240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 |
    
    263 | 
    264 |
    265 | 266 | 267 | 268 | 269 |
    
    270 | 
    271 |
    272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 |
    280 |              
    281 |             
    282 |             
    283 |             
    284 |         
    285 |
    286 |
    287 |
    288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | width: 299 | 300 | ; 301 | 302 | 303 | height: 304 | ; 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | width: 318 | 319 | ; 320 | 321 | 322 | height: 323 | ; 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | ##### SOURCE BEGIN ##### 344 | 345 | ##### SOURCE END ##### 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 391 | 392 | 394 | 395 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 |
    406 | -------------------------------------------------------------------------------- /docs/ncg_doc.m: -------------------------------------------------------------------------------- 1 | %% Poblano Toolbox for MATLAB: Nonlinear Conjugate Gradient Optimization 2 | % Nonlinear conjugate gradient (NCG) methods [3] are used to solve 3 | % unconstrained nonlinear optimization problems. They are extensions of the 4 | % conjugate gradient iterative method for solving linear systems adapted to 5 | % solve unconstrained nonlinear optimization problems. 6 | % 7 | % The Poblano function for the nonlinear conjugate gradient methods is 8 | % called |ncg|. 9 | %% 10 | % 11 | %
    12 | %% Method Description 13 | % 14 | % The general steps of NCG methods are given below in high-level 15 | % pseudo-code: 16 | %% 17 | % 18 | % 19 | % $$ 20 | % \begin{tabular}{ll} 21 | % \emph{1.} & Input: $x_0$, a starting point\\ 22 | % \emph{2.} & Evaluate $f_0 = f(x_0), g_0 = \nabla f(x_0)$\\ 23 | % \emph{3.} & Set $p_0 = -g_0, i = 0$\\ 24 | % \emph{4.} & \textbf{while} $\|g_i\| > 0$\\ 25 | % \emph{5.} & \qquad Compute a step length $\alpha_i$ and set $x_{i+1} = x_i + \alpha_i p_i$\\ 26 | % \emph{6.} & \qquad Set $g_i = \nabla f(x_{i+1})$\\ 27 | % \emph{7.} & \qquad Compute $\beta_{i+1}$\\ 28 | % \emph{8.} & \qquad Set $p_{i+1} = -g_{i+1} + \beta_{i+1} p_i$\\ 29 | % \emph{9.} & \qquad Set $i = i + 1$\\ 30 | % \emph{10.} & \textbf{end while}\\ 31 | % \emph{11.} & Output: $x_i \approx x^*$\\ 32 | % \end{tabular} 33 | % $$ 34 | % 35 | %% 36 | % *Conjugate direction updates* 37 | % 38 | % The computation of $\beta_{i+1}$ in Step 7 above leads to different NCG 39 | % methods. The update methods for $\beta_{i+1}$ available in Poblano are 40 | % listed below. Note that the special case of $\beta_{i+1} = 0$ leads to 41 | % the steepest descent method [3], which is available by specifying this 42 | % update using the input parameter described in the section below. 44 | %% 45 | % 46 | % _Fletcher-Reeves_ [1]: 47 | % 48 | % $$\beta_{i+1} = \frac{g_{i+1}^T {-} g_{i+1}}{g_{i}^T g_{i}} $$ 49 | % 50 | % _Polak-Ribiere_ [4]: 51 | % 52 | % $$\beta_{i+1} = \frac{g_{i+1}^T (g_{i+1} {-} g_{i})}{g_{i}^T g_{i}} $$ 53 | % 54 | % _Hestenes-Stiefel_ [2]: 55 | % 56 | % $$\beta_{i+1} = \frac{g_{i+1}^T (g_{i+1} {-} g_{i})}{p_i^T (g_{i+1} {-} g_{i})} $$ 57 | % 58 | %% 59 | % *Negative update coefficients* 60 | % 61 | % In cases where the update coefficient $\beta_{i+1} < 0$, it is 62 | % set to $0$ to avoid directions that are not descent directions [3]. 63 | %% 64 | % *Restart procedures* 65 | % 66 | % The NCG iterations are restarted every _n_ iterations, where _n_ is 67 | % specified by user by setting the |RestartIters| parameter. 68 | % 69 | % Another restart modification available in |ncg| that was suggested by 70 | % Nocedal and Wright [3] is taking a step in the direction of steepest 71 | % descent when two consecutive gradients are far from orthogonal. 72 | % Specifically, a steepest descent step is taking when 73 | % 74 | % $$\frac{|g_{i+1}^T g_{i}|}{\|g_{i+1}\|} \geq \nu $$ 75 | % 76 | % where $\nu$ is specified by the user by setting the |RestartNWTol| 77 | % parameter. This modification is off by default, but can be 78 | % used by setting the |RestartNW| parameter to "true". 79 | %% 80 | % 81 | %
    82 | %% Method Specific Input Parameters 83 | % 84 | % The input parameters specific to the |ncg| method are presented below. 85 | % See the 86 | % documentation for more details on the Poblano parameters shared across 87 | % all methods. 88 | % 89 | % Update Conjugate direction update {'PR'} 90 | % 'FR' Fletcher-Reeves 91 | % 'PR' Polak-Ribiere 92 | % 'HS' Hestenes-Stiefel 93 | % 'SD' Steepest Decsent 94 | % 95 | % RestartIters Number of iterations to run before conjugate direction restart {20} 96 | % 97 | % RestartNW Flag to use restart heuristic of Nocedal and Wright {false} 98 | % 99 | % RestartNWTol Tolerance for Nocedal and Wright restart heuristic {0.1} 100 | %% 101 | % 102 | %
    103 | %% Default Input Parameters 104 | % The default input parameters are returned with the following call to 105 | % |ncg|: 106 | params = ncg('defaults') 107 | %% 108 | % 109 | % See the 110 | % documentation for more details on the Poblano parameters shared across 111 | % all methods. 112 | %% 113 | % 114 | %
    115 | %% Examples 116 | %% 117 | % *Example 1* (from ) 118 | % 119 | % In this example, we have $x \in R^{10}$ and $a = 3$, and use a random 120 | % starting point. 121 | randn('state',0); 122 | x0 = randn(10,1) 123 | out = ncg(@(x) example1(x,3), x0) 124 | %% 125 | % *Example 2* (from ) 126 | % 127 | % In this example, we compute a rank-4 approximation to a $4 \times 4$ 128 | % Pascal matrix (generated using the Matlab function |pascal(4)|). The 129 | % starting point is random vector. Note that in the interest of space, 130 | % Poblano is set to display only the final iteration is this example. 131 | m = 4; n = 4; k = 4; 132 | Data.rank = k; 133 | Data.A = pascal(m); 134 | randn('state',0); 135 | x0 = randn((m+n)*k,1); 136 | out = ncg(@(x) example2(x,Data), x0, 'Display', 'final') 137 | %% 138 | % 139 | % The fact that |out.ExitFlag| > 0 indicates that the method did not 140 | % converge to the specified tolerance (i.e., using the default |StopTol| 141 | % input parameter value of |1e-5|). This exit flag indicates that the 142 | % maximum number of function evaluations was exceeded. (See the 143 | % documentation 144 | % for more details.) Increasing the number of maximum numbers of function 145 | % evaluations and iterations allowed, the optimizer converges to a solution 146 | % within the specified tolerance. 147 | out = ncg(@(x) example2(x,Data), x0, 'MaxIters',1000, ... 148 | 'MaxFuncEvals',10000,'Display','final') 149 | %% 150 | % 151 | % Verifying the solution, we see that we find a matrix decomposition which 152 | % fits the matrix with very small relative error (given the stopping 153 | % tolerance of |1e-5| used by the optimizer). 154 | [U,V] = example2_extract(m,n,k,out.X); 155 | norm(Data.A-U*V')/norm(Data.A) 156 | %% 157 | % 158 | %
    159 | %% References 160 | % 161 | % [1] Fletcher, R. & Reeves, C.M. (1964). 162 | % Function minimization by conjugate gradients. 163 | % _The Computer Journal_, 7, 149-154. 164 | % 165 | % [2] Hestenes, M. R. and Stiefel, E. (1952). 166 | % Methods of conjugate gradients for solving linear systems. 167 | % _J. Res. Nat. Bur. Standards Sec. B._, 48, 409-436. 168 | % 169 | % [3] Nocedal, J. and Wright S. J. (1999). 170 | % _Numerical Optimization_. Springer. 171 | % 172 | % [4] Polak, E. and Ribiere, G. (1969). 173 | % Note sur la convergence de methods de directions conjugres. 174 | % _Revue Francaise Informat. Recherche Operationnelle_, 16, 35-43. 175 | -------------------------------------------------------------------------------- /docs/overview_doc.m: -------------------------------------------------------------------------------- 1 | %% Poblano Toolbox for MATLAB: Overview 2 | % Poblano is a toolbox of large-scale algorithms for nonlinear 3 | % optimization. The algorithms in Poblano require only first-order 4 | % derivative information (e.g., gradients for scalar-valued objective 5 | % functions). 6 | %% 7 | % 8 | %
    9 | %% Introduction 10 | % Poblano optimizers find local minimizers of scalar-valued objective 11 | % functions taking vector inputs. Specifically, the problems solved by 12 | % Poblano optimizers are of the following form: 13 | %% 14 | % 15 | % $$\min_{x}f(x), \quad \mbox{where} \quad f:R^n \rightarrow R$$ 16 | %% 17 | % 18 | % The gradient of the objective function, $\nabla f(x)$, is required 19 | % for all Poblano optimizers. The optimizers converge to a stationary point, 20 | % $x^*$, where 21 | %% 22 | % 23 | % $$\nabla f(x^*) \approx 0$$ 24 | %% 25 | % A line search satisfying the strong Wolfe conditions is used to 26 | % guarantee global convergence of the Poblano optimizers. 27 | %% 28 | % 29 | %
    30 | %% Optimization Methods 31 | % The following optimization methods are available in Poblano. 32 | % 33 | % *Nonlinear conjugate gradient method* (|ncg|) [4] 34 | % 35 | % * Uses Fletcher-Reeves, Polak-Ribiere, and Hestenes-Stiefel conjugate direction updates 36 | % * Restart strategies based on number of iterations or orthogonality of gradients across iterations 37 | % * Steepest descent method is a special case 38 | % 39 | % *Limited-memory BFGS method* (|lbfgs|) [4] 40 | % 41 | % * Uses a two-loop recursion for approximate Hessian-gradient products 42 | % 43 | % *Truncated Newton method* (|tn|) [1] 44 | % 45 | % * Uses finite differencing for approximate Hessian-vector products 46 | % 47 | %% 48 | % 49 | %
    50 | %% Globalization Strategies 51 | % 52 | % *Line search methods* 53 | % 54 | % * More-Thuente cubic interpolation line search (|cvsrch|) [3] 55 | %% 56 | % 57 | %
    58 | %% Optimization Input Parameters 59 | % 60 | % Input parameters are passed to the different optimization methods using 61 | % Matlab |inputParser| objects. Some parameters are shared across all 62 | % methods and others are specific to a particular method. Below are 63 | % descriptions of the shared input parameters and examples of how to set 64 | % and use these parameters in the optimization methods. The Poblano 65 | % function |poblano_params| is used by the optimization methods to set the 66 | % input parameters. See the documentation for more details. 68 | %% 69 | % 70 | %
    71 | %% Optimization Output Parameters 72 | % 73 | % Each of the optimization methods in Poblano outputs a single structure 74 | % containing fields for the approximate solution, function and gradient 75 | % values at the solution, and various information about the optimization 76 | % run (e.g., number of function evaluations, etc.). The Poblano function 77 | % |poblano_out| is used by the optimization methods to set the output 78 | % parameters. See the documentation for more details. 80 | %% 81 | % 82 | %
    83 | 84 | %% Checking Gradient Calculations 85 | % 86 | % Analytic gradients can be checked using finite difference approximations. 87 | % The Poblano function |gradientcheck| computes the gradient approximations 88 | % and compares the results to the analytic gradient using a user-supplied 89 | % objective function/gradient M-file. The user can choose one of several 90 | % difference formulas as well as the difference step used in the 91 | % computations. See the documentation for more details. 93 | %% 94 | % 95 | %
    96 | %% Examples 97 | % 98 | % Poblano provides two example function/gradient M-files: 99 | % 100 | % * |example1|: simple multivariate function 101 | % * |example2|: more complicated function of a matrix variable 102 | % 103 | % See the documentation 104 | % for more details. 105 | %% 106 | % 107 | %
    108 | %% Calling a Poblano Optimizer 109 | % All Poblano methods are called using the name of the method along with 110 | % two required arguments and one or more optional arguments. The required 111 | % arguments are 1) a handle to the function being minimized, and 2) the 112 | % initial guess of the solution (as a scalar or column vector). For 113 | % example, the following is a call to the |ncg| method to minimize the 114 | % |example1| function distributed with Poblano starting with an initial 115 | % guess of $x = \pi/4$ and using the default |ncg| parameters. 116 | ncg(@example1, pi/4); 117 | %% 118 | % Parameterize functions can be optimized using Poblano as well. For such 119 | % functions, the function handle can be used to specify the function 120 | % parameters. For example, Poblano's |example1| function takes an optional 121 | % scalar parameter as follows. 122 | ncg(@(x) example1(x,3), pi/4); 123 | %% 124 | % Functions taking vectors as inputs can be optimized using Poblano as 125 | % well. For functions which can take input vectors of arbitrary sizes 126 | % (e.g., Matlab functions such as |sin|, |fft|, etc.), the 127 | % size of the initial guess (as a scalar or column vector) determines the 128 | % size of the problem to be solved. For example, Poblano's 129 | % \texttt{example1} function can take as input a vector (in this case a 130 | % vector in $R^3$) as follows. 131 | ncg(@(x) example1(x,3), [pi/5 pi/4 pi/3]'); 132 | %% 133 | % The optional arguments are input parameters specifying how the 134 | % optimization method is to be run. See the documentation for details about the input parameters.) 136 | %% 137 | % 138 | %
    139 | %% Acknowledgments 140 | % 141 | % * Dianne O'Leary of the University of Maryland, College Park, provided 142 | % the Matlab translation of the MINPACK implementation of the More-Thuente 143 | % line search. 144 | % * This product includes software developed by the University of Chicago, 145 | % as Operator of Argonne National Laboratory. 146 | % 147 | %% 148 | % 149 | %
    150 | %% References 151 | % 152 | % [1] Dembo, R.S. and and Steihaug, T. (1983). Truncated-Newton algorithms 153 | % for large-scale unconstrained minimization. _Mathematical Programming_, 154 | % 26, 190-212. 155 | % 156 | % [2] Golub, G. H. and Loan, C. F. V. (1996). _Matrix Computations_. 157 | % Johns Hopkins University Press. 158 | % 159 | % [3] More, J. J. and Thuente, D. J. (1994). Line search algorithms with 160 | % guaranteed sufficient decrease. _ACM Trans. Math. Softw._, 20, 286-307. 161 | % 162 | % [4] Nocedal, J. and Wright S. J. (1999). 163 | % _Numerical Optimization_. Springer. 164 | % 165 | -------------------------------------------------------------------------------- /docs/parameters_input_doc.m: -------------------------------------------------------------------------------- 1 | %% Poblano Toolbox for MATLAB: Optimization Input Parameters 2 | % Input parameters are passed to the different optimization methods using 3 | % Matlab |inputParser| objects. Some parameters are shared across all 4 | % methods and others are specific to a particular method. Below are 5 | % descriptions of the shared input parameters and examples of how to set 6 | % and use these parameters in the optimization methods. The Poblano 7 | % function |poblano_params| is used by the optimization methods to set the 8 | % input parameters. 9 | %% 10 | % 11 | %
    12 | %% Parameters Shared Across All Methods 13 | % 14 | %% 15 | % *Display Parameters* 16 | % 17 | % The following parameters control the information that is displayed during 18 | % a run of a Poblano optimizer. 19 | % 20 | % Display Controls amount of printed output {'iter'} 21 | % 'iter' Display information every iteration 22 | % 'final' Display information only after final iteration 23 | % 'off' Display no information 24 | % 25 | % DisplayIters Number of iterations performed before displaying 26 | % printed out when Display parameter is set to 27 | % 'iter' {1} 28 | % 29 | % Example using default value of |'iter'|: 30 | ncg(@example1, pi/4); 31 | %% 32 | % The per iteration information displayed contains the iteration number 33 | % (|Iter|), total number of function evaluations performed thus far 34 | % (|FuncEvals|), the function value at the current iterate 35 | % (|F(X)|), and the norm of the gradient at the current iterate 36 | % scaled by the problem size (|||||G(X)||||||/N|). After the final 37 | % iteration is performed, the total iteration and function evaluations, 38 | % along with the function value and scaled gradient norm at the solution 39 | % found is displayed. Below is an example of the information displayed 40 | % using the default parameters. 41 | %% 42 | % *Stopping Criteria Parameters* 43 | % 44 | % The following parameters control the stopping criteria of the 45 | % optimization methods. 46 | % 47 | % MaxIters Maximum number of iterations allowed {1000} 48 | % 49 | % MaxFuncEvals Maximum number of function evaluations allowed {10000} 50 | % 51 | % StopTol Gradient norm stopping tolerance, i.e., the 52 | % method stops when the norm of the gradient is less 53 | % than StopTol times the number of variables {1e-5} 54 | % 55 | % RelFuncTol Relative function value change stopping tolerance, 56 | % i.e., the method stops when the relative change 57 | % of the function value from one iteration to the 58 | % next is less than RelFuncTol {1e-6} 59 | % 60 | %% 61 | % *Trace Parameters* 62 | % 63 | % The following parameters control the information that is saved and output 64 | % for each iteration. 65 | % 66 | % TraceX Flag to save a history of X (iterates) {false} 67 | % 68 | % TraceFunc Flag to save a history of the function values 69 | % of the iterates {false} 70 | % 71 | % TraceRelFunc Flag to save a history of the relative difference 72 | % between the function values at the current and 73 | % previous iterates {false} 74 | % 75 | % TraceGrad Flag to save a history of the gradients of the 76 | % iterates {false} 77 | % 78 | % TraceGradNorm Flag to save a history of the norm of the 79 | % gradients of the iterates {false} 80 | % 81 | % TraceFuncEvals Flag to save a history of the number of function 82 | % evaluations performed at each iteration {false} 83 | %% 84 | % *Line Search Parameters* 85 | % 86 | % The following parameters control the behavior of the line search method 87 | % used in the optimization methods. 88 | % 89 | % LineSearch_xtol Stopping tolerance for minimum change input 90 | % variable {1e-15} 91 | % 92 | % LineSearch_ftol Stopping tolerance for sufficient decrease 93 | % condition {1e-4} 94 | % 95 | % LineSearch_gtol Stopping tolerance for directional derivative 96 | % condition {1e-2} 97 | % 98 | % LineSearch_stpmin Minimum step to take {1e-15} 99 | % 100 | % LineSearch_stpmax Maximum step to take {1e15} 101 | % 102 | % LineSearch_maxfev Maximum number of iterations {20} 103 | % 104 | % LineSearch_initialstep Initial step to be taken in the line search {1} 105 | % 106 | %% 107 | % 108 | %
    109 | %% Method-Specific Parameters 110 | % 111 | % *Nonlinear Conjugate Gradent Minimization* (|ncg|) 112 | % 113 | % Update Conjugate direction update {'PR'} 114 | % 'FR' Fletcher-Reeves 115 | % 'PR' Polak-Ribiere 116 | % 'HS' Hestenes-Stiefel 117 | % 'SD' Steepest Decsent 118 | % 119 | % RestartIters Number of iterations to run before conjugate 120 | % direction restart {20} 121 | % 122 | % RestartNW Flag to use restart heuristic of Nocedal and 123 | % Wright {false} 124 | % 125 | % RestartNWTol Tolerance for Nocedal and Wright restart 126 | % heuristic {0.1} 127 | % 128 | % 129 | % *Limited-memory BFGS Minimization* (|lbfgs|) 130 | % 131 | % M Limited memory parameter {5} 132 | % 133 | % 134 | % *Truncated Newton Minimization* (|tn|) 135 | % 136 | % CGSolver Matlab conjugate gradient (CG) method used to 137 | % solve for search direction {'symmlq'} 138 | % 'symmlq' Symmetric LQ method 139 | % 'pcg' Classical CG method 140 | % 141 | % CGIters Maximum number of CG iterations allowed {5} 142 | % 143 | % CGTolType CG stopping tolerance type used {'quadratic'} 144 | % 'quadratic' ||R|| / ||G|| < min(0.5,||G||) 145 | % 'superlinear' ||R|| / ||G|| < min(0.5,sqrt(||G||)) 146 | % 'fixed' ||R|| < CGTol 147 | % where R is the residual and G is the gradient 148 | % of FUN at X 149 | % 150 | % CGTol CG stopping tolerance when CGTolType is 151 | % 'fixed' {1e-6} 152 | % 153 | % HessVecFDStep Hessian vector product finite difference step {1e-10} 154 | % 0 Use iterate-based step: 1e-8*(1+||X||) 155 | % >0 Fixed value to use as the difference step 156 | %% 157 | % 158 | %
    159 | %% Default Parameters 160 | % The default input parameters are returned using the sole input of 161 | % |'defaults'| to one of the Poblano optimization methods: 162 | ncg_default_params = ncg('defaults') 163 | lbfgs_default_params = lbfgs('defaults') 164 | tn_default_params = tn('defaults') 165 | %% 166 | % 167 | %
    168 | %% Passing Parameters to Methods 169 | % 170 | % As mentioned above, input parameters are passed to the Poblano 171 | % optimization methods using Matlab |inputParser| objects. Below are 172 | % several examples of passing parameters to Poblano methods. For more 173 | % detailed description of |inputParser| objects, see the Matlab 174 | % documentation. 175 | %% 176 | % *Case 1: Using default input parameters.* 177 | % 178 | % To use the default methods, simply pass the function handle to the 179 | % function/gradient method and a starting point to the optimization method 180 | % (i.e., do not pass any input parameters into the method). 181 | ncg(@(x) example1(x,3), pi/4); 182 | %% 183 | % *Case 2: Passing parameter-value pairs into a method.* 184 | % 185 | % Instead of passing a structure of input parameters, pairs of parameters 186 | % and values may be passed as well. In this case, all parameters not 187 | % specified as input use their default values. Below is an example of 188 | % specifying a parameter in this way. 189 | ncg(@(x) example1(x,3), pi/4,'Display','final'); 190 | %% 191 | % *Case 3: Passing input parameters as fields in a structure.* 192 | % 193 | % Input parameters can be passed as fields in a structure. Note, though, 194 | % that since each optimization method uses method specific parameters, it 195 | % is suggested to start from a structure of the default parameters for a 196 | % particular method. Once the structure of default parameters has been 197 | % created, the individual parameters (i.e., fields in the structure) can be 198 | % changed. Below is an example of this for the |ncg| method. 199 | params = ncg('defaults'); 200 | params.MaxIters = 1; 201 | ncg(@(x) example1(x,3), pi/4, params); 202 | %% 203 | % *Case 4: Using parameters from one run in another run.* 204 | % 205 | % One of the outputs returned by the Poblano optimization methods is the 206 | % |inputParser| object of the input parameters used in that run. That 207 | % object contains a field called |Results|, which can be passed as the 208 | % input parameters to another run. For example, this is helpful when 209 | % running comparisons of methods where only one parameter is changed. Shown 210 | % below is such an example, where default parameters are used in one run, 211 | % and the same parameters with just a single change are used in another 212 | % run. 213 | out = ncg(@(x) example1(x,3), pi./[4 5 6]'); 214 | params = out.Params.Results; 215 | params.Display = 'final'; 216 | ncg(@(x) example1(x,3), pi./[4 5 6]',params); 217 | -------------------------------------------------------------------------------- /docs/parameters_output_doc.m: -------------------------------------------------------------------------------- 1 | %% Poblano Toolbox for MATLAB: Optimization Output Parameters 2 | % 3 | % Each of the optimization methods in Poblano outputs a single structure 4 | % containing fields for the approximate solution, function and gradient 5 | % values at the solution, and various information about the optimization 6 | % run (e.g., number of function evaluations, etc.). The Poblano function 7 | % |poblano_out| is used by the optimization methods to set the output 8 | % parameters. 9 | %% 10 | % 11 | %
    12 | % 13 | %% Output Parameters 14 | % 15 | % Each of the optimization methods in Poblano outputs a single structure 16 | % containing fields described below. 17 | % 18 | % X Final iterate 19 | % 20 | % F Function value at X 21 | % 22 | % G Gradient at X 23 | % 24 | % Params Input parameters used for the minimization method (as 25 | % parsed Matlab inputParser object) 26 | % 27 | % FuncEvals Number of function evaluations performed 28 | % 29 | % Iters Number of iterations performed (see individual minimization 30 | % routines for details on what each iteration consists of 31 | % 32 | % ExitFlag Termination flag, with one of the following values 33 | % 0 : scaled gradient norm < StopTol input parameter) 34 | % 1 : maximum number of iterations exceeded 35 | % 2 : maximum number of function values exceeded 36 | % 3 : relative change in function value < RelFuncTol input parameter 37 | % 4 : NaNs found in F, G, or ||G|| 38 | % 39 | % ExitDescription Text description of the termination flag 40 | %% 41 | % 42 | %
    43 | %% Optional Trace Output Parameters 44 | % Additional output parameters returned by the Poblano optimization methods 45 | % are presented below. The histories (i.e., traces) of different variables 46 | % and parameters at each iteration are returned as output parameters if the 47 | % corresponding input parameters are set to |true| (see the 48 | % documentation 49 | % for more details on the input parameters). 50 | % 51 | % TraceX History of X (iterates) 52 | % 53 | % TraceFunc History of the function values of the iterates 54 | % 55 | % TraceRelFunc History of the relative difference between the function 56 | % values at the current and previous iterates 57 | % 58 | % TraceGrad History of the gradients of the iterates 59 | % 60 | % TraceGradNorm History of the norm of the gradients of the iterates 61 | % 62 | % TraceFuncEvals History of the number of function evaluations performed 63 | % at each iteration 64 | %% 65 | % 66 | %
    67 | %% Example Output 68 | % 69 | %% 70 | % The following example shows the output produced when the default 71 | % input parameters are used. 72 | out = ncg(@(x) example1(x,3), pi/4) 73 | %% 74 | % The following example presents an example where a method terminates 75 | % before convergence (due to a limit on the number of iterations allowed). 76 | out = ncg(@(x) example1(x,3), pi/4,'MaxIters',1) 77 | %% 78 | % The following shows the ability to save traces of the different 79 | % information for each iteration. 80 | out = ncg(@(x) example1(x,3), [1 2 3]','TraceX',true,'TraceFunc', true, ... 81 | 'TraceRelFunc',true,'TraceGrad',true,'TraceGradNorm',true,'TraceFuncEvals',true) 82 | %% 83 | % We can examine the final solution and its gradient (which list only their 84 | % sizes when viewing the output structure): 85 | X = out.X 86 | G = out.G 87 | %% 88 | % We can also see the values of |X| (current iterate) and its 89 | % gradient |G| for each iteration (including iteration 0, which just 90 | % computes the function and gradient values of the initial point): 91 | out.TraceX 92 | out.TraceGrad 93 | -------------------------------------------------------------------------------- /docs/tn_doc.m: -------------------------------------------------------------------------------- 1 | %% Poblano Toolbox for MATLAB: Truncated Newton Optimization 2 | % Truncated Newton (TN) methods for minimization are Newton methods in 3 | % which the Newton direction is only approximated at each iteration (thus 4 | % reducing computation). Furthermore, the Poblano implementation of the 5 | % truncated Newton method does not require an explicit Hessian matrix in 6 | % the computation of the approximate Newton direction (thus reducing 7 | % storage requirements). 8 | % 9 | % The Poblano function for the truncated Newton method is called |tn|. 10 | %% 11 | % 12 | %
    13 | %% Introduction 14 | % 15 | % The general steps of the TN method in Poblano is given below in high-level 16 | % pseudo-code [1]: 17 | %% 18 | % 19 | % 20 | % $$ 21 | % \begin{tabular}{ll} 22 | % \emph{1.} & Input: $x_0$, a starting point\\ 23 | % \emph{2.} & Evaluate $f_0 = f(x_0), g_0 = \nabla f(x_0)$\\ 24 | % \emph{3.} & Set $i=0$\\ 25 | % \emph{4.} & \textbf{while} $\|g_i\| > 0$\\ 26 | % \emph{5.} & \qquad Compute the conjugate gradient stopping tolerance, $\eta_i$\\ 27 | % \emph{6.} & \qquad Compute $p_i$ by solving $\nabla^2f(x_i)p = -g_i$ using a linear conjugate gradient (CG) method\\ 28 | % \emph{7.} & \qquad Compute a step length $\alpha_i$ and set $x_{i+1} = x_i + \alpha_i p_i$\\ 29 | % \emph{8.} & \qquad Set $g_i = \nabla f(x_{i+1})$\\ 30 | % \emph{9.} & \qquad Set $i = i + 1$\\ 31 | % \emph{10.} & \textbf{end while}\\ 32 | % \emph{11.} & Output: $x_i \approx x^*$\\ 33 | % \end{tabular} 34 | % $$ 35 | %% 36 | % *Notes* 37 | % 38 | % In Step 5, the linear conjugate gradient (CG) method stopping tolerance 39 | % is allowed to change at each iteration. The input parameter 40 | % |CGTolType| determines how $\eta_i$ is computed. 41 | %% 42 | % 43 | % In Step 6, 44 | % 45 | % * One of Matlab's CG methods is used to solve for $p_i$: |symmlq| 46 | % (designed for symmetric indefinite systems) or |pcg| (the classical CG 47 | % method for symmetric positive definite systems). The input parameter 48 | % |CGSolver| controls the choice of CG method to use. 49 | % 50 | % * The maximum number of CG iterations is specified using the input 51 | % parameter |CGIters|. 52 | % 53 | % * The CG method stops when $\|-g_i - \nabla^2f(x_i)p_i\| \leq \eta_i\|g_i\|$ . 54 | % 55 | % * In the CG method, matrix-vector products involving $\nabla^2f(x_i)$ times 56 | % a vector $v$ are approximated using the following finite difference 57 | % approximation [1]: 58 | % $\nabla^2f(x_i)v \approx \frac{\nabla f(x_i + \sigma v) - \nabla f(x_i)}{\sigma}$ 59 | % 60 | % * The difference step, $\sigma$, is specified using the input parameter 61 | % |HessVecFDStep|. The computation of the finite difference approximation 62 | % is performed using the |hessvec_fd| provided with Poblano. 63 | %% 64 | % 65 | %
    66 | %% Method Specific Input Parameters 67 | % 68 | % The input parameters specific to the |tn| method are presented below. 69 | % See the 70 | % documentation for more details on the Poblano parameters shared across 71 | % all methods. 72 | % 73 | % CGSolver Matlab CG method to use {'symmlq'} 74 | % 'symmlq' symmlq (designed for symmetric indefinite systems) 75 | % 'pcg' pcg (designed for symmetric positive definite systems) 76 | % 77 | % CGIters Maximum number of conjugate gradient iterations allowed {5} 78 | % 79 | % CGTolType CG stopping tolerance type used {'quadratic'} 80 | % 'quadratic' ||R|| / ||G|| < min(0.5,||G||) 81 | % 'superlinear' ||R|| / ||G|| < min(0.5,sqrt(||G||)) 82 | % 'fixed' ||R|| < CGTol 83 | % where R is the residual and G is the gradient of FUN at X 84 | % 85 | % CGTol CG stopping tolerance when CGTolType is 'fixed' {1e-6} 86 | % 87 | % HessVecFDStep Hessian vector product finite difference step {1e-10} 88 | % 0 Use iterate-based step: 1e-8*(1+||X||) 89 | % >0 Fixed value to use at the difference step 90 | %% 91 | % 92 | %
    93 | %% Default Input Parameters 94 | % The default input parameters are returned with the following call to 95 | % |tn|: 96 | params = tn('defaults') 97 | %% 98 | % 99 | % See the 100 | % documentation for more details on the Poblano parameters shared across 101 | % all methods. 102 | %% 103 | % 104 | %
    105 | %% Examples 106 | % Below are the results of using the |tn| method in Poblano to solve 107 | % example problems solved using the |ncg| method in the and |lbfgs| method in the 109 | % documentation. 110 | %% 111 | % *Example 1* (from ) 112 | % 113 | % In this example, we have $x \in R^{10}$ and $a = 3$, and use a random 114 | % starting point. 115 | randn('state',0); 116 | x0 = randn(10,1) 117 | out = tn(@(x) example1(x,3), x0) 118 | %% 119 | % 120 | % Note that in this example the line search in |tn| method displays a 121 | % warning during iterations 1, 3 and 5, indicating that the norm of the 122 | % search direction is nearly 0. In those cases, the steepest descent 123 | % direction is used for the search direction during those iterations. 124 | %% 125 | % *Example 2* 126 | % 127 | % In this example, we compute a rank 2 approximation to a $4 \times 4$ 128 | % Pascal matrix (generated using the Matlab function |pascal(4)|). The 129 | % starting point is a random vector. Note that in the interest of space, 130 | % Poblano is set to display only the final iteration is this example. 131 | m = 4; n = 4; k = 4; 132 | Data.rank = k; 133 | Data.A = pascal(m); 134 | randn('state',0); 135 | x0 = randn((m+n)*k,1); 136 | out = tn(@(x) example2(x,Data), x0, 'Display', 'final') 137 | %% 138 | % 139 | % As for the |ncg| and |lbfgs| methods, the fact that 140 | % |out.ExitFlag| > 0 indicates that the method did not 141 | % converge to the specified tolerance (i.e., the default |StopTol| 142 | % input parameter value of |1e-5|). Since the maximum number of function 143 | % evaluations was exceeded, we can increasing the number of maximum numbers 144 | % of function evaluations and iterations allowed, and the optimizer 145 | % converges to a solution within the specified tolerance. 146 | out = tn(@(x) example2(x,Data), x0, 'MaxIters',1000, ... 147 | 'MaxFuncEvals',10000,'Display','final') 148 | %% 149 | % 150 | % Verifying the solution, we see that we find a matrix decomposition which 151 | % fits the matrix with very small relative error (given the stopping 152 | % tolerance of |1e-5| used by the optimizer). 153 | [U,V] = example2_extract(m,n,k,out.X); 154 | norm(Data.A-U*V')/norm(Data.A) 155 | %% 156 | % 157 | % Again, in Example 2, we see that |tn| exhibits different behavior from 158 | % that of the |ncg| and |lbfgs| methods. Thus, it is 159 | % recommended that several test runs on smaller problems are performed 160 | % initially using the different methods to help decide which method and set 161 | % of parameters works best for a particular class of problems. 162 | %% 163 | % 164 | %
    165 | %% References 166 | % 167 | % [1] Dembo, R.S. and and Steihaug, T. (1983). Truncated-Newton Algorithms for 168 | % Large-Scale Unconstrained Minimization., _Mathematical Programming_, 26, 190-212. 169 | % 170 | -------------------------------------------------------------------------------- /example1.m: -------------------------------------------------------------------------------- 1 | function [f,g]=example1(x,a) 2 | %EXAMPLE1 Example function for Poblano Toolbox routines. 3 | % 4 | % [F,G] = EXAMPLE1(X) computes F = sin(X(N)) + ... sin(X(N)) and G, the 5 | % gradient of F at X, where N is the number of variables in X. 6 | % 7 | % [F,G] = EXAMPLE1(X,A) computes F = sin(A*X(1)) + ... + sin(A*X(N)) and G, 8 | % the gradient of F at X. 9 | % 10 | %Poblano Toolbox for MATLAB 11 | % 12 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 13 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 14 | %U.S. Government retains certain rights in this software. 15 | 16 | if (nargin < 2), a = 1; end 17 | f = sum(sin(a*x)); 18 | g = a*cos(a*x); 19 | 20 | -------------------------------------------------------------------------------- /example2.m: -------------------------------------------------------------------------------- 1 | function [f,g] = example2(x,Data) 2 | %EXAMPLE2 Matrix factorization. 3 | % Compute Frobenius norm residual of an approximate 4 | % (rank-reduced) two-factor decomposition of a matrix: 5 | % 6 | % 1/2 * ( || A - U*V' ||_F )^2 7 | % 8 | % Derivatives are computed using the matrix form. 9 | % 10 | % Input 11 | % Data.A: matrix being approximated, A 12 | % Data.rank: rank of approximation factors 13 | % x: approximation encoded as follows: 14 | % U = reshape(x(1:m*k),m,k); 15 | % V = reshape(x(m*k+1:m*k+n*k),n,k); 16 | % 17 | % Output 18 | % f: function value (residual) 19 | % g: first derivatives of f w.r.t. x (i.e., U & V) 20 | % 21 | % Matrix dimensions 22 | % A is m x n 23 | % U is m x k 24 | % V is n x k 25 | % 26 | % Examples 27 | % % Defaults for example2_init: m = 10; n = 8; k = 6; 28 | % [x0,Data] = example2_init; 29 | % [f,g] = example2(x0,Data) 30 | % 31 | % % Larger problem 32 | % m = 100; n = 80; k = 10; 33 | % [x0,Data] = example2_init(m,n,k); 34 | % out = ncg(@(x) example2(x,Data), x0) 35 | % 36 | % % Increase amount of computation allowed 37 | % params = out.Params.Results; 38 | % params.MaxIters = 1000; 39 | % params.MaxFuncEvals = 2000; 40 | % out = ncg(@(x) example2(x,Data), x0, params) 41 | % 42 | %Poblano Toolbox for MATLAB 43 | % 44 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 45 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 46 | %U.S. Government retains certain rights in this software. 47 | 48 | % Notes: 49 | % - Derivatives are computed using matrix form. 50 | % 51 | 52 | 53 | %% Data setup 54 | % Data.A should have matrix being modeled 55 | [m,n] = size(Data.A); 56 | k = Data.rank; 57 | U = reshape(x(1:m*k),m,k); 58 | V = reshape(x(m*k+1:m*k+n*k),n,k); 59 | 60 | %% Function value (residual) 61 | AmUVt = Data.A-U*V'; 62 | f = 0.5*norm(AmUVt,'fro')^2; 63 | 64 | %% First derivatives computed in matrix form 65 | g = zeros((m+n)*k,1); 66 | g(1:m*k) = -reshape(AmUVt*V,m*k,1); 67 | g(m*k+1:end) = -reshape(AmUVt'*U,n*k,1); 68 | 69 | -------------------------------------------------------------------------------- /example2_extract.m: -------------------------------------------------------------------------------- 1 | function [U,V] = example2_extract(m,n,k,x) 2 | %EXAMPLE2_EXTRACT Matrix factorization example: extraction. 3 | % 4 | % Helper function to extract factors of an approximate 5 | % (rank-reduced) two-factor decomposition of a matrix: 6 | % 7 | % A \approx U*V' 8 | % 9 | % Input 10 | % m: number of rows of U to be approximated 11 | % n: number of rows of V to be approximated 12 | % k: rank of approximation factors 13 | % x: vector of variables in two-factor approximation 14 | % 15 | % Output 16 | % U: first factor 17 | % V: second factor 18 | % 19 | % Matrix dimensions 20 | % A is m x n 21 | % U is m x k 22 | % V is n x k 23 | % 24 | % Example 25 | % m = 3; n = 2; k = 2; 26 | % [x,Data] = example2_init(m,n,k); 27 | % [U,V] = example_extract(m,n,k,x) 28 | % norm(Data.A-U*V') 29 | % 30 | %Poblano Toolbox for MATLAB 31 | % 32 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 33 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 34 | %U.S. Government retains certain rights in this software. 35 | 36 | %% Perform check on sizes of inputs 37 | [x_m, x_n] = size(x); 38 | if ( (x_n ~= 1) || (x_m ~= (m+n)*k) ) 39 | error('Dimensions do not agree!'); 40 | end 41 | 42 | %% Extract approximation factors from x 43 | U = reshape(x(1:m*k),m,k); 44 | V = reshape(x(m*k+1:m*k+n*k),n,k); 45 | -------------------------------------------------------------------------------- /example2_init.m: -------------------------------------------------------------------------------- 1 | function [x0,Data] = example_init(m,n,k,A) 2 | %EXAMPLE2_INIT Matrix factorization example: initialization. 3 | % 4 | % Initialize data for use in computing an approximate 5 | % (rank-reduced) two-factor decomposition of a matrix: 6 | % 7 | % A \approx U*V' 8 | % 9 | % Input 10 | % m: number of rows of matrix to be approximated (10) 11 | % n: number of columns of matrix to be approximated (8) 12 | % k: rank of approximation factors (6) 13 | % A: matrix to be approximated (optional) 14 | % 15 | % Output 16 | % x0: vector of variables in two-factor approximation 17 | % Data: A: matrix to be approximated 18 | % rank: rank of approximation factors (== k) 19 | % 20 | % Matrix dimensions 21 | % Data.A is m x n 22 | % x0 is (m+n)*k x 1 23 | % 24 | % Example 25 | % m = 3; n = 4; k = 2; 26 | % [x0,Data] = example2_init(m,n,k) 27 | % 28 | %Poblano Toolbox for MATLAB 29 | % 30 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 31 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 32 | %U.S. Government retains certain rights in this software. 33 | 34 | % Notes: 35 | % - Ordering of vector of variables must be consistent across uses 36 | % 37 | 38 | %% Check parameters 39 | if (nargin < 3) 40 | k = 6; 41 | end 42 | if (nargin < 2) 43 | n = 8; 44 | end 45 | if (nargin < 1) 46 | m = 10; 47 | end 48 | 49 | %% Check to see if dimensions are valid 50 | if (k > min(m,n)) 51 | error('The value of k is too large for the values of m and n!'); 52 | end 53 | 54 | %% Data matrix and rank of approximation 55 | Data.rank = k; 56 | if (nargin < 4) 57 | Data.A = randn(m,n); 58 | elseif ( (m ~= size(A,1)) && (n ~= size(A,2)) ) 59 | error('A is not an m x n matrix'); 60 | else 61 | Data.A = A; 62 | end 63 | 64 | %% Initial two-factor approximation 65 | U0 = randn(m,k); 66 | V0 = randn(n,k); 67 | U0 = reshape(U0,m*k,1); 68 | V0 = reshape(V0,n*k,1); 69 | x0 = [U0;V0]; 70 | -------------------------------------------------------------------------------- /gradientcheck.m: -------------------------------------------------------------------------------- 1 | function out = gradientcheck(FUN,x0,varargin) 2 | %GRADIENTCHECK Finite difference verification of analytic gradients. 3 | % 4 | % OUT = GRADIENTCHECK(FUN,X0) computes the finite difference gradient 5 | % approximations to the gradient of FUN at X0. FUN is a handle for a 6 | % function that takes a single vector input and returns two arguments --- 7 | % the scalar function value and the vector-valued gradient. The output 8 | % contains the following informationfields: 9 | % 10 | % OUT.G : analytic gradient (G) 11 | % OUT.GFD : FD approximation of gradient (GFD) 12 | % OUT.MaxDiff : maximum difference between G and GFD 13 | % OUT.MaxDiffInd : index of maximum difference between G and GFD 14 | % OUT.NormGradientDiffs : 2-norm of G - GFD 15 | % OUT.GradientDiffs : G - GFD 16 | % OUT.Params : paramters used to compute FD approximations 17 | % 18 | % OUT = GRADIENTCHECK(FUN,X0,'param,value) specifies a 19 | % parameters and its value. Available parameters are as follows: 20 | % 21 | % DifferenceType - difference scheme to use {'forward'} 22 | % 'forward' : g_i = (f(x+h*e_i) - f(x))/h 23 | % 'backward' : g_i = (f(x) - f(x-h*e_i))/h 24 | % 'centered' : g_i = (f(x+h*e_i) - f(x-h*e_i))/(2h) 25 | % 26 | % DifferenceStep - value of h in difference formulae {1e-8} 27 | % 28 | % 29 | % EXAMPLE 30 | % 31 | % Suppose the function and gradient of the objective function are 32 | % specified in an mfile named mysin.m: 33 | % 34 | % function [f,g]=example1(x,a) 35 | % if nargin < 2, a = 1; end 36 | % f = sum(sin(a*x)); 37 | % g = a*cos(a*x); 38 | % 39 | % We can call the gradient checker (using its default 40 | % parameters) using the command: 41 | % 42 | % out = gradientcheck(@(x) example1(x,3), pi/4) 43 | % 44 | % To change a parameter, we can specify a param/value input pair 45 | % as follows: 46 | % 47 | % out = gradientcheck(@(x) example1(x,3), pi/4, 'DifferenceType', 'centered') 48 | % 49 | % Alternatively, we can use a structure to define the parameters: 50 | % 51 | % params.DifferenceStep = 1e-6; 52 | % out = gradientcheck(@(x) example1(x,3), pi/4, params) 53 | % 54 | %Poblano Toolbox for MATLAB 55 | % 56 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 57 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 58 | %U.S. Government retains certain rights in this software. 59 | 60 | %% Parse parameters 61 | 62 | % Create parser 63 | params = inputParser; 64 | 65 | % Set parameters for this method 66 | params.addParamValue('DifferenceType','forward', @(x) ismember(x,{'forward','backward','centered'})); 67 | params.addParamValue('DifferenceStep',1e-8, @(x) x >= 1e-16); 68 | 69 | % Parse input 70 | params.parse(varargin{:}); 71 | 72 | %% Compute finite difference approximation of gradient 73 | fdtype = params.Results.DifferenceType; 74 | h = params.Results.DifferenceStep; 75 | 76 | % setup variables 77 | N = length(x0); 78 | ei = zeros(size(x0)); 79 | g = zeros(N,1); 80 | if ~(strcmp(fdtype,'backward')), fxp = g; end 81 | if ~(strcmp(fdtype,'forward')), fxm = g; end 82 | 83 | % function value at x0 84 | [f,g] = feval(FUN,x0); 85 | 86 | for i = 1:N 87 | ei(i) = h; 88 | if i>1 89 | ei(i-1)=0; 90 | end 91 | if ~(strcmp(fdtype,'backward')), fxp(i) = feval(FUN,x0+ei); end 92 | if ~(strcmp(fdtype,'forward')), fxm(i) = feval(FUN,x0-ei); end 93 | end 94 | 95 | % Compute the FD approximations 96 | switch fdtype 97 | case 'forward' 98 | gfd = (fxp-f)/h; % foward difference 99 | case 'backward' 100 | gfd = (f-fxm)/h; % backward difference 101 | case 'centered' 102 | gfd = (fxp-fxm)/2/h; % central difference 103 | end 104 | g_gfd = g-gfd; 105 | 106 | %% Output 107 | 108 | % Analytic gradient and FD approximation 109 | out.G = g; 110 | out.GFD = gfd; 111 | 112 | % Maximum difference (in magnitude) 113 | [m,ind] = max(abs(g_gfd)); 114 | out.MaxDiff = g_gfd(ind); 115 | out.MaxDiffInd = ind; 116 | 117 | % Differences between gradient and finite difference gradient 118 | out.NormGradientDiffs = norm(g_gfd); 119 | out.GradientDiffs = g_gfd; 120 | 121 | % Parameters used to compute the differences 122 | out.Params = params.Results; 123 | -------------------------------------------------------------------------------- /hessvec_fd.m: -------------------------------------------------------------------------------- 1 | function Hv = hessvec_fd(v,FUN,x,gx,s) 2 | %HESSVEC_FD Hessian vector product finite difference approximation. 3 | % 4 | % HV = HESSVEC_FD(V,FUN,X) computes a forward finite difference 5 | % approximation of the Hessian vector product, H(X)*V, where V is given 6 | % and H(X) is the Hessian of the function FUN at the point X. The 7 | % approximation is given by 8 | % 9 | % G(X+S*V) - G(X) 10 | % H(X)*V = --------------- 11 | % S 12 | % 13 | % where G(X) is the gradient of the function FUN at the point X and 14 | % h is the difference step (default = 1e-8*(1+norm(X)). 15 | % 16 | % HV = HESSVEC(V,FUN,X,GX) uses GX for the value of G(X), in the 17 | % case that is has already been computed outside of this method. 18 | % 19 | % HV = HESSVEC(V,FUN,X,GX,S) uses S as the difference step. 20 | % 21 | % This method should not be called directly, but only by Poblano Toolbox 22 | % optimization algorithms. 23 | % 24 | % The number of calls to FUN is kept track of in the global variable 25 | % NFEV_HESSVEC_FD to facilitate using this method as a callback function 26 | % (e.g., from an iterative linear solver such as Matlab's SYMMLQ). 27 | % 28 | % See also TN. 29 | % 30 | %Poblano Toolbox for MATLAB 31 | % 32 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 33 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 34 | %U.S. Government retains certain rights in this software. 35 | 36 | %% Number of calls to FUN 37 | global nfev_hessvec_fd; 38 | 39 | %% Check input 40 | if nargin < 3 41 | error('HESSVEC_FD => at least two input arguments are required (V,FUN,X)'); 42 | end 43 | 44 | % Compute gradient if not given 45 | if nargin < 4 46 | [f,gx] = feval(FUN,x); 47 | nfev_hessvec_fd = nfev_hessvec_fd + 1; 48 | end 49 | 50 | % Compute a difference step if not given 51 | if nargin < 5 52 | s = 1e-8*(1+norm(x)); 53 | end 54 | 55 | % Compute the gradient at the new point 56 | [f,gxsv] = feval(FUN,x+s*v); 57 | nfev_hessvec_fd = nfev_hessvec_fd + 1; 58 | 59 | %% Hessian vector product approximation 60 | Hv = (gxsv-gx)/s; 61 | -------------------------------------------------------------------------------- /info.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | R2018a 7 | Poblano 8 | toolbox 9 | $toolbox/matlab/icons/unknownicon.gif 10 | docs 11 | 12 | 13 | 14 | 15 | doc poblano_toolbox/ 16 | $toolbox/matlab/icons/webicon.gif 17 | 18 | 19 | 20 | web http://software.sandia.gov/poblano 21 | $toolbox/matlab/icons/webicon.gif 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /install_poblano.m: -------------------------------------------------------------------------------- 1 | %INSTALL_POBLANO Script to install Poblano path. 2 | % 3 | %Poblano Toolbox for MATLAB 4 | % 5 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 6 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 7 | %U.S. Government retains certain rights in this software. 8 | 9 | addpath(pwd) 10 | savepath 11 | 12 | 13 | -------------------------------------------------------------------------------- /lbfgs.m: -------------------------------------------------------------------------------- 1 | function out = lbfgs(FUN,x0,varargin) 2 | %LBFGS Limited-memory BFGS minimization (vector-based). 3 | % 4 | % OUT = LBFGS(FUN,X0) minimizes FUN starting at the point X0 using L-BFGS. 5 | % FUN is a handle for a function that takes a single vector input and 6 | % returns two arguments --- the scalar function value and the 7 | % vector-valued gradient. See POBLANO_OUT for details of the output 8 | % parameters. 9 | % 10 | % OUT = LBFGS(FUN,X0,'param',value,...) specifies a parameters and its 11 | % value. See POBLANO_PARAMS for further details on standard parameters. 12 | % Additionally, LBFGS requires 13 | % 14 | % 'M' - Limited memory parameter {5}. 15 | % 16 | % PARAMS = LBFGS('defaults') returns a structure containing the 17 | % default parameters for the particular Poblano method. 18 | % 19 | % 20 | % Examples 21 | % 22 | % Suppose the function and gradient of the objective function are 23 | % specified in an mfile named example1.m: 24 | % 25 | % function [f,g]=example1(x,a) 26 | % if nargin < 2, a = 1; end 27 | % f = sin(a*x); 28 | % g = a*cos(a*x); 29 | % 30 | % We can call the optimization method (using its default 31 | % parameters) using the command: 32 | % 33 | % out = lbfgs(@(x) example1(x,3), pi/4); 34 | % 35 | % To change a parameter, we can specify a param/value input pair 36 | % as follows: 37 | % 38 | % out = lbfgs(@(x) example1(x,3), pi/4, 'Display', 'final'); 39 | % 40 | % Alternatively, we can use a structure to define the parameters: 41 | % 42 | % params.MaxIters = 2; 43 | % out = lbfgs(@(x) example1(x,3), pi/4, params); 44 | % 45 | % See also POBLANO_OUT, POBLANO_PARAMS, FUNCTION_HANDLE. 46 | % 47 | %Poblano Toolbox for MATLAB 48 | % 49 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 50 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 51 | %U.S. Government retains certain rights in this software. 52 | 53 | %% Parse parameters 54 | 55 | % Create parser 56 | params = inputParser; 57 | 58 | % Set Poblano parameters 59 | params = poblano_params(params); 60 | 61 | % Set parameters for this method 62 | params.addParamValue('M',5,@(x) x > 0); 63 | 64 | % Parse input 65 | params.parse(varargin{:}); 66 | 67 | %% Check input arguments 68 | if (nargin == 1) && isequal(FUN,'defaults') && (nargout == 1) 69 | out = params.Results; 70 | return; 71 | elseif (nargin < 2) 72 | error('Error: invalid input arguments'); 73 | end 74 | 75 | %% Initialize 76 | 77 | xk = x0; 78 | [fk,gk] = feval(FUN,xk); 79 | out = poblano_out(xk,fk,gk,1,params); 80 | 81 | %% Main loop 82 | while out.ExitFlag == -1 83 | 84 | if out.Iters == 0 85 | % Initialize quantities before first iteration 86 | pk = -gk; 87 | ak = 1.0; 88 | S = []; 89 | Y = []; 90 | rho = []; 91 | else 92 | % Precompute quantites used in this iteration 93 | sk = xk - xkold; 94 | yk = gk - gkold; 95 | skyk = yk'*sk; 96 | ykyk = yk'*yk; 97 | rhok = 1 / skyk; 98 | gamma = skyk/ykyk; 99 | 100 | % Use information from last M iterations only 101 | if out.Iters <= params.Results.M 102 | S = [sk S]; 103 | Y = [yk Y]; 104 | rho = [rhok rho]; 105 | else 106 | S = [sk S(:,1:end-1)]; 107 | Y = [yk Y(:,1:end-1)]; 108 | rho = [rhok rho(1:end-1)]; 109 | end 110 | 111 | % Adjust M to available number of iterations 112 | m = size(S,2); 113 | 114 | % L-BFGS two-loop recursion 115 | q = gk; 116 | for i = 1:m 117 | alpha(i) = rho(i)*S(:,i)'*q; 118 | q = q - alpha(i)*Y(:,i); 119 | end 120 | r = gamma*q; 121 | for i = m:-1:1 122 | beta = rho(i)*Y(:,i)'*r; 123 | r = r + (alpha(i) - beta)*S(:,i); 124 | end 125 | 126 | % r contains H_k * g_k (Hessian approximation at iteration k times 127 | % the gradient at iteration k 128 | pk = -r; 129 | end 130 | xkold = xk; 131 | gkold = gk; 132 | 133 | % Compute step length 134 | [xk,fk,gk,ak,lsinfo,nfev] = poblano_linesearch(FUN,xk,fk,gk,ak,pk,params.Results); 135 | if (lsinfo ~= 1) && strcmp(params.Results.Display, 'iter') 136 | fprintf(1,[mfilename,': line search warning = %d\n'],lsinfo); 137 | end 138 | 139 | % Update counts, check exit conditions, etc. 140 | out = poblano_out(xk,fk,gk,nfev,params,out); 141 | end 142 | 143 | -------------------------------------------------------------------------------- /ncg.m: -------------------------------------------------------------------------------- 1 | function out = ncg(FUN,x0,varargin) 2 | %NCG Nonlinear conjugate gradient minimization. 3 | % 4 | % OUT = NCG(FUN,X0) minimizes FUN starting at the point 5 | % X0 using nonlinear conjugate gradients. FUN is a handle for a 6 | % function that takes a single vector input and returns two arguments 7 | % --- the scalar function value and the vector-valued gradient. 8 | % See POBLANO_OUT for details of the output parameters. 9 | % 10 | % OUT = NCG(FUN,X0,'param',value,...) specifies a 11 | % parameters and its value. See POBLANO_PARAMS for further details on 12 | % standard parameters. Additionally, POBLANO_NCG requires 13 | % 14 | % 'Update' - conjugate direction update {'PR'} 15 | % 'FR' Fletcher-Reeves NCG 16 | % 'PR' Polak-Ribiere NCG 17 | % 'HS' Hestenes-Stiefel NCG 18 | % 'SD' Steepest Decsent 19 | % 20 | % 'RestartIters' - number of iterations to run before conjugate direction 21 | % restart {20} 22 | % 23 | % 'RestartNW' - flag to use restart heuristic of Nocedal and Wright {false} 24 | % 25 | % 'RestartNWTol' - tolerance for Nocedal and Wright restart heuristic {0.1} 26 | % 27 | % PARAMS = NCG('defaults') returns a structure containing the 28 | % default parameters for the particular Poblano method. 29 | % 30 | % 31 | % Examples 32 | % 33 | % Suppose the function and gradient of the objective function are 34 | % specified in an mfile named mysin.m: 35 | % 36 | % function [f,g]=example1(x,a) 37 | % if nargin < 2, a = 1; end 38 | % f = sin(a*x); 39 | % g = a*cos(a*x); 40 | % 41 | % We can call the optimization method (using its default 42 | % parameters) using the command: 43 | % 44 | % out = ncg(@(x) example1(x,3), pi/4); 45 | % 46 | % To change a parameter, we can specify a param/value input pair 47 | % as follows: 48 | % 49 | % out = ncg(@(x) example1(x,3), pi/4, 'Display', 'final'); 50 | % 51 | % Alternatively, we can use a structure to define the parameters: 52 | % 53 | % params.MaxIters = 2; 54 | % out = ncg(@(x) example1(x,3), pi/4, params); 55 | % 56 | % See also POBLANO_OUT, POBLANO_PARAMS, POBLANO_LINESEARCH, FUNCTION_HANDLE. 57 | % 58 | %Poblano Toolbox for MATLAB 59 | % 60 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 61 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 62 | %U.S. Government retains certain rights in this software. 63 | 64 | %% Parse parameters 65 | 66 | % Create parser 67 | params = inputParser; 68 | 69 | % Set Poblano parameters 70 | params = poblano_params(params); 71 | 72 | % Set parameters for this method 73 | params.addParamValue('RestartIters',20,@(x) x > 0); 74 | params.addParamValue('Update','PR',@(x) ismember(x,{'FR','PR','HS','SD'})); 75 | params.addParamValue('RestartNW',false,@islogical); 76 | params.addParamValue('RestartNWTol',0.1,@(x) x > 0); 77 | 78 | % Parse input 79 | params.parse(varargin{:}); 80 | 81 | %% Check input arguments 82 | if (nargin == 1) && isequal(FUN,'defaults') && (nargout == 1) 83 | out = params.Results; 84 | return; 85 | elseif (nargin < 2) 86 | error('Error: invalid input arguments'); 87 | end 88 | 89 | %% Initialize 90 | xk = x0; 91 | [fk,gk] = feval(FUN,xk); 92 | out = poblano_out(xk,fk,gk,1,params); 93 | 94 | %% Main loop 95 | while out.ExitFlag == -1 96 | 97 | if out.Iters == 0 98 | pk = -gk; 99 | ak = 1.0; 100 | gkTgk = gk'*gk; 101 | else 102 | % Compute next direction 103 | if mod(out.Iters,params.Results.RestartIters) == 0 104 | % restart to prevent stagnation 105 | bk = 0; 106 | pk = -gk; 107 | else 108 | % direction update 109 | switch (params.Results.Update) 110 | case 'FR' 111 | % Fletcher-Reeves 112 | gkTgk = gk'*gk; 113 | if gkTgkold > 0 114 | bk = gkTgk/gkTgkold; 115 | else 116 | fprintf(1,[mfilename,': warning: bk set to 0\n']); 117 | bk = 0; 118 | end 119 | case 'PR' 120 | % Polak-Ribiere 121 | gkTgk = gk'*gk; 122 | gkMgkold = gk-gkold; 123 | if gkTgkold > 0 124 | bk = (gk'*gkMgkold)/gkTgkold; 125 | else 126 | fprintf(1,[mfilename,': warning: bk set to 0\n']); 127 | bk = 0; 128 | end 129 | case 'HS' 130 | % Hestenes-Stiefel 131 | gkMgkold = gk-gkold; 132 | denom = pkold'*gkMgkold; 133 | if denom > 0 134 | bk = (gk'*gkMgkold)/denom; 135 | else 136 | fprintf(1,[mfilename,': warning: bk set to 0\n']); 137 | bk = 0; 138 | end 139 | case 'SD' 140 | % Steepest Descent 141 | bk = 0; 142 | otherwise 143 | error('Error: options.Update is not valid. Choices are {FR, PR, HS}'); 144 | end 145 | % do not allow negative conjugate direction weights 146 | if bk < 0 147 | bk = max(0,bk); 148 | end 149 | 150 | % restart method from Nocedal and Wright 151 | if params.Results.RestartNW 152 | v = params.Results.RestartNWTol; 153 | if ((gk'*gkold)/(gkTgkold^2) >= v) 154 | bk = 0; 155 | end 156 | end 157 | 158 | % new direction 159 | pk = -gk + bk*pkold; 160 | end 161 | end 162 | xkold = xk; 163 | fkold = fk; 164 | gkold = gk; 165 | pkold = pk; 166 | gkTgkold = gkTgk; 167 | 168 | % Compute step length 169 | [xk,fk,gk,ak,lsinfo,nfev] = poblano_linesearch(FUN,xk,fk,gk,ak,pk,params.Results); 170 | if (lsinfo ~= 1) && strcmp(params.Results.Display, 'iter') 171 | fprintf(1,[mfilename,': line search warning = %d\n'],lsinfo); 172 | end 173 | 174 | % Update counts, check exit conditions, etc. 175 | out = poblano_out(xk,fk,gk,nfev,params,out); 176 | end 177 | -------------------------------------------------------------------------------- /poblano_linesearch.m: -------------------------------------------------------------------------------- 1 | function [x,f,g,a,lsinfo,lsnfev] = poblano_linesearch(FUN,x0,f0,g0,a0,d0,params) 2 | %POBLANO_LINESEARCH Line search methods in the Poblano Toolbox. 3 | % 4 | % [X,F,G,A] = POBLANO_LINESEARCH(FUN,X0,F0,G0,A0,D0,PARAMS) performs a 5 | % line search to find a point X such that FUN(X0+A*D0) is minimized. The 6 | % method returns X = X0+A*D0 with the function (F) and gradient (G) at 7 | % that point, along with the step length (A) found by the line search. The 8 | % parameters used in the line search are specified in PARAMS (see 9 | % POBLANO_PARAMS for more details). 10 | % 11 | % If PARAMS.LineSearch_initialstep equals 0, the initial step used in the 12 | % line search is A0, which is 1 for the first iteration and the step 13 | % length computed in the previous iteration for all subsequent iterations 14 | % If PARAMS.LineSearch_initialstep > 0, the initial step is the value of 15 | % PARAMS.LineSearch_initialstep. 16 | % 17 | % [X,F,G,A,LSINFO] = POBLANO_LINESEARCH(FUN,...) also provides an exit 18 | % code from the line search method. See the particular line search method 19 | % for an explanation of the exit codes. 20 | % 21 | % [X,F,G,A,LSINFO,LSNFEV] = POBLANO_LINESEARCH(FUN,...) also provides the 22 | % number of function/gradient calculations performed during the line search. 23 | % 24 | % Parameters (PARAMS) 25 | % 26 | % PARAMS.LineSearch_method 27 | % 28 | % 'more-thuente' : More-Thuente line search from MINPACK, adapted for 29 | % Matlab by Dianne O'Leary 30 | % 31 | % See CVSRCH for details on the use of parameters. Below are how the 32 | % Poblano parameters map to the CVSRCH parameters: 33 | % 34 | % LineSearch_xtol: xtol 35 | % LineSearch_ftol: ftol 36 | % LineSearch_gtol: gtol 37 | % LineSearch_stpmin: stpmin 38 | % LineSearch_stpmax: stpmax 39 | % LineSearch_maxfev: maxfev 40 | % 41 | % PARAMS.LineSearch_initialstep 42 | % 0 : use step length provided in A0 43 | % >0 : use step length provided in PARAMS.Linesearch_initialstep 44 | % 45 | % See also POBLANO_PARAMS, CVSRCH. 46 | % 47 | %Poblano Toolbox for MATLAB 48 | % 49 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 50 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 51 | %U.S. Government retains certain rights in this software. 52 | 53 | if nargin < 7 54 | error('POBLANO_LINESEARCH: too few arguments'); 55 | end 56 | 57 | switch (params.LineSearch_method) 58 | case 'more-thuente' 59 | minfun = 'cvsrch'; 60 | end 61 | 62 | % Check whether user specified an initial step 63 | if params.LineSearch_initialstep > 0 64 | a0 = params.LineSearch_initialstep; 65 | end 66 | 67 | [x,f,g,a,lsinfo,lsnfev] = feval(minfun,FUN,x0,f0,g0,a0,d0,params); 68 | -------------------------------------------------------------------------------- /poblano_out.m: -------------------------------------------------------------------------------- 1 | function out = poblano_out(x,f,g,nfev,params,out) 2 | %POBLANO_OUT Standard output parameters for the Poblano Toolbox. 3 | % 4 | % Standard ouptut parameters for Poblano are as follows: 5 | % 6 | % OUT.X : solution (i.e., best point found so far) 7 | % OUT.F : function value at solution 8 | % OUT.G : gradient at solution 9 | % OUT.Params : input parameters 10 | % OUT.FuncEvals: number of function evaluations 11 | % OUT.Iters : number of iterations 12 | % OUT.ExitFlag : termination flag 13 | % -1 = no termination condition has been statisfied 14 | % 0 = successful termination (based on StopTol) 15 | % 1 = maximum number of iterations exceeded 16 | % 2 = maximum number of function values exceeded 17 | % 3 = relative change in function value < RelFuncTol 18 | % 4 = NaNs found in f, g, or norm(g) 19 | % OUT.ExitDescription : text description of termination flag 20 | % 21 | % Additional fields may be included if a trace is requested. 22 | % 23 | % This method should not be called directly, but only by Poblano 24 | % optimization algorithms. 25 | % 26 | % See also POBLANO_PARAMS. 27 | % 28 | %Poblano Toolbox for MATLAB 29 | % 30 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 31 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 32 | %U.S. Government retains certain rights in this software. 33 | 34 | %% Check if this is the first call 35 | if ~exist('out','var') 36 | out.Params = params; 37 | out.ExitFlag = -1; 38 | out.ExitDescription = ''; 39 | out.X = x; 40 | out.F = f; 41 | out.G = g; 42 | out.FuncEvals = nfev; 43 | out.Iters = 0; 44 | if params.Results.TraceX, out.TraceX=[]; end 45 | if params.Results.TraceFunc, out.TraceFunc=[]; end 46 | if params.Results.TraceRelFunc, out.TraceRelFunc=[]; end 47 | if params.Results.TraceGrad, out.TraceGrad=[]; end 48 | if params.Results.TraceGradNorm, out.TraceGradNorm=[]; end 49 | if params.Results.TraceFuncEvals, out.TraceFuncEvals=[]; end 50 | else 51 | oldf = out.F; 52 | if (f <= out.F) 53 | out.X = x; 54 | out.F = f; 55 | out.G = g; 56 | end 57 | if abs(oldf) < eps 58 | relfit = abs(f - oldf); 59 | else 60 | relfit = abs((f - oldf) / oldf); 61 | end 62 | out.FuncEvals = out.FuncEvals + nfev; 63 | out.Iters = out.Iters + 1; 64 | end 65 | 66 | %% Initialize variables 67 | nx = length(x); 68 | g2norm = norm(g); 69 | g2normnx = g2norm/nx; 70 | 71 | %% Record trace info 72 | if params.Results.TraceX 73 | out.TraceX(:,end+1) = x; 74 | end 75 | 76 | if params.Results.TraceFunc 77 | out.TraceFunc(end+1) = f; 78 | end 79 | 80 | if params.Results.TraceFunc && params.Results.TraceRelFunc 81 | if (out.Iters > 0) 82 | out.TraceRelFunc(end+1) = relfit; 83 | end 84 | end 85 | 86 | if params.Results.TraceGrad 87 | out.TraceGrad(:,end+1) = g; 88 | end 89 | 90 | if params.Results.TraceGradNorm 91 | out.TraceGradNorm(end+1) = g2norm; 92 | end 93 | 94 | if params.Results.TraceFuncEvals 95 | out.TraceFuncEvals(end+1) = nfev; 96 | end 97 | 98 | %% Check termination conditions 99 | if g2normnx < params.Results.StopTol 100 | % solution found 101 | out.ExitFlag = 0; 102 | out.ExitDescription = 'Successful termination based on StopTol'; 103 | elseif out.Iters >= params.Results.MaxIters 104 | % maximum iterations exceeded 105 | out.ExitFlag = 1; 106 | out.ExitDescription = 'Maximum number of iterations exceeded'; 107 | elseif out.FuncEvals >= params.Results.MaxFuncEvals, 108 | % maximum function evaluations exceeded 109 | out.ExitFlag = 2; 110 | out.ExitDescription = 'Maximum number of function evaluations exceeded'; 111 | elseif (out.Iters > 0) && (relfit <= params.Results.RelFuncTol) 112 | % relative fit tolerance is reached 113 | out.ExitFlag=3; 114 | out.ExitDescription = 'Relative change in F < RelFuncTol'; 115 | elseif ~isfinite(f) || sum(~isfinite(g)) || ~isfinite(g2norm) 116 | % new point found results in NaNs 117 | out.ExitFlag = 4; 118 | out.ExitDescription = 'NaN/Inf found in F, G, or norm(G)'; 119 | end 120 | 121 | %% Display iteration information 122 | 123 | % First Iteration 124 | if (out.Iters == 0) && ~strcmp(params.Results.Display, 'off') 125 | fprintf(1,' Iter FuncEvals F(X) ||G(X)||/N \n'); 126 | fprintf(1,'------ --------- ---------------- ----------------\n'); 127 | end 128 | 129 | % Iteration info 130 | if strcmp(params.Results.Display, 'iter') && (mod(out.Iters,params.Results.DisplayIters)==0 || out.ExitFlag>=0) 131 | fprintf(1,'%6d %9d %16.8f %16.8f\n', out.Iters, out.FuncEvals, ... 132 | f, g2normnx); 133 | end 134 | 135 | % Final Iteration 136 | if ((out.ExitFlag >= 0) && strcmp(params.Results.Display, 'final')) 137 | % if strcmp(params.Results.Display, 'iter') 138 | % fprintf(1,'------ --------- ---------------- ----------------\n'); 139 | % end 140 | fprintf(1,'%6d %9d %16.8f %16.8f\n', out.Iters, out.FuncEvals, ... 141 | out.F, g2normnx); 142 | end 143 | -------------------------------------------------------------------------------- /poblano_params.m: -------------------------------------------------------------------------------- 1 | function params = poblano_params(params) 2 | %POBLANO_PARAMS Standard input parameters for the Poblano Toolbox. 3 | % 4 | % Standard input parameters for Poblano are as follows, with defaults in 5 | % parentheses: 6 | % 7 | % Display - controls printed output {'iter'} 8 | % 'iter' : display every iteration 9 | % 'final': display only after final iteration 10 | % 'off' : no display 11 | % 12 | % DisplayIters - number of iterations to display printed output {1} 13 | % 14 | % MaxIters - maximum number of iterations {1000} 15 | % 16 | % MaxFuncEvals - maximum number of function evaluations {10000} 17 | % 18 | % StopTol - gradient norm stopping tolerance {1e-5}, i.e., the 19 | % method stops when the norm of the gradient is less than StopTol 20 | % times the number of variables. 21 | % 22 | % RelFuncTol - relative function value change stopping tolerance {1e-5}, 23 | % i.e., the method stops when the relative change of the function value 24 | % from one iteration to the next is less than RelFuncTol. 25 | % 26 | % TraceX, TraceFunc, TraceRelFunc, TraceGrad, TraceGradNorm, 27 | % TraceFuncEvals {false} - booleans specifying whether or not to 28 | % record X, F(X), G(X), ||G(X)||, and number of function 29 | % evaluations, respectively, at each iteration. 30 | % 31 | % LineSearch_method {'more-thuente'} - string specifying the line search 32 | % method to use 33 | % 'more-thuente': find a step which satisfies a sufficient decrease 34 | % condition and a curvature condition (CVSRCH) 35 | % 36 | % See POBLANO_LINESEARCH for details on the line search parameters for 37 | % each available method. The available parameters: 38 | % LineSearch_xtol {1e-15} 39 | % LineSearch_ftol {1e-4} 40 | % LineSearch_gtol {1e-2} 41 | % LineSearch_stpmin {1e-15} 42 | % LineSearch_stpmax {1e15} 43 | % LineSearch_maxfev {20} 44 | % LineSearch_initialstep {1} 45 | % 46 | % This function should not be called directly, but only by Poblano 47 | % optimization algorithms. 48 | % 49 | % 50 | % Examples 51 | % 52 | % The default options can be retrieved from any Poblano method by 53 | % 54 | % % params = POBLANO_METHOD('defaults') 55 | % params = ncg('defaults') 56 | % 57 | % Also, it is possible to pass in parameters to any Poblano method as 58 | % a strucure. 59 | % 60 | % See also POBLANO_LINESEARCH. 61 | % 62 | %Poblano Toolbox for MATLAB 63 | % 64 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 65 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 66 | %U.S. Government retains certain rights in this software. 67 | 68 | %% Standard parameters across Poblano methods 69 | params.addParamValue('Display','iter', @(x) ismember(x,{'iter','final','off'})); 70 | params.addParamValue('DisplayIters', 1, @(x) x >= 1); 71 | params.addParamValue('MaxIters', 1000, @(x) x >= 0); 72 | params.addParamValue('MaxFuncEvals', 10000, @(x) x >= 0); 73 | params.addParamValue('StopTol', 1e-5, @(x) x >= 0); 74 | params.addParamValue('RelFuncTol', 1e-6, @isnumeric); 75 | params.addParamValue('TraceX', false, @islogical); 76 | params.addParamValue('TraceFunc', false, @islogical); 77 | params.addParamValue('TraceRelFunc', false, @islogical); 78 | params.addParamValue('TraceGrad', false, @islogical); 79 | params.addParamValue('TraceGradNorm', false, @islogical); 80 | params.addParamValue('TraceFuncEvals', false, @islogical); 81 | params.addParamValue('LineSearch_method', 'more-thuente', @(x) ismember(x,{'more-thuente'})); 82 | params.addParamValue('LineSearch_initialstep', 1, @(x) x >= 0); 83 | params.addParamValue('LineSearch_xtol', 1e-15, @(x) x >= 0); 84 | params.addParamValue('LineSearch_ftol', 1e-4, @(x) x >= 0); 85 | params.addParamValue('LineSearch_gtol', 1e-2, @(x) x >= 0); 86 | params.addParamValue('LineSearch_stpmin', 1e-15, @(x) x >= 0); 87 | params.addParamValue('LineSearch_stpmax', 1e15, @(x) x >= 0); 88 | params.addParamValue('LineSearch_maxfev', 20, @(x) x >= 0); 89 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Tests for Poblano Toolbox for MATLAB 2 | 3 | Name each test file as `test_.m`, replacing `` 4 | with the name of the method being tested. 5 | 6 | ## Running the Tests 7 | ``` matlab 8 | r=runtests; % Runs all tests 9 | table(r) % View results 10 | r=runtests('test_gradientcheck.m'); % Runs just the tests in that file 11 | ``` 12 | 13 | ## Creating New Tests 14 | Copy one of the existing tests as a guide and modify to do tests 15 | relvant for the m-files being created. See the MATLAB documentation on 16 | `Script-Based Unit Tests` for more information. 17 | 18 | ## Coverage Report 19 | You can run a coverage report associated with the tests in this 20 | directory by running `poblano_coverage`. -------------------------------------------------------------------------------- /tests/matlab.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/poblano_toolbox/2bc59e21132d51eac7336777a2fc634b391af648/tests/matlab.mat -------------------------------------------------------------------------------- /tests/poblano_coverage.m: -------------------------------------------------------------------------------- 1 | import matlab.unittest.TestSuite 2 | import matlab.unittest.TestRunner 3 | import matlab.unittest.plugins.CodeCoveragePlugin 4 | 5 | suite = TestSuite.fromFolder('.'); 6 | 7 | runner = TestRunner.withTextOutput; 8 | 9 | runner.addPlugin(CodeCoveragePlugin.forFolder('../')) 10 | result = runner.run(suite); 11 | -------------------------------------------------------------------------------- /tests/test_cvsrch.m: -------------------------------------------------------------------------------- 1 | % fixtures 2 | n = 10; 3 | fun1 = @(x) unittest_sum_squares(x); 4 | fun1_true_F = 0; 5 | fun2 = @(x) unittest_sum_sin(x,1); 6 | fun2_true_F = -1.0 * n; 7 | close_tol = 1e-8; 8 | 9 | %% Test 1: cvsrch defaults 10 | params = ncg('defaults'); 11 | params.Display = 'off'; 12 | rng(1); x0 = randn(n,1); 13 | x0 = 2 * ones(n,1); 14 | [f0,g0] = fun2(x0); 15 | a0 = params.LineSearch_initialstep; 16 | d0 = -g0; 17 | [x,f,g,a,lsinfo,lsnfev] = cvsrch(fun2,x0,f0,g0,a0,d0,params); 18 | assert(all(abs(x - 4.712236570537566*ones(n,1)) < close_tol)); 19 | assert(abs(f - (-9.999999883856193)) < close_tol); 20 | assert(all(abs(g - (-0.000152409846533415)*ones(n,1)) < close_tol)); 21 | assert(abs(a - (6.517498950710673)) < close_tol); 22 | assert(lsinfo == 1); 23 | assert(lsnfev == 5); 24 | 25 | %% Test 2: cvsrch incorrect inputs 26 | params = ncg('defaults'); 27 | params.Display = 'off'; 28 | rng(1); x0 = randn(n,1); 29 | x0 = 2 * ones(n,1); 30 | [f0,g0] = fun2(x0); 31 | a0 = params.LineSearch_initialstep; 32 | d0 = -g0; 33 | % empty x0 34 | [x,f,g,a,lsinfo,lsnfev] = cvsrch(fun2,[],f0,g0,a0,d0,params); 35 | assert(lsinfo == 0); 36 | % negative step length 37 | [x,f,g,a,lsinfo,lsnfev] = cvsrch(fun2,x0,f0,g0,-1,d0,params); 38 | assert(lsinfo == 0); 39 | % negative xtol 40 | params = ncg('defaults'); 41 | params.Display = 'off'; 42 | params.LineSearch_xtol = -1; 43 | [x,f,g,a,lsinfo,lsnfev] = cvsrch(fun2,x0,f0,g0,a0,d0,params); 44 | assert(lsinfo == 0); 45 | % negative ftol 46 | params = ncg('defaults'); 47 | params.Display = 'off'; 48 | params.LineSearch_ftol = -1; 49 | [x,f,g,a,lsinfo,lsnfev] = cvsrch(fun2,x0,f0,g0,a0,d0,params); 50 | assert(lsinfo == 0); 51 | % negative gtol 52 | params = ncg('defaults'); 53 | params.Display = 'off'; 54 | params.LineSearch_gtol = -1; 55 | [x,f,g,a,lsinfo,lsnfev] = cvsrch(fun2,x0,f0,g0,a0,d0,params); 56 | assert(lsinfo == 0); 57 | % negative stpmin 58 | params = ncg('defaults'); 59 | params.Display = 'off'; 60 | params.LineSearch_stpmin = -1; 61 | [x,f,g,a,lsinfo,lsnfev] = cvsrch(fun2,x0,f0,g0,a0,d0,params); 62 | assert(lsinfo == 0); 63 | % stpmax < stpmin 64 | params = ncg('defaults'); 65 | params.Display = 'off'; 66 | params.LineSearch_stpmin = 1e-3; 67 | params.LineSearch_stpmax = 1e-4; 68 | [x,f,g,a,lsinfo,lsnfev] = cvsrch(fun2,x0,f0,g0,a0,d0,params); 69 | assert(lsinfo == 0); 70 | % zero max evals 71 | params = ncg('defaults'); 72 | params.Display = 'off'; 73 | params.LineSearch_maxfev = 0; 74 | [x,f,g,a,lsinfo,lsnfev] = cvsrch(fun2,x0,f0,g0,a0,d0,params); 75 | assert(lsinfo == 0); 76 | 77 | -------------------------------------------------------------------------------- /tests/test_gradientcheck.m: -------------------------------------------------------------------------------- 1 | % fixtures 2 | x0 = ones(100,1); 3 | true_MaxDiff = -8.274037099909037e-08; 4 | true_NormGradientDiffs = 8.274037099909037e-07; 5 | fun = @(x) unittest_sum_squares(x); 6 | 7 | %% Test 1: gradientcheck forward differences 8 | out = gradientcheck(fun, x0, 'DifferenceType', 'forward'); 9 | assert(abs(out.MaxDiff) < length(x0) * out.Params.DifferenceStep) 10 | assert(abs(out.MaxDiff-true_MaxDiff) < out.Params.DifferenceStep); 11 | assert(abs(out.NormGradientDiffs-true_NormGradientDiffs) < out.Params.DifferenceStep); 12 | 13 | %% Test 2: gradientcheck backward differences 14 | out = gradientcheck(fun, x0, 'DifferenceType', 'backward'); 15 | assert(abs(out.MaxDiff) < length(x0) * out.Params.DifferenceStep) 16 | assert(abs(out.MaxDiff-true_MaxDiff) < out.Params.DifferenceStep); 17 | assert(abs(out.NormGradientDiffs-true_NormGradientDiffs) < out.Params.DifferenceStep); 18 | 19 | %% Test 3: gradientcheck backward differences 20 | out = gradientcheck(fun, x0, 'DifferenceType', 'centered'); 21 | assert(abs(out.MaxDiff) < length(x0) * out.Params.DifferenceStep) 22 | assert(abs(out.MaxDiff-true_MaxDiff) < out.Params.DifferenceStep); 23 | assert(abs(out.NormGradientDiffs-true_NormGradientDiffs) < out.Params.DifferenceStep); 24 | 25 | %% Test 4: gradientcheck using parameters struct 26 | params = struct('DifferenceType', 'backward'); 27 | out = gradientcheck(fun, x0, params); 28 | assert(strcmp(out.Params.DifferenceType, params.DifferenceType)); 29 | -------------------------------------------------------------------------------- /tests/test_hessvec_fd.m: -------------------------------------------------------------------------------- 1 | % fixtures 2 | n = 10; 3 | fun1 = @(x) unittest_sum_squares(x); 4 | fun1_true_F = 0; 5 | fun2 = @(x) unittest_sum_sin(x,1); 6 | fun2_true_F = -1.0 * n; 7 | close_tol = 1e-8; 8 | 9 | %% Test 1: hessvec_fd 3 parameters 10 | v = (1:n)'; 11 | x0 = 2 * ones(n,1); 12 | hv = hessvec_fd(v, fun1, x0); 13 | assert(all(abs(hv - v) < close_tol)); 14 | 15 | %% Test 2: hessvec_fd 4 parameters 16 | v = (1:n)'; 17 | x0 = 2 * ones(n,1); 18 | [fx0,gx0] = fun1(x0); 19 | hv = hessvec_fd(v, fun1, x0, gx0); 20 | assert(all(abs(hv - v) < close_tol)); 21 | 22 | %% Test 3: hessvec_fd 5 parameters 23 | v = (1:n)'; 24 | x0 = 2 * ones(n,1); 25 | [fx0,gx0] = fun1(x0); 26 | small_step = 1e-10; 27 | hv = hessvec_fd(v, fun1, x0, gx0, small_step); 28 | assert(all(abs(hv - v) < sqrt(small_step))); 29 | 30 | %% Test 4: hessvec_fd too few arguments 31 | % should result in an error 32 | try 33 | v = (1:n)'; 34 | x0 = 2 * ones(n,1); 35 | [fx0,gx0] = fun1(x0); 36 | hv = hessvec_fd(v, fun1); 37 | catch ME 38 | assert(strcmp(ME.message, 'HESSVEC_FD => at least two input arguments are required (V,FUN,X)')); 39 | end -------------------------------------------------------------------------------- /tests/test_lbfgs.m: -------------------------------------------------------------------------------- 1 | % fixtures 2 | n = 10; 3 | fun1 = @(x) unittest_sum_squares(x); 4 | fun1_true_F = 0; 5 | fun2 = @(x) unittest_sum_sin(x,1); 6 | fun2_true_F = -1.0 * n; 7 | close_tol = 1e-8; 8 | 9 | %% Test 1: lbfgs default parameters fun1 10 | params = lbfgs('defaults'); 11 | params.Display = 'off'; 12 | rng(1); x0 = randn(n,1); 13 | out = lbfgs(fun1, x0, params); 14 | assert(abs(out.F - fun1_true_F) < close_tol); 15 | 16 | %% Test 2: lbfgs default parameters fun2 17 | params = lbfgs('defaults'); 18 | params.Display = 'off'; 19 | rng(1); x0 = randn(n,1); 20 | out = lbfgs(fun2, x0, params); 21 | assert(abs(out.F - fun2_true_F) < close_tol); 22 | 23 | %% Test 3: lbfgs too few arguments 24 | % should result in an error 25 | try 26 | out = lbfgs(fun2); 27 | catch ME 28 | assert(strcmp(ME.message, 'Error: invalid input arguments')); 29 | end -------------------------------------------------------------------------------- /tests/test_ncg.m: -------------------------------------------------------------------------------- 1 | % fixtures 2 | n = 10; 3 | fun1 = @(x) unittest_sum_squares(x); 4 | fun1_true_F = 0; 5 | fun2 = @(x) unittest_sum_sin(x,1); 6 | fun2_true_F = -1.0 * n; 7 | close_tol = 1e-8; 8 | 9 | %% Test 1: ncg default parameters fun1 10 | params = ncg('defaults'); 11 | params.Display = 'off'; 12 | rng(1); x0 = randn(n,1); 13 | out = ncg(fun1, x0, params); 14 | assert(abs(out.F - fun1_true_F) < close_tol); 15 | 16 | %% Test 2: ncg default parameters fun2 17 | params = ncg('defaults'); 18 | params.Display = 'off'; 19 | rng(1); x0 = randn(n,1); 20 | out = ncg(fun2, x0, params); 21 | assert(abs(out.F - fun2_true_F) < close_tol); 22 | 23 | %% Test 3: ncg too few arguments 24 | % should result in an error 25 | try 26 | out = ncg(fun2); 27 | catch ME 28 | assert(strcmp(ME.message, 'Error: invalid input arguments')); 29 | end 30 | 31 | %% Test 4: ncg flectcher-reeves update 32 | % method does not converge to close_tol, but does to sqrt(close_tol) 33 | params = ncg('defaults'); 34 | params.Display = 'off'; 35 | params.Update = 'FR'; 36 | rng(1); x0 = randn(n,1); 37 | out = ncg(fun2, x0, params); 38 | assert(abs(out.F - fun2_true_F) < sqrt(close_tol)); 39 | 40 | %% Test 5: ncg polak-ribiere update 41 | params = ncg('defaults'); 42 | params.Display = 'off'; 43 | params.Update = 'PR'; 44 | rng(1); x0 = randn(n,1); 45 | out = ncg(fun2, x0, params); 46 | assert(abs(out.F - fun2_true_F) < close_tol); 47 | 48 | %% Test 6: ncg hestenes-stiefel update 49 | params = ncg('defaults'); 50 | params.Display = 'off'; 51 | params.Update = 'HS'; 52 | rng(1); x0 = randn(n,1); 53 | out = ncg(fun2, x0, params); 54 | assert(abs(out.F - fun2_true_F) < close_tol); 55 | 56 | %% Test 7: ncg steepest descent update 57 | params = ncg('defaults'); 58 | params.Display = 'off'; 59 | params.Update = 'SD'; 60 | rng(1); x0 = randn(n,1); 61 | out = ncg(fun2, x0, params); 62 | assert(abs(out.F - fun2_true_F) < close_tol); 63 | 64 | %% Test 8: ncg invalid update 65 | % should result in an error 66 | try 67 | params = ncg('defaults'); 68 | params.Display = 'off'; 69 | params.Update = 'FOO'; 70 | rng(1); x0 = randn(n,1); 71 | out = ncg(fun2, x0, params); 72 | catch ME 73 | assert(strcmp(ME.message, 'The value of ''Update'' is invalid. It must satisfy the function: @(x)ismember(x,{''FR'',''PR'',''HS'',''SD''}).')); 74 | end 75 | 76 | %% Test 9: ncg RestartIters 1 77 | params = ncg('defaults'); 78 | params.Display = 'off'; 79 | params.RestartNW = true; 80 | params.RestartIters = 1; 81 | rng(1); x0 = randn(n,1); 82 | out = ncg(fun2, x0, params); 83 | assert(abs(out.F - fun2_true_F) < close_tol); 84 | 85 | %% Test 10: ncg RestartIters 20 86 | params = ncg('defaults'); 87 | params.Display = 'off'; 88 | params.RestartNW = true; 89 | params.RestartNWTol = 1e-18; 90 | params.RestartIters = 20; 91 | rng(1); x0 = randn(n,1); 92 | out = ncg(fun2, x0, params); 93 | assert(abs(out.F - fun2_true_F) < close_tol); 94 | 95 | %% Test 11: ncg invalid RestartNWTol 96 | % should result in an error 97 | try 98 | params = ncg('defaults'); 99 | params.Display = 'off'; 100 | params.RestartNW = true; 101 | params.RestartNWTol = 0; 102 | rng(1); x0 = randn(n,1); 103 | out = ncg(fun2, x0, params); 104 | catch ME 105 | assert(strcmp(ME.message, 'The value of ''RestartNWTol'' is invalid. It must satisfy the function: @(x)x>0.')); 106 | end 107 | 108 | %% Test 12: ncg linesearch nonzero return 109 | params = ncg('defaults'); 110 | params.Display = 'iter'; 111 | params.LineSearch_maxfev = 1; 112 | rng(1); x0 = randn(n,1); 113 | S = evalc('out = ncg(fun2, x0, params);'); 114 | assert(contains(S,'ncg: line search warning')); 115 | -------------------------------------------------------------------------------- /tests/test_poblano_linsearch.m: -------------------------------------------------------------------------------- 1 | % fixtures 2 | n = 10; 3 | fun1 = @(x) unittest_sum_squares(x); 4 | fun1_true_F = 0; 5 | fun2 = @(x) unittest_sum_sin(x,1); 6 | fun2_true_F = -1.0 * n; 7 | close_tol = 1e-8; 8 | 9 | %% Test 1: poblano_linesearch default parameters fun1 10 | params = ncg('defaults'); 11 | params.Display = 'off'; 12 | x0 = 2 * ones(n,1); 13 | [fx0,gx0] = fun1(x0); 14 | [x,f,g,a] = poblano_linesearch(fun1, x0, fx0, gx0, 2, -gx0, params); 15 | assert(all(abs(x - zeros(n,1)) < close_tol)); 16 | assert(abs(f - 0) < close_tol); 17 | assert(all(abs(g - zeros(n,1)) < close_tol)); 18 | assert(abs(a - 1) < close_tol); 19 | 20 | %% Test 2: poblano_linsearch too few arguments 21 | params = ncg('defaults'); 22 | params.Display = 'off'; 23 | x0 = 2 * ones(n,1); 24 | [fx0,gx0] = fun1(x0); 25 | try 26 | [x,f,g,a] = poblano_linesearch(fun1, x0, fx0, gx0, 2, -gx0) 27 | catch ME 28 | assert(strcmp(ME.message, 'POBLANO_LINESEARCH: too few arguments')); 29 | end -------------------------------------------------------------------------------- /tests/test_poblano_out.m: -------------------------------------------------------------------------------- 1 | % fixtures 2 | n = 10; 3 | fun1 = @(x) unittest_sum_squares(x); 4 | fun1_true_F = 0; 5 | fun2 = @(x) unittest_sum_sin(x,1); 6 | fun2_true_F = -1.0 * n; 7 | close_tol = 1e-8; 8 | 9 | %% Test 1: poblano_out default parameters fun1 10 | params = inputParser; 11 | params = poblano_params(params); 12 | params.parse('Display','off'); 13 | x0 = 2 * ones(n,1); 14 | [fx0,gx0] = fun1(x0); 15 | out = poblano_out(x0, fx0, gx0, 1, params); 16 | 17 | %% Test 2: poblano_out using previous result 18 | params = inputParser; 19 | params = poblano_params(params); 20 | params.parse('Display','off'); 21 | x0 = 2 * ones(n,1); 22 | [fx0,gx0] = fun1(x0); 23 | out = poblano_out(x0, fx0, gx0, 1, params); 24 | out2 = poblano_out(x0, fx0, gx0, 1, params, out); 25 | 26 | %% Test 3: poblano_out using 0 old F 27 | params = inputParser; 28 | params = poblano_params(params); 29 | params.parse('Display','off'); 30 | x0 = 2 * ones(n,1); 31 | [fx0,gx0] = fun1(x0); 32 | out = poblano_out(x0, fx0, gx0, 1, params); 33 | out.F = 0; 34 | out2 = poblano_out(x0, fx0, gx0, 1, params, out); 35 | 36 | %% Test 4: poblano_out trace iterations 37 | params = inputParser; 38 | params = poblano_params(params); 39 | params.parse(... 40 | 'Display','off', ... 41 | 'TraceX', true, ... 42 | 'TraceFunc', true, ... 43 | 'TraceRelFunc', true, ... 44 | 'TraceGrad', true, ... 45 | 'TraceGradNorm', true, ... 46 | 'TraceFuncEvals', true ... 47 | ); 48 | x0 = 2 * ones(n,1); 49 | [fx0,gx0] = fun2(x0); 50 | out = poblano_out(x0, fx0, gx0, 1, params); 51 | out = poblano_out(x0, fx0, gx0, 1, params, out); 52 | assert(all(all(abs(out.TraceX - [x0 x0]) < close_tol))); 53 | assert(all(abs(out.TraceFunc - 9.092974268256818 * [1 1]) < close_tol)); 54 | assert(out.TraceRelFunc == 0); 55 | assert(all(all(abs(out.TraceGrad - (-0.416146836547142 * ones(n,2))) < close_tol))); 56 | assert(all(abs(out.TraceGradNorm - 1.315971844562770 * [1 1]) < close_tol)); 57 | assert(all(out.TraceFuncEvals == [1 1])); 58 | 59 | %% Test 5: poblano_out MaxIters reached 60 | params = inputParser; 61 | params = poblano_params(params); 62 | params.parse(... 63 | 'Display','off', ... 64 | 'MaxIters', 1 ... 65 | ); 66 | x0 = 2 * ones(n,1); 67 | [fx0,gx0] = fun2(x0); 68 | out = poblano_out(x0, fx0, gx0, 1, params); 69 | out.Iters = 1; 70 | out = poblano_out(x0, fx0, gx0, 1, params, out); 71 | assert(out.ExitFlag == 1); 72 | assert(strcmp(out.ExitDescription, 'Maximum number of iterations exceeded')); 73 | 74 | %% Test 6: poblano_out MaxFuncEvals reached 75 | params = inputParser; 76 | params = poblano_params(params); 77 | params.parse(... 78 | 'Display','off', ... 79 | 'MaxFuncEvals', 1 ... 80 | ); 81 | x0 = 2 * ones(n,1); 82 | [fx0,gx0] = fun2(x0); 83 | out = poblano_out(x0, fx0, gx0, 1, params); 84 | out.MaxFuncEvals = 1; 85 | out = poblano_out(x0, fx0, gx0, 1, params, out); 86 | assert(out.ExitFlag == 2); 87 | assert(strcmp(out.ExitDescription, 'Maximum number of function evaluations exceeded')); 88 | 89 | %% Test 7: poblano_out NANs found F 90 | params = inputParser; 91 | params = poblano_params(params); 92 | params.parse(... 93 | 'Display','off' ... 94 | ); 95 | x0 = 2 * ones(n,1); 96 | [fx0,gx0] = fun2(x0); 97 | out = poblano_out(x0, Inf, gx0, 1, params); 98 | assert(out.ExitFlag == 4); 99 | assert(strcmp(out.ExitDescription, 'NaN/Inf found in F, G, or norm(G)')); 100 | 101 | %% Test 8: poblano_out NANs found G 102 | params = inputParser; 103 | params = poblano_params(params); 104 | params.parse(... 105 | 'Display','off' ... 106 | ); 107 | x0 = 2 * ones(n,1); 108 | [fx0,gx0] = fun2(x0); 109 | gx0(end) = nan; 110 | out = poblano_out(x0, fx0, gx0, 1, params); 111 | assert(out.ExitFlag == 4); 112 | assert(strcmp(out.ExitDescription, 'NaN/Inf found in F, G, or norm(G)')); 113 | 114 | %% Test 9: poblano_out NANs found GXNORM 115 | params = inputParser; 116 | params = poblano_params(params); 117 | params.parse(... 118 | 'Display','off' ... 119 | ); 120 | x0 = 2 * ones(n,1); 121 | [fx0,gx0] = fun2(x0); 122 | out = poblano_out(x0, fx0, realmax*ones(n,1), 1, params); 123 | assert(out.ExitFlag == 4); 124 | assert(strcmp(out.ExitDescription, 'NaN/Inf found in F, G, or norm(G)')); 125 | 126 | %% Test 10: poblano_out default parameters fun1 127 | params = inputParser; 128 | params = poblano_params(params); 129 | params.parse(... 130 | 'Display','final', ... 131 | 'StopTol', realmax ... 132 | ); 133 | x0 = 2 * ones(n,1); 134 | [fx0,gx0] = fun1(x0); 135 | T = evalc('out = poblano_out(x0, fx0, gx0, 1, params);'); 136 | T2 = sprintf('%s\n%s\n%s\n', ... 137 | ' Iter FuncEvals F(X) ||G(X)||/N ', ... 138 | '------ --------- ---------------- ----------------', ... 139 | ' 0 1 20.00000000 0.63245553'); 140 | assert(strcmp(T, T2)); 141 | 142 | %% Test 11: poblano_out default parameters fun1 143 | params = inputParser; 144 | params = poblano_params(params); 145 | params.parse(... 146 | 'Display','iter', ... 147 | 'DisplayIters', 1 ... 148 | ); 149 | x0 = 2 * ones(n,1); 150 | [fx0,gx0] = fun1(x0); 151 | T = evalc('out = poblano_out(x0, fx0, gx0, 1, params);'); 152 | out.Iters = 2; 153 | T = evalc('out = poblano_out(x0, fx0, gx0, 1, params, out);'); 154 | T2 = sprintf('%s\n', ... 155 | ' 3 2 20.00000000 0.63245553' ... 156 | ); 157 | assert(strcmp(T, T2)); 158 | -------------------------------------------------------------------------------- /tests/test_tn.m: -------------------------------------------------------------------------------- 1 | % fixtures 2 | n = 10; 3 | fun1 = @(x) unittest_sum_squares(x); 4 | fun1_true_F = 0; 5 | fun2 = @(x) unittest_sum_sin(x,1); 6 | fun2_true_F = -1.0 * n; 7 | close_tol = 1e-8; 8 | 9 | %% Test 1: tn default parameters fun1 10 | params = tn('defaults'); 11 | params.Display = 'off'; 12 | rng(1); x0 = randn(n,1); 13 | out = tn(fun1, x0, params); 14 | assert(abs(out.F - fun1_true_F) < close_tol); 15 | 16 | %% Test 2: tn default parameters fun2 17 | params = tn('defaults'); 18 | params.Display = 'off'; 19 | rng(1); x0 = randn(n,1); 20 | out = tn(fun2, x0, params); 21 | assert(abs(out.F - fun2_true_F) < close_tol); 22 | 23 | %% Test 3: tn too few arguments 24 | % should result in an error 25 | try 26 | out = tn(fun2); 27 | catch ME 28 | assert(strcmp(ME.message, 'Error: invalid input arguments')); 29 | end 30 | 31 | %% Test 4: tn CGTolType superlinear 32 | params = tn('defaults'); 33 | params.Display = 'off'; 34 | params.CGTolType = 'superlinear'; 35 | rng(1); x0 = randn(n,1); 36 | out = tn(fun2, x0, params); 37 | assert(abs(out.F - fun2_true_F) < close_tol); 38 | 39 | %% Test 5: tn HessVecFDStep default 40 | params = tn('defaults'); 41 | params.Display = 'off'; 42 | params.HessVecFDStep = 0; 43 | rng(1); x0 = randn(n,1); 44 | out = tn(fun2, x0, params); 45 | assert(abs(out.F - fun2_true_F) < close_tol); 46 | 47 | %% Test 6: tn linesearch nonzero return 48 | params = tn('defaults'); 49 | params.Display = 'iter'; 50 | params.LineSearch_maxfev = 1; 51 | rng(1); x0 = randn(n,1); 52 | S = evalc('out = tn(fun2, x0, params);'); 53 | assert(contains(S,'tn: line search warning')); 54 | 55 | -------------------------------------------------------------------------------- /tests/unittest_sum_sin.m: -------------------------------------------------------------------------------- 1 | % fixture: function 2 | function [f,g] = sum_sin(x,a) 3 | 4 | if nargin < 2, a = 1; end 5 | f = sum(sin(a*x)); 6 | g = a*cos(a*x); 7 | 8 | end 9 | -------------------------------------------------------------------------------- /tests/unittest_sum_squares.m: -------------------------------------------------------------------------------- 1 | function [f,g] = unittest_function_1(x) 2 | 3 | f = 0.5 * sum(x.^2); 4 | g = x; 5 | 6 | end -------------------------------------------------------------------------------- /tn.m: -------------------------------------------------------------------------------- 1 | function out = tn(FUN,x0,varargin) 2 | %TN Truncated Newton minimization. 3 | % 4 | % OUT = TN(FUN,X0) minimizes FUN starting at the point X0 using a 5 | % Hessian-free truncated Newton method. The outer loop is Newton's 6 | % method, and the inner loop uses a conjugate gradient method to compute 7 | % an approximation to the Newton direction. Furthermore, the Hessian 8 | % vector product is approximated using forward finite differences using 9 | % HESSVEC_FD. FUN is a handle for a function that takes a single vector 10 | % input and returns two arguments --- the scalar function value and the 11 | % vector-valued gradient. See POBLANO_OUT for details of the output 12 | % parameters. 13 | % 14 | % OUT = TN(FUN,X0,'param',value,...) specifies a 15 | % parameters and its value. See POBLANO_PARAMS for further details on 16 | % standard parameters. Additionally, TN requires 17 | % 18 | % 'CGSolver' - Matlab CG method to use {'symmlq'} 19 | % 'symmlq' : symmlq (designed for symmetric indefinite systems) 20 | % 'pcg' : pcg (designed for symmetric positive definite systems) 21 | % 22 | % 'CGIters' - maximum number of conjugate gradient iterations allowed {5} 23 | % 24 | % 'CGTolType' - CG stopping tolerance type used ('quadratic') 25 | % 'quadratic' : || R || / || G || < min(0.5,|| G ||) 26 | % 'superlinear' : || R || / || G || < min(0.5,sqrt(|| G ||)) 27 | % 'fixed' : || R || < CGTol 28 | % where R is the residual and G is the gradient of FUN at X. 29 | % 30 | % 'CGTol' - CG stopping tolerance when CGTolType is 'fixed' {1e-6} 31 | % 32 | % 'HessVecFDStep' - Hessian vector product finite difference step {1e-10} 33 | % 0 - use the default step given in HESSVEC_FD 34 | % >0 - fixed value to use at the difference step 35 | % 36 | % PARAMS = TN('defaults') returns a structure containing the 37 | % default parameters for the particular Poblano method. 38 | % 39 | % 40 | % Examples 41 | % 42 | % Suppose the function and gradient of the objective function are 43 | % specified in an mfile named mysin.m: 44 | % 45 | % function [f,g]=example1(x,a) 46 | % if nargin < 2, a = 1; end 47 | % f = sin(a*x); 48 | % g = a*cos(a*x); 49 | % 50 | % We can call the optimization method (using its default 51 | % parameters) using the command: 52 | % 53 | % out = tn(@(x) example1(x,3), pi/4); 54 | % 55 | % To change a parameter, we can specify a param/value input pair 56 | % as follows: 57 | % 58 | % out = tn(@(x) example1(x,3), pi/4, 'Display', 'final'); 59 | % 60 | % Alternatively, we can use a structure to define the parameters: 61 | % 62 | % params.MaxIters = 2; 63 | % out = tn(@(x) example1(x,3), pi/4, params); 64 | % 65 | % See also POBLANO_OUT, POBLANO_PARAMS, HESSVEC_FD, FUNCTION_HANDLE. 66 | % 67 | %Poblano Toolbox for MATLAB 68 | % 69 | %Copyright 2009 National Technology & Engineering Solutions of Sandia, 70 | %LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the 71 | %U.S. Government retains certain rights in this software. 72 | 73 | %% Parse parameters 74 | 75 | % Create parser 76 | params = inputParser; 77 | 78 | % Set Poblano parameters 79 | params = poblano_params(params); 80 | 81 | % Set parameters for this method 82 | params.addParamValue('CGIters',5,@(x) x > 0); 83 | params.addParamValue('CGTolType','quadratic',@(x) ismember(x,{'quadratic','superlinear','fixed'})); 84 | params.addParamValue('CGTol',1e-6, @(x) x > 0); 85 | params.addParamValue('HessVecFDStep',1e-10, @(x) x >= 0); 86 | params.addParamValue('CGSolver','symmlq',@(x) ismember(x,{'symmlq','pcg'})); 87 | 88 | % Parse input 89 | params.parse(varargin{:}); 90 | 91 | %% Check input arguments 92 | if (nargin == 1) && isequal(FUN,'defaults') && (nargout == 1) 93 | out = params.Results; 94 | return; 95 | elseif (nargin < 2) 96 | error('Error: invalid input arguments'); 97 | end 98 | 99 | %% Initialize 100 | xk = x0; 101 | [fk,gk] = feval(FUN,xk); 102 | out = poblano_out(xk,fk,gk,1,params); 103 | ak = 1.0; 104 | 105 | cgIters = params.Results.CGIters; 106 | tolType = params.Results.CGTolType; 107 | cgTol = params.Results.CGTol; 108 | hessvecFDstep = params.Results.HessVecFDStep; 109 | solver = params.Results.CGSolver; 110 | 111 | %% Main loop 112 | while out.ExitFlag == -1 113 | 114 | % Compute step direction, pk 115 | ngk = norm(gk); 116 | switch tolType 117 | case 'quadratic' 118 | cgTol = min(1-eps,min(0.5,ngk)*ngk); % quadratic convergence 119 | case 'superlinear' 120 | cgTol = min(1-eps,min(0.5,sqrt(ngk))*ngk); % superlinear convergence 121 | end 122 | 123 | % Setup Hessian vector product finite difference approximation 124 | if hessvecFDstep > 0 125 | hv = @(v) hessvec_fd(v,FUN,xk,gk,hessvecFDstep); 126 | else 127 | hv = @(v) hessvec_fd(v,FUN,xk,gk); 128 | end 129 | 130 | % Keep track of number of function calls are made using hessvec_fd 131 | global nfev_hessvec_fd; 132 | nfev_hessvec_fd = 0; 133 | 134 | % Compute step direction, pk 135 | [pk,cg_flag,cg_relres,ncgfev]= feval(solver, hv, -gk, cgTol, cgIters); 136 | 137 | % Compute step length 138 | [xk,fk,gk,ak,lsinfo,nfev] = poblano_linesearch(FUN,xk,fk,gk,ak,pk,params.Results); 139 | if (lsinfo ~= 1) 140 | if strcmp(params.Results.Display, 'iter') 141 | fprintf(1,[mfilename,': line search warning = %d\n'],lsinfo); 142 | end 143 | pk = -gk; 144 | [xk,fk,gk,ak,lsinfo,nfev] = poblano_linesearch(FUN,xk,fk,gk,ak,pk,params.Results); 145 | end 146 | 147 | % Update counts, check exit conditions, etc. 148 | out = poblano_out(xk,fk,gk,nfev+nfev_hessvec_fd,params,out); 149 | end 150 | 151 | clear nfev_hessvec_fd; 152 | --------------------------------------------------------------------------------