├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .travis.yml ├── .travis └── test-coverage.sh ├── README.md ├── backtracking.go ├── bfgs.go ├── bisection.go ├── cg.go ├── convex └── lp │ ├── convert.go │ ├── simplex.go │ ├── simplex_test.go │ └── simplexexample_test.go ├── doc.go ├── errors.go ├── functionconvergence.go ├── functions ├── functions.go ├── functions_test.go ├── minsurf.go ├── minsurf_test.go └── validate.go ├── global.go ├── gradientdescent.go ├── guessandcheck.go ├── guessandcheck_test.go ├── interfaces.go ├── lbfgs.go ├── linesearch.go ├── linesearcher_test.go ├── local.go ├── local_example_test.go ├── minimize.go ├── morethuente.go ├── neldermead.go ├── newton.go ├── printer.go ├── stepsizers.go ├── termination.go ├── types.go └── unconstrained_test.go /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### This repository is no longer actively maintained. 2 | 3 | Development of the packages in this repository has moved to https://github.com/gonum/gonum. 4 | Please file issues [there](https://github.com/gonum/gonum/issues) after having checked that your issue has not been fixed. 5 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### This repository is no longer actively maintained. 2 | 3 | Development of the packages in this repository has moved to https://github.com/gonum/gonum. 4 | Please send pull requests [there](https://github.com/gonum/gonum/pulls) after having checked that your addition has not already been made. 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | language: go 4 | 5 | # Versions of go that are explicitly supported by gonum. 6 | go: 7 | - 1.5.4 8 | - 1.6.3 9 | - 1.7.3 10 | 11 | # Required for coverage. 12 | before_install: 13 | - go get golang.org/x/tools/cmd/cover 14 | - go get github.com/mattn/goveralls 15 | 16 | # Get deps, build, test, and ensure the code is gofmt'ed. 17 | # If we are building as gonum, then we have access to the coveralls api key, so we can run coverage as well. 18 | script: 19 | - go get -d -t -v ./... 20 | - go build -v ./... 21 | - go test -v ./... 22 | - test -z "$(gofmt -d .)" 23 | - if [[ $TRAVIS_SECURE_ENV_VARS = "true" ]]; then bash ./.travis/test-coverage.sh; fi 24 | -------------------------------------------------------------------------------- /.travis/test-coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PROFILE_OUT=$PWD/profile.out 4 | ACC_OUT=$PWD/acc.out 5 | 6 | testCover() { 7 | # set the return value to 0 (succesful) 8 | retval=0 9 | # get the directory to check from the parameter. Default to '.' 10 | d=${1:-.} 11 | # skip if there are no Go files here 12 | ls $d/*.go &> /dev/null || return $retval 13 | # switch to the directory to check 14 | pushd $d > /dev/null 15 | # create the coverage profile 16 | coverageresult=`go test -v -coverprofile=$PROFILE_OUT` 17 | # output the result so we can check the shell output 18 | echo ${coverageresult} 19 | # append the results to acc.out if coverage didn't fail, else set the retval to 1 (failed) 20 | ( [[ ${coverageresult} == *FAIL* ]] && retval=1 ) || ( [ -f $PROFILE_OUT ] && grep -v "mode: set" $PROFILE_OUT >> $ACC_OUT ) 21 | # return to our working dir 22 | popd > /dev/null 23 | # return our return value 24 | return $retval 25 | } 26 | 27 | # Init acc.out 28 | echo "mode: set" > $ACC_OUT 29 | 30 | # Run test coverage on all directories containing go files 31 | find . -maxdepth 10 -type d | while read d; do testCover $d || exit; done 32 | 33 | # Upload the coverage profile to coveralls.io 34 | [ -n "$COVERALLS_TOKEN" ] && goveralls -coverprofile=$ACC_OUT -service=travis-ci -repotoken $COVERALLS_TOKEN 35 | 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gonum Optimize [![Build Status](https://travis-ci.org/gonum/optimize.svg?branch=master)](https://travis-ci.org/gonum/optimize) [![Coverage Status](https://coveralls.io/repos/gonum/optimize/badge.svg?branch=master&service=github)](https://coveralls.io/github/gonum/optimize?branch=master) [![GoDoc](https://godoc.org/github.com/gonum/optimize?status.svg)](https://godoc.org/github.com/gonum/optimize) 2 | 3 | # This repository is no longer maintained. Development has moved to https://github.com/gonum/gonum. 4 | 5 | This is an optimization package for the Go language. More documentation can be seen at godoc.org/github.com/gonum/optimize 6 | 7 | ## Issues 8 | 9 | If you find any bugs, feel free to file an issue on the github [issue tracker for gonum/gonum](https://github.com/gonum/gonum/issues) if the bug exists in that reposity; no code changes will be made to this repository. Other dicussions should be taken to the gonum-dev Google Group. 10 | 11 | https://groups.google.com/forum/#!forum/gonum-dev 12 | 13 | ## License 14 | 15 | Please see github.com/gonum/license for general license information, contributors, authors, etc on the Gonum suite of packages. 16 | -------------------------------------------------------------------------------- /backtracking.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | const ( 8 | defaultBacktrackingContraction = 0.5 9 | defaultBacktrackingDecrease = 1e-4 10 | minimumBacktrackingStepSize = 1e-20 11 | ) 12 | 13 | // Backtracking is a Linesearcher that uses backtracking to find a point that 14 | // satisfies the Armijo condition with the given decrease factor. If the Armijo 15 | // condition has not been met, the step size is decreased by ContractionFactor. 16 | // 17 | // The Armijo condition only requires the gradient at the beginning of each 18 | // major iteration (not at successive step locations), and so Backtracking may 19 | // be a good linesearch for functions with expensive gradients. Backtracking is 20 | // not appropriate for optimizers that require the Wolfe conditions to be met, 21 | // such as BFGS. 22 | // 23 | // Both DecreaseFactor and ContractionFactor must be between zero and one, and 24 | // Backtracking will panic otherwise. If either DecreaseFactor or 25 | // ContractionFactor are zero, it will be set to a reasonable default. 26 | type Backtracking struct { 27 | DecreaseFactor float64 // Constant factor in the sufficient decrease (Armijo) condition. 28 | ContractionFactor float64 // Step size multiplier at each iteration (step *= ContractionFactor). 29 | 30 | stepSize float64 31 | initF float64 32 | initG float64 33 | 34 | lastOp Operation 35 | } 36 | 37 | func (b *Backtracking) Init(f, g float64, step float64) Operation { 38 | if step <= 0 { 39 | panic("backtracking: bad step size") 40 | } 41 | if g >= 0 { 42 | panic("backtracking: initial derivative is non-negative") 43 | } 44 | 45 | if b.ContractionFactor == 0 { 46 | b.ContractionFactor = defaultBacktrackingContraction 47 | } 48 | if b.DecreaseFactor == 0 { 49 | b.DecreaseFactor = defaultBacktrackingDecrease 50 | } 51 | if b.ContractionFactor <= 0 || b.ContractionFactor >= 1 { 52 | panic("backtracking: ContractionFactor must be between 0 and 1") 53 | } 54 | if b.DecreaseFactor <= 0 || b.DecreaseFactor >= 1 { 55 | panic("backtracking: DecreaseFactor must be between 0 and 1") 56 | } 57 | 58 | b.stepSize = step 59 | b.initF = f 60 | b.initG = g 61 | 62 | b.lastOp = FuncEvaluation 63 | return b.lastOp 64 | } 65 | 66 | func (b *Backtracking) Iterate(f, _ float64) (Operation, float64, error) { 67 | if b.lastOp != FuncEvaluation { 68 | panic("backtracking: Init has not been called") 69 | } 70 | 71 | if ArmijoConditionMet(f, b.initF, b.initG, b.stepSize, b.DecreaseFactor) { 72 | b.lastOp = MajorIteration 73 | return b.lastOp, b.stepSize, nil 74 | } 75 | b.stepSize *= b.ContractionFactor 76 | if b.stepSize < minimumBacktrackingStepSize { 77 | b.lastOp = NoOperation 78 | return b.lastOp, b.stepSize, ErrLinesearcherFailure 79 | } 80 | b.lastOp = FuncEvaluation 81 | return b.lastOp, b.stepSize, nil 82 | } 83 | -------------------------------------------------------------------------------- /bfgs.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "math" 9 | 10 | "github.com/gonum/matrix/mat64" 11 | ) 12 | 13 | // BFGS implements the Broyden–Fletcher–Goldfarb–Shanno optimization method. It 14 | // is a quasi-Newton method that performs successive rank-one updates to an 15 | // estimate of the inverse Hessian of the objective function. It exhibits 16 | // super-linear convergence when in proximity to a local minimum. It has memory 17 | // cost that is O(n^2) relative to the input dimension. 18 | type BFGS struct { 19 | // Linesearcher selects suitable steps along the descent direction. 20 | // Accepted steps should satisfy the strong Wolfe conditions. 21 | // If Linesearcher == nil, an appropriate default is chosen. 22 | Linesearcher Linesearcher 23 | 24 | ls *LinesearchMethod 25 | 26 | dim int 27 | x mat64.Vector // Location of the last major iteration. 28 | grad mat64.Vector // Gradient at the last major iteration. 29 | s mat64.Vector // Difference between locations in this and the previous iteration. 30 | y mat64.Vector // Difference between gradients in this and the previous iteration. 31 | tmp mat64.Vector 32 | 33 | invHess *mat64.SymDense 34 | 35 | first bool // Indicator of the first iteration. 36 | } 37 | 38 | func (b *BFGS) Init(loc *Location) (Operation, error) { 39 | if b.Linesearcher == nil { 40 | b.Linesearcher = &Bisection{} 41 | } 42 | if b.ls == nil { 43 | b.ls = &LinesearchMethod{} 44 | } 45 | b.ls.Linesearcher = b.Linesearcher 46 | b.ls.NextDirectioner = b 47 | 48 | return b.ls.Init(loc) 49 | } 50 | 51 | func (b *BFGS) Iterate(loc *Location) (Operation, error) { 52 | return b.ls.Iterate(loc) 53 | } 54 | 55 | func (b *BFGS) InitDirection(loc *Location, dir []float64) (stepSize float64) { 56 | dim := len(loc.X) 57 | b.dim = dim 58 | b.first = true 59 | 60 | x := mat64.NewVector(dim, loc.X) 61 | grad := mat64.NewVector(dim, loc.Gradient) 62 | b.x.CloneVec(x) 63 | b.grad.CloneVec(grad) 64 | 65 | b.y.Reset() 66 | b.s.Reset() 67 | b.tmp.Reset() 68 | 69 | if b.invHess == nil || cap(b.invHess.RawSymmetric().Data) < dim*dim { 70 | b.invHess = mat64.NewSymDense(dim, nil) 71 | } else { 72 | b.invHess = mat64.NewSymDense(dim, b.invHess.RawSymmetric().Data[:dim*dim]) 73 | } 74 | // The values of the inverse Hessian are initialized in the first call to 75 | // NextDirection. 76 | 77 | // Initial direction is just negative of the gradient because the Hessian 78 | // is an identity matrix. 79 | d := mat64.NewVector(dim, dir) 80 | d.ScaleVec(-1, grad) 81 | return 1 / mat64.Norm(d, 2) 82 | } 83 | 84 | func (b *BFGS) NextDirection(loc *Location, dir []float64) (stepSize float64) { 85 | dim := b.dim 86 | if len(loc.X) != dim { 87 | panic("bfgs: unexpected size mismatch") 88 | } 89 | if len(loc.Gradient) != dim { 90 | panic("bfgs: unexpected size mismatch") 91 | } 92 | if len(dir) != dim { 93 | panic("bfgs: unexpected size mismatch") 94 | } 95 | 96 | x := mat64.NewVector(dim, loc.X) 97 | grad := mat64.NewVector(dim, loc.Gradient) 98 | 99 | // s = x_{k+1} - x_{k} 100 | b.s.SubVec(x, &b.x) 101 | // y = g_{k+1} - g_{k} 102 | b.y.SubVec(grad, &b.grad) 103 | 104 | sDotY := mat64.Dot(&b.s, &b.y) 105 | 106 | if b.first { 107 | // Rescale the initial Hessian. 108 | // From: Nocedal, J., Wright, S.: Numerical Optimization (2nd ed). 109 | // Springer (2006), page 143, eq. 6.20. 110 | yDotY := mat64.Dot(&b.y, &b.y) 111 | scale := sDotY / yDotY 112 | for i := 0; i < dim; i++ { 113 | for j := i; j < dim; j++ { 114 | if i == j { 115 | b.invHess.SetSym(i, i, scale) 116 | } else { 117 | b.invHess.SetSym(i, j, 0) 118 | } 119 | } 120 | } 121 | b.first = false 122 | } 123 | 124 | if math.Abs(sDotY) != 0 { 125 | // Update the inverse Hessian according to the formula 126 | // 127 | // B_{k+1}^-1 = B_k^-1 128 | // + (s_k^T y_k + y_k^T B_k^-1 y_k) / (s_k^T y_k)^2 * (s_k s_k^T) 129 | // - (B_k^-1 y_k s_k^T + s_k y_k^T B_k^-1) / (s_k^T y_k). 130 | // 131 | // Note that y_k^T B_k^-1 y_k is a scalar, and that the third term is a 132 | // rank-two update where B_k^-1 y_k is one vector and s_k is the other. 133 | yBy := mat64.Inner(&b.y, b.invHess, &b.y) 134 | b.tmp.MulVec(b.invHess, &b.y) 135 | scale := (1 + yBy/sDotY) / sDotY 136 | b.invHess.SymRankOne(b.invHess, scale, &b.s) 137 | b.invHess.RankTwo(b.invHess, -1/sDotY, &b.tmp, &b.s) 138 | } 139 | 140 | // Update the stored BFGS data. 141 | b.x.CopyVec(x) 142 | b.grad.CopyVec(grad) 143 | 144 | // New direction is stored in dir. 145 | d := mat64.NewVector(dim, dir) 146 | d.MulVec(b.invHess, grad) 147 | d.ScaleVec(-1, d) 148 | 149 | return 1 150 | } 151 | 152 | func (*BFGS) Needs() struct { 153 | Gradient bool 154 | Hessian bool 155 | } { 156 | return struct { 157 | Gradient bool 158 | Hessian bool 159 | }{true, false} 160 | } 161 | -------------------------------------------------------------------------------- /bisection.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import "math" 8 | 9 | const ( 10 | defaultBisectionCurvature = 0.9 11 | ) 12 | 13 | // Bisection is a Linesearcher that uses a bisection to find a point that 14 | // satisfies the strong Wolfe conditions with the given curvature factor and 15 | // a decrease factor of zero. 16 | type Bisection struct { 17 | // CurvatureFactor is the constant factor in the curvature condition. 18 | // Smaller values result in a more exact line search. 19 | // A set value must be in the interval (0, 1), otherwise Init will panic. 20 | // If it is zero, it will be defaulted to 0.9. 21 | CurvatureFactor float64 22 | 23 | minStep float64 24 | maxStep float64 25 | currStep float64 26 | 27 | initF float64 28 | minF float64 29 | maxF float64 30 | lastF float64 31 | 32 | initGrad float64 33 | 34 | lastOp Operation 35 | } 36 | 37 | func (b *Bisection) Init(f, g float64, step float64) Operation { 38 | if step <= 0 { 39 | panic("bisection: bad step size") 40 | } 41 | if g >= 0 { 42 | panic("bisection: initial derivative is non-negative") 43 | } 44 | 45 | if b.CurvatureFactor == 0 { 46 | b.CurvatureFactor = defaultBisectionCurvature 47 | } 48 | if b.CurvatureFactor <= 0 || b.CurvatureFactor >= 1 { 49 | panic("bisection: CurvatureFactor not between 0 and 1") 50 | } 51 | 52 | b.minStep = 0 53 | b.maxStep = math.Inf(1) 54 | b.currStep = step 55 | 56 | b.initF = f 57 | b.minF = f 58 | b.maxF = math.NaN() 59 | 60 | b.initGrad = g 61 | 62 | // Only evaluate the gradient when necessary. 63 | b.lastOp = FuncEvaluation 64 | return b.lastOp 65 | } 66 | 67 | func (b *Bisection) Iterate(f, g float64) (Operation, float64, error) { 68 | if b.lastOp != FuncEvaluation && b.lastOp != GradEvaluation { 69 | panic("bisection: Init has not been called") 70 | } 71 | minF := b.initF 72 | if b.maxF < minF { 73 | minF = b.maxF 74 | } 75 | if b.minF < minF { 76 | minF = b.minF 77 | } 78 | if b.lastOp == FuncEvaluation { 79 | // See if the function value is good enough to make progress. If it is, 80 | // evaluate the gradient. If not, set it to the upper bound if the bound 81 | // has not yet been found, otherwise iterate toward the minimum location. 82 | if f <= minF { 83 | b.lastF = f 84 | b.lastOp = GradEvaluation 85 | return b.lastOp, b.currStep, nil 86 | } 87 | if math.IsInf(b.maxStep, 1) { 88 | b.maxStep = b.currStep 89 | b.maxF = f 90 | return b.nextStep((b.minStep + b.maxStep) / 2) 91 | } 92 | if b.minF <= b.maxF { 93 | b.maxStep = b.currStep 94 | b.maxF = f 95 | } else { 96 | b.minStep = b.currStep 97 | b.minF = f 98 | } 99 | return b.nextStep((b.minStep + b.maxStep) / 2) 100 | } 101 | f = b.lastF 102 | // The function value was lower. Check if this location is sufficient to 103 | // converge the linesearch, otherwise iterate. 104 | if StrongWolfeConditionsMet(f, g, minF, b.initGrad, b.currStep, 0, b.CurvatureFactor) { 105 | b.lastOp = MajorIteration 106 | return b.lastOp, b.currStep, nil 107 | } 108 | if math.IsInf(b.maxStep, 1) { 109 | // The function value is lower. If the gradient is positive, an upper bound 110 | // of the minimum been found. If the gradient is negative, search farther 111 | // in that direction. 112 | if g > 0 { 113 | b.maxStep = b.currStep 114 | b.maxF = f 115 | return b.nextStep((b.minStep + b.maxStep) / 2) 116 | } 117 | b.minStep = b.currStep 118 | b.minF = f 119 | return b.nextStep(b.currStep * 2) 120 | } 121 | // The interval has been bounded, and we have found a new lowest value. Use 122 | // the gradient to decide which direction. 123 | if g < 0 { 124 | b.minStep = b.currStep 125 | b.minF = f 126 | } else { 127 | b.maxStep = b.currStep 128 | b.maxF = f 129 | } 130 | return b.nextStep((b.minStep + b.maxStep) / 2) 131 | } 132 | 133 | // nextStep checks if the new step is equal to the old step. 134 | // This can happen if min and max are the same, or if the step size is infinity, 135 | // both of which indicate the minimization must stop. If the steps are different, 136 | // it sets the new step size and returns the evaluation type and the step. If the steps 137 | // are the same, it returns an error. 138 | func (b *Bisection) nextStep(step float64) (Operation, float64, error) { 139 | if b.currStep == step { 140 | b.lastOp = NoOperation 141 | return b.lastOp, b.currStep, ErrLinesearcherFailure 142 | } 143 | b.currStep = step 144 | b.lastOp = FuncEvaluation 145 | return b.lastOp, b.currStep, nil 146 | } 147 | -------------------------------------------------------------------------------- /cg.go: -------------------------------------------------------------------------------- 1 | package optimize 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/gonum/floats" 7 | ) 8 | 9 | const ( 10 | iterationRestartFactor = 6 11 | angleRestartThreshold = -0.9 12 | ) 13 | 14 | // CGVariant calculates the scaling parameter, β, used for updating the 15 | // conjugate direction in the nonlinear conjugate gradient (CG) method. 16 | type CGVariant interface { 17 | // Init is called at the first iteration and provides a way to initialize 18 | // any internal state. 19 | Init(loc *Location) 20 | // Beta returns the value of the scaling parameter that is computed 21 | // according to the particular variant of the CG method. 22 | Beta(grad, gradPrev, dirPrev []float64) float64 23 | } 24 | 25 | // CG implements the nonlinear conjugate gradient method for solving nonlinear 26 | // unconstrained optimization problems. It is a line search method that 27 | // generates the search directions d_k according to the formula 28 | // d_{k+1} = -∇f_{k+1} + β_k*d_k, d_0 = -∇f_0. 29 | // Variants of the conjugate gradient method differ in the choice of the 30 | // parameter β_k. The conjugate gradient method usually requires fewer function 31 | // evaluations than the gradient descent method and no matrix storage, but 32 | // L-BFGS is usually more efficient. 33 | // 34 | // CG implements a restart strategy that takes the steepest descent direction 35 | // (i.e., d_{k+1} = -∇f_{k+1}) whenever any of the following conditions holds: 36 | // 37 | // - A certain number of iterations has elapsed without a restart. This number 38 | // is controllable via IterationRestartFactor and if equal to 0, it is set to 39 | // a reasonable default based on the problem dimension. 40 | // - The angle between the gradients at two consecutive iterations ∇f_k and 41 | // ∇f_{k+1} is too large. 42 | // - The direction d_{k+1} is not a descent direction. 43 | // - β_k returned from CGVariant.Beta is equal to zero. 44 | // 45 | // The line search for CG must yield step sizes that satisfy the strong Wolfe 46 | // conditions at every iteration, otherwise the generated search direction 47 | // might fail to be a descent direction. The line search should be more 48 | // stringent compared with those for Newton-like methods, which can be achieved 49 | // by setting the gradient constant in the strong Wolfe conditions to a small 50 | // value. 51 | // 52 | // See also William Hager, Hongchao Zhang, A survey of nonlinear conjugate 53 | // gradient methods. Pacific Journal of Optimization, 2 (2006), pp. 35-58, and 54 | // references therein. 55 | type CG struct { 56 | // Linesearcher must satisfy the strong Wolfe conditions at every iteration. 57 | // If Linesearcher == nil, an appropriate default is chosen. 58 | Linesearcher Linesearcher 59 | // Variant implements the particular CG formula for computing β_k. 60 | // If Variant is nil, an appropriate default is chosen. 61 | Variant CGVariant 62 | // InitialStep estimates the initial line search step size, because the CG 63 | // method does not generate well-scaled search directions. 64 | // If InitialStep is nil, an appropriate default is chosen. 65 | InitialStep StepSizer 66 | 67 | // IterationRestartFactor determines the frequency of restarts based on the 68 | // problem dimension. The negative gradient direction is taken whenever 69 | // ceil(IterationRestartFactor*(problem dimension)) iterations have elapsed 70 | // without a restart. For medium and large-scale problems 71 | // IterationRestartFactor should be set to 1, low-dimensional problems a 72 | // larger value should be chosen. Note that if the ceil function returns 1, 73 | // CG will be identical to gradient descent. 74 | // If IterationRestartFactor is 0, it will be set to 6. 75 | // CG will panic if IterationRestartFactor is negative. 76 | IterationRestartFactor float64 77 | // AngleRestartThreshold sets the threshold angle for restart. The method 78 | // is restarted if the cosine of the angle between two consecutive 79 | // gradients is smaller than or equal to AngleRestartThreshold, that is, if 80 | // ∇f_k·∇f_{k+1} / (|∇f_k| |∇f_{k+1}|) <= AngleRestartThreshold. 81 | // A value of AngleRestartThreshold closer to -1 (successive gradients in 82 | // exact opposite directions) will tend to reduce the number of restarts. 83 | // If AngleRestartThreshold is 0, it will be set to -0.9. 84 | // CG will panic if AngleRestartThreshold is not in the interval [-1, 0]. 85 | AngleRestartThreshold float64 86 | 87 | ls *LinesearchMethod 88 | 89 | restartAfter int 90 | iterFromRestart int 91 | 92 | dirPrev []float64 93 | gradPrev []float64 94 | gradPrevNorm float64 95 | } 96 | 97 | func (cg *CG) Init(loc *Location) (Operation, error) { 98 | if cg.IterationRestartFactor < 0 { 99 | panic("cg: IterationRestartFactor is negative") 100 | } 101 | if cg.AngleRestartThreshold < -1 || cg.AngleRestartThreshold > 0 { 102 | panic("cg: AngleRestartThreshold not in [-1, 0]") 103 | } 104 | 105 | if cg.Linesearcher == nil { 106 | cg.Linesearcher = &MoreThuente{CurvatureFactor: 0.1} 107 | } 108 | if cg.Variant == nil { 109 | cg.Variant = &HestenesStiefel{} 110 | } 111 | if cg.InitialStep == nil { 112 | cg.InitialStep = &FirstOrderStepSize{} 113 | } 114 | 115 | if cg.IterationRestartFactor == 0 { 116 | cg.IterationRestartFactor = iterationRestartFactor 117 | } 118 | if cg.AngleRestartThreshold == 0 { 119 | cg.AngleRestartThreshold = angleRestartThreshold 120 | } 121 | 122 | if cg.ls == nil { 123 | cg.ls = &LinesearchMethod{} 124 | } 125 | cg.ls.Linesearcher = cg.Linesearcher 126 | cg.ls.NextDirectioner = cg 127 | 128 | return cg.ls.Init(loc) 129 | } 130 | 131 | func (cg *CG) Iterate(loc *Location) (Operation, error) { 132 | return cg.ls.Iterate(loc) 133 | } 134 | 135 | func (cg *CG) InitDirection(loc *Location, dir []float64) (stepSize float64) { 136 | dim := len(loc.X) 137 | 138 | cg.restartAfter = int(math.Ceil(cg.IterationRestartFactor * float64(dim))) 139 | cg.iterFromRestart = 0 140 | 141 | // The initial direction is always the negative gradient. 142 | copy(dir, loc.Gradient) 143 | floats.Scale(-1, dir) 144 | 145 | cg.dirPrev = resize(cg.dirPrev, dim) 146 | copy(cg.dirPrev, dir) 147 | cg.gradPrev = resize(cg.gradPrev, dim) 148 | copy(cg.gradPrev, loc.Gradient) 149 | cg.gradPrevNorm = floats.Norm(loc.Gradient, 2) 150 | 151 | cg.Variant.Init(loc) 152 | return cg.InitialStep.Init(loc, dir) 153 | } 154 | 155 | func (cg *CG) NextDirection(loc *Location, dir []float64) (stepSize float64) { 156 | copy(dir, loc.Gradient) 157 | floats.Scale(-1, dir) 158 | 159 | cg.iterFromRestart++ 160 | var restart bool 161 | if cg.iterFromRestart == cg.restartAfter { 162 | // Restart because too many iterations have been taken without a restart. 163 | restart = true 164 | } 165 | 166 | gDot := floats.Dot(loc.Gradient, cg.gradPrev) 167 | gNorm := floats.Norm(loc.Gradient, 2) 168 | if gDot <= cg.AngleRestartThreshold*gNorm*cg.gradPrevNorm { 169 | // Restart because the angle between the last two gradients is too large. 170 | restart = true 171 | } 172 | 173 | // Compute the scaling factor β_k even when restarting, because cg.Variant 174 | // may be keeping an inner state that needs to be updated at every iteration. 175 | beta := cg.Variant.Beta(loc.Gradient, cg.gradPrev, cg.dirPrev) 176 | if beta == 0 { 177 | // β_k == 0 means that the steepest descent direction will be taken, so 178 | // indicate that the method is in fact being restarted. 179 | restart = true 180 | } 181 | if !restart { 182 | // The method is not being restarted, so update the descent direction. 183 | floats.AddScaled(dir, beta, cg.dirPrev) 184 | if floats.Dot(loc.Gradient, dir) >= 0 { 185 | // Restart because the new direction is not a descent direction. 186 | restart = true 187 | copy(dir, loc.Gradient) 188 | floats.Scale(-1, dir) 189 | } 190 | } 191 | 192 | // Get the initial line search step size from the StepSizer even if the 193 | // method was restarted, because StepSizers need to see every iteration. 194 | stepSize = cg.InitialStep.StepSize(loc, dir) 195 | if restart { 196 | // The method was restarted and since the steepest descent direction is 197 | // not related to the previous direction, discard the estimated step 198 | // size from cg.InitialStep and use step size of 1 instead. 199 | stepSize = 1 200 | // Reset to 0 the counter of iterations taken since the last restart. 201 | cg.iterFromRestart = 0 202 | } 203 | 204 | copy(cg.gradPrev, loc.Gradient) 205 | copy(cg.dirPrev, dir) 206 | cg.gradPrevNorm = gNorm 207 | return stepSize 208 | } 209 | 210 | func (*CG) Needs() struct { 211 | Gradient bool 212 | Hessian bool 213 | } { 214 | return struct { 215 | Gradient bool 216 | Hessian bool 217 | }{true, false} 218 | } 219 | 220 | // FletcherReeves implements the Fletcher-Reeves variant of the CG method that 221 | // computes the scaling parameter β_k according to the formula 222 | // β_k = |∇f_{k+1}|^2 / |∇f_k|^2. 223 | type FletcherReeves struct { 224 | prevNorm float64 225 | } 226 | 227 | func (fr *FletcherReeves) Init(loc *Location) { 228 | fr.prevNorm = floats.Norm(loc.Gradient, 2) 229 | } 230 | 231 | func (fr *FletcherReeves) Beta(grad, _, _ []float64) (beta float64) { 232 | norm := floats.Norm(grad, 2) 233 | beta = (norm / fr.prevNorm) * (norm / fr.prevNorm) 234 | fr.prevNorm = norm 235 | return beta 236 | } 237 | 238 | // PolakRibierePolyak implements the Polak-Ribiere-Polyak variant of the CG 239 | // method that computes the scaling parameter β_k according to the formula 240 | // β_k = max(0, ∇f_{k+1}·y_k / |∇f_k|^2), 241 | // where y_k = ∇f_{k+1} - ∇f_k. 242 | type PolakRibierePolyak struct { 243 | prevNorm float64 244 | } 245 | 246 | func (pr *PolakRibierePolyak) Init(loc *Location) { 247 | pr.prevNorm = floats.Norm(loc.Gradient, 2) 248 | } 249 | 250 | func (pr *PolakRibierePolyak) Beta(grad, gradPrev, _ []float64) (beta float64) { 251 | norm := floats.Norm(grad, 2) 252 | dot := floats.Dot(grad, gradPrev) 253 | beta = (norm*norm - dot) / (pr.prevNorm * pr.prevNorm) 254 | pr.prevNorm = norm 255 | return math.Max(0, beta) 256 | } 257 | 258 | // HestenesStiefel implements the Hestenes-Stiefel variant of the CG method 259 | // that computes the scaling parameter β_k according to the formula 260 | // β_k = max(0, ∇f_{k+1}·y_k / d_k·y_k), 261 | // where y_k = ∇f_{k+1} - ∇f_k. 262 | type HestenesStiefel struct { 263 | y []float64 264 | } 265 | 266 | func (hs *HestenesStiefel) Init(loc *Location) { 267 | hs.y = resize(hs.y, len(loc.Gradient)) 268 | } 269 | 270 | func (hs *HestenesStiefel) Beta(grad, gradPrev, dirPrev []float64) (beta float64) { 271 | floats.SubTo(hs.y, grad, gradPrev) 272 | beta = floats.Dot(grad, hs.y) / floats.Dot(dirPrev, hs.y) 273 | return math.Max(0, beta) 274 | } 275 | 276 | // DaiYuan implements the Dai-Yuan variant of the CG method that computes the 277 | // scaling parameter β_k according to the formula 278 | // β_k = |∇f_{k+1}|^2 / d_k·y_k, 279 | // where y_k = ∇f_{k+1} - ∇f_k. 280 | type DaiYuan struct { 281 | y []float64 282 | } 283 | 284 | func (dy *DaiYuan) Init(loc *Location) { 285 | dy.y = resize(dy.y, len(loc.Gradient)) 286 | } 287 | 288 | func (dy *DaiYuan) Beta(grad, gradPrev, dirPrev []float64) (beta float64) { 289 | floats.SubTo(dy.y, grad, gradPrev) 290 | norm := floats.Norm(grad, 2) 291 | return norm * norm / floats.Dot(dirPrev, dy.y) 292 | } 293 | 294 | // HagerZhang implements the Hager-Zhang variant of the CG method that computes the 295 | // scaling parameter β_k according to the formula 296 | // β_k = (y_k - 2 d_k |y_k|^2/(d_k·y_k))·∇f_{k+1} / (d_k·y_k), 297 | // where y_k = ∇f_{k+1} - ∇f_k. 298 | type HagerZhang struct { 299 | y []float64 300 | } 301 | 302 | func (hz *HagerZhang) Init(loc *Location) { 303 | hz.y = resize(hz.y, len(loc.Gradient)) 304 | } 305 | 306 | func (hz *HagerZhang) Beta(grad, gradPrev, dirPrev []float64) (beta float64) { 307 | floats.SubTo(hz.y, grad, gradPrev) 308 | dirDotY := floats.Dot(dirPrev, hz.y) 309 | gDotY := floats.Dot(grad, hz.y) 310 | gDotDir := floats.Dot(grad, dirPrev) 311 | yNorm := floats.Norm(hz.y, 2) 312 | return (gDotY - 2*gDotDir*yNorm*yNorm/dirDotY) / dirDotY 313 | } 314 | -------------------------------------------------------------------------------- /convex/lp/convert.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2016 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package lp 6 | 7 | import ( 8 | "github.com/gonum/floats" 9 | "github.com/gonum/matrix/mat64" 10 | ) 11 | 12 | // TODO(btracey): Have some sort of preprocessing step for helping to fix A to make it 13 | // full rank? 14 | // TODO(btracey): Reduce rows? Get rid of all zeros, places where only one variable 15 | // is there, etc. Could be implemented with a Reduce function. 16 | // TODO(btracey): Provide method of artificial variables for help when problem 17 | // is infeasible? 18 | // TODO(btracey): Add an lp.Solve that solves an LP in non-standard form. 19 | 20 | // Convert converts a General-form LP into a standard form LP. 21 | // The general form of an LP is: 22 | // minimize c^T * x 23 | // s.t G * x <= h 24 | // A * x = b 25 | // And the standard form is: 26 | // minimize cNew^T * x 27 | // s.t aNew * x = bNew 28 | // x >= 0 29 | // If there are no constraints of the given type, the inputs may be nil. 30 | func Convert(c []float64, g mat64.Matrix, h []float64, a mat64.Matrix, b []float64) (cNew []float64, aNew *mat64.Dense, bNew []float64) { 31 | nVar := len(c) 32 | nIneq := len(h) 33 | 34 | // Check input sizes. 35 | if g == nil { 36 | if nIneq != 0 { 37 | panic(badShape) 38 | } 39 | } else { 40 | gr, gc := g.Dims() 41 | if gr != nIneq { 42 | panic(badShape) 43 | } 44 | if gc != nVar { 45 | panic(badShape) 46 | } 47 | } 48 | 49 | nEq := len(b) 50 | if a == nil { 51 | if nEq != 0 { 52 | panic(badShape) 53 | } 54 | } else { 55 | ar, ac := a.Dims() 56 | if ar != nEq { 57 | panic(badShape) 58 | } 59 | if ac != nVar { 60 | panic(badShape) 61 | } 62 | } 63 | 64 | // Convert the general form LP. 65 | // Derivation: 66 | // 0. Start with general form 67 | // min. c^T * x 68 | // s.t. G * x <= h 69 | // A * x = b 70 | // 1. Introduce slack variables for each constraint 71 | // min. c^T * x 72 | // s.t. G * x + s = h 73 | // A * x = b 74 | // s >= 0 75 | // 2. Add non-negativity constraints for x by splitting x 76 | // into positive and negative components. 77 | // x = xp - xn 78 | // xp >= 0, xn >= 0 79 | // This makes the LP 80 | // min. c^T * xp - c^T xn 81 | // s.t. G * xp - G * xn + s = h 82 | // A * xp - A * xn = b 83 | // xp >= 0, xn >= 0, s >= 0 84 | // 3. Write the above in standard form: 85 | // xt = [xp 86 | // xn 87 | // s ] 88 | // min. [c^T, -c^T, 0] xt 89 | // s.t. [G, -G, I] xt = h 90 | // [A, -A, 0] xt = b 91 | // x >= 0 92 | 93 | // In summary: 94 | // Original LP: 95 | // min. c^T * x 96 | // s.t. G * x <= h 97 | // A * x = b 98 | // Standard Form: 99 | // xt = [xp; xn; s] 100 | // min. [c^T, -c^T, 0] xt 101 | // s.t. [G, -G, I] xt = h 102 | // [A, -A, 0] xt = b 103 | // x >= 0 104 | 105 | // New size of x is [xp, xn, s] 106 | nNewVar := nVar + nVar + nIneq 107 | 108 | // Construct cNew = [c; -c; 0] 109 | cNew = make([]float64, nNewVar) 110 | copy(cNew, c) 111 | copy(cNew[nVar:], c) 112 | floats.Scale(-1, cNew[nVar:2*nVar]) 113 | 114 | // New number of equality constraints is the number of total constraints. 115 | nNewEq := nIneq + nEq 116 | 117 | // Construct bNew = [h, b]. 118 | bNew = make([]float64, nNewEq) 119 | copy(bNew, h) 120 | copy(bNew[nIneq:], b) 121 | 122 | // Construct aNew = [G, -G, I; A, -A, 0]. 123 | aNew = mat64.NewDense(nNewEq, nNewVar, nil) 124 | if nIneq != 0 { 125 | aView := (aNew.View(0, 0, nIneq, nVar)).(*mat64.Dense) 126 | aView.Copy(g) 127 | aView = (aNew.View(0, nVar, nIneq, nVar)).(*mat64.Dense) 128 | aView.Scale(-1, g) 129 | aView = (aNew.View(0, 2*nVar, nIneq, nIneq)).(*mat64.Dense) 130 | for i := 0; i < nIneq; i++ { 131 | aView.Set(i, i, 1) 132 | } 133 | } 134 | if nEq != 0 { 135 | aView := (aNew.View(nIneq, 0, nEq, nVar)).(*mat64.Dense) 136 | aView.Copy(a) 137 | aView = (aNew.View(nIneq, nVar, nEq, nVar)).(*mat64.Dense) 138 | aView.Scale(-1, a) 139 | } 140 | return cNew, aNew, bNew 141 | } 142 | -------------------------------------------------------------------------------- /convex/lp/simplex.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2016 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // This repository is no longer maintained. 6 | // Development has moved to https://github.com/gonum/gonum. 7 | // 8 | // package lp implements routines for solving linear programs. 9 | package lp 10 | 11 | import ( 12 | "errors" 13 | "fmt" 14 | "math" 15 | 16 | "github.com/gonum/floats" 17 | "github.com/gonum/matrix/mat64" 18 | ) 19 | 20 | // TODO(btracey): Could have a solver structure with an abstract factorizer. With 21 | // this transformation the same high-level code could handle both Dense and Sparse. 22 | // TODO(btracey): Need to improve error handling. Only want to panic if condition number inf. 23 | // TODO(btracey): Performance enhancements. There are currently lots of linear 24 | // solves that can be improved by doing rank-one updates. For example, the swap 25 | // step is just a rank-one update. 26 | // TODO(btracey): Better handling on the linear solve errors. If the condition 27 | // number is not inf and the equation solved "well", should keep moving. 28 | 29 | var ( 30 | ErrBland = errors.New("lp: bland: all replacements are negative or cause ill-conditioned ab") 31 | ErrInfeasible = errors.New("lp: problem is infeasible") 32 | ErrLinSolve = errors.New("lp: linear solve failure") 33 | ErrUnbounded = errors.New("lp: problem is unbounded") 34 | ErrSingular = errors.New("lp: A is singular") 35 | ErrZeroColumn = errors.New("lp: A has a column of all zeros") 36 | ErrZeroRow = errors.New("lp: A has a row of all zeros") 37 | ) 38 | 39 | var ( 40 | badShape = "lp: size mismatch" 41 | ) 42 | 43 | // TODO(btracey): Should these tolerances be part of a settings struct? 44 | 45 | const ( 46 | // initPosTol is the tolerance on the initial condition being feasible. Strictly, 47 | // the x should be positive, but instead it must be greater than -initPosTol. 48 | initPosTol = 1e-13 49 | // blandNegTol is the tolerance on the value being greater than 0 in the bland test. 50 | blandNegTol = 1e-14 51 | // rRoundTol is the tolerance for rounding values to zero when testing if 52 | // constraints are met. 53 | rRoundTol = 1e-13 54 | // dRoundTol is the tolerance for testing if values are zero for the problem 55 | // being unbounded. 56 | dRoundTol = 1e-13 57 | // phaseIZeroTol tests if the Phase I problem returned a feasible solution. 58 | phaseIZeroTol = 1e-12 59 | // blandZeroTol is the tolerance on testing if the bland solution can move. 60 | blandZeroTol = 1e-12 61 | ) 62 | 63 | // Simplex solves a linear program in standard form using Danzig's Simplex 64 | // algorithm. The standard form of a linear program is: 65 | // minimize c^T x 66 | // s.t. A*x = b 67 | // x >= 0 . 68 | // The input tol sets how close to the optimal solution is found (specifically, 69 | // when the maximal reduced cost is below tol). An error will be returned if the 70 | // problem is infeasible or unbounded. In rare cases, numeric errors can cause 71 | // the Simplex to fail. In this case, an error will be returned along with the 72 | // most recently found feasible solution. 73 | // 74 | // The Convert function can be used to transform a general LP into standard form. 75 | // 76 | // The input matrix A must have full rank and may not contain any columns with 77 | // all zeros. Furthermore, len(c) must equal the number of columns of A, and len(b) 78 | // must equal the number of rows of A. Simplex will panic if these conditions are 79 | // not met. 80 | // 81 | // initialBasic can be used to set the initial set of indices for a feasible 82 | // solution to the LP. If an initial feasible solution is not known, initialBasic 83 | // may be nil. If initialBasic is non-nil, len(initialBasic) must equal the number 84 | // of rows of A and must be an actual feasible solution to the LP, otherwise 85 | // Simplex will panic. 86 | // 87 | // A description of the Simplex algorithm can be found in Ch. 8 of 88 | // Strang, Gilbert. "Linear Algebra and Applications." Academic, New York (1976). 89 | // For a detailed video introduction, see lectures 11-13 of UC Math 352 90 | // https://www.youtube.com/watch?v=ESzYPFkY3og&index=11&list=PLh464gFUoJWOmBYla3zbZbc4nv2AXez6X. 91 | func Simplex(c []float64, A mat64.Matrix, b []float64, tol float64, initialBasic []int) (optF float64, optX []float64, err error) { 92 | ans, x, _, err := simplex(initialBasic, c, A, b, tol) 93 | return ans, x, err 94 | } 95 | 96 | func simplex(initialBasic []int, c []float64, A mat64.Matrix, b []float64, tol float64) (float64, []float64, []int, error) { 97 | err := verifyInputs(initialBasic, c, A, b) 98 | if err != nil { 99 | if err == ErrUnbounded { 100 | return math.Inf(-1), nil, nil, ErrUnbounded 101 | } 102 | return math.NaN(), nil, nil, err 103 | } 104 | m, n := A.Dims() 105 | 106 | // There is at least one optimal solution to the LP which is at the intersection 107 | // to a set of constraint boundaries. For a standard form LP with m variables 108 | // and n equality constraints, at least m-n elements of x must equal zero 109 | // at optimality. The Simplex algorithm solves the standard-form LP by starting 110 | // at an initial constraint vertex and successively moving to adjacent constraint 111 | // vertices. At every vertex, the set of non-zero x values is the "basic 112 | // feasible solution". The list of non-zero x's are maintained in basicIdxs, 113 | // the respective columns of A are in ab, and the actual non-zero values of 114 | // x are in xb. 115 | // 116 | // The LP is equality constrained such that A * x = b. This can be expanded 117 | // to 118 | // ab * xb + an * xn = b 119 | // where ab are the columns of a in the basic set, and an are all of the 120 | // other columns. Since each element of xn is zero by definition, this means 121 | // that for all feasible solutions xb = ab^-1 * b. 122 | // 123 | // Before the simplex algorithm can start, an initial feasible solution must 124 | // be found. If initialBasic is non-nil a feasible solution has been supplied. 125 | // Otherwise the "Phase I" problem must be solved to find an initial feasible 126 | // solution. 127 | 128 | var basicIdxs []int // The indices of the non-zero x values. 129 | var ab *mat64.Dense // The subset of columns of A listed in basicIdxs. 130 | var xb []float64 // The non-zero elements of x. xb = ab^-1 b 131 | 132 | if initialBasic != nil { 133 | // InitialBasic supplied. Panic if incorrect length or infeasible. 134 | if len(initialBasic) != m { 135 | panic("lp: incorrect number of initial vectors") 136 | } 137 | ab = mat64.NewDense(m, len(initialBasic), nil) 138 | extractColumns(ab, A, initialBasic) 139 | xb = make([]float64, m) 140 | err = initializeFromBasic(xb, ab, b) 141 | if err != nil { 142 | panic(err) 143 | } 144 | basicIdxs = make([]int, len(initialBasic)) 145 | copy(basicIdxs, initialBasic) 146 | } else { 147 | // No inital basis supplied. Solve the PhaseI problem. 148 | basicIdxs, ab, xb, err = findInitialBasic(A, b) 149 | if err != nil { 150 | return math.NaN(), nil, nil, err 151 | } 152 | } 153 | 154 | // basicIdxs contains the indexes for an initial feasible solution, 155 | // ab contains the extracted columns of A, and xb contains the feasible 156 | // solution. All x not in the basic set are 0 by construction. 157 | 158 | // nonBasicIdx is the set of nonbasic variables. 159 | nonBasicIdx := make([]int, 0, n-m) 160 | inBasic := make(map[int]struct{}) 161 | for _, v := range basicIdxs { 162 | inBasic[v] = struct{}{} 163 | } 164 | for i := 0; i < n; i++ { 165 | _, ok := inBasic[i] 166 | if !ok { 167 | nonBasicIdx = append(nonBasicIdx, i) 168 | } 169 | } 170 | 171 | // cb is the subset of c for the basic variables. an and cn 172 | // are the equivalents to ab and cb but for the nonbasic variables. 173 | cb := make([]float64, len(basicIdxs)) 174 | for i, idx := range basicIdxs { 175 | cb[i] = c[idx] 176 | } 177 | cn := make([]float64, len(nonBasicIdx)) 178 | for i, idx := range nonBasicIdx { 179 | cn[i] = c[idx] 180 | } 181 | an := mat64.NewDense(m, len(nonBasicIdx), nil) 182 | extractColumns(an, A, nonBasicIdx) 183 | 184 | bVec := mat64.NewVector(len(b), b) 185 | cbVec := mat64.NewVector(len(cb), cb) 186 | 187 | // Temporary data needed each iteration. (Described later) 188 | r := make([]float64, n-m) 189 | move := make([]float64, m) 190 | 191 | // Solve the linear program starting from the initial feasible set. This is 192 | // the "Phase 2" problem. 193 | // 194 | // Algorithm: 195 | // 1) Compute the "reduced costs" for the non-basic variables. The reduced 196 | // costs are the lagrange multipliers of the constraints. 197 | // r = cn - an^T * ab^-T * cb 198 | // 2) If all of the reduced costs are positive, no improvement is possible, 199 | // and the solution is optimal (xn can only increase because of 200 | // non-negativity constraints). Otherwise, the solution can be improved and 201 | // one element will be exchanged in the basic set. 202 | // 3) Choose the x_n with the most negative value of r. Call this value xe. 203 | // This variable will be swapped into the basic set. 204 | // 4) Increase xe until the next constraint boundary is met. This will happen 205 | // when the first element in xb becomes 0. The distance xe can increase before 206 | // a given element in xb becomes negative can be found from 207 | // xb = Ab^-1 b - Ab^-1 An xn 208 | // = Ab^-1 b - Ab^-1 Ae xe 209 | // = bhat + d x_e 210 | // xe = bhat_i / - d_i 211 | // where Ae is the column of A corresponding to xe. 212 | // The constraining basic index is the first index for which this is true, 213 | // so remove the element which is min_i (bhat_i / -d_i), assuming d_i is negative. 214 | // If no d_i is less than 0, then the problem is unbounded. 215 | // 5) If the new xe is 0 (that is, bhat_i == 0), then this location is at 216 | // the intersection of several constraints. Use the Bland rule instead 217 | // of the rule in step 4 to avoid cycling. 218 | for { 219 | // Compute reduced costs -- r = cn - an^T ab^-T cb 220 | var tmp mat64.Vector 221 | err = tmp.SolveVec(ab.T(), cbVec) 222 | if err != nil { 223 | break 224 | } 225 | data := make([]float64, n-m) 226 | tmp2 := mat64.NewVector(n-m, data) 227 | tmp2.MulVec(an.T(), &tmp) 228 | floats.SubTo(r, cn, data) 229 | 230 | // Replace the most negative element in the simplex. If there are no 231 | // negative entries then the optimal solution has been found. 232 | minIdx := floats.MinIdx(r) 233 | if r[minIdx] >= -tol { 234 | break 235 | } 236 | 237 | for i, v := range r { 238 | if math.Abs(v) < rRoundTol { 239 | r[i] = 0 240 | } 241 | } 242 | 243 | // Compute the moving distance. 244 | err = computeMove(move, minIdx, A, ab, xb, nonBasicIdx) 245 | if err != nil { 246 | if err == ErrUnbounded { 247 | return math.Inf(-1), nil, nil, ErrUnbounded 248 | } 249 | break 250 | } 251 | 252 | // Replace the basic index along the tightest constraint. 253 | replace := floats.MinIdx(move) 254 | if move[replace] <= 0 { 255 | replace, minIdx, err = replaceBland(A, ab, xb, basicIdxs, nonBasicIdx, r, move) 256 | if err != nil { 257 | if err == ErrUnbounded { 258 | return math.Inf(-1), nil, nil, ErrUnbounded 259 | } 260 | break 261 | } 262 | } 263 | 264 | // Replace the constrained basicIdx with the newIdx. 265 | basicIdxs[replace], nonBasicIdx[minIdx] = nonBasicIdx[minIdx], basicIdxs[replace] 266 | cb[replace], cn[minIdx] = cn[minIdx], cb[replace] 267 | tmpCol1 := mat64.Col(nil, replace, ab) 268 | tmpCol2 := mat64.Col(nil, minIdx, an) 269 | ab.SetCol(replace, tmpCol2) 270 | an.SetCol(minIdx, tmpCol1) 271 | 272 | // Compute the new xb. 273 | xbVec := mat64.NewVector(len(xb), xb) 274 | err = xbVec.SolveVec(ab, bVec) 275 | if err != nil { 276 | break 277 | } 278 | } 279 | // Found the optimum successfully or died trying. The basic variables get 280 | // their values, and the non-basic variables are all zero. 281 | opt := floats.Dot(cb, xb) 282 | xopt := make([]float64, n) 283 | for i, v := range basicIdxs { 284 | xopt[v] = xb[i] 285 | } 286 | return opt, xopt, basicIdxs, err 287 | } 288 | 289 | // computeMove computes how far can be moved replacing each index. The results 290 | // are stored into move. 291 | func computeMove(move []float64, minIdx int, A mat64.Matrix, ab *mat64.Dense, xb []float64, nonBasicIdx []int) error { 292 | // Find ae. 293 | col := mat64.Col(nil, nonBasicIdx[minIdx], A) 294 | aCol := mat64.NewVector(len(col), col) 295 | 296 | // d = - Ab^-1 Ae 297 | nb, _ := ab.Dims() 298 | d := make([]float64, nb) 299 | dVec := mat64.NewVector(nb, d) 300 | err := dVec.SolveVec(ab, aCol) 301 | if err != nil { 302 | return ErrLinSolve 303 | } 304 | floats.Scale(-1, d) 305 | 306 | for i, v := range d { 307 | if math.Abs(v) < dRoundTol { 308 | d[i] = 0 309 | } 310 | } 311 | 312 | // If no di < 0, then problem is unbounded. 313 | if floats.Min(d) >= 0 { 314 | return ErrUnbounded 315 | } 316 | 317 | // move = bhat_i / - d_i, assuming d is negative. 318 | bHat := xb // ab^-1 b 319 | for i, v := range d { 320 | if v >= 0 { 321 | move[i] = math.Inf(1) 322 | } else { 323 | move[i] = bHat[i] / math.Abs(v) 324 | } 325 | } 326 | return nil 327 | } 328 | 329 | // replaceBland uses the Bland rule to find the indices to swap if the minimum 330 | // move is 0. The indices to be swapped are replace and minIdx (following the 331 | // nomenclature in the main routine). 332 | func replaceBland(A mat64.Matrix, ab *mat64.Dense, xb []float64, basicIdxs, nonBasicIdx []int, r, move []float64) (replace, minIdx int, err error) { 333 | m, _ := A.Dims() 334 | // Use the traditional bland rule, except don't replace a constraint which 335 | // causes the new ab to be singular. 336 | for i, v := range r { 337 | if v > -blandNegTol { 338 | continue 339 | } 340 | minIdx = i 341 | err = computeMove(move, minIdx, A, ab, xb, nonBasicIdx) 342 | if err != nil { 343 | // Either unbounded or something went wrong. 344 | return -1, -1, err 345 | } 346 | replace = floats.MinIdx(move) 347 | if math.Abs(move[replace]) > blandZeroTol { 348 | // Large enough that it shouldn't be a problem 349 | return replace, minIdx, nil 350 | } 351 | // Find a zero index where replacement is non-singular. 352 | biCopy := make([]int, len(basicIdxs)) 353 | for replace, v := range move { 354 | if v > blandZeroTol { 355 | continue 356 | } 357 | copy(biCopy, basicIdxs) 358 | biCopy[replace] = nonBasicIdx[minIdx] 359 | abTmp := mat64.NewDense(m, len(biCopy), nil) 360 | extractColumns(abTmp, A, biCopy) 361 | // If the condition number is reasonable, use this index. 362 | if mat64.Cond(abTmp, 1) < 1e16 { 363 | return replace, minIdx, nil 364 | } 365 | } 366 | } 367 | return -1, -1, ErrBland 368 | } 369 | 370 | func verifyInputs(initialBasic []int, c []float64, A mat64.Matrix, b []float64) error { 371 | m, n := A.Dims() 372 | if len(c) != n { 373 | panic("lp: c vector incorrect length") 374 | } 375 | if len(b) != m { 376 | panic("lp: b vector incorrect length") 377 | } 378 | if len(c) != n { 379 | panic("lp: c vector incorrect length") 380 | } 381 | if len(initialBasic) != 0 && len(initialBasic) != m { 382 | panic("lp: initialBasic incorrect length") 383 | } 384 | 385 | // Do some sanity checks so that ab does not become singular during the 386 | // simplex solution. If the ZeroRow checks are removed then the code for 387 | // finding a set of linearly indepent columns must be improved. 388 | 389 | // Check that if a row of A only has zero elements that corresponding 390 | // element in b is zero, otherwise the problem is infeasible. 391 | // Otherwise return ErrZeroRow. 392 | for i := 0; i < m; i++ { 393 | isZero := true 394 | for j := 0; j < n; j++ { 395 | if A.At(i, j) != 0 { 396 | isZero = false 397 | break 398 | } 399 | } 400 | if isZero && b[i] != 0 { 401 | // Infeasible 402 | return ErrInfeasible 403 | } else if isZero { 404 | return ErrZeroRow 405 | } 406 | } 407 | // Check that if a column only has zero elements that the respective C vector 408 | // is positive (otherwise unbounded). Otherwise return ErrZeroColumn. 409 | for j := 0; j < n; j++ { 410 | isZero := true 411 | for i := 0; i < m; i++ { 412 | if A.At(i, j) != 0 { 413 | isZero = false 414 | break 415 | } 416 | } 417 | if isZero && c[j] < 0 { 418 | return ErrUnbounded 419 | } else if isZero { 420 | return ErrZeroColumn 421 | } 422 | } 423 | return nil 424 | } 425 | 426 | // initializeFromBasic initializes the basic feasible solution given a set of 427 | // basic indices. It extracts the columns of A specified by basicIdxs and finds 428 | // the x values at that location. These are stored into xb. 429 | // 430 | // If the columns of A are not linearly independent or if the initial set is not 431 | // feasible, an error is returned. 432 | func initializeFromBasic(xb []float64, ab *mat64.Dense, b []float64) error { 433 | m, _ := ab.Dims() 434 | if len(xb) != m { 435 | panic("simplex: bad xb length") 436 | } 437 | xbMat := mat64.NewVector(m, xb) 438 | 439 | err := xbMat.SolveVec(ab, mat64.NewVector(m, b)) 440 | if err != nil { 441 | return errors.New("lp: subcolumns of A for supplied initial basic singular") 442 | } 443 | // The solve ensures that the equality constraints are met (ab * xb = b). 444 | // Thus, the solution is feasible if and only if all of the x's are positive. 445 | allPos := true 446 | for _, v := range xb { 447 | if v < -initPosTol { 448 | allPos = false 449 | break 450 | } 451 | } 452 | if !allPos { 453 | return errors.New("lp: supplied subcolumns not a feasible solution") 454 | } 455 | return nil 456 | } 457 | 458 | // extractColumns copies the columns specified by cols into the columns of dst. 459 | func extractColumns(dst *mat64.Dense, A mat64.Matrix, cols []int) { 460 | r, c := dst.Dims() 461 | ra, _ := A.Dims() 462 | if ra != r { 463 | panic("simplex: row mismatch") 464 | } 465 | if c != len(cols) { 466 | panic("simplex: column mismatch") 467 | } 468 | col := make([]float64, r) 469 | for j, idx := range cols { 470 | mat64.Col(col, idx, A) 471 | dst.SetCol(j, col) 472 | } 473 | } 474 | 475 | // findInitialBasic finds an initial basic solution, and returns the basic 476 | // indices, ab, and xb. 477 | func findInitialBasic(A mat64.Matrix, b []float64) ([]int, *mat64.Dense, []float64, error) { 478 | m, n := A.Dims() 479 | basicIdxs := findLinearlyIndependent(A) 480 | if len(basicIdxs) != m { 481 | return nil, nil, nil, ErrSingular 482 | } 483 | 484 | // It may be that this linearly independent basis is also a feasible set. If 485 | // so, the Phase I problem can be avoided. 486 | ab := mat64.NewDense(m, len(basicIdxs), nil) 487 | extractColumns(ab, A, basicIdxs) 488 | xb := make([]float64, m) 489 | err := initializeFromBasic(xb, ab, b) 490 | if err == nil { 491 | return basicIdxs, ab, xb, nil 492 | } 493 | 494 | // This set was not feasible. Instead the "Phase I" problem must be solved 495 | // to find an initial feasible set of basis. 496 | // 497 | // Method: Construct an LP whose optimal solution is a feasible solution 498 | // to the original LP. 499 | // 1) Introduce an artificial variable x_{n+1}. 500 | // 2) Let x_j be the most negative element of x_b (largest constraint violation). 501 | // 3) Add the artificial variable to A with: 502 | // a_{n+1} = b - \sum_{i in basicIdxs} a_i + a_j 503 | // swap j with n+1 in the basicIdxs. 504 | // 4) Define a new LP: 505 | // minimize x_{n+1} 506 | // subject to [A A_{n+1}][x_1 ... x_{n+1}] = b 507 | // x, x_{n+1} >= 0 508 | // 5) Solve this LP. If x_{n+1} != 0, then the problem is infeasible, otherwise 509 | // the found basis can be used as an initial basis for phase II. 510 | // 511 | // The extra column in Step 3 is defined such that the vector of 1s is an 512 | // initial feasible solution. 513 | 514 | // Find the largest constraint violator. 515 | // Compute a_{n+1} = b - \sum{i in basicIdxs}a_i + a_j. j is in basicIDx, so 516 | // instead just subtract the basicIdx columns that are not minIDx. 517 | minIdx := floats.MinIdx(xb) 518 | aX1 := make([]float64, m) 519 | copy(aX1, b) 520 | col := make([]float64, m) 521 | for i, v := range basicIdxs { 522 | if i == minIdx { 523 | continue 524 | } 525 | mat64.Col(col, v, A) 526 | floats.Sub(aX1, col) 527 | } 528 | 529 | // Construct the new LP. 530 | // aNew = [A, a_{n+1}] 531 | // bNew = b 532 | // cNew = 1 for x_{n+1} 533 | aNew := mat64.NewDense(m, n+1, nil) 534 | aNew.Copy(A) 535 | aNew.SetCol(n, aX1) 536 | basicIdxs[minIdx] = n // swap minIdx with n in the basic set. 537 | c := make([]float64, n+1) 538 | c[n] = 1 539 | 540 | // Solve the Phase I linear program. 541 | _, xOpt, newBasic, err := simplex(basicIdxs, c, aNew, b, 1e-10) 542 | if err != nil { 543 | return nil, nil, nil, errors.New(fmt.Sprintf("lp: error finding feasible basis: %s", err)) 544 | } 545 | 546 | // The original LP is infeasible if the added variable has non-zero value 547 | // in the optimal solution to the Phase I problem. 548 | if math.Abs(xOpt[n]) > phaseIZeroTol { 549 | return nil, nil, nil, ErrInfeasible 550 | } 551 | 552 | // The basis found in Phase I is a feasible solution to the original LP if 553 | // the added variable is not in the basis. 554 | addedIdx := -1 555 | for i, v := range newBasic { 556 | if v == n { 557 | addedIdx = i 558 | } 559 | xb[i] = xOpt[v] 560 | } 561 | if addedIdx == -1 { 562 | extractColumns(ab, A, newBasic) 563 | return newBasic, ab, xb, nil 564 | } 565 | 566 | // The value of the added variable is in the basis, but it has a zero value. 567 | // See if exchanging another variable into the basic set finds a feasible 568 | // solution. 569 | basicMap := make(map[int]struct{}) 570 | for _, v := range newBasic { 571 | basicMap[v] = struct{}{} 572 | } 573 | var set bool 574 | for i := range xOpt { 575 | if _, inBasic := basicMap[i]; inBasic { 576 | continue 577 | } 578 | newBasic[addedIdx] = i 579 | if set { 580 | mat64.Col(col, i, A) 581 | ab.SetCol(addedIdx, col) 582 | } else { 583 | extractColumns(ab, A, newBasic) 584 | set = true 585 | } 586 | err := initializeFromBasic(xb, ab, b) 587 | if err == nil { 588 | return newBasic, ab, xb, nil 589 | } 590 | } 591 | return nil, nil, nil, ErrInfeasible 592 | } 593 | 594 | // findLinearlyIndependnt finds a set of linearly independent columns of A, and 595 | // returns the column indexes of the linearly independent columns. 596 | func findLinearlyIndependent(A mat64.Matrix) []int { 597 | m, n := A.Dims() 598 | idxs := make([]int, 0, m) 599 | columns := mat64.NewDense(m, m, nil) 600 | newCol := make([]float64, m) 601 | // Walk in reverse order because slack variables are typically the last columns 602 | // of A. 603 | for i := n - 1; i >= 0; i-- { 604 | if len(idxs) == m { 605 | break 606 | } 607 | mat64.Col(newCol, i, A) 608 | columns.SetCol(len(idxs), newCol) 609 | if len(idxs) == 0 { 610 | // A column is linearly independent from the null set. 611 | // If all-zero column of A are allowed, this code needs to be adjusted. 612 | idxs = append(idxs, i) 613 | continue 614 | } 615 | if mat64.Cond(columns.View(0, 0, m, len(idxs)+1), 1) > 1e12 { 616 | // Not linearly independent. 617 | continue 618 | } 619 | idxs = append(idxs, i) 620 | } 621 | return idxs 622 | } 623 | -------------------------------------------------------------------------------- /convex/lp/simplex_test.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2016 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package lp 6 | 7 | import ( 8 | "fmt" 9 | "math/rand" 10 | "testing" 11 | 12 | "github.com/gonum/floats" 13 | "github.com/gonum/matrix/mat64" 14 | ) 15 | 16 | const convergenceTol = 1e-10 17 | 18 | func TestSimplex(t *testing.T) { 19 | // First test specific inputs. These were collected from failures 20 | // during randomized testing. 21 | // TODO(btracey): Test specific problems with known solutions. 22 | for _, test := range []struct { 23 | A mat64.Matrix 24 | b []float64 25 | c []float64 26 | tol float64 27 | initialBasic []int 28 | }{ 29 | { 30 | // Basic feasible LP 31 | A: mat64.NewDense(2, 4, []float64{ 32 | -1, 2, 1, 0, 33 | 3, 1, 0, 1, 34 | }), 35 | b: []float64{4, 9}, 36 | c: []float64{-1, -2, 0, 0}, 37 | //initialBasic: nil, 38 | tol: 0, 39 | }, 40 | { 41 | // Zero row that caused linear solver failure 42 | A: mat64.NewDense(3, 5, []float64{0.09917822373225804, 0, 0, -0.2588175087223661, -0.5935518220870567, 1.301111422556007, 0.12220247487326946, 0, 0, -1.9194869979254463, 0, 0, 0, 0, -0.8588221231396473}), 43 | b: []float64{0, 0, 0}, 44 | c: []float64{0, 0.598992624019304, 0, 0, 0}, 45 | }, 46 | { 47 | // Case that caused linear solver failure 48 | A: mat64.NewDense(13, 26, []float64{-0.7001209024399848, -0.7027502615621812, -0, 0.7354444798695736, -0, 0.476457578966189, 0.7001209024399848, 0.7027502615621812, 0, -0.7354444798695736, 0, -0.476457578966189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1.8446087238391438, -0, -0, 0.7705609478497938, -0, -0, -2.7311218710244463, 0, 0, -0.7705609478497938, 0, 0, 2.7311218710244463, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -0, 0.8332519091897401, 0.7762132098737671, -0, -0, -0.052470638647269585, 0, -0.8332519091897401, -0.7762132098737671, 0, 0, 0.052470638647269585, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.9898577208292023, 0.31653724289408824, -0, -0, 0.17797227766447388, 1.2702427184954932, -0.7998764021535656, -0.31653724289408824, 0, 0, -0.17797227766447388, -1.2702427184954932, 0.7998764021535656, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -0, -0, -0, -0, 0.4206278126213235, -0.7253374879437113, 0, 0, 0, 0, -0.4206278126213235, 0.7253374879437113, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, -1, -0, -0.7567988418466963, 0.3304567624749696, 0.8385927625193501, -0.0021606686026376387, -0, 0, 0.7567988418466963, -0.3304567624749696, -0.8385927625193501, 0.0021606686026376387, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, -2.230107839590404, -0.9897104202085316, -0, 0.24703471683023603, -0, -2.382860345431941, 0.6206871648345162, 0.9897104202085316, 0, -0.24703471683023603, 0, 2.382860345431941, -0.6206871648345162, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, -1, 1.4350469221322282, -0.9730343818431852, -0, 2.326429855201535, -0, -0.14347849887004038, -1.4350469221322282, 0.9730343818431852, 0, -2.326429855201535, 0, 0.14347849887004038, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, -0.7943912888763849, -0.13735037357335078, -0.5101161104860161, -0, -0, -1.4790634590370297, 0.050911195996747316, 0.13735037357335078, 0.5101161104860161, 0, 0, 1.4790634590370297, -0.050911195996747316, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1, -0, -0.2515400440591492, 0.2058339272568599, -0, -0, -1.314023802253438, 0, 0.2515400440591492, -0.2058339272568599, 0, 0, 1.314023802253438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, -1, 0.08279503413614919, -0.16669071891829756, -0, -0.6208413721884664, -0, -0.6348258970402827, -0.08279503413614919, 0.16669071891829756, 0, 0.6208413721884664, 0, 0.6348258970402827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, -1, -0, -0, 0.49634739711260845, -0, -0, -0, 0, 0, -0.49634739711260845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, -1, -0.2797437186631715, -0.8356683570259136, 1.8970426594969672, -0.4095711945594497, 0.45831284820623924, -0.6109615338552246, 0.2797437186631715, 0.8356683570259136, -1.8970426594969672, 0.4095711945594497, -0.45831284820623924, 0.6109615338552246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, -1}), 49 | b: []float64{-0.8446087238391436, 0, 1.9898577208292023, 0, 0, -2.230107839590404, 0, 0.20560871112361512, 0, 0, 0, 0, 0}, 50 | c: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, 51 | }, 52 | { 53 | // Phase 1 of the above that panicked 54 | A: mat64.NewDense(26, 52, []float64{0.7001209024399848, -0, -0, -0.31653724289408824, -0, -0, 0.9897104202085316, -1.4350469221322282, 0.13735037357335078, -0, -0.08279503413614919, -0, 0.2797437186631715, -0.7001209024399848, 0, 0, 0.31653724289408824, 0, 0, -0.9897104202085316, 1.4350469221322282, -0.13735037357335078, 0, 0.08279503413614919, 0, -0.2797437186631715, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.7027502615621812, -0, -0.8332519091897401, -0, -0, 0.7567988418466963, -0, 0.9730343818431852, 0.5101161104860161, 0.2515400440591492, 0.16669071891829756, -0, 0.8356683570259136, -0.7027502615621812, 0, 0.8332519091897401, 0, 0, -0.7567988418466963, 0, -0.9730343818431852, -0.5101161104860161, -0.2515400440591492, -0.16669071891829756, 0, -0.8356683570259136, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, -0.7705609478497938, -0.7762132098737671, -0, -0, -0.3304567624749696, -0.24703471683023603, -0, -0, -0.2058339272568599, -0, -0.49634739711260845, -1.8970426594969672, 0, 0.7705609478497938, 0.7762132098737671, 0, 0, 0.3304567624749696, 0.24703471683023603, 0, 0, 0.2058339272568599, 0, 0.49634739711260845, 1.8970426594969672, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0.7354444798695736, -0, -0, -0.17797227766447388, -0, -0.8385927625193501, -0, -2.326429855201535, -0, -0, 0.6208413721884664, -0, 0.4095711945594497, 0.7354444798695736, 0, 0, 0.17797227766447388, 0, 0.8385927625193501, 0, 2.326429855201535, 0, 0, -0.6208413721884664, 0, -0.4095711945594497, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, -0, -0, -1.2702427184954932, -0.4206278126213235, 0.0021606686026376387, 2.382860345431941, -0, 1.4790634590370297, -0, -0, -0, -0.45831284820623924, 0, 0, 0, 1.2702427184954932, 0.4206278126213235, -0.0021606686026376387, -2.382860345431941, 0, -1.4790634590370297, 0, 0, 0, 0.45831284820623924, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0.476457578966189, 2.7311218710244463, 0.052470638647269585, 0.7998764021535656, 0.7253374879437113, -0, -0.6206871648345162, 0.14347849887004038, -0.050911195996747316, 1.314023802253438, 0.6348258970402827, -0, 0.6109615338552246, 0.476457578966189, -2.7311218710244463, -0.052470638647269585, -0.7998764021535656, -0.7253374879437113, 0, 0.6206871648345162, -0.14347849887004038, 0.050911195996747316, -1.314023802253438, -0.6348258970402827, 0, -0.6109615338552246, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0.7001209024399848, -0, -0, 0.31653724289408824, -0, -0, -0.9897104202085316, 1.4350469221322282, -0.13735037357335078, -0, 0.08279503413614919, -0, -0.2797437186631715, 0.7001209024399848, 0, 0, -0.31653724289408824, 0, 0, 0.9897104202085316, -1.4350469221322282, 0.13735037357335078, 0, -0.08279503413614919, 0, 0.2797437186631715, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0.7027502615621812, -0, 0.8332519091897401, -0, -0, -0.7567988418466963, -0, -0.9730343818431852, -0.5101161104860161, -0.2515400440591492, -0.16669071891829756, -0, -0.8356683570259136, 0.7027502615621812, 0, -0.8332519091897401, 0, 0, 0.7567988418466963, 0, 0.9730343818431852, 0.5101161104860161, 0.2515400440591492, 0.16669071891829756, 0, 0.8356683570259136, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, 0.7705609478497938, 0.7762132098737671, -0, -0, 0.3304567624749696, 0.24703471683023603, -0, -0, 0.2058339272568599, -0, 0.49634739711260845, 1.8970426594969672, 0, -0.7705609478497938, -0.7762132098737671, 0, 0, -0.3304567624749696, -0.24703471683023603, 0, 0, -0.2058339272568599, 0, -0.49634739711260845, -1.8970426594969672, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.7354444798695736, -0, -0, 0.17797227766447388, -0, 0.8385927625193501, -0, 2.326429855201535, -0, -0, -0.6208413721884664, -0, -0.4095711945594497, -0.7354444798695736, 0, 0, -0.17797227766447388, 0, -0.8385927625193501, 0, -2.326429855201535, 0, 0, 0.6208413721884664, 0, 0.4095711945594497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, -0, -0, 1.2702427184954932, 0.4206278126213235, -0.0021606686026376387, -2.382860345431941, -0, -1.4790634590370297, -0, -0, -0, 0.45831284820623924, 0, 0, 0, -1.2702427184954932, -0.4206278126213235, 0.0021606686026376387, 2.382860345431941, 0, 1.4790634590370297, 0, 0, 0, -0.45831284820623924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.476457578966189, -2.7311218710244463, -0.052470638647269585, -0.7998764021535656, -0.7253374879437113, -0, 0.6206871648345162, -0.14347849887004038, 0.050911195996747316, -1.314023802253438, -0.6348258970402827, -0, -0.6109615338552246, -0.476457578966189, 2.7311218710244463, 0.052470638647269585, 0.7998764021535656, 0.7253374879437113, 0, -0.6206871648345162, 0.14347849887004038, -0.050911195996747316, 1.314023802253438, 0.6348258970402827, 0, 0.6109615338552246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, -1, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, -0, -1, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, -0, -0, -1, -0, -0, -0, -0, -0, -0, -0, -0, -0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, -0, -0, -0, -1, -0, -0, -0, -0, -0, -0, -0, -0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0, -0, -0, -0, -0, -1, -0, -0, -0, -0, -0, -0, -0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, -0, -0, -0, -0, -0, -0, -1, -0, -0, -0, -0, -0, -0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, -0, -0, -0, -0, -0, -0, -0, -1, -0, -0, -0, -0, -0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, -0, -0, -0, -0, -0, -0, -0, -0, -1, -0, -0, -0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -1, -0, -0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -1, -0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -1, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1.8446087238391438, 1, -0.9898577208292023, 1, 1, 2.230107839590404, 1, 0.7943912888763849, 1, 1, 1, 1, 1, -1.8446087238391438, -1, 0.9898577208292023, -1, -1, -2.230107839590404, -1, -0.7943912888763849, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}), 55 | b: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, 56 | c: []float64{-0.8446087238391436, 0, 1.9898577208292023, 0, 0, -2.230107839590404, 0, 0.20560871112361512, 0, 0, 0, 0, 0, 0.8446087238391436, -0, -1.9898577208292023, -0, -0, 2.230107839590404, -0, -0.20560871112361512, -0, -0, -0, -0, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 57 | }, 58 | { 59 | // Dense case that panicked. 60 | A: mat64.NewDense(6, 15, []float64{0.3279477313560112, 0.04126296122557327, 0.24121743535067522, -0.8676933623438741, -0.3279477313560112, -0.04126296122557327, -0.24121743535067522, 0.8676933623438741, 1, 0, 0, 0, 0, 0, 1.3702148909442915, 0.43713186538468607, 0.8613818492485417, -0.9298615442657688, -0.037784779008231184, -0.43713186538468607, -0.8613818492485417, 0.9298615442657688, 0.037784779008231184, 0, 1, 0, 0, 0, 0, 0.3478112701177931, -0, 0.748352668598051, -0.4294796840343912, -0, 0, -0.748352668598051, 0.4294796840343912, 0, 0, 0, 1, 0, 0, 0, -1, -0, 1.1913912184457485, 1.732132186658447, 0.4026384828544584, 0, -1.1913912184457485, -1.732132186658447, -0.4026384828544584, 0, 0, 0, 1, 0, 0, -0.4598555419763902, -0, 1.2088959976921831, -0.7297794575275871, 1.9835614149566971, 0, -1.2088959976921831, 0.7297794575275871, -1.9835614149566971, 0, 0, 0, 0, 1, 0, 0.5986809560819324, 0.19738159369304414, -1.0647198575836367, -0, -0.7264943883762761, -0.19738159369304414, 1.0647198575836367, 0, 0.7264943883762761, 0, 0, 0, 0, 0, 1, 0.36269644970561576}), 61 | b: []float64{2.3702148909442915, 1.3478112701177931, 0, -0.4598555419763902, 1.5986809560819324, 1.3626964497056158}, 62 | c: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, 63 | }, 64 | { 65 | A: mat64.NewDense(6, 11, []float64{-0.036551083288733854, -0.8967234664797694, 0.036551083288733854, 0.8967234664797694, 1, 0, 0, 0, 0, 0, -0.719908817815329, -1.9043311904524263, -0, 1.9043311904524263, 0, 0, 1, 0, 0, 0, 0, -1.142213296802784, -0, 0.17584914855696687, 0, -0.17584914855696687, 0, 0, 1, 0, 0, 0, -0.5423586338987796, -0.21663357118058713, -0.4815354890024489, 0.21663357118058713, 0.4815354890024489, 0, 0, 0, 1, 0, 0, -0.6864090947259134, -0, -0, 0, 0, 0, 0, 0, 0, 1, 0, 0.4091621839837596, -1.1853040616164046, -0.11374085137543871, 1.1853040616164046, 0.11374085137543871, 0, 0, 0, 0, 0, 1, -1.7416078575675549}), 66 | b: []float64{0.28009118218467105, -0.14221329680278405, 0.4576413661012204, 0.3135909052740866, 1.4091621839837596, -1.7416078575675549}, 67 | c: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, 68 | initialBasic: []int{10, 8, 7, 6, 5, 4}, 69 | }, 70 | { 71 | A: mat64.NewDense(6, 10, []float64{-0.036551083288733854, -0.8967234664797694, 0.036551083288733854, 0.8967234664797694, 1, 0, 0, 0, 0, 0, -1.9043311904524263, -0, 1.9043311904524263, 0, 0, 1, 0, 0, 0, 0, -0, 0.17584914855696687, 0, -0.17584914855696687, 0, 0, 1, 0, 0, 0, -0.21663357118058713, -0.4815354890024489, 0.21663357118058713, 0.4815354890024489, 0, 0, 0, 1, 0, 0, -0, -0, 0, 0, 0, 0, 0, 0, 1, 0, -1.1853040616164046, -0.11374085137543871, 1.1853040616164046, 0.11374085137543871, 0, 0, 0, 0, 0, 1}), 72 | b: []float64{0.28009118218467105, -0.14221329680278405, 0.4576413661012204, 0.3135909052740866, 1.4091621839837596, -1.7416078575675549}, 73 | c: []float64{-1.1951160054922971, -1.354633418345746, 1.1951160054922971, 1.354633418345746, 0, 0, 0, 0, 0, 0}, 74 | initialBasic: []int{0, 8, 7, 6, 5, 4}, 75 | }, 76 | { 77 | A: mat64.NewDense(6, 14, []float64{-0.4398035705048233, -0, -1.1190414559968929, -0, 0.4398035705048233, 0, 1.1190414559968929, 0, 1, 0, 0, 0, 0, 0, -0, 0.45892918156139395, -0, -0, 0, -0.45892918156139395, 0, 0, 0, 1, 0, 0, 0, 0, -0, -0, -0.3163051515958635, -0, 0, 0, 0.3163051515958635, 0, 0, 0, 1, 0, 0, 0, -0, -0, -1.8226051692445888, -0.8154477101733032, 0, 0, 1.8226051692445888, 0.8154477101733032, 0, 0, 0, 1, 0, 0, -0, 1.0020104354806922, -2.80863692523519, -0.8493721031516384, 0, -1.0020104354806922, 2.80863692523519, 0.8493721031516384, 0, 0, 0, 0, 1, 0, -0.8292937871394104, -1.4615144665021647, -0, -0, 0.8292937871394104, 1.4615144665021647, 0, 0, 0, 0, 0, 0, 0, 1}), 78 | b: []float64{0, -1.0154749704172474, 0, 0, 0, -1.5002324315812783}, 79 | c: []float64{1.0665389045026794, 0.097366273706136, 0, 2.7928153636989954, -1.0665389045026794, -0.097366273706136, -0, -2.7928153636989954, 0, 0, 0, 0, 0, 0}, 80 | initialBasic: []int{5, 12, 11, 10, 0, 8}, 81 | }, 82 | { 83 | // Bad Phase I setup. 84 | A: mat64.NewDense(6, 7, []float64{1.4009742075419371, 0, 0.05737255493210325, -2.5954004393412915, 0, 1.561789236911904, 0, 0.17152506517602673, 0, 0, 0, 0, 0, -0.3458126550149948, 1.900744052464951, -0.32773164134097343, -0.9648201331251137, 0, 0, 0, 0, -1.3229549190526497, 0.0692227703722903, 0, 0, -0.1024297720479933, 0.4550740188869777, 0, 0.013599438965679167, 0, 0, 0, 0, 0, -0.1164365105021209, 0, 0, 0.4077091957443405, 1.5682816151954875, 0.8411734682369051, 0.22379142247562167, 1.2206581060250778}), 85 | b: []float64{0.3293809220378252, 0, -0.5688424847664554, 0, 0, 1.4832526082339592}, 86 | c: []float64{0.5246370956983506, -0.36608819899109946, 1.5854141981237713, 0.5170486527020665, 0, 1.4006819866163691, 0.7733814538809437}, 87 | }, 88 | { 89 | // The problem is feasible, but the PhaseI problem keeps the last 90 | // variable in the basis. 91 | A: mat64.NewDense(2, 3, []float64{0.7171320440380402, 0, 0.22818288617480836, 0, -0.10030202006494793, -0.3282372661549324}), 92 | b: []float64{0.8913013436978257, 0}, 93 | c: []float64{0, 0, 1.16796158316812}, 94 | initialBasic: nil, 95 | }, 96 | { 97 | // Case where primal was returned as feasible, but the dual was returned 98 | // as infeasible. This is the dual. 99 | // Here, the phase I problem returns the value in the basis but equal 100 | // to epsilon and not 0. 101 | A: mat64.NewDense(5, 11, []float64{0.48619717875196006, 0.5089083769874058, 1.4064796473022745, -0.48619717875196006, -0.5089083769874058, -1.4064796473022745, 1, 0, 0, 0, 0, 1.5169837857318682, -0, -0, -1.5169837857318682, 0, 0, 0, 1, 0, 0, 0, -1.3096160896447528, 0.12600426735917414, 0.296082394213142, 1.3096160896447528, -0.12600426735917414, -0.296082394213142, 0, 0, 1, 0, 0, -0, -0, 1.9870800277141467, 0, 0, -1.9870800277141467, 0, 0, 0, 1, 0, -0.3822356988571877, -0, -0.1793908926957139, 0.3822356988571877, 0, 0.1793908926957139, 0, 0, 0, 0, 1}), 102 | b: []float64{0.6015865977347667, 0, -1.5648780993757594, 0, 0}, 103 | c: []float64{-0.642801659201449, -0.5412741400343285, -1.4634460998530177, 0.642801659201449, 0.5412741400343285, 1.4634460998530177, 0, 0, 0, 0, 0}, 104 | }, 105 | { 106 | // Caused linear solve error. The error is because replacing the minimum 107 | // index in Bland causes the new basis to be singular. This 108 | // necessitates the ending loop in bland over possible moves. 109 | A: mat64.NewDense(9, 23, []float64{-0.898219823758102, -0, -0, -0, 1.067555075209233, 1.581598470243863, -1.0656096883610071, 0.898219823758102, 0, 0, 0, -1.067555075209233, -1.581598470243863, 1.0656096883610071, 1, 0, 0, 0, 0, 0, 0, 0, 0, -1.5657353278668433, 0.5798888118401012, -0, 0.14560553520321928, -0, -0, -0, 1.5657353278668433, -0.5798888118401012, 0, -0.14560553520321928, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, -0, -0, -1.5572250142582087, -0, -0, -0, -0, 0, 0, 1.5572250142582087, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, -0, -0, -0, -1.1266215512973428, -0, 1.0661059397023553, -0, 0, 0, 0, 1.1266215512973428, 0, -1.0661059397023553, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, -0, -2.060232129551813, 1.756900609902372, -0, -0, -0, -0, 0, 2.060232129551813, -1.756900609902372, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1.0628806512935949, -0, -0, 0.3306985942820342, -0, 0.5013194822231914, -0, -1.0628806512935949, 0, 0, -0.3306985942820342, 0, -0.5013194822231914, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, -0.02053916418367785, 2.0967009672108627, -0, 1.276296057052031, -0, -0.8396554873675388, -0, 0.02053916418367785, -2.0967009672108627, 0, -1.276296057052031, 0, 0.8396554873675388, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, -1.5173172721095745, -0, -0, -0, -0, -0.7781977786718928, -0.08927683907374018, 1.5173172721095745, 0, 0, 0, 0, 0.7781977786718928, 0.08927683907374018, 0, 0, 0, 0, 0, 0, 0, 1, 0, -0, -0, -0, -0, -0, 0.39773149008355624, -0, 0, 0, 0, 0, 0, -0.39773149008355624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}), 110 | b: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0}, 111 | c: []float64{0.24547850255842107, -0.9373919913433648, 0, 0, 0, 0.2961224049153204, 0, -0.24547850255842107, 0.9373919913433648, -0, -0, -0, -0.2961224049153204, -0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 112 | }, 113 | { 114 | // Caused error because ALL of the possible replacements in Bland cause. 115 | // ab to be singular. This necessitates outer loop in bland over possible 116 | // moves. 117 | A: mat64.NewDense(9, 23, []float64{0.6595219196440785, -0, -0, -1.8259394918781682, -0, -0, 0.005457361044175046, -0.6595219196440785, 0, 0, 1.8259394918781682, 0, 0, -0.005457361044175046, 1, 0, 0, 0, 0, 0, 0, 0, 0, -0, -0.10352878714214864, -0, -0, -0, -0, 0.5945016966696087, 0, 0.10352878714214864, 0, 0, 0, 0, -0.5945016966696087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0.31734882842876444, -0, -0, -0, -0, -0, -0.716633126367685, -0.31734882842876444, 0, 0, 0, 0, 0, 0.716633126367685, 0, 0, 1, 0, 0, 0, 0, 0, 0, -0.7769812182932578, -0, -0.17370050158829553, 0.19405062263734607, -0, 1.1472330031002533, -0.6776631768730962, 0.7769812182932578, 0, 0.17370050158829553, -0.19405062263734607, 0, -1.1472330031002533, 0.6776631768730962, 0, 0, 0, 1, 0, 0, 0, 0, 0, -0, 0.8285611486611473, -0, -0, -0, -0, -0, 0, -0.8285611486611473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, -2.088953453647358, 1.3286488791152795, -0, -0, -0, -0, 0.9147833235021142, 2.088953453647358, -1.3286488791152795, 0, 0, 0, 0, -0.9147833235021142, 0, 0, 0, 0, 0, 1, 0, 0, 0, -0, -0, -0, -0, 0.6560365621262937, -0, -0, 0, 0, 0, 0, -0.6560365621262937, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0.8957188338098074, -0, -0, -0, -0, -0, -0, -0.8957188338098074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, -0.2761381891117365, -0, -0, -0, 1.1154921426237823, 0.06429872020552618, -0, 0.2761381891117365, 0, 0, 0, -1.1154921426237823, -0.06429872020552618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}), 118 | b: []float64{0, 0, 0, 0, 0.5046208538522362, 1.0859412982429362, -2.066283584195025, 0, -0.2604305274353169}, 119 | c: []float64{0, 0, 0, 0, 0, 0, -0.05793762969330718, -0, -0, -0, -0, -0, -0, 0.05793762969330718, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 120 | initialBasic: []int{22, 11, 7, 19, 18, 17, 16, 15, 14}, 121 | }, 122 | { 123 | // Caused initial supplied basis of Phase I to be singular. 124 | A: mat64.NewDense(7, 11, []float64{0, 0, 0, 0, 0, 0.6667874223914787, -0.04779440888372957, -0.810020924434026, 0, 1.4190243477163373, 0, 0, 1.0452496826112936, 1.1966134226828076, 0, 0, 0, 0, -0.676136041089015, 0, 0, 0, 0, 0, 0, 0, 0, 0, -2.123232807871834, 0.2795467733707712, 0.21997115467272987, 0, -0.1572003980840453, 0, 0, 0, 0, 0.5130196002804861, 0, -0.005957174211761673, 0.3262874931735277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1.5582052881594286, 0, 0.3544026193217651, 0, -1.0761986709145068, 0, 0.2438593072108347, 0, 0, 0, 0, 1.387509848081664, 0, 0, 0.3958750570508226, 1.6281679612990678, 0, 0, -0.24638311667922103, 0, 0, 0, 0, 0, 0, -0.628850893994423}), 125 | b: []float64{0.4135281763629115, 0, 0, 0, 0, 0, 0}, 126 | c: []float64{0.5586772876113472, 0, 0.14261332948424457, 0, -0.016394076753000086, -0.506087285562544, 0, 0.37619482505459145, 1.2943822852419233, 0.5887960293578207, 0}, 127 | }, 128 | } { 129 | testSimplex(t, test.initialBasic, test.c, test.A, test.b, convergenceTol) 130 | } 131 | 132 | rnd := rand.New(rand.NewSource(1)) 133 | // Randomized tests 134 | testRandomSimplex(t, 20000, 0.7, 10, rnd) 135 | testRandomSimplex(t, 20000, 0, 10, rnd) 136 | testRandomSimplex(t, 200, 0, 100, rnd) 137 | testRandomSimplex(t, 2, 0, 400, rnd) 138 | } 139 | 140 | func testRandomSimplex(t *testing.T, nTest int, pZero float64, maxN int, rnd *rand.Rand) { 141 | // Try a bunch of random LPs 142 | for i := 0; i < nTest; i++ { 143 | n := rnd.Intn(maxN) + 2 // n must be at least two. 144 | m := rnd.Intn(n-1) + 1 // m must be between 1 and n 145 | if m == 0 || n == 0 { 146 | continue 147 | } 148 | randValue := func() float64 { 149 | //var pZero float64 150 | v := rnd.Float64() 151 | if v < pZero { 152 | return 0 153 | } 154 | return rnd.NormFloat64() 155 | } 156 | a := mat64.NewDense(m, n, nil) 157 | for i := 0; i < m; i++ { 158 | for j := 0; j < n; j++ { 159 | a.Set(i, j, randValue()) 160 | } 161 | } 162 | b := make([]float64, m) 163 | for i := range b { 164 | b[i] = randValue() 165 | } 166 | 167 | c := make([]float64, n) 168 | for i := range c { 169 | c[i] = randValue() 170 | } 171 | 172 | testSimplex(t, nil, c, a, b, convergenceTol) 173 | } 174 | } 175 | 176 | func testSimplex(t *testing.T, initialBasic []int, c []float64, a mat64.Matrix, b []float64, convergenceTol float64) error { 177 | primalOpt, primalX, _, errPrimal := simplex(initialBasic, c, a, b, convergenceTol) 178 | if errPrimal == nil { 179 | // No error solving the simplex, check that the solution is feasible. 180 | var bCheck mat64.Vector 181 | bCheck.MulVec(a, mat64.NewVector(len(primalX), primalX)) 182 | if !mat64.EqualApprox(&bCheck, mat64.NewVector(len(b), b), 1e-10) { 183 | t.Errorf("No error in primal but solution infeasible") 184 | } 185 | } 186 | 187 | primalInfeasible := errPrimal == ErrInfeasible 188 | primalUnbounded := errPrimal == ErrUnbounded 189 | primalBounded := errPrimal == nil 190 | primalASingular := errPrimal == ErrSingular 191 | primalZeroRow := errPrimal == ErrZeroRow 192 | primalZeroCol := errPrimal == ErrZeroColumn 193 | 194 | primalBad := !primalInfeasible && !primalUnbounded && !primalBounded && !primalASingular && !primalZeroRow && !primalZeroCol 195 | 196 | // It's an error if it's not one of the known returned errors. If it's 197 | // singular the problem is undefined and so the result cannot be compared 198 | // to the dual. 199 | if errPrimal == ErrSingular || primalBad { 200 | if primalBad { 201 | t.Errorf("non-known error returned: %s", errPrimal) 202 | } 203 | return errPrimal 204 | } 205 | 206 | // Compare the result to the answer found from solving the dual LP. 207 | 208 | // Construct and solve the dual LP. 209 | // Standard Form: 210 | // minimize c^T * x 211 | // subject to A * x = b, x >= 0 212 | // The dual of this problem is 213 | // maximize -b^T * nu 214 | // subject to A^T * nu + c >= 0 215 | // Which is 216 | // minimize b^T * nu 217 | // subject to -A^T * nu <= c 218 | 219 | negAT := &mat64.Dense{} 220 | negAT.Clone(a.T()) 221 | negAT.Scale(-1, negAT) 222 | cNew, aNew, bNew := Convert(b, negAT, c, nil, nil) 223 | 224 | dualOpt, dualX, _, errDual := simplex(nil, cNew, aNew, bNew, convergenceTol) 225 | if errDual == nil { 226 | // Check that the dual is feasible 227 | var bCheck mat64.Vector 228 | bCheck.MulVec(aNew, mat64.NewVector(len(dualX), dualX)) 229 | if !mat64.EqualApprox(&bCheck, mat64.NewVector(len(bNew), bNew), 1e-10) { 230 | t.Errorf("No error in dual but solution infeasible") 231 | } 232 | } 233 | 234 | // Check about the zero status. 235 | if errPrimal == ErrZeroRow || errPrimal == ErrZeroColumn { 236 | return errPrimal 237 | } 238 | 239 | // If the primal problem is feasible, then the primal and the dual should 240 | // be the same answer. We have flopped the sign in the dual (minimizing 241 | // b^T *nu instead of maximizing -b^T*nu), so flip it back. 242 | if errPrimal == nil { 243 | if errDual != nil { 244 | fmt.Println("errDual", errDual) 245 | panic("here") 246 | t.Errorf("Primal feasible but dual errored: %s", errDual) 247 | } 248 | dualOpt *= -1 249 | if !floats.EqualWithinAbsOrRel(dualOpt, primalOpt, convergenceTol, convergenceTol) { 250 | t.Errorf("Primal and dual value mismatch. Primal %v, dual %v.", primalOpt, dualOpt) 251 | } 252 | } 253 | // If the primal problem is unbounded, then the dual should be infeasible. 254 | if errPrimal == ErrUnbounded && errDual != ErrInfeasible { 255 | t.Errorf("Primal unbounded but dual not infeasible. ErrDual = %s", errDual) 256 | } 257 | 258 | // If the dual is unbounded, then the primal should be infeasible. 259 | if errDual == ErrUnbounded && errPrimal != ErrInfeasible { 260 | t.Errorf("Dual unbounded but primal not infeasible. ErrDual = %s", errPrimal) 261 | } 262 | 263 | // If the primal is infeasible, then the dual should be either infeasible 264 | // or unbounded. 265 | if errPrimal == ErrInfeasible { 266 | if errDual != ErrUnbounded && errDual != ErrInfeasible && errDual != ErrZeroColumn { 267 | t.Errorf("Primal infeasible but dual not infeasible or unbounded: %s", errDual) 268 | } 269 | } 270 | 271 | return errPrimal 272 | } 273 | -------------------------------------------------------------------------------- /convex/lp/simplexexample_test.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2016 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package lp_test 6 | 7 | import ( 8 | "fmt" 9 | "log" 10 | 11 | "github.com/gonum/matrix/mat64" 12 | "github.com/gonum/optimize/convex/lp" 13 | ) 14 | 15 | func ExampleSimplex() { 16 | c := []float64{-1, -2, 0, 0} 17 | A := mat64.NewDense(2, 4, []float64{-1, 2, 1, 0, 3, 1, 0, 1}) 18 | b := []float64{4, 9} 19 | 20 | opt, x, err := lp.Simplex(c, A, b, 0, nil) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | fmt.Printf("opt: %v\n", opt) 25 | fmt.Printf("x: %v\n", x) 26 | // Output: 27 | // opt: -8 28 | // x: [2 3 0 0] 29 | } 30 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // This repository is no longer maintained. 6 | // Development has moved to https://github.com/gonum/gonum. 7 | // 8 | // Package optimize implements algorithms for finding the optimum value of functions. 9 | package optimize 10 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "math" 11 | ) 12 | 13 | var ( 14 | // ErrZeroDimensional signifies an optimization was called with an input of length 0. 15 | ErrZeroDimensional = errors.New("optimize: zero dimensional input") 16 | 17 | // ErrLinesearcherFailure signifies that a Linesearcher has iterated too 18 | // many times. This may occur if the gradient tolerance is set too low. 19 | ErrLinesearcherFailure = errors.New("linesearch: failed to converge") 20 | 21 | // ErrNonDescentDirection signifies that LinesearchMethod has received a 22 | // search direction from a NextDirectioner in which the function is not 23 | // decreasing. 24 | ErrNonDescentDirection = errors.New("linesearch: non-descent search direction") 25 | 26 | // ErrNoProgress signifies that LinesearchMethod cannot make further 27 | // progress because there is no change in location after Linesearcher step 28 | // due to floating-point arithmetic. 29 | ErrNoProgress = errors.New("linesearch: no change in location after Linesearcher step") 30 | 31 | // ErrLinesearcherBound signifies that a Linesearcher reached a step that 32 | // lies out of allowed bounds. 33 | ErrLinesearcherBound = errors.New("linesearch: step out of bounds") 34 | ) 35 | 36 | // ErrFunc is returned when an initial function value is invalid. The error 37 | // state may be either +Inf or NaN. ErrFunc satisfies the error interface. 38 | type ErrFunc float64 39 | 40 | func (err ErrFunc) Error() string { 41 | switch { 42 | case math.IsInf(float64(err), 1): 43 | return "optimize: initial function value is infinite" 44 | case math.IsNaN(float64(err)): 45 | return "optimize: initial function value is NaN" 46 | default: 47 | panic("optimize: bad ErrFunc") 48 | } 49 | } 50 | 51 | // ErrGrad is returned when an initial gradient is invalid. The error gradient 52 | // may be either ±Inf or NaN. ErrGrad satisfies the error interface. 53 | type ErrGrad struct { 54 | Grad float64 // Grad is the invalid gradient value. 55 | Index int // Index is the position at which the invalid gradient was found. 56 | } 57 | 58 | func (err ErrGrad) Error() string { 59 | switch { 60 | case math.IsInf(err.Grad, 0): 61 | return fmt.Sprintf("optimize: initial gradient is infinite at position %d", err.Index) 62 | case math.IsNaN(err.Grad): 63 | return fmt.Sprintf("optimize: initial gradient is NaN at position %d", err.Index) 64 | default: 65 | panic("optimize: bad ErrGrad") 66 | } 67 | } 68 | 69 | // List of shared panic strings 70 | var ( 71 | badProblem = "optimize: objective function is undefined" 72 | ) 73 | -------------------------------------------------------------------------------- /functionconvergence.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import "math" 8 | 9 | // FunctionConverge tests for the convergence of function values. See comment 10 | // in Settings. 11 | type FunctionConverge struct { 12 | Absolute float64 13 | Relative float64 14 | Iterations int 15 | 16 | best float64 17 | iter int 18 | } 19 | 20 | func (fc *FunctionConverge) Init(f float64) { 21 | fc.best = f 22 | fc.iter = 0 23 | } 24 | 25 | func (fc *FunctionConverge) FunctionConverged(f float64) Status { 26 | if fc.Iterations == 0 { 27 | return NotTerminated 28 | } 29 | maxAbs := math.Max(math.Abs(f), math.Abs(fc.best)) 30 | if f < fc.best && fc.best-f > fc.Relative*maxAbs+fc.Absolute { 31 | fc.best = f 32 | fc.iter = 0 33 | return NotTerminated 34 | } 35 | fc.iter++ 36 | if fc.iter < fc.Iterations { 37 | return NotTerminated 38 | } 39 | return FunctionConvergence 40 | } 41 | -------------------------------------------------------------------------------- /functions/functions_test.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package functions 6 | 7 | import "testing" 8 | 9 | func TestBeale(t *testing.T) { 10 | tests := []funcTest{ 11 | { 12 | X: []float64{1, 1}, 13 | F: 14.203125, 14 | Gradient: []float64{0, 27.75}, 15 | }, 16 | { 17 | X: []float64{1, 4}, 18 | F: 4624.453125, 19 | Gradient: []float64{8813.25, 6585}, 20 | }, 21 | } 22 | testFunction(Beale{}, tests, t) 23 | } 24 | 25 | func TestBiggsEXP2(t *testing.T) { 26 | tests := []funcTest{ 27 | { 28 | X: []float64{1, 2}, 29 | F: 32.26255055084012, 30 | Gradient: []float64{8.308203800550878, -25.32607145221645}, 31 | }, 32 | } 33 | testFunction(BiggsEXP2{}, tests, t) 34 | } 35 | 36 | func TestBiggsEXP3(t *testing.T) { 37 | tests := []funcTest{ 38 | { 39 | X: []float64{1, 2, 1}, 40 | F: 1.598844540607779, 41 | Gradient: []float64{1.0633795027631927, -0.5196392672262664, -0.3180919155433357}, 42 | }, 43 | } 44 | testFunction(BiggsEXP3{}, tests, t) 45 | } 46 | 47 | func TestBiggsEXP4(t *testing.T) { 48 | tests := []funcTest{ 49 | { 50 | X: []float64{1, 2, 1, 1}, 51 | F: 1.598844540607779, 52 | Gradient: []float64{1.0633795027631927, -0.5196392672262664, 53 | -0.44245622408151464, -0.3180919155433357}, 54 | }, 55 | } 56 | testFunction(BiggsEXP4{}, tests, t) 57 | } 58 | 59 | func TestBiggsEXP5(t *testing.T) { 60 | tests := []funcTest{ 61 | { 62 | X: []float64{1, 2, 1, 1, 1}, 63 | F: 13.386420552801937, 64 | Gradient: []float64{-6.54665204477596, 3.5259856535515293, 65 | 14.36984212995392, -9.522506150695783, -19.639956134327882}, 66 | }, 67 | } 68 | testFunction(BiggsEXP5{}, tests, t) 69 | } 70 | 71 | func TestBiggsEXP6(t *testing.T) { 72 | tests := []funcTest{ 73 | { 74 | X: []float64{1, 2, 1, 1, 1, 1}, 75 | F: 0.77907007565597, 76 | Gradient: []float64{-0.149371887533426, -0.183163468182936, 77 | -1.483958013575642, 1.428277503849742, -0.149371887533426, 78 | -1.483958013575642}, 79 | }, 80 | } 81 | testFunction(BiggsEXP6{}, tests, t) 82 | } 83 | 84 | func TestBox3D(t *testing.T) { 85 | tests := []funcTest{ 86 | { 87 | X: []float64{0, 10, 20}, 88 | F: 1031.1538106093985, 89 | Gradient: []float64{98.22343149849218, -2.11937420675874, 112.38817362220350}, 90 | }, 91 | } 92 | testFunction(Box3D{}, tests, t) 93 | } 94 | 95 | func TestBrownBadlyScaled(t *testing.T) { 96 | tests := []funcTest{ 97 | { 98 | X: []float64{1, 1}, 99 | F: 999998000003, 100 | Gradient: []float64{-2e+6, -4e-6}, 101 | }, 102 | } 103 | testFunction(BrownBadlyScaled{}, tests, t) 104 | } 105 | 106 | // TODO(vladimir-ch): The minimum of BrownAndDennis is not known accurately 107 | // enough, which would force defaultGradTol to be unnecessarily large for the 108 | // tests to pass. This is the only function that causes problems, so disable 109 | // this test until the minimum is more accurate. 110 | // func TestBrownAndDennis(t *testing.T) { 111 | // tests := []funcTest{ 112 | // { 113 | // X: []float64{25, 5, -5, -1}, 114 | // F: 7926693.33699744, 115 | // Gradient: []float64{1149322.836365895, 1779291.674339785, -254579.585463521, -173400.429253115}, 116 | // }, 117 | // } 118 | // testFunction(BrownAndDennis{}, tests, t) 119 | // } 120 | 121 | func TestExtendedPowellSingular(t *testing.T) { 122 | tests := []funcTest{ 123 | { 124 | X: []float64{3, -1, 0, 3}, 125 | F: 95, 126 | Gradient: []float64{-14, -144, -22, 30}, 127 | }, 128 | { 129 | X: []float64{3, -1, 0, 3, 3, -1, 0, 3}, 130 | F: 190, 131 | Gradient: []float64{-14, -144, -22, 30, -14, -144, -22, 30}, 132 | }, 133 | } 134 | testFunction(ExtendedPowellSingular{}, tests, t) 135 | } 136 | 137 | func TestExtendedRosenbrock(t *testing.T) { 138 | tests := []funcTest{ 139 | { 140 | X: []float64{-1.2, 1}, 141 | F: 24.2, 142 | Gradient: []float64{-215.6, -88}, 143 | }, 144 | { 145 | X: []float64{-1.2, 1, -1.2}, 146 | F: 508.2, 147 | Gradient: []float64{-215.6, 792, -440}, 148 | }, 149 | { 150 | X: []float64{-1.2, 1, -1.2, 1}, 151 | F: 532.4, 152 | Gradient: []float64{-215.6, 792, -655.6, -88}, 153 | }, 154 | } 155 | testFunction(ExtendedRosenbrock{}, tests, t) 156 | } 157 | 158 | func TestGaussian(t *testing.T) { 159 | tests := []funcTest{ 160 | { 161 | X: []float64{0.4, 1, 0}, 162 | F: 3.88810699116688e-06, 163 | Gradient: []float64{7.41428466839991e-03, -7.44126392165149e-04, -5.30189685421989e-20}, 164 | }, 165 | } 166 | testFunction(Gaussian{}, tests, t) 167 | } 168 | 169 | func TestGulfResearchAndDevelopment(t *testing.T) { 170 | tests := []funcTest{ 171 | { 172 | X: []float64{5, 2.5, 0.15}, 173 | F: 12.11070582556949, 174 | Gradient: []float64{2.0879783574289799, 0.0345792619697154, -39.6766801029386400}, 175 | }, 176 | } 177 | testFunction(GulfResearchAndDevelopment{}, tests, t) 178 | } 179 | 180 | func TestHelicalValley(t *testing.T) { 181 | tests := []funcTest{ 182 | { 183 | X: []float64{-1, 0, 0}, 184 | F: 2500, 185 | Gradient: []float64{0, -1.59154943091895e+03, -1e+03}, 186 | }, 187 | } 188 | testFunction(HelicalValley{}, tests, t) 189 | } 190 | 191 | func TestPenaltyI(t *testing.T) { 192 | tests := []funcTest{ 193 | { 194 | X: []float64{1, 2, 3, 4}, 195 | F: 885.06264, 196 | Gradient: []float64{119, 238.00002, 357.00004, 476.00006}, 197 | }, 198 | { 199 | X: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 200 | F: 148032.56535, 201 | Gradient: []float64{1539, 3078.00002, 4617.00004, 6156.00006, 202 | 7695.00008, 9234.0001, 10773.00012, 12312.00014, 13851.00016, 15390.00018}, 203 | }, 204 | } 205 | testFunction(PenaltyI{}, tests, t) 206 | } 207 | 208 | func TestPenaltyII(t *testing.T) { 209 | tests := []funcTest{ 210 | { 211 | X: []float64{0.5, 0.5, 0.5, 0.5}, 212 | F: 2.34000880546302, 213 | Gradient: []float64{12.59999952896435, 8.99999885134508, 214 | 5.99999776830493, 2.99999875380719}, 215 | }, 216 | { 217 | X: []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5}, 218 | F: 162.65277656596712, 219 | Gradient: []float64{255.5999995289644, 229.4999988513451, 220 | 203.9999977683049, 178.4999965713605, 152.9999952485322, 221 | 127.4999937865809, 101.9999921708749, 76.4999903852436, 222 | 50.9999884118158, 25.4999938418451}, 223 | }, 224 | } 225 | testFunction(PenaltyII{}, tests, t) 226 | } 227 | 228 | func TestPowelBadlyScaled(t *testing.T) { 229 | tests := []funcTest{ 230 | { 231 | X: []float64{0, 1}, 232 | F: 1.13526171734838, 233 | Gradient: []float64{-2.00007355588823e+04, -2.70596990584991e-01}, 234 | }, 235 | } 236 | testFunction(PowellBadlyScaled{}, tests, t) 237 | } 238 | 239 | func TestTrigonometric(t *testing.T) { 240 | tests := []funcTest{ 241 | { 242 | X: []float64{0.5, 0.5}, 243 | F: 0.0126877761614045, 244 | Gradient: []float64{-0.00840962732040673, -0.09606967736232540}, 245 | }, 246 | { 247 | X: []float64{0.2, 0.2, 0.2, 0.2, 0.2}, 248 | F: 0.0116573789904718, 249 | Gradient: []float64{0.04568602319608119, -0.00896259022885634, 250 | -0.04777056509084983, -0.07073790138989976, -0.07786459912600564}, 251 | }, 252 | { 253 | X: []float64{0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1}, 254 | F: 0.00707575946622261, 255 | Gradient: []float64{0.03562782195259399, 0.01872017956076182, 256 | 0.00380754216611998, -0.00911009023133202, -0.02003271763159338, 257 | -0.02896034003466506, -0.03589295744054654, -0.04083056984923782, 258 | -0.04377317726073873, -0.04472077967504980}, 259 | }, 260 | } 261 | testFunction(Trigonometric{}, tests, t) 262 | } 263 | 264 | func TestVariablyDimensioned(t *testing.T) { 265 | tests := []funcTest{ 266 | { 267 | X: []float64{0.5, 0}, 268 | F: 46.5625, 269 | Gradient: []float64{-68.5, -137}, 270 | }, 271 | { 272 | X: []float64{2.0 / 3, 1.0 / 3, 0}, 273 | F: 497.60493827160514, 274 | Gradient: []float64{-416.518518518519, -833.037037037037, -1249.555555555556}, 275 | }, 276 | { 277 | X: []float64{0.75, 0.5, 0.25, 0}, 278 | F: 3222.1875, 279 | Gradient: []float64{-1703, -3406, -5109, -6812}, 280 | }, 281 | } 282 | testFunction(VariablyDimensioned{}, tests, t) 283 | } 284 | 285 | func TestWatson(t *testing.T) { 286 | tests := []funcTest{ 287 | { 288 | X: []float64{0, 0}, 289 | F: 30, 290 | Gradient: []float64{0, -60}, 291 | }, 292 | { 293 | X: []float64{0, 0, 0, 0, 0, 0}, 294 | F: 30, 295 | Gradient: []float64{0, -60, -60, -61.034482758620697, 296 | -62.068965517241381, -63.114928861371936}, 297 | }, 298 | { 299 | X: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0}, 300 | F: 30, 301 | Gradient: []float64{0, -60, -60, -61.034482758620697, 302 | -62.068965517241381, -63.114928861371936, -64.172372791012350, 303 | -65.241283655050239, -66.321647802373235}, 304 | }, 305 | { 306 | X: []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 307 | F: 30, 308 | Gradient: []float64{0, -60, -60, -61.034482758620697, 309 | -62.068965517241381, -63.114928861371936, -64.172372791012350, 310 | -65.241283655050239, -66.321647802373235, -67.413448880864095, 311 | -68.516667837400661, -69.631282933991471}, 312 | }, 313 | } 314 | testFunction(Watson{}, tests, t) 315 | } 316 | 317 | func TestWood(t *testing.T) { 318 | tests := []funcTest{ 319 | { 320 | X: []float64{-3, -1, -3, -1}, 321 | F: 19192, 322 | Gradient: []float64{-12008, -2080, -10808, -1880}, 323 | }, 324 | } 325 | testFunction(Wood{}, tests, t) 326 | } 327 | -------------------------------------------------------------------------------- /functions/minsurf.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package functions 6 | 7 | import ( 8 | "fmt" 9 | "math" 10 | ) 11 | 12 | // MinimalSurface implements a finite element approximation to a minimal 13 | // surface problem: determine the surface with minimal area and given boundary 14 | // values in a unit square centered at the origin. 15 | // 16 | // References: 17 | // Averick, M.B., Carter, R.G., Moré, J.J., Xue, G.-L.: The Minpack-2 Test 18 | // Problem Collection. Preprint MCS-P153-0692, Argonne National Laboratory (1992) 19 | type MinimalSurface struct { 20 | bottom, top []float64 21 | left, right []float64 22 | origin, step [2]float64 23 | } 24 | 25 | // NewMinimalSurface creates a new discrete minimal surface problem and 26 | // precomputes its boundary values. The problem is discretized on a rectilinear 27 | // grid with nx×ny nodes which means that the problem dimension is (nx-2)(ny-2). 28 | func NewMinimalSurface(nx, ny int) *MinimalSurface { 29 | ms := &MinimalSurface{ 30 | bottom: make([]float64, nx), 31 | top: make([]float64, nx), 32 | left: make([]float64, ny), 33 | right: make([]float64, ny), 34 | origin: [2]float64{-0.5, -0.5}, 35 | step: [2]float64{1 / float64(nx-1), 1 / float64(ny-1)}, 36 | } 37 | 38 | ms.initBoundary(ms.bottom, ms.origin[0], ms.origin[1], ms.step[0], 0) 39 | startY := ms.origin[1] + float64(ny-1)*ms.step[1] 40 | ms.initBoundary(ms.top, ms.origin[0], startY, ms.step[0], 0) 41 | ms.initBoundary(ms.left, ms.origin[0], ms.origin[1], 0, ms.step[1]) 42 | startX := ms.origin[0] + float64(nx-1)*ms.step[0] 43 | ms.initBoundary(ms.right, startX, ms.origin[1], 0, ms.step[1]) 44 | 45 | return ms 46 | } 47 | 48 | // Func returns the area of the surface represented by the vector x. 49 | func (ms *MinimalSurface) Func(x []float64) (area float64) { 50 | nx, ny := ms.Dims() 51 | if len(x) != (nx-2)*(ny-2) { 52 | panic("functions: problem size mismatch") 53 | } 54 | 55 | hx, hy := ms.Steps() 56 | for j := 0; j < ny-1; j++ { 57 | for i := 0; i < nx-1; i++ { 58 | vLL := ms.at(i, j, x) 59 | vLR := ms.at(i+1, j, x) 60 | vUL := ms.at(i, j+1, x) 61 | vUR := ms.at(i+1, j+1, x) 62 | 63 | dvLdx := (vLR - vLL) / hx 64 | dvLdy := (vUL - vLL) / hy 65 | dvUdx := (vUR - vUL) / hx 66 | dvUdy := (vUR - vLR) / hy 67 | 68 | fL := math.Sqrt(1 + dvLdx*dvLdx + dvLdy*dvLdy) 69 | fU := math.Sqrt(1 + dvUdx*dvUdx + dvUdy*dvUdy) 70 | area += fL + fU 71 | } 72 | } 73 | area *= 0.5 * hx * hy 74 | return area 75 | } 76 | 77 | // Grad evaluates the area gradient of the surface represented by the vector. 78 | func (ms *MinimalSurface) Grad(grad, x []float64) { 79 | nx, ny := ms.Dims() 80 | if len(x) != (nx-2)*(ny-2) { 81 | panic("functions: problem size mismatch") 82 | } 83 | if grad != nil && len(x) != len(grad) { 84 | panic("functions: unexpected size mismatch") 85 | } 86 | 87 | for i := range grad { 88 | grad[i] = 0 89 | } 90 | hx, hy := ms.Steps() 91 | for j := 0; j < ny-1; j++ { 92 | for i := 0; i < nx-1; i++ { 93 | vLL := ms.at(i, j, x) 94 | vLR := ms.at(i+1, j, x) 95 | vUL := ms.at(i, j+1, x) 96 | vUR := ms.at(i+1, j+1, x) 97 | 98 | dvLdx := (vLR - vLL) / hx 99 | dvLdy := (vUL - vLL) / hy 100 | dvUdx := (vUR - vUL) / hx 101 | dvUdy := (vUR - vLR) / hy 102 | 103 | fL := math.Sqrt(1 + dvLdx*dvLdx + dvLdy*dvLdy) 104 | fU := math.Sqrt(1 + dvUdx*dvUdx + dvUdy*dvUdy) 105 | 106 | if grad != nil { 107 | if i > 0 { 108 | if j > 0 { 109 | grad[ms.index(i, j)] -= (dvLdx/hx + dvLdy/hy) / fL 110 | } 111 | if j < ny-2 { 112 | grad[ms.index(i, j+1)] += (dvLdy/hy)/fL - (dvUdx/hx)/fU 113 | } 114 | } 115 | if i < nx-2 { 116 | if j > 0 { 117 | grad[ms.index(i+1, j)] += (dvLdx/hx)/fL - (dvUdy/hy)/fU 118 | } 119 | if j < ny-2 { 120 | grad[ms.index(i+1, j+1)] += (dvUdx/hx + dvUdy/hy) / fU 121 | } 122 | } 123 | } 124 | } 125 | 126 | } 127 | cellSize := 0.5 * hx * hy 128 | for i := range grad { 129 | grad[i] *= cellSize 130 | } 131 | } 132 | 133 | // InitX returns a starting location for the minimization problem. Length of 134 | // the returned slice is (nx-2)(ny-2). 135 | func (ms *MinimalSurface) InitX() []float64 { 136 | nx, ny := ms.Dims() 137 | x := make([]float64, (nx-2)*(ny-2)) 138 | for j := 1; j < ny-1; j++ { 139 | for i := 1; i < nx-1; i++ { 140 | x[ms.index(i, j)] = (ms.left[j] + ms.bottom[i]) / 2 141 | } 142 | } 143 | return x 144 | } 145 | 146 | // ExactX returns the exact solution to the _continuous_ minimization problem 147 | // projected on the interior nodes of the grid. Length of the returned slice is 148 | // (nx-2)(ny-2). 149 | func (ms *MinimalSurface) ExactX() []float64 { 150 | nx, ny := ms.Dims() 151 | v := make([]float64, (nx-2)*(ny-2)) 152 | for j := 1; j < ny-1; j++ { 153 | for i := 1; i < nx-1; i++ { 154 | v[ms.index(i, j)] = ms.ExactSolution(ms.x(i), ms.y(j)) 155 | } 156 | } 157 | return v 158 | } 159 | 160 | // ExactSolution returns the value of the exact solution to the minimal surface 161 | // problem at (x,y). The exact solution is 162 | // F_exact(x,y) = U^2(x,y) - V^2(x,y), 163 | // where U and V are the unique solutions to the equations 164 | // x = u + uv^2 - u^3/3, 165 | // y = -v - u^2v + v^3/3. 166 | func (ms *MinimalSurface) ExactSolution(x, y float64) float64 { 167 | var u = [2]float64{x, -y} 168 | var f [2]float64 169 | var jac [2][2]float64 170 | for k := 0; k < 100; k++ { 171 | f[0] = u[0] + u[0]*u[1]*u[1] - u[0]*u[0]*u[0]/3 - x 172 | f[1] = -u[1] - u[0]*u[0]*u[1] + u[1]*u[1]*u[1]/3 - y 173 | fNorm := math.Hypot(f[0], f[1]) 174 | if fNorm < 1e-13 { 175 | break 176 | } 177 | jac[0][0] = 1 + u[1]*u[1] - u[0]*u[0] 178 | jac[0][1] = 2 * u[0] * u[1] 179 | jac[1][0] = -2 * u[0] * u[1] 180 | jac[1][1] = -1 - u[0]*u[0] + u[1]*u[1] 181 | det := jac[0][0]*jac[1][1] - jac[0][1]*jac[1][0] 182 | u[0] -= (jac[1][1]*f[0] - jac[0][1]*f[1]) / det 183 | u[1] -= (jac[0][0]*f[1] - jac[1][0]*f[0]) / det 184 | } 185 | return u[0]*u[0] - u[1]*u[1] 186 | } 187 | 188 | // Dims returns the size of the underlying rectilinear grid. 189 | func (ms *MinimalSurface) Dims() (nx, ny int) { 190 | return len(ms.bottom), len(ms.left) 191 | } 192 | 193 | // Steps returns the spatial step sizes of the underlying rectilinear grid. 194 | func (ms *MinimalSurface) Steps() (hx, hy float64) { 195 | return ms.step[0], ms.step[1] 196 | } 197 | 198 | func (ms *MinimalSurface) x(i int) float64 { 199 | return ms.origin[0] + float64(i)*ms.step[0] 200 | } 201 | 202 | func (ms *MinimalSurface) y(j int) float64 { 203 | return ms.origin[1] + float64(j)*ms.step[1] 204 | } 205 | 206 | func (ms *MinimalSurface) at(i, j int, x []float64) float64 { 207 | nx, ny := ms.Dims() 208 | if i < 0 || i >= nx { 209 | panic(fmt.Sprintf("node [%v,%v] not on grid", i, j)) 210 | } 211 | if j < 0 || j >= ny { 212 | panic(fmt.Sprintf("node [%v,%v] not on grid", i, j)) 213 | } 214 | 215 | if i == 0 { 216 | return ms.left[j] 217 | } 218 | if j == 0 { 219 | return ms.bottom[i] 220 | } 221 | if i == nx-1 { 222 | return ms.right[j] 223 | } 224 | if j == ny-1 { 225 | return ms.top[i] 226 | } 227 | return x[ms.index(i, j)] 228 | } 229 | 230 | // index maps an interior grid node (i, j) to a one-dimensional index and 231 | // returns it. 232 | func (ms *MinimalSurface) index(i, j int) int { 233 | nx, ny := ms.Dims() 234 | if i <= 0 || i >= nx-1 { 235 | panic(fmt.Sprintf("[%v,%v] is not an interior node", i, j)) 236 | } 237 | if j <= 0 || j >= ny-1 { 238 | panic(fmt.Sprintf("[%v,%v] is not an interior node", i, j)) 239 | } 240 | 241 | return i - 1 + (j-1)*(nx-2) 242 | } 243 | 244 | // initBoundary initializes with the exact solution the boundary b whose i-th 245 | // element b[i] is located at [startX+i×hx, startY+i×hy]. 246 | func (ms *MinimalSurface) initBoundary(b []float64, startX, startY, hx, hy float64) { 247 | for i := range b { 248 | x := startX + float64(i)*hx 249 | y := startY + float64(i)*hy 250 | b[i] = ms.ExactSolution(x, y) 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /functions/minsurf_test.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package functions 6 | 7 | import ( 8 | "math" 9 | "testing" 10 | 11 | "github.com/gonum/diff/fd" 12 | "github.com/gonum/floats" 13 | ) 14 | 15 | func TestMinimalSurface(t *testing.T) { 16 | for _, size := range [][2]int{ 17 | {20, 30}, 18 | {30, 30}, 19 | {50, 40}, 20 | } { 21 | f := NewMinimalSurface(size[0], size[1]) 22 | x0 := f.InitX() 23 | grad := make([]float64, len(x0)) 24 | f.Grad(grad, x0) 25 | fdGrad := fd.Gradient(nil, f.Func, x0, &fd.Settings{Formula: fd.Central}) 26 | 27 | // Test that the numerical and analytical gradients agree. 28 | dist := floats.Distance(grad, fdGrad, math.Inf(1)) 29 | if dist > 1e-9 { 30 | t.Errorf("grid %v x %v: numerical and analytical gradient do not match. |fdGrad - grad|_∞ = %v", 31 | size[0], size[1], dist) 32 | } 33 | 34 | // Test that the gradient at the minimum is small enough. 35 | // In some sense this test is not completely correct because ExactX 36 | // returns the exact solution to the continuous problem projected on the 37 | // grid, not the exact solution to the discrete problem which we are 38 | // solving. This is the reason why a relatively loose tolerance 1e-4 39 | // must be used. 40 | xSol := f.ExactX() 41 | f.Grad(grad, xSol) 42 | norm := floats.Norm(grad, math.Inf(1)) 43 | if norm > 1e-4 { 44 | t.Errorf("grid %v x %v: gradient at the minimum not small enough. |grad|_∞ = %v", 45 | size[0], size[1], norm) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /functions/validate.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // This repository is no longer maintained. 6 | // Development has moved to https://github.com/gonum/gonum. 7 | // 8 | // Package functions provides objective functions for testing optimization 9 | // algorithms. 10 | // 11 | // We encourage outside contributions of additional test functions that exhibit 12 | // properties not already covered in the testing suite or that have 13 | // significance due to prior use as benchmark cases. 14 | package functions 15 | 16 | import ( 17 | "math" 18 | "testing" 19 | 20 | "github.com/gonum/diff/fd" 21 | "github.com/gonum/floats" 22 | ) 23 | 24 | // function represents an objective function. 25 | type function interface { 26 | Func(x []float64) float64 27 | } 28 | 29 | type gradient interface { 30 | Grad(grad, x []float64) 31 | } 32 | 33 | // minimumer is an objective function that can also provide information about 34 | // its minima. 35 | type minimumer interface { 36 | function 37 | 38 | // Minima returns _known_ minima of the function. 39 | Minima() []Minimum 40 | } 41 | 42 | // Minimum represents information about an optimal location of a function. 43 | type Minimum struct { 44 | // X is the location of the minimum. X may not be nil. 45 | X []float64 46 | // F is the value of the objective function at X. 47 | F float64 48 | // Global indicates if the location is a global minimum. 49 | Global bool 50 | } 51 | 52 | type funcTest struct { 53 | X []float64 54 | 55 | // F is the expected function value at X. 56 | F float64 57 | // Gradient is the expected gradient at X. If nil, it is not evaluated. 58 | Gradient []float64 59 | } 60 | 61 | // TODO(vladimir-ch): Decide and implement an exported testing function: 62 | // func Test(f Function, ??? ) ??? { 63 | // } 64 | 65 | const ( 66 | defaultTol = 1e-12 67 | defaultGradTol = 1e-9 68 | defaultFDGradTol = 1e-5 69 | ) 70 | 71 | // testFunction checks that the function can evaluate itself (and its gradient) 72 | // correctly. 73 | func testFunction(f function, ftests []funcTest, t *testing.T) { 74 | // Make a copy of tests because we may append to the slice. 75 | tests := make([]funcTest, len(ftests)) 76 | copy(tests, ftests) 77 | 78 | // Get information about the function. 79 | fMinima, isMinimumer := f.(minimumer) 80 | fGradient, isGradient := f.(gradient) 81 | 82 | // If the function is a Minimumer, append its minima to the tests. 83 | if isMinimumer { 84 | for _, minimum := range fMinima.Minima() { 85 | // Allocate gradient only if the function can evaluate it. 86 | var grad []float64 87 | if isGradient { 88 | grad = make([]float64, len(minimum.X)) 89 | } 90 | tests = append(tests, funcTest{ 91 | X: minimum.X, 92 | F: minimum.F, 93 | Gradient: grad, 94 | }) 95 | } 96 | } 97 | 98 | for i, test := range tests { 99 | F := f.Func(test.X) 100 | 101 | // Check that the function value is as expected. 102 | if math.Abs(F-test.F) > defaultTol { 103 | t.Errorf("Test #%d: function value given by Func is incorrect. Want: %v, Got: %v", 104 | i, test.F, F) 105 | } 106 | 107 | if test.Gradient == nil { 108 | continue 109 | } 110 | 111 | // Evaluate the finite difference gradient. 112 | fdGrad := fd.Gradient(nil, f.Func, test.X, &fd.Settings{ 113 | Formula: fd.Central, 114 | Step: 1e-6, 115 | }) 116 | 117 | // Check that the finite difference and expected gradients match. 118 | if !floats.EqualApprox(fdGrad, test.Gradient, defaultFDGradTol) { 119 | dist := floats.Distance(fdGrad, test.Gradient, math.Inf(1)) 120 | t.Errorf("Test #%d: numerical and expected gradients do not match. |fdGrad - WantGrad|_∞ = %v", 121 | i, dist) 122 | } 123 | 124 | // If the function is a Gradient, check that it computes the gradient correctly. 125 | if isGradient { 126 | grad := make([]float64, len(test.Gradient)) 127 | fGradient.Grad(grad, test.X) 128 | 129 | if !floats.EqualApprox(grad, test.Gradient, defaultGradTol) { 130 | dist := floats.Distance(grad, test.Gradient, math.Inf(1)) 131 | t.Errorf("Test #%d: gradient given by Grad is incorrect. |grad - WantGrad|_∞ = %v", 132 | i, dist) 133 | } 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /global.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2016 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "math" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | // GlobalMethod is a global optimizer. Typically will require more function 14 | // evaluations and no sense of local convergence 15 | type GlobalMethod interface { 16 | // Global tells method the max number of tasks, method returns how many it wants. 17 | // This is needed to sync the Global goroutines and inside goroutines. 18 | InitGlobal(dim, tasks int) int 19 | // Global method may assume that the same task id always has the same pointer with it. 20 | IterateGlobal(task int, loc *Location) (Operation, error) 21 | Needser 22 | // Done communicates to the optimization method that the optimization has 23 | // concluded to allow for shutdown. 24 | Done() 25 | } 26 | 27 | // Global uses a global optimizer to search for the gloabl minimum of a 28 | // function. A maximization problem can be transformed into a 29 | // minimization problem by multiplying the function by -1. 30 | // 31 | // The first argument represents the problem to be minimized. Its fields are 32 | // routines that evaluate the objective function, gradient, and other 33 | // quantities related to the problem. The objective function, p.Func, must not 34 | // be nil. The optimization method used may require other fields to be non-nil 35 | // as specified by method.Needs. Global will panic if these are not met. The 36 | // method can be determined automatically from the supplied problem which is 37 | // described below. 38 | // 39 | // If p.Status is not nil, it is called before every evaluation. If the 40 | // returned Status is not NotTerminated or the error is not nil, the 41 | // optimization run is terminated. 42 | // 43 | // The third argument contains the settings for the minimization. The 44 | // DefaultGlobalSettings function can be called for a Settings struct with the 45 | // default values initialized. If settings == nil, the default settings are used. 46 | // Global optimization methods typically do not make assumptions about the number 47 | // and location of local minima. Thus, the only convergence metric used is the 48 | // function values found at major iterations of the optimization. Bounds on the 49 | // length of optimization are obeyed, such as the number of allowed function 50 | // evaluations. 51 | // 52 | // The final argument is the optimization method to use. If method == nil, then 53 | // an appropriate default is chosen based on the properties of the other arguments 54 | // (dimension, gradient-free or gradient-based, etc.). 55 | // 56 | // If method implements Statuser, method.Status is called before every call 57 | // to method.Iterate. If the returned Status is not NotTerminated or the 58 | // error is non-nil, the optimization run is terminated. 59 | // 60 | // Global returns a Result struct and any error that occurred. See the 61 | // documentation of Result for more information. 62 | // 63 | // Be aware that the default behavior of Global is to find the minimum. 64 | // For certain functions and optimization methods, this process can take many 65 | // function evaluations. If you would like to put limits on this, for example 66 | // maximum runtime or maximum function evaluations, modify the Settings 67 | // input struct. 68 | // 69 | // Something about Global cannot guarantee strict bounds on function evaluations, 70 | // iterations, etc. in the precense of concurrency. 71 | func Global(p Problem, dim int, settings *Settings, method GlobalMethod) (*Result, error) { 72 | startTime := time.Now() 73 | if method == nil { 74 | method = &GuessAndCheck{} 75 | } 76 | if settings == nil { 77 | settings = DefaultSettingsGlobal() 78 | } 79 | stats := &Stats{} 80 | err := checkOptimization(p, dim, method, settings.Recorder) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | optLoc := newLocation(dim, method) 86 | optLoc.F = math.Inf(1) 87 | 88 | if settings.FunctionConverge != nil { 89 | settings.FunctionConverge.Init(optLoc.F) 90 | } 91 | 92 | stats.Runtime = time.Since(startTime) 93 | 94 | // Send initial location to Recorder 95 | if settings.Recorder != nil { 96 | err = settings.Recorder.Record(optLoc, InitIteration, stats) 97 | if err != nil { 98 | return nil, err 99 | } 100 | } 101 | 102 | // Run optimization 103 | var status Status 104 | status, err = minimizeGlobal(&p, method, settings, stats, optLoc, startTime) 105 | 106 | // Cleanup and collect results 107 | if settings.Recorder != nil && err == nil { 108 | err = settings.Recorder.Record(optLoc, PostIteration, stats) 109 | } 110 | stats.Runtime = time.Since(startTime) 111 | return &Result{ 112 | Location: *optLoc, 113 | Stats: *stats, 114 | Status: status, 115 | }, err 116 | } 117 | 118 | func minimizeGlobal(p *Problem, method GlobalMethod, settings *Settings, stats *Stats, optLoc *Location, startTime time.Time) (status Status, err error) { 119 | dim := len(optLoc.X) 120 | statuser, _ := method.(Statuser) 121 | gs := &globalStatus{ 122 | mux: &sync.RWMutex{}, 123 | stats: stats, 124 | status: NotTerminated, 125 | p: p, 126 | startTime: startTime, 127 | optLoc: optLoc, 128 | settings: settings, 129 | statuser: statuser, 130 | } 131 | 132 | nTasks := settings.Concurrent 133 | nTasks = method.InitGlobal(dim, nTasks) 134 | 135 | // Launch optimization workers 136 | var wg sync.WaitGroup 137 | for task := 0; task < nTasks; task++ { 138 | wg.Add(1) 139 | go func(task int) { 140 | defer wg.Done() 141 | loc := newLocation(dim, method) 142 | x := make([]float64, dim) 143 | globalWorker(task, method, gs, loc, x) 144 | }(task) 145 | } 146 | wg.Wait() 147 | method.Done() 148 | return gs.status, gs.err 149 | } 150 | 151 | type globalStatus struct { 152 | mux *sync.RWMutex 153 | stats *Stats 154 | status Status 155 | p *Problem 156 | startTime time.Time 157 | optLoc *Location 158 | settings *Settings 159 | method GlobalMethod 160 | statuser Statuser 161 | err error 162 | } 163 | 164 | func globalWorker(task int, m GlobalMethod, g *globalStatus, loc *Location, x []float64) { 165 | for { 166 | // Find Evaluation location 167 | op, err := m.IterateGlobal(task, loc) 168 | if err != nil { 169 | // TODO(btracey): Figure out how to handle errors properly. Shut 170 | // everything down? Pass to globalStatus so it can shut everything down? 171 | g.mux.Lock() 172 | g.err = err 173 | g.status = Failure 174 | g.mux.Unlock() 175 | break 176 | } 177 | 178 | // Evaluate location and/or update stats. 179 | status := g.globalOperation(op, loc, x) 180 | if status != NotTerminated { 181 | break 182 | } 183 | } 184 | } 185 | 186 | // globalOperation updates handles the status received by an individual worker. 187 | // It uses a mutex to protect updates where necessary. 188 | func (g *globalStatus) globalOperation(op Operation, loc *Location, x []float64) Status { 189 | // Do a quick check to see if one of the other workers converged in the meantime. 190 | var status Status 191 | var err error 192 | g.mux.RLock() 193 | status = g.status 194 | g.mux.RUnlock() 195 | if status != NotTerminated { 196 | return status 197 | } 198 | switch op { 199 | case NoOperation: 200 | case InitIteration: 201 | panic("optimize: Method returned InitIteration") 202 | case PostIteration: 203 | panic("optimize: Method returned PostIteration") 204 | case MajorIteration: 205 | g.mux.Lock() 206 | g.stats.MajorIterations++ 207 | copyLocation(g.optLoc, loc) 208 | g.mux.Unlock() 209 | 210 | g.mux.RLock() 211 | status = checkConvergence(g.optLoc, g.settings, false) 212 | g.mux.RUnlock() 213 | default: // Any of the Evaluation operations. 214 | status, err = evaluate(g.p, loc, op, x) 215 | g.mux.Lock() 216 | updateStats(g.stats, op) 217 | g.mux.Unlock() 218 | } 219 | 220 | g.mux.Lock() 221 | status, err = iterCleanup(status, err, g.stats, g.settings, g.statuser, g.startTime, loc, op) 222 | // Update the termination status if it hasn't already terminated. 223 | if g.status == NotTerminated { 224 | g.status = status 225 | g.err = err 226 | } 227 | g.mux.Unlock() 228 | 229 | return status 230 | } 231 | 232 | func DefaultSettingsGlobal() *Settings { 233 | return &Settings{ 234 | FunctionThreshold: math.Inf(-1), 235 | FunctionConverge: &FunctionConverge{ 236 | Absolute: 1e-10, 237 | Iterations: 100, 238 | }, 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /gradientdescent.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import "github.com/gonum/floats" 8 | 9 | // GradientDescent implements the steepest descent optimization method that 10 | // performs successive steps along the direction of the negative gradient. 11 | type GradientDescent struct { 12 | // Linesearcher selects suitable steps along the descent direction. 13 | // If Linesearcher is nil, a reasonable default will be chosen. 14 | Linesearcher Linesearcher 15 | // StepSizer determines the initial step size along each direction. 16 | // If StepSizer is nil, a reasonable default will be chosen. 17 | StepSizer StepSizer 18 | 19 | ls *LinesearchMethod 20 | } 21 | 22 | func (g *GradientDescent) Init(loc *Location) (Operation, error) { 23 | if g.Linesearcher == nil { 24 | g.Linesearcher = &Backtracking{} 25 | } 26 | if g.StepSizer == nil { 27 | g.StepSizer = &QuadraticStepSize{} 28 | } 29 | 30 | if g.ls == nil { 31 | g.ls = &LinesearchMethod{} 32 | } 33 | g.ls.Linesearcher = g.Linesearcher 34 | g.ls.NextDirectioner = g 35 | 36 | return g.ls.Init(loc) 37 | } 38 | 39 | func (g *GradientDescent) Iterate(loc *Location) (Operation, error) { 40 | return g.ls.Iterate(loc) 41 | } 42 | 43 | func (g *GradientDescent) InitDirection(loc *Location, dir []float64) (stepSize float64) { 44 | copy(dir, loc.Gradient) 45 | floats.Scale(-1, dir) 46 | return g.StepSizer.Init(loc, dir) 47 | } 48 | 49 | func (g *GradientDescent) NextDirection(loc *Location, dir []float64) (stepSize float64) { 50 | copy(dir, loc.Gradient) 51 | floats.Scale(-1, dir) 52 | return g.StepSizer.StepSize(loc, dir) 53 | } 54 | 55 | func (*GradientDescent) Needs() struct { 56 | Gradient bool 57 | Hessian bool 58 | } { 59 | return struct { 60 | Gradient bool 61 | Hessian bool 62 | }{true, false} 63 | } 64 | -------------------------------------------------------------------------------- /guessandcheck.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2016 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "math" 9 | "sync" 10 | 11 | "github.com/gonum/stat/distmv" 12 | ) 13 | 14 | // GuessAndCheck is a global optimizer that evaluates the function at random 15 | // locations. Not a good optimizer, but useful for comparison and debugging. 16 | type GuessAndCheck struct { 17 | Rander distmv.Rander 18 | 19 | eval []bool 20 | 21 | mux *sync.Mutex 22 | bestF float64 23 | bestX []float64 24 | } 25 | 26 | func (g *GuessAndCheck) Needs() struct{ Gradient, Hessian bool } { 27 | return struct{ Gradient, Hessian bool }{false, false} 28 | } 29 | 30 | func (g *GuessAndCheck) Done() { 31 | // No cleanup needed 32 | } 33 | 34 | func (g *GuessAndCheck) InitGlobal(dim, tasks int) int { 35 | g.eval = make([]bool, tasks) 36 | g.bestF = math.Inf(1) 37 | g.bestX = resize(g.bestX, dim) 38 | g.mux = &sync.Mutex{} 39 | return tasks 40 | } 41 | 42 | func (g *GuessAndCheck) IterateGlobal(task int, loc *Location) (Operation, error) { 43 | // Task is true if it contains a new function evaluation. 44 | if g.eval[task] { 45 | g.eval[task] = false 46 | g.mux.Lock() 47 | if loc.F < g.bestF { 48 | g.bestF = loc.F 49 | copy(g.bestX, loc.X) 50 | } else { 51 | loc.F = g.bestF 52 | copy(loc.X, g.bestX) 53 | } 54 | g.mux.Unlock() 55 | return MajorIteration, nil 56 | } 57 | g.eval[task] = true 58 | g.Rander.Rand(loc.X) 59 | return FuncEvaluation, nil 60 | } 61 | -------------------------------------------------------------------------------- /guessandcheck_test.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2016 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/gonum/matrix/mat64" 11 | "github.com/gonum/optimize/functions" 12 | "github.com/gonum/stat/distmv" 13 | ) 14 | 15 | func TestGuessAndCheck(t *testing.T) { 16 | dim := 3000 17 | problem := Problem{ 18 | Func: functions.ExtendedRosenbrock{}.Func, 19 | } 20 | mu := make([]float64, dim) 21 | sigma := mat64.NewSymDense(dim, nil) 22 | for i := 0; i < dim; i++ { 23 | sigma.SetSym(i, i, 1) 24 | } 25 | d, ok := distmv.NewNormal(mu, sigma, nil) 26 | if !ok { 27 | panic("bad test") 28 | } 29 | Global(problem, dim, nil, &GuessAndCheck{Rander: d}) 30 | settings := DefaultSettingsGlobal() 31 | settings.Concurrent = 5 32 | settings.MajorIterations = 15 33 | Global(problem, dim, settings, &GuessAndCheck{Rander: d}) 34 | } 35 | -------------------------------------------------------------------------------- /interfaces.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | // A Method can optimize an objective function. 8 | // 9 | // It uses a reverse-communication interface between the optimization method 10 | // and the caller. Method acts as a client that asks the caller to perform 11 | // needed operations via Operation returned from Init and Iterate methods. 12 | // This provides independence of the optimization algorithm on user-supplied 13 | // data and their representation, and enables automation of common operations 14 | // like checking for (various types of) convergence and maintaining statistics. 15 | // 16 | // A Method can command an Evaluation, a MajorIteration or NoOperation operations. 17 | // 18 | // An evaluation operation is one or more of the Evaluation operations 19 | // (FuncEvaluation, GradEvaluation, etc.) which can be combined with 20 | // the bitwise or operator. In an evaluation operation, the requested fields of 21 | // Problem will be evaluated at the point specified in Location.X. 22 | // The corresponding fields of Location will be filled with the results that 23 | // can be retrieved upon the next call to Iterate. The Method interface 24 | // requires that entries of Location are not modified aside from the commanded 25 | // evaluations. Thus, the type implementing Method may use multiple Operations 26 | // to set the Location fields at a particular x value. 27 | // 28 | // Instead of an Evaluation, a Method may declare MajorIteration. In 29 | // a MajorIteration, the values in the fields of Location are treated as 30 | // a potential optimizer. The convergence of the optimization routine 31 | // (GradientThreshold, etc.) is checked at this new best point. In 32 | // a MajorIteration, the fields of Location must be valid and consistent. 33 | // 34 | // A Method must not return InitIteration and PostIteration operations. These are 35 | // reserved for the clients to be passed to Recorders. A Method must also not 36 | // combine the Evaluation operations with the Iteration operations. 37 | type Method interface { 38 | // Init initializes the method based on the initial data in loc, updates it 39 | // and returns the first operation to be carried out by the caller. 40 | // The initial location must be valid as specified by Needs. 41 | Init(loc *Location) (Operation, error) 42 | 43 | // Iterate retrieves data from loc, performs one iteration of the method, 44 | // updates loc and returns the next operation. 45 | Iterate(loc *Location) (Operation, error) 46 | 47 | Needser 48 | } 49 | 50 | type Needser interface { 51 | // Needs specifies information about the objective function needed by the 52 | // optimizer beyond just the function value. The information is used 53 | // internally for initialization and must match evaluation types returned 54 | // by Init and Iterate during the optimization process. 55 | Needs() struct { 56 | Gradient bool 57 | Hessian bool 58 | } 59 | } 60 | 61 | // Statuser can report the status and any error. It is intended for methods as 62 | // an additional error reporting mechanism apart from the errors returned from 63 | // Init and Iterate. 64 | type Statuser interface { 65 | Status() (Status, error) 66 | } 67 | 68 | // Linesearcher is a type that can perform a line search. It tries to find an 69 | // (approximate) minimum of the objective function along the search direction 70 | // dir_k starting at the most recent location x_k, i.e., it tries to minimize 71 | // the function 72 | // φ(step) := f(x_k + step * dir_k) where step > 0. 73 | // Typically, a Linesearcher will be used in conjuction with LinesearchMethod 74 | // for performing gradient-based optimization through sequential line searches. 75 | type Linesearcher interface { 76 | // Init initializes the Linesearcher and a new line search. Value and 77 | // derivative contain φ(0) and φ'(0), respectively, and step contains the 78 | // first trial step length. It returns an Operation that must be one of 79 | // FuncEvaluation, GradEvaluation, FuncEvaluation|GradEvaluation. The 80 | // caller must evaluate φ(step), φ'(step), or both, respectively, and pass 81 | // the result to Linesearcher in value and derivative arguments to Iterate. 82 | Init(value, derivative float64, step float64) Operation 83 | 84 | // Iterate takes in the values of φ and φ' evaluated at the previous step 85 | // and returns the next operation. 86 | // 87 | // If op is one of FuncEvaluation, GradEvaluation, 88 | // FuncEvaluation|GradEvaluation, the caller must evaluate φ(step), 89 | // φ'(step), or both, respectively, and pass the result to Linesearcher in 90 | // value and derivative arguments on the next call to Iterate. 91 | // 92 | // If op is MajorIteration, a sufficiently accurate minimum of φ has been 93 | // found at the previous step and the line search has concluded. Init must 94 | // be called again to initialize a new line search. 95 | // 96 | // If err is nil, op must not specify another operation. If err is not nil, 97 | // the values of op and step are undefined. 98 | Iterate(value, derivative float64) (op Operation, step float64, err error) 99 | } 100 | 101 | // NextDirectioner implements a strategy for computing a new line search 102 | // direction at each major iteration. Typically, a NextDirectioner will be 103 | // used in conjuction with LinesearchMethod for performing gradient-based 104 | // optimization through sequential line searches. 105 | type NextDirectioner interface { 106 | // InitDirection initializes the NextDirectioner at the given starting location, 107 | // putting the initial direction in place into dir, and returning the initial 108 | // step size. InitDirection must not modify Location. 109 | InitDirection(loc *Location, dir []float64) (step float64) 110 | 111 | // NextDirection updates the search direction and step size. Location is 112 | // the location seen at the conclusion of the most recent linesearch. The 113 | // next search direction is put in place into dir, and the next step size 114 | // is returned. NextDirection must not modify Location. 115 | NextDirection(loc *Location, dir []float64) (step float64) 116 | } 117 | 118 | // StepSizer can set the next step size of the optimization given the last Location. 119 | // Returned step size must be positive. 120 | type StepSizer interface { 121 | Init(loc *Location, dir []float64) float64 122 | StepSize(loc *Location, dir []float64) float64 123 | } 124 | 125 | // A Recorder can record the progress of the optimization, for example to print 126 | // the progress to StdOut or to a log file. A Recorder must not modify any data. 127 | type Recorder interface { 128 | Init() error 129 | Record(*Location, Operation, *Stats) error 130 | } 131 | -------------------------------------------------------------------------------- /lbfgs.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "github.com/gonum/floats" 9 | ) 10 | 11 | // LBFGS implements the limited-memory BFGS method for gradient-based 12 | // unconstrained minimization. 13 | // 14 | // It stores a modified version of the inverse Hessian approximation H 15 | // implicitly from the last Store iterations while the normal BFGS method 16 | // stores and manipulates H directly as a dense matrix. Therefore LBFGS is more 17 | // appropriate than BFGS for large problems as the cost of LBFGS scales as 18 | // O(Store * dim) while BFGS scales as O(dim^2). The "forgetful" nature of 19 | // LBFGS may also make it perform better than BFGS for functions with Hessians 20 | // that vary rapidly spatially. 21 | type LBFGS struct { 22 | // Linesearcher selects suitable steps along the descent direction. 23 | // Accepted steps should satisfy the strong Wolfe conditions. 24 | // If Linesearcher is nil, a reasonable default will be chosen. 25 | Linesearcher Linesearcher 26 | // Store is the size of the limited-memory storage. 27 | // If Store is 0, it will be defaulted to 15. 28 | Store int 29 | 30 | ls *LinesearchMethod 31 | 32 | dim int // Dimension of the problem 33 | x []float64 // Location at the last major iteration 34 | grad []float64 // Gradient at the last major iteration 35 | 36 | // History 37 | oldest int // Index of the oldest element of the history 38 | y [][]float64 // Last Store values of y 39 | s [][]float64 // Last Store values of s 40 | rho []float64 // Last Store values of rho 41 | a []float64 // Cache of Hessian updates 42 | } 43 | 44 | func (l *LBFGS) Init(loc *Location) (Operation, error) { 45 | if l.Linesearcher == nil { 46 | l.Linesearcher = &Bisection{} 47 | } 48 | if l.Store == 0 { 49 | l.Store = 15 50 | } 51 | 52 | if l.ls == nil { 53 | l.ls = &LinesearchMethod{} 54 | } 55 | l.ls.Linesearcher = l.Linesearcher 56 | l.ls.NextDirectioner = l 57 | 58 | return l.ls.Init(loc) 59 | } 60 | 61 | func (l *LBFGS) Iterate(loc *Location) (Operation, error) { 62 | return l.ls.Iterate(loc) 63 | } 64 | 65 | func (l *LBFGS) InitDirection(loc *Location, dir []float64) (stepSize float64) { 66 | dim := len(loc.X) 67 | l.dim = dim 68 | l.oldest = 0 69 | 70 | l.a = resize(l.a, l.Store) 71 | l.rho = resize(l.rho, l.Store) 72 | l.y = l.initHistory(l.y) 73 | l.s = l.initHistory(l.s) 74 | 75 | l.x = resize(l.x, dim) 76 | copy(l.x, loc.X) 77 | 78 | l.grad = resize(l.grad, dim) 79 | copy(l.grad, loc.Gradient) 80 | 81 | copy(dir, loc.Gradient) 82 | floats.Scale(-1, dir) 83 | return 1 / floats.Norm(dir, 2) 84 | } 85 | 86 | func (l *LBFGS) initHistory(hist [][]float64) [][]float64 { 87 | c := cap(hist) 88 | if c < l.Store { 89 | n := make([][]float64, l.Store-c) 90 | hist = append(hist[:c], n...) 91 | } 92 | hist = hist[:l.Store] 93 | for i := range hist { 94 | hist[i] = resize(hist[i], l.dim) 95 | for j := range hist[i] { 96 | hist[i][j] = 0 97 | } 98 | } 99 | return hist 100 | } 101 | 102 | func (l *LBFGS) NextDirection(loc *Location, dir []float64) (stepSize float64) { 103 | // Uses two-loop correction as described in 104 | // Nocedal, J., Wright, S.: Numerical Optimization (2nd ed). Springer (2006), chapter 7, page 178. 105 | 106 | if len(loc.X) != l.dim { 107 | panic("lbfgs: unexpected size mismatch") 108 | } 109 | if len(loc.Gradient) != l.dim { 110 | panic("lbfgs: unexpected size mismatch") 111 | } 112 | if len(dir) != l.dim { 113 | panic("lbfgs: unexpected size mismatch") 114 | } 115 | 116 | y := l.y[l.oldest] 117 | floats.SubTo(y, loc.Gradient, l.grad) 118 | s := l.s[l.oldest] 119 | floats.SubTo(s, loc.X, l.x) 120 | sDotY := floats.Dot(s, y) 121 | l.rho[l.oldest] = 1 / sDotY 122 | 123 | l.oldest = (l.oldest + 1) % l.Store 124 | 125 | copy(l.x, loc.X) 126 | copy(l.grad, loc.Gradient) 127 | copy(dir, loc.Gradient) 128 | 129 | // Start with the most recent element and go backward, 130 | for i := 0; i < l.Store; i++ { 131 | idx := l.oldest - i - 1 132 | if idx < 0 { 133 | idx += l.Store 134 | } 135 | l.a[idx] = l.rho[idx] * floats.Dot(l.s[idx], dir) 136 | floats.AddScaled(dir, -l.a[idx], l.y[idx]) 137 | } 138 | 139 | // Scale the initial Hessian. 140 | gamma := sDotY / floats.Dot(y, y) 141 | floats.Scale(gamma, dir) 142 | 143 | // Start with the oldest element and go forward. 144 | for i := 0; i < l.Store; i++ { 145 | idx := i + l.oldest 146 | if idx >= l.Store { 147 | idx -= l.Store 148 | } 149 | beta := l.rho[idx] * floats.Dot(l.y[idx], dir) 150 | floats.AddScaled(dir, l.a[idx]-beta, l.s[idx]) 151 | } 152 | 153 | // dir contains H^{-1} * g, so flip the direction for minimization. 154 | floats.Scale(-1, dir) 155 | 156 | return 1 157 | } 158 | 159 | func (*LBFGS) Needs() struct { 160 | Gradient bool 161 | Hessian bool 162 | } { 163 | return struct { 164 | Gradient bool 165 | Hessian bool 166 | }{true, false} 167 | } 168 | -------------------------------------------------------------------------------- /linesearch.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "math" 9 | 10 | "github.com/gonum/floats" 11 | ) 12 | 13 | // LinesearchMethod represents an abstract optimization method in which a 14 | // function is optimized through successive line search optimizations. 15 | type LinesearchMethod struct { 16 | // NextDirectioner specifies the search direction of each linesearch. 17 | NextDirectioner NextDirectioner 18 | // Linesearcher performs a linesearch along the search direction. 19 | Linesearcher Linesearcher 20 | 21 | x []float64 // Starting point for the current iteration. 22 | dir []float64 // Search direction for the current iteration. 23 | 24 | first bool // Indicator of the first iteration. 25 | nextMajor bool // Indicates that MajorIteration must be commanded at the next call to Iterate. 26 | eval Operation // Indicator of valid fields in Location. 27 | 28 | lastStep float64 // Step taken from x in the previous call to Iterate. 29 | lastOp Operation // Operation returned from the previous call to Iterate. 30 | } 31 | 32 | func (ls *LinesearchMethod) Init(loc *Location) (Operation, error) { 33 | if loc.Gradient == nil { 34 | panic("linesearch: gradient is nil") 35 | } 36 | 37 | dim := len(loc.X) 38 | ls.x = resize(ls.x, dim) 39 | ls.dir = resize(ls.dir, dim) 40 | 41 | ls.first = true 42 | ls.nextMajor = false 43 | 44 | // Indicate that all fields of loc are valid. 45 | ls.eval = FuncEvaluation | GradEvaluation 46 | if loc.Hessian != nil { 47 | ls.eval |= HessEvaluation 48 | } 49 | 50 | ls.lastStep = math.NaN() 51 | ls.lastOp = NoOperation 52 | 53 | return ls.initNextLinesearch(loc) 54 | } 55 | 56 | func (ls *LinesearchMethod) Iterate(loc *Location) (Operation, error) { 57 | switch ls.lastOp { 58 | case NoOperation: 59 | // TODO(vladimir-ch): Either Init has not been called, or the caller is 60 | // trying to resume the optimization run after Iterate previously 61 | // returned with an error. Decide what is the proper thing to do. See also #125. 62 | 63 | case MajorIteration: 64 | // The previous updated location did not converge the full 65 | // optimization. Initialize a new Linesearch. 66 | return ls.initNextLinesearch(loc) 67 | 68 | default: 69 | // Update the indicator of valid fields of loc. 70 | ls.eval |= ls.lastOp 71 | 72 | if ls.nextMajor { 73 | ls.nextMajor = false 74 | 75 | // Linesearcher previously finished, and the invalid fields of loc 76 | // have now been validated. Announce MajorIteration. 77 | ls.lastOp = MajorIteration 78 | return ls.lastOp, nil 79 | } 80 | } 81 | 82 | // Continue the linesearch. 83 | 84 | f := math.NaN() 85 | if ls.eval&FuncEvaluation != 0 { 86 | f = loc.F 87 | } 88 | projGrad := math.NaN() 89 | if ls.eval&GradEvaluation != 0 { 90 | projGrad = floats.Dot(loc.Gradient, ls.dir) 91 | } 92 | op, step, err := ls.Linesearcher.Iterate(f, projGrad) 93 | if err != nil { 94 | return ls.error(err) 95 | } 96 | 97 | switch op { 98 | case MajorIteration: 99 | // Linesearch has been finished. 100 | 101 | ls.lastOp = complementEval(loc, ls.eval) 102 | if ls.lastOp == NoOperation { 103 | // loc is complete, MajorIteration can be declared directly. 104 | ls.lastOp = MajorIteration 105 | } else { 106 | // Declare MajorIteration on the next call to Iterate. 107 | ls.nextMajor = true 108 | } 109 | 110 | case FuncEvaluation, GradEvaluation, FuncEvaluation | GradEvaluation: 111 | if step != ls.lastStep { 112 | // We are moving to a new location, and not, say, evaluating extra 113 | // information at the current location. 114 | 115 | // Compute the next evaluation point and store it in loc.X. 116 | floats.AddScaledTo(loc.X, ls.x, step, ls.dir) 117 | if floats.Equal(ls.x, loc.X) { 118 | // Step size has become so small that the next evaluation point is 119 | // indistinguishable from the starting point for the current 120 | // iteration due to rounding errors. 121 | return ls.error(ErrNoProgress) 122 | } 123 | ls.lastStep = step 124 | ls.eval = NoOperation // Indicate all invalid fields of loc. 125 | } 126 | ls.lastOp = op 127 | 128 | default: 129 | panic("linesearch: Linesearcher returned invalid operation") 130 | } 131 | 132 | return ls.lastOp, nil 133 | } 134 | 135 | func (ls *LinesearchMethod) error(err error) (Operation, error) { 136 | ls.lastOp = NoOperation 137 | return ls.lastOp, err 138 | } 139 | 140 | // initNextLinesearch initializes the next linesearch using the previous 141 | // complete location stored in loc. It fills loc.X and returns an evaluation 142 | // to be performed at loc.X. 143 | func (ls *LinesearchMethod) initNextLinesearch(loc *Location) (Operation, error) { 144 | copy(ls.x, loc.X) 145 | 146 | var step float64 147 | if ls.first { 148 | ls.first = false 149 | step = ls.NextDirectioner.InitDirection(loc, ls.dir) 150 | } else { 151 | step = ls.NextDirectioner.NextDirection(loc, ls.dir) 152 | } 153 | 154 | projGrad := floats.Dot(loc.Gradient, ls.dir) 155 | if projGrad >= 0 { 156 | return ls.error(ErrNonDescentDirection) 157 | } 158 | 159 | op := ls.Linesearcher.Init(loc.F, projGrad, step) 160 | switch op { 161 | case FuncEvaluation, GradEvaluation, FuncEvaluation | GradEvaluation: 162 | default: 163 | panic("linesearch: Linesearcher returned invalid operation") 164 | } 165 | 166 | floats.AddScaledTo(loc.X, ls.x, step, ls.dir) 167 | if floats.Equal(ls.x, loc.X) { 168 | // Step size is so small that the next evaluation point is 169 | // indistinguishable from the starting point for the current iteration 170 | // due to rounding errors. 171 | return ls.error(ErrNoProgress) 172 | } 173 | 174 | ls.lastStep = step 175 | ls.eval = NoOperation // Invalidate all fields of loc. 176 | 177 | ls.lastOp = op 178 | return ls.lastOp, nil 179 | } 180 | 181 | // ArmijoConditionMet returns true if the Armijo condition (aka sufficient 182 | // decrease) has been met. Under normal conditions, the following should be 183 | // true, though this is not enforced: 184 | // - initGrad < 0 185 | // - step > 0 186 | // - 0 < decrease < 1 187 | func ArmijoConditionMet(currObj, initObj, initGrad, step, decrease float64) bool { 188 | return currObj <= initObj+decrease*step*initGrad 189 | } 190 | 191 | // StrongWolfeConditionsMet returns true if the strong Wolfe conditions have been met. 192 | // The strong Wolfe conditions ensure sufficient decrease in the function 193 | // value, and sufficient decrease in the magnitude of the projected gradient. 194 | // Under normal conditions, the following should be true, though this is not 195 | // enforced: 196 | // - initGrad < 0 197 | // - step > 0 198 | // - 0 <= decrease < curvature < 1 199 | func StrongWolfeConditionsMet(currObj, currGrad, initObj, initGrad, step, decrease, curvature float64) bool { 200 | if currObj > initObj+decrease*step*initGrad { 201 | return false 202 | } 203 | return math.Abs(currGrad) < curvature*math.Abs(initGrad) 204 | } 205 | 206 | // WeakWolfeConditionsMet returns true if the weak Wolfe conditions have been met. 207 | // The weak Wolfe conditions ensure sufficient decrease in the function value, 208 | // and sufficient decrease in the value of the projected gradient. Under normal 209 | // conditions, the following should be true, though this is not enforced: 210 | // - initGrad < 0 211 | // - step > 0 212 | // - 0 <= decrease < curvature< 1 213 | func WeakWolfeConditionsMet(currObj, currGrad, initObj, initGrad, step, decrease, curvature float64) bool { 214 | if currObj > initObj+decrease*step*initGrad { 215 | return false 216 | } 217 | return currGrad >= curvature*initGrad 218 | } 219 | -------------------------------------------------------------------------------- /linesearcher_test.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "fmt" 9 | "math" 10 | "reflect" 11 | "testing" 12 | 13 | "github.com/gonum/optimize/functions" 14 | ) 15 | 16 | func TestMoreThuente(t *testing.T) { 17 | d := 0.001 18 | c := 0.001 19 | ls := &MoreThuente{ 20 | DecreaseFactor: d, 21 | CurvatureFactor: c, 22 | } 23 | testLinesearcher(t, ls, d, c, true) 24 | } 25 | 26 | func TestBisection(t *testing.T) { 27 | c := 0.1 28 | ls := &Bisection{ 29 | CurvatureFactor: c, 30 | } 31 | testLinesearcher(t, ls, 0, c, true) 32 | } 33 | 34 | func TestBacktracking(t *testing.T) { 35 | d := 0.001 36 | ls := &Backtracking{ 37 | DecreaseFactor: d, 38 | } 39 | testLinesearcher(t, ls, d, 0, false) 40 | } 41 | 42 | type funcGrader interface { 43 | Func([]float64) float64 44 | Grad([]float64, []float64) 45 | } 46 | 47 | type linesearcherTest struct { 48 | name string 49 | f func(float64) float64 50 | g func(float64) float64 51 | } 52 | 53 | func newLinesearcherTest(name string, fg funcGrader) linesearcherTest { 54 | grad := make([]float64, 1) 55 | return linesearcherTest{ 56 | name: name, 57 | f: func(x float64) float64 { 58 | return fg.Func([]float64{x}) 59 | }, 60 | g: func(x float64) float64 { 61 | fg.Grad(grad, []float64{x}) 62 | return grad[0] 63 | }, 64 | } 65 | } 66 | 67 | func testLinesearcher(t *testing.T, ls Linesearcher, decrease, curvature float64, strongWolfe bool) { 68 | for i, prob := range []linesearcherTest{ 69 | newLinesearcherTest("Concave-to-the-right function", functions.ConcaveRight{}), 70 | newLinesearcherTest("Concave-to-the-left function", functions.ConcaveLeft{}), 71 | newLinesearcherTest("Plassmann wiggly function (l=39, beta=0.01)", functions.Plassmann{39, 0.01}), 72 | newLinesearcherTest("Yanai-Ozawa-Kaneko function (beta1=0.001, beta2=0.001)", functions.YanaiOzawaKaneko{0.001, 0.001}), 73 | newLinesearcherTest("Yanai-Ozawa-Kaneko function (beta1=0.01, beta2=0.001)", functions.YanaiOzawaKaneko{0.01, 0.001}), 74 | newLinesearcherTest("Yanai-Ozawa-Kaneko function (beta1=0.001, beta2=0.01)", functions.YanaiOzawaKaneko{0.001, 0.01}), 75 | } { 76 | for _, initStep := range []float64{0.001, 0.1, 1, 10, 1000} { 77 | prefix := fmt.Sprintf("test %d (%v started from %v)", i, prob.name, initStep) 78 | 79 | f0 := prob.f(0) 80 | g0 := prob.g(0) 81 | if g0 >= 0 { 82 | panic("bad test function") 83 | } 84 | 85 | op := ls.Init(f0, g0, initStep) 86 | if !op.isEvaluation() { 87 | t.Errorf("%v: Linesearcher.Init returned non-evaluating operation %v", op) 88 | continue 89 | } 90 | 91 | var ( 92 | err error 93 | k int 94 | f, g float64 95 | step float64 96 | ) 97 | loop: 98 | for { 99 | switch op { 100 | case MajorIteration: 101 | if f > f0+step*decrease*g0 { 102 | t.Errorf("%v: %v found step %v that does not satisfy the sufficient decrease condition", 103 | prefix, reflect.TypeOf(ls), step) 104 | } 105 | if strongWolfe && math.Abs(g) > curvature*(-g0) { 106 | t.Errorf("%v: %v found step %v that does not satisfy the curvature condition", 107 | prefix, reflect.TypeOf(ls), step) 108 | } 109 | break loop 110 | case FuncEvaluation: 111 | f = prob.f(step) 112 | case GradEvaluation: 113 | g = prob.g(step) 114 | case FuncEvaluation | GradEvaluation: 115 | f = prob.f(step) 116 | g = prob.g(step) 117 | default: 118 | t.Errorf("%v: Linesearcher returned an invalid operation %v", op) 119 | break loop 120 | } 121 | 122 | k++ 123 | if k == 1000 { 124 | t.Errorf("%v: %v did not finish", prefix, reflect.TypeOf(ls)) 125 | break 126 | } 127 | 128 | op, step, err = ls.Iterate(f, g) 129 | if err != nil { 130 | t.Errorf("%v: %v failed at step %v with %v", prefix, reflect.TypeOf(ls), step, err) 131 | break 132 | } 133 | } 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /local.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "math" 9 | "time" 10 | ) 11 | 12 | // Local finds a local minimum of a minimization problem using a sequential 13 | // algorithm. A maximization problem can be transformed into a minimization 14 | // problem by multiplying the function by -1. 15 | // 16 | // The first argument represents the problem to be minimized. Its fields are 17 | // routines that evaluate the objective function, gradient, and other 18 | // quantities related to the problem. The objective function, p.Func, must not 19 | // be nil. The optimization method used may require other fields to be non-nil 20 | // as specified by method.Needs. Local will panic if these are not met. The 21 | // method can be determined automatically from the supplied problem which is 22 | // described below. 23 | // 24 | // If p.Status is not nil, it is called before every evaluation. If the 25 | // returned Status is not NotTerminated or the error is not nil, the 26 | // optimization run is terminated. 27 | // 28 | // The second argument is the initial location at which to start the minimization. 29 | // The initial location must be supplied, and must have a length equal to the 30 | // problem dimension. 31 | // 32 | // The third argument contains the settings for the minimization. It is here that 33 | // gradient tolerance, etc. are specified. The DefaultSettings function 34 | // can be called for a Settings struct with the default values initialized. 35 | // If settings == nil, the default settings are used. See the documentation 36 | // for the Settings structure for more information. The optimization Method used 37 | // may also contain settings, see documentation for the appropriate optimizer. 38 | // 39 | // The final argument is the optimization method to use. If method == nil, then 40 | // an appropriate default is chosen based on the properties of the other arguments 41 | // (dimension, gradient-free or gradient-based, etc.). The optimization 42 | // methods in this package are designed such that reasonable defaults occur 43 | // if options are not specified explicitly. For example, the code 44 | // method := &optimize.BFGS{} 45 | // creates a pointer to a new BFGS struct. When Local is called, the settings 46 | // in the method will be populated with default values. The methods are also 47 | // designed such that they can be reused in future calls to Local. 48 | // 49 | // If method implements Statuser, method.Status is called before every call 50 | // to method.Iterate. If the returned Status is not NotTerminated or the 51 | // error is non-nil, the optimization run is terminated. 52 | // 53 | // Local returns a Result struct and any error that occurred. See the 54 | // documentation of Result for more information. 55 | // 56 | // Be aware that the default behavior of Local is to find the minimum. 57 | // For certain functions and optimization methods, this process can take many 58 | // function evaluations. If you would like to put limits on this, for example 59 | // maximum runtime or maximum function evaluations, modify the Settings 60 | // input struct. 61 | func Local(p Problem, initX []float64, settings *Settings, method Method) (*Result, error) { 62 | startTime := time.Now() 63 | dim := len(initX) 64 | if method == nil { 65 | method = getDefaultMethod(&p) 66 | } 67 | if settings == nil { 68 | settings = DefaultSettings() 69 | } 70 | 71 | stats := &Stats{} 72 | 73 | err := checkOptimization(p, dim, method, settings.Recorder) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | optLoc, err := getStartingLocation(&p, method, initX, stats, settings) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | if settings.FunctionConverge != nil { 84 | settings.FunctionConverge.Init(optLoc.F) 85 | } 86 | 87 | stats.Runtime = time.Since(startTime) 88 | 89 | // Send initial location to Recorder 90 | if settings.Recorder != nil { 91 | err = settings.Recorder.Record(optLoc, InitIteration, stats) 92 | if err != nil { 93 | return nil, err 94 | } 95 | } 96 | 97 | // Check if the starting location satisfies the convergence criteria. 98 | status := checkConvergence(optLoc, settings, true) 99 | 100 | // Run optimization 101 | if status == NotTerminated && err == nil { 102 | // The starting location is not good enough, we need to perform a 103 | // minimization. The optimal location will be stored in-place in 104 | // optLoc. 105 | status, err = minimize(&p, method, settings, stats, optLoc, startTime) 106 | } 107 | 108 | // Cleanup and collect results 109 | if settings.Recorder != nil && err == nil { 110 | // Send the optimal location to Recorder. 111 | err = settings.Recorder.Record(optLoc, PostIteration, stats) 112 | } 113 | stats.Runtime = time.Since(startTime) 114 | return &Result{ 115 | Location: *optLoc, 116 | Stats: *stats, 117 | Status: status, 118 | }, err 119 | } 120 | 121 | func minimize(p *Problem, method Method, settings *Settings, stats *Stats, optLoc *Location, startTime time.Time) (status Status, err error) { 122 | loc := &Location{} 123 | copyLocation(loc, optLoc) 124 | x := make([]float64, len(loc.X)) 125 | 126 | statuser, _ := method.(Statuser) 127 | 128 | var op Operation 129 | op, err = method.Init(loc) 130 | if err != nil { 131 | status = Failure 132 | return 133 | } 134 | 135 | for { 136 | // Sequentially call method.Iterate, performing the operations it has 137 | // commanded, until convergence. 138 | 139 | switch op { 140 | case NoOperation: 141 | case InitIteration: 142 | panic("optimize: Method returned InitIteration") 143 | case PostIteration: 144 | panic("optimize: Method returned PostIteration") 145 | case MajorIteration: 146 | copyLocation(optLoc, loc) 147 | stats.MajorIterations++ 148 | status = checkConvergence(optLoc, settings, true) 149 | default: // Any of the Evaluation operations. 150 | status, err = evaluate(p, loc, op, x) 151 | updateStats(stats, op) 152 | } 153 | 154 | status, err = iterCleanup(status, err, stats, settings, statuser, startTime, loc, op) 155 | if status != NotTerminated || err != nil { 156 | return 157 | } 158 | 159 | op, err = method.Iterate(loc) 160 | if err != nil { 161 | status = Failure 162 | return 163 | } 164 | } 165 | panic("optimize: unreachable") 166 | } 167 | 168 | func getDefaultMethod(p *Problem) Method { 169 | if p.Grad != nil { 170 | return &BFGS{} 171 | } 172 | return &NelderMead{} 173 | } 174 | 175 | // getStartingLocation allocates and initializes the starting location for the minimization. 176 | func getStartingLocation(p *Problem, method Method, initX []float64, stats *Stats, settings *Settings) (*Location, error) { 177 | dim := len(initX) 178 | loc := newLocation(dim, method) 179 | copy(loc.X, initX) 180 | 181 | if settings.UseInitialData { 182 | loc.F = settings.InitialValue 183 | if loc.Gradient != nil { 184 | initG := settings.InitialGradient 185 | if initG == nil { 186 | panic("optimize: initial gradient is nil") 187 | } 188 | if len(initG) != dim { 189 | panic("optimize: initial gradient size mismatch") 190 | } 191 | copy(loc.Gradient, initG) 192 | } 193 | if loc.Hessian != nil { 194 | initH := settings.InitialHessian 195 | if initH == nil { 196 | panic("optimize: initial Hessian is nil") 197 | } 198 | if initH.Symmetric() != dim { 199 | panic("optimize: initial Hessian size mismatch") 200 | } 201 | loc.Hessian.CopySym(initH) 202 | } 203 | } else { 204 | eval := FuncEvaluation 205 | if loc.Gradient != nil { 206 | eval |= GradEvaluation 207 | } 208 | if loc.Hessian != nil { 209 | eval |= HessEvaluation 210 | } 211 | x := make([]float64, len(loc.X)) 212 | evaluate(p, loc, eval, x) 213 | updateStats(stats, eval) 214 | } 215 | 216 | if math.IsInf(loc.F, 1) || math.IsNaN(loc.F) { 217 | return loc, ErrFunc(loc.F) 218 | } 219 | for i, v := range loc.Gradient { 220 | if math.IsInf(v, 0) || math.IsNaN(v) { 221 | return loc, ErrGrad{Grad: v, Index: i} 222 | } 223 | } 224 | 225 | return loc, nil 226 | } 227 | -------------------------------------------------------------------------------- /local_example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2016 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize_test 6 | 7 | import ( 8 | "fmt" 9 | "log" 10 | 11 | "github.com/gonum/optimize" 12 | "github.com/gonum/optimize/functions" 13 | ) 14 | 15 | func ExampleLocal() { 16 | p := optimize.Problem{ 17 | Func: functions.ExtendedRosenbrock{}.Func, 18 | Grad: functions.ExtendedRosenbrock{}.Grad, 19 | } 20 | 21 | x := []float64{1.3, 0.7, 0.8, 1.9, 1.2} 22 | settings := optimize.DefaultSettings() 23 | settings.Recorder = nil 24 | settings.GradientThreshold = 1e-12 25 | settings.FunctionConverge = nil 26 | 27 | result, err := optimize.Local(p, x, settings, &optimize.BFGS{}) 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | if err = result.Status.Err(); err != nil { 32 | log.Fatal(err) 33 | } 34 | fmt.Printf("result.Status: %v\n", result.Status) 35 | fmt.Printf("result.X: %v\n", result.X) 36 | fmt.Printf("result.F: %v\n", result.F) 37 | fmt.Printf("result.Stats.FuncEvaluations: %d\n", result.Stats.FuncEvaluations) 38 | // Output: 39 | // result.Status: GradientThreshold 40 | // result.X: [1 1 1 1 1] 41 | // result.F: 0 42 | // result.Stats.FuncEvaluations: 35 43 | } 44 | -------------------------------------------------------------------------------- /minimize.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2016 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "fmt" 9 | "math" 10 | "time" 11 | 12 | "github.com/gonum/floats" 13 | "github.com/gonum/matrix/mat64" 14 | ) 15 | 16 | // newLocation allocates a new locatian structure of the appropriate size. It 17 | // allocates memory based on the dimension and the values in Needs. The initial 18 | // function value is set to math.Inf(1). 19 | func newLocation(dim int, method Needser) *Location { 20 | // TODO(btracey): combine this with Local. 21 | loc := &Location{ 22 | X: make([]float64, dim), 23 | } 24 | loc.F = math.Inf(1) 25 | if method.Needs().Gradient { 26 | loc.Gradient = make([]float64, dim) 27 | } 28 | if method.Needs().Hessian { 29 | loc.Hessian = mat64.NewSymDense(dim, nil) 30 | } 31 | return loc 32 | } 33 | 34 | func copyLocation(dst, src *Location) { 35 | dst.X = resize(dst.X, len(src.X)) 36 | copy(dst.X, src.X) 37 | 38 | dst.F = src.F 39 | 40 | dst.Gradient = resize(dst.Gradient, len(src.Gradient)) 41 | copy(dst.Gradient, src.Gradient) 42 | 43 | if src.Hessian != nil { 44 | if dst.Hessian == nil || dst.Hessian.Symmetric() != len(src.X) { 45 | dst.Hessian = mat64.NewSymDense(len(src.X), nil) 46 | } 47 | dst.Hessian.CopySym(src.Hessian) 48 | } 49 | } 50 | 51 | func checkOptimization(p Problem, dim int, method Needser, recorder Recorder) error { 52 | if p.Func == nil { 53 | panic("optimize: objective function is undefined") 54 | } 55 | if dim <= 0 { 56 | panic("optimize: impossible problem dimension") 57 | } 58 | if err := p.satisfies(method); err != nil { 59 | return err 60 | } 61 | if p.Status != nil { 62 | _, err := p.Status() 63 | if err != nil { 64 | return err 65 | } 66 | } 67 | if recorder != nil { 68 | err := recorder.Init() 69 | if err != nil { 70 | return err 71 | } 72 | } 73 | return nil 74 | } 75 | 76 | // evaluate evaluates the routines specified by the Operation at loc.X, and stores 77 | // the answer into loc. loc.X is copied into x before 78 | // evaluating in order to prevent the routines from modifying it. 79 | func evaluate(p *Problem, loc *Location, op Operation, x []float64) (Status, error) { 80 | if !op.isEvaluation() { 81 | panic(fmt.Sprintf("optimize: invalid evaluation %v", op)) 82 | } 83 | if p.Status != nil { 84 | status, err := p.Status() 85 | if err != nil || status != NotTerminated { 86 | return status, err 87 | } 88 | } 89 | copy(x, loc.X) 90 | if op&FuncEvaluation != 0 { 91 | loc.F = p.Func(x) 92 | } 93 | if op&GradEvaluation != 0 { 94 | p.Grad(loc.Gradient, x) 95 | } 96 | if op&HessEvaluation != 0 { 97 | p.Hess(loc.Hessian, x) 98 | } 99 | return NotTerminated, nil 100 | } 101 | 102 | // checkConvergence returns NotTerminated if the Location does not satisfy the 103 | // convergence criteria given by settings. Otherwise a corresponding status is 104 | // returned. 105 | // Unlike checkLimits, checkConvergence is called only at MajorIterations. 106 | // 107 | // If local is true, gradient convergence is also checked. 108 | func checkConvergence(loc *Location, settings *Settings, local bool) Status { 109 | if local && loc.Gradient != nil { 110 | norm := floats.Norm(loc.Gradient, math.Inf(1)) 111 | if norm < settings.GradientThreshold { 112 | return GradientThreshold 113 | } 114 | } 115 | if loc.F < settings.FunctionThreshold { 116 | return FunctionThreshold 117 | } 118 | if settings.FunctionConverge != nil { 119 | return settings.FunctionConverge.FunctionConverged(loc.F) 120 | } 121 | return NotTerminated 122 | } 123 | 124 | // updateStats updates the statistics based on the operation. 125 | func updateStats(stats *Stats, op Operation) { 126 | if op&FuncEvaluation != 0 { 127 | stats.FuncEvaluations++ 128 | } 129 | if op&GradEvaluation != 0 { 130 | stats.GradEvaluations++ 131 | } 132 | if op&HessEvaluation != 0 { 133 | stats.HessEvaluations++ 134 | } 135 | } 136 | 137 | // checkLimits returns NotTerminated status if the various limits given by 138 | // settings have not been reached. Otherwise it returns a corresponding status. 139 | // Unlike checkConvergence, checkLimits is called by Local and Global at _every_ 140 | // iteration. 141 | func checkLimits(loc *Location, stats *Stats, settings *Settings) Status { 142 | // Check the objective function value for negative infinity because it 143 | // could break the linesearches and -inf is the best we can do anyway. 144 | if math.IsInf(loc.F, -1) { 145 | return FunctionNegativeInfinity 146 | } 147 | 148 | if settings.MajorIterations > 0 && stats.MajorIterations >= settings.MajorIterations { 149 | return IterationLimit 150 | } 151 | 152 | if settings.FuncEvaluations > 0 && stats.FuncEvaluations >= settings.FuncEvaluations { 153 | return FunctionEvaluationLimit 154 | } 155 | 156 | if settings.GradEvaluations > 0 && stats.GradEvaluations >= settings.GradEvaluations { 157 | return GradientEvaluationLimit 158 | } 159 | 160 | if settings.HessEvaluations > 0 && stats.HessEvaluations >= settings.HessEvaluations { 161 | return HessianEvaluationLimit 162 | } 163 | 164 | // TODO(vladimir-ch): It would be nice to update Runtime here. 165 | if settings.Runtime > 0 && stats.Runtime >= settings.Runtime { 166 | return RuntimeLimit 167 | } 168 | 169 | return NotTerminated 170 | } 171 | 172 | // TODO(btracey): better name 173 | func iterCleanup(status Status, err error, stats *Stats, settings *Settings, statuser Statuser, startTime time.Time, loc *Location, op Operation) (Status, error) { 174 | if status != NotTerminated || err != nil { 175 | return status, err 176 | } 177 | 178 | if settings.Recorder != nil { 179 | stats.Runtime = time.Since(startTime) 180 | err = settings.Recorder.Record(loc, op, stats) 181 | if err != nil { 182 | if status == NotTerminated { 183 | status = Failure 184 | } 185 | return status, err 186 | } 187 | } 188 | 189 | stats.Runtime = time.Since(startTime) 190 | status = checkLimits(loc, stats, settings) 191 | if status != NotTerminated { 192 | return status, nil 193 | } 194 | 195 | if statuser != nil { 196 | status, err = statuser.Status() 197 | if err != nil || status != NotTerminated { 198 | return status, err 199 | } 200 | } 201 | return status, nil 202 | } 203 | -------------------------------------------------------------------------------- /morethuente.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import "math" 8 | 9 | // MoreThuente is a Linesearcher that finds steps that satisfy both the 10 | // sufficient decrease and curvature conditions (the strong Wolfe conditions). 11 | // 12 | // References: 13 | // - More, J.J. and D.J. Thuente: Line Search Algorithms with Guaranteed Sufficient 14 | // Decrease. ACM Transactions on Mathematical Software 20(3) (1994), 286-307 15 | type MoreThuente struct { 16 | // DecreaseFactor is the constant factor in the sufficient decrease 17 | // (Armijo) condition. 18 | // It must be in the interval [0, 1). The default value is 0. 19 | DecreaseFactor float64 20 | // CurvatureFactor is the constant factor in the Wolfe conditions. Smaller 21 | // values result in a more exact line search. 22 | // A set value must be in the interval (0, 1). If it is zero, it will be 23 | // defaulted to 0.9. 24 | CurvatureFactor float64 25 | // StepTolerance sets the minimum acceptable width for the linesearch 26 | // interval. If the relative interval length is less than this value, 27 | // ErrLinesearcherFailure is returned. 28 | // It must be non-negative. If it is zero, it will be defaulted to 1e-10. 29 | StepTolerance float64 30 | 31 | // MinimumStep is the minimum step that the linesearcher will take. 32 | // It must be non-negative and less than MaximumStep. Defaults to no 33 | // minimum (a value of 0). 34 | MinimumStep float64 35 | // MaximumStep is the maximum step that the linesearcher will take. 36 | // It must be greater than MinimumStep. If it is zero, it will be defaulted 37 | // to 1e20. 38 | MaximumStep float64 39 | 40 | bracketed bool // Indicates if a minimum has been bracketed. 41 | fInit float64 // Function value at step = 0. 42 | gInit float64 // Derivative value at step = 0. 43 | 44 | // When stage is 1, the algorithm updates the interval given by x and y 45 | // so that it contains a minimizer of the modified function 46 | // psi(step) = f(step) - f(0) - DecreaseFactor * step * f'(0). 47 | // When stage is 2, the interval is updated so that it contains a minimizer 48 | // of f. 49 | stage int 50 | 51 | step float64 // Current step. 52 | lower, upper float64 // Lower and upper bounds on the next step. 53 | x float64 // Endpoint of the interval with a lower function value. 54 | fx, gx float64 // Data at x. 55 | y float64 // The other endpoint. 56 | fy, gy float64 // Data at y. 57 | width [2]float64 // Width of the interval at two previous iterations. 58 | } 59 | 60 | const ( 61 | mtMinGrowthFactor float64 = 1.1 62 | mtMaxGrowthFactor float64 = 4 63 | ) 64 | 65 | func (mt *MoreThuente) Init(f, g float64, step float64) Operation { 66 | // Based on the original Fortran code that is available, for example, from 67 | // http://ftp.mcs.anl.gov/pub/MINPACK-2/csrch/ 68 | // as part of 69 | // MINPACK-2 Project. November 1993. 70 | // Argonne National Laboratory and University of Minnesota. 71 | // Brett M. Averick, Richard G. Carter, and Jorge J. Moré. 72 | 73 | if g >= 0 { 74 | panic("morethuente: initial derivative is non-negative") 75 | } 76 | if step <= 0 { 77 | panic("morethuente: invalid initial step") 78 | } 79 | 80 | if mt.CurvatureFactor == 0 { 81 | mt.CurvatureFactor = 0.9 82 | } 83 | if mt.StepTolerance == 0 { 84 | mt.StepTolerance = 1e-10 85 | } 86 | if mt.MaximumStep == 0 { 87 | mt.MaximumStep = 1e20 88 | } 89 | 90 | if mt.MinimumStep < 0 { 91 | panic("morethuente: minimum step is negative") 92 | } 93 | if mt.MaximumStep <= mt.MinimumStep { 94 | panic("morethuente: maximum step is not greater than minimum step") 95 | } 96 | if mt.DecreaseFactor < 0 || mt.DecreaseFactor >= 1 { 97 | panic("morethuente: invalid decrease factor") 98 | } 99 | if mt.CurvatureFactor <= 0 || mt.CurvatureFactor >= 1 { 100 | panic("morethuente: invalid curvature factor") 101 | } 102 | if mt.StepTolerance <= 0 { 103 | panic("morethuente: step tolerance is not positive") 104 | } 105 | 106 | if step < mt.MinimumStep { 107 | step = mt.MinimumStep 108 | } 109 | if step > mt.MaximumStep { 110 | step = mt.MaximumStep 111 | } 112 | 113 | mt.bracketed = false 114 | mt.stage = 1 115 | mt.fInit = f 116 | mt.gInit = g 117 | 118 | mt.x, mt.fx, mt.gx = 0, f, g 119 | mt.y, mt.fy, mt.gy = 0, f, g 120 | 121 | mt.lower = 0 122 | mt.upper = step + mtMaxGrowthFactor*step 123 | 124 | mt.width[0] = mt.MaximumStep - mt.MinimumStep 125 | mt.width[1] = 2 * mt.width[0] 126 | 127 | mt.step = step 128 | return FuncEvaluation | GradEvaluation 129 | } 130 | 131 | func (mt *MoreThuente) Iterate(f, g float64) (Operation, float64, error) { 132 | if mt.stage == 0 { 133 | panic("morethuente: Init has not been called") 134 | } 135 | 136 | gTest := mt.DecreaseFactor * mt.gInit 137 | fTest := mt.fInit + mt.step*gTest 138 | 139 | if mt.bracketed { 140 | if mt.step <= mt.lower || mt.step >= mt.upper || mt.upper-mt.lower <= mt.StepTolerance*mt.upper { 141 | // step contains the best step found (see below). 142 | return NoOperation, mt.step, ErrLinesearcherFailure 143 | } 144 | } 145 | if mt.step == mt.MaximumStep && f <= fTest && g <= gTest { 146 | return NoOperation, mt.step, ErrLinesearcherBound 147 | } 148 | if mt.step == mt.MinimumStep && (f > fTest || g >= gTest) { 149 | return NoOperation, mt.step, ErrLinesearcherFailure 150 | } 151 | 152 | // Test for convergence. 153 | if f <= fTest && math.Abs(g) <= mt.CurvatureFactor*(-mt.gInit) { 154 | mt.stage = 0 155 | return MajorIteration, mt.step, nil 156 | } 157 | 158 | if mt.stage == 1 && f <= fTest && g >= 0 { 159 | mt.stage = 2 160 | } 161 | 162 | if mt.stage == 1 && f <= mt.fx && f > fTest { 163 | // Lower function value but the decrease is not sufficient . 164 | 165 | // Compute values and derivatives of the modified function at step, x, y. 166 | fm := f - mt.step*gTest 167 | fxm := mt.fx - mt.x*gTest 168 | fym := mt.fy - mt.y*gTest 169 | gm := g - gTest 170 | gxm := mt.gx - gTest 171 | gym := mt.gy - gTest 172 | // Update x, y and step. 173 | mt.nextStep(fxm, gxm, fym, gym, fm, gm) 174 | // Recover values and derivates of the non-modified function at x and y. 175 | mt.fx = fxm + mt.x*gTest 176 | mt.fy = fym + mt.y*gTest 177 | mt.gx = gxm + gTest 178 | mt.gy = gym + gTest 179 | } else { 180 | // Update x, y and step. 181 | mt.nextStep(mt.fx, mt.gx, mt.fy, mt.gy, f, g) 182 | } 183 | 184 | if mt.bracketed { 185 | // Monitor the length of the bracketing interval. If the interval has 186 | // not been reduced sufficiently after two steps, use bisection to 187 | // force its length to zero. 188 | width := mt.y - mt.x 189 | if math.Abs(width) >= 2.0/3*mt.width[1] { 190 | mt.step = mt.x + 0.5*width 191 | } 192 | mt.width[0], mt.width[1] = math.Abs(width), mt.width[0] 193 | } 194 | 195 | if mt.bracketed { 196 | mt.lower = math.Min(mt.x, mt.y) 197 | mt.upper = math.Max(mt.x, mt.y) 198 | } else { 199 | mt.lower = mt.step + mtMinGrowthFactor*(mt.step-mt.x) 200 | mt.upper = mt.step + mtMaxGrowthFactor*(mt.step-mt.x) 201 | } 202 | 203 | // Force the step to be in [MinimumStep, MaximumStep]. 204 | mt.step = math.Max(mt.MinimumStep, math.Min(mt.step, mt.MaximumStep)) 205 | 206 | if mt.bracketed { 207 | if mt.step <= mt.lower || mt.step >= mt.upper || mt.upper-mt.lower <= mt.StepTolerance*mt.upper { 208 | // If further progress is not possible, set step to the best step 209 | // obtained during the search. 210 | mt.step = mt.x 211 | } 212 | } 213 | 214 | return FuncEvaluation | GradEvaluation, mt.step, nil 215 | } 216 | 217 | // nextStep computes the next safeguarded step and updates the interval that 218 | // contains a step that satisfies the sufficient decrease and curvature 219 | // conditions. 220 | func (mt *MoreThuente) nextStep(fx, gx, fy, gy, f, g float64) { 221 | x := mt.x 222 | y := mt.y 223 | step := mt.step 224 | 225 | gNeg := g < 0 226 | if gx < 0 { 227 | gNeg = !gNeg 228 | } 229 | 230 | var next float64 231 | var bracketed bool 232 | switch { 233 | case f > fx: 234 | // A higher function value. The minimum is bracketed between x and step. 235 | // We want the next step to be closer to x because the function value 236 | // there is lower. 237 | 238 | theta := 3*(fx-f)/(step-x) + gx + g 239 | s := math.Max(math.Abs(gx), math.Abs(g)) 240 | s = math.Max(s, math.Abs(theta)) 241 | gamma := s * math.Sqrt((theta/s)*(theta/s)-(gx/s)*(g/s)) 242 | if step < x { 243 | gamma *= -1 244 | } 245 | p := gamma - gx + theta 246 | q := gamma - gx + gamma + g 247 | r := p / q 248 | stpc := x + r*(step-x) 249 | stpq := x + gx/((fx-f)/(step-x)+gx)/2*(step-x) 250 | 251 | if math.Abs(stpc-x) < math.Abs(stpq-x) { 252 | // The cubic step is closer to x than the quadratic step. 253 | // Take the cubic step. 254 | next = stpc 255 | } else { 256 | // If f is much larger than fx, then the quadratic step may be too 257 | // close to x. Therefore heuristically take the average of the 258 | // cubic and quadratic steps. 259 | next = stpc + (stpq-stpc)/2 260 | } 261 | bracketed = true 262 | 263 | case gNeg: 264 | // A lower function value and derivatives of opposite sign. The minimum 265 | // is bracketed between x and step. If we choose a step that is far 266 | // from step, the next iteration will also likely fall in this case. 267 | 268 | theta := 3*(fx-f)/(step-x) + gx + g 269 | s := math.Max(math.Abs(gx), math.Abs(g)) 270 | s = math.Max(s, math.Abs(theta)) 271 | gamma := s * math.Sqrt((theta/s)*(theta/s)-(gx/s)*(g/s)) 272 | if step > x { 273 | gamma *= -1 274 | } 275 | p := gamma - g + theta 276 | q := gamma - g + gamma + gx 277 | r := p / q 278 | stpc := step + r*(x-step) 279 | stpq := step + g/(g-gx)*(x-step) 280 | 281 | if math.Abs(stpc-step) > math.Abs(stpq-step) { 282 | // The cubic step is farther from x than the quadratic step. 283 | // Take the cubic step. 284 | next = stpc 285 | } else { 286 | // Take the quadratic step. 287 | next = stpq 288 | } 289 | bracketed = true 290 | 291 | case math.Abs(g) < math.Abs(gx): 292 | // A lower function value, derivatives of the same sign, and the 293 | // magnitude of the derivative decreases. Extrapolate function values 294 | // at x and step so that the next step lies between step and y. 295 | 296 | theta := 3*(fx-f)/(step-x) + gx + g 297 | s := math.Max(math.Abs(gx), math.Abs(g)) 298 | s = math.Max(s, math.Abs(theta)) 299 | gamma := s * math.Sqrt(math.Max(0, (theta/s)*(theta/s)-(gx/s)*(g/s))) 300 | if step > x { 301 | gamma *= -1 302 | } 303 | p := gamma - g + theta 304 | q := gamma + gx - g + gamma 305 | r := p / q 306 | var stpc float64 307 | switch { 308 | case r < 0 && gamma != 0: 309 | stpc = step + r*(x-step) 310 | case step > x: 311 | stpc = mt.upper 312 | default: 313 | stpc = mt.lower 314 | } 315 | stpq := step + g/(g-gx)*(x-step) 316 | 317 | if mt.bracketed { 318 | // We are extrapolating so be cautious and take the step that 319 | // is closer to step. 320 | if math.Abs(stpc-step) < math.Abs(stpq-step) { 321 | next = stpc 322 | } else { 323 | next = stpq 324 | } 325 | // Modify next if it is close to or beyond y. 326 | if step > x { 327 | next = math.Min(step+2.0/3*(y-step), next) 328 | } else { 329 | next = math.Max(step+2.0/3*(y-step), next) 330 | } 331 | } else { 332 | // Minimum has not been bracketed so take the larger step... 333 | if math.Abs(stpc-step) > math.Abs(stpq-step) { 334 | next = stpc 335 | } else { 336 | next = stpq 337 | } 338 | // ...but within reason. 339 | next = math.Max(mt.lower, math.Min(next, mt.upper)) 340 | } 341 | 342 | default: 343 | // A lower function value, derivatives of the same sign, and the 344 | // magnitude of the derivative does not decrease. The function seems to 345 | // decrease rapidly in the direction of the step. 346 | 347 | switch { 348 | case mt.bracketed: 349 | theta := 3*(f-fy)/(y-step) + gy + g 350 | s := math.Max(math.Abs(gy), math.Abs(g)) 351 | s = math.Max(s, math.Abs(theta)) 352 | gamma := s * math.Sqrt((theta/s)*(theta/s)-(gy/s)*(g/s)) 353 | if step > y { 354 | gamma *= -1 355 | } 356 | p := gamma - g + theta 357 | q := gamma - g + gamma + gy 358 | r := p / q 359 | next = step + r*(y-step) 360 | case step > x: 361 | next = mt.upper 362 | default: 363 | next = mt.lower 364 | } 365 | } 366 | 367 | if f > fx { 368 | // x is still the best step. 369 | mt.y = step 370 | mt.fy = f 371 | mt.gy = g 372 | } else { 373 | // step is the new best step. 374 | if gNeg { 375 | mt.y = x 376 | mt.fy = fx 377 | mt.gy = gx 378 | } 379 | mt.x = step 380 | mt.fx = f 381 | mt.gx = g 382 | } 383 | mt.bracketed = bracketed 384 | mt.step = next 385 | } 386 | -------------------------------------------------------------------------------- /neldermead.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "sort" 9 | 10 | "github.com/gonum/floats" 11 | ) 12 | 13 | // nmIterType is a Nelder-Mead evaluation kind 14 | type nmIterType int 15 | 16 | const ( 17 | nmReflected = iota 18 | nmExpanded 19 | nmContractedInside 20 | nmContractedOutside 21 | nmInitialize 22 | nmShrink 23 | nmMajor 24 | ) 25 | 26 | type nmVertexSorter struct { 27 | vertices [][]float64 28 | values []float64 29 | } 30 | 31 | func (n nmVertexSorter) Len() int { 32 | return len(n.values) 33 | } 34 | 35 | func (n nmVertexSorter) Less(i, j int) bool { 36 | return n.values[i] < n.values[j] 37 | } 38 | 39 | func (n nmVertexSorter) Swap(i, j int) { 40 | n.values[i], n.values[j] = n.values[j], n.values[i] 41 | n.vertices[i], n.vertices[j] = n.vertices[j], n.vertices[i] 42 | } 43 | 44 | // NelderMead is an implementation of the Nelder-Mead simplex algorithm for 45 | // gradient-free nonlinear optimization (not to be confused with Danzig's 46 | // simplex algorithm for linear programming). The implementation follows the 47 | // algorithm described in 48 | // 49 | // http://epubs.siam.org/doi/pdf/10.1137/S1052623496303470 50 | // 51 | // If an initial simplex is provided, it is used and initLoc is ignored. If 52 | // InitialVertices and InitialValues are both nil, an initial simplex will be 53 | // generated automatically using the initial location as one vertex, and each 54 | // additional vertex as SimplexSize away in one dimension. 55 | // 56 | // If the simplex update parameters (Reflection, etc.) 57 | // are zero, they will be set automatically based on the dimension according to 58 | // the recommendations in 59 | // 60 | // http://www.webpages.uidaho.edu/~fuchang/res/ANMS.pdf 61 | type NelderMead struct { 62 | InitialVertices [][]float64 63 | InitialValues []float64 64 | Reflection float64 // Reflection parameter (>0) 65 | Expansion float64 // Expansion parameter (>1) 66 | Contraction float64 // Contraction parameter (>0, <1) 67 | Shrink float64 // Shrink parameter (>0, <1) 68 | SimplexSize float64 // size of auto-constructed initial simplex 69 | 70 | reflection float64 71 | expansion float64 72 | contraction float64 73 | shrink float64 74 | 75 | vertices [][]float64 // location of the vertices sorted in ascending f 76 | values []float64 // function values at the vertices sorted in ascending f 77 | centroid []float64 // centroid of all but the worst vertex 78 | 79 | fillIdx int // index for filling the simplex during initialization and shrinking 80 | lastIter nmIterType // Last iteration 81 | reflectedPoint []float64 // Storage of the reflected point location 82 | reflectedValue float64 // Value at the last reflection point 83 | } 84 | 85 | func (n *NelderMead) Init(loc *Location) (Operation, error) { 86 | dim := len(loc.X) 87 | if cap(n.vertices) < dim+1 { 88 | n.vertices = make([][]float64, dim+1) 89 | } 90 | n.vertices = n.vertices[:dim+1] 91 | for i := range n.vertices { 92 | n.vertices[i] = resize(n.vertices[i], dim) 93 | } 94 | n.values = resize(n.values, dim+1) 95 | n.centroid = resize(n.centroid, dim) 96 | n.reflectedPoint = resize(n.reflectedPoint, dim) 97 | 98 | if n.SimplexSize == 0 { 99 | n.SimplexSize = 0.05 100 | } 101 | 102 | // Default parameter choices are chosen in a dimension-dependent way 103 | // from http://www.webpages.uidaho.edu/~fuchang/res/ANMS.pdf 104 | n.reflection = n.Reflection 105 | if n.reflection == 0 { 106 | n.reflection = 1 107 | } 108 | n.expansion = n.Expansion 109 | if n.expansion == 0 { 110 | n.expansion = 1 + 2/float64(dim) 111 | } 112 | n.contraction = n.Contraction 113 | if n.contraction == 0 { 114 | n.contraction = 0.75 - 1/(2*float64(dim)) 115 | } 116 | n.shrink = n.Shrink 117 | if n.shrink == 0 { 118 | n.shrink = 1 - 1/float64(dim) 119 | } 120 | 121 | if n.InitialVertices != nil { 122 | // Initial simplex provided. Copy the locations and values, and sort them. 123 | if len(n.InitialVertices) != dim+1 { 124 | panic("neldermead: incorrect number of vertices in initial simplex") 125 | } 126 | if len(n.InitialValues) != dim+1 { 127 | panic("neldermead: incorrect number of values in initial simplex") 128 | } 129 | for i := range n.InitialVertices { 130 | if len(n.InitialVertices[i]) != dim { 131 | panic("neldermead: vertex size mismatch") 132 | } 133 | copy(n.vertices[i], n.InitialVertices[i]) 134 | } 135 | copy(n.values, n.InitialValues) 136 | sort.Sort(nmVertexSorter{n.vertices, n.values}) 137 | computeCentroid(n.vertices, n.centroid) 138 | return n.returnNext(nmMajor, loc) 139 | } 140 | 141 | // No simplex provided. Begin initializing initial simplex. First simplex 142 | // entry is the initial location, then step 1 in every direction. 143 | copy(n.vertices[dim], loc.X) 144 | n.values[dim] = loc.F 145 | n.fillIdx = 0 146 | loc.X[n.fillIdx] += n.SimplexSize 147 | n.lastIter = nmInitialize 148 | return FuncEvaluation, nil 149 | } 150 | 151 | // computeCentroid computes the centroid of all the simplex vertices except the 152 | // final one 153 | func computeCentroid(vertices [][]float64, centroid []float64) { 154 | dim := len(centroid) 155 | for i := range centroid { 156 | centroid[i] = 0 157 | } 158 | for i := 0; i < dim; i++ { 159 | vertex := vertices[i] 160 | for j, v := range vertex { 161 | centroid[j] += v 162 | } 163 | } 164 | for i := range centroid { 165 | centroid[i] /= float64(dim) 166 | } 167 | } 168 | 169 | func (n *NelderMead) Iterate(loc *Location) (Operation, error) { 170 | dim := len(loc.X) 171 | switch n.lastIter { 172 | case nmInitialize: 173 | n.values[n.fillIdx] = loc.F 174 | copy(n.vertices[n.fillIdx], loc.X) 175 | n.fillIdx++ 176 | if n.fillIdx == dim { 177 | // Successfully finished building initial simplex. 178 | sort.Sort(nmVertexSorter{n.vertices, n.values}) 179 | computeCentroid(n.vertices, n.centroid) 180 | return n.returnNext(nmMajor, loc) 181 | } 182 | copy(loc.X, n.vertices[dim]) 183 | loc.X[n.fillIdx] += n.SimplexSize 184 | return FuncEvaluation, nil 185 | case nmMajor: 186 | // Nelder Mead iterations start with Reflection step 187 | return n.returnNext(nmReflected, loc) 188 | case nmReflected: 189 | n.reflectedValue = loc.F 190 | switch { 191 | case loc.F >= n.values[0] && loc.F < n.values[dim-1]: 192 | n.replaceWorst(loc.X, loc.F) 193 | return n.returnNext(nmMajor, loc) 194 | case loc.F < n.values[0]: 195 | return n.returnNext(nmExpanded, loc) 196 | default: 197 | if loc.F < n.values[dim] { 198 | return n.returnNext(nmContractedOutside, loc) 199 | } 200 | return n.returnNext(nmContractedInside, loc) 201 | } 202 | case nmExpanded: 203 | if loc.F < n.reflectedValue { 204 | n.replaceWorst(loc.X, loc.F) 205 | } else { 206 | n.replaceWorst(n.reflectedPoint, n.reflectedValue) 207 | } 208 | return n.returnNext(nmMajor, loc) 209 | case nmContractedOutside: 210 | if loc.F <= n.reflectedValue { 211 | n.replaceWorst(loc.X, loc.F) 212 | return n.returnNext(nmMajor, loc) 213 | } 214 | n.fillIdx = 1 215 | return n.returnNext(nmShrink, loc) 216 | case nmContractedInside: 217 | if loc.F < n.values[dim] { 218 | n.replaceWorst(loc.X, loc.F) 219 | return n.returnNext(nmMajor, loc) 220 | } 221 | n.fillIdx = 1 222 | return n.returnNext(nmShrink, loc) 223 | case nmShrink: 224 | copy(n.vertices[n.fillIdx], loc.X) 225 | n.values[n.fillIdx] = loc.F 226 | n.fillIdx++ 227 | if n.fillIdx != dim+1 { 228 | return n.returnNext(nmShrink, loc) 229 | } 230 | sort.Sort(nmVertexSorter{n.vertices, n.values}) 231 | computeCentroid(n.vertices, n.centroid) 232 | return n.returnNext(nmMajor, loc) 233 | default: 234 | panic("unreachable") 235 | } 236 | } 237 | 238 | // returnNext updates the location based on the iteration type and the current 239 | // simplex, and returns the next operation. 240 | func (n *NelderMead) returnNext(iter nmIterType, loc *Location) (Operation, error) { 241 | n.lastIter = iter 242 | switch iter { 243 | case nmMajor: 244 | // Fill loc with the current best point and value, 245 | // and command a convergence check. 246 | copy(loc.X, n.vertices[0]) 247 | loc.F = n.values[0] 248 | return MajorIteration, nil 249 | case nmReflected, nmExpanded, nmContractedOutside, nmContractedInside: 250 | // x_new = x_centroid + scale * (x_centroid - x_worst) 251 | var scale float64 252 | switch iter { 253 | case nmReflected: 254 | scale = n.reflection 255 | case nmExpanded: 256 | scale = n.reflection * n.expansion 257 | case nmContractedOutside: 258 | scale = n.reflection * n.contraction 259 | case nmContractedInside: 260 | scale = -n.contraction 261 | } 262 | dim := len(loc.X) 263 | floats.SubTo(loc.X, n.centroid, n.vertices[dim]) 264 | floats.Scale(scale, loc.X) 265 | floats.Add(loc.X, n.centroid) 266 | if iter == nmReflected { 267 | copy(n.reflectedPoint, loc.X) 268 | } 269 | return FuncEvaluation, nil 270 | case nmShrink: 271 | // x_shrink = x_best + delta * (x_i + x_best) 272 | floats.SubTo(loc.X, n.vertices[n.fillIdx], n.vertices[0]) 273 | floats.Scale(n.shrink, loc.X) 274 | floats.Add(loc.X, n.vertices[0]) 275 | return FuncEvaluation, nil 276 | default: 277 | panic("unreachable") 278 | } 279 | } 280 | 281 | // replaceWorst removes the worst location in the simplex and adds the new 282 | // {x, f} pair maintaining sorting. 283 | func (n *NelderMead) replaceWorst(x []float64, f float64) { 284 | dim := len(x) 285 | if f >= n.values[dim] { 286 | panic("increase in simplex value") 287 | } 288 | copy(n.vertices[dim], x) 289 | n.values[dim] = f 290 | 291 | // Sort the newly-added value. 292 | for i := dim - 1; i >= 0; i-- { 293 | if n.values[i] < f { 294 | break 295 | } 296 | n.vertices[i], n.vertices[i+1] = n.vertices[i+1], n.vertices[i] 297 | n.values[i], n.values[i+1] = n.values[i+1], n.values[i] 298 | } 299 | 300 | // Update the location of the centroid. Only one point has been replaced, so 301 | // subtract the worst point and add the new one. 302 | floats.AddScaled(n.centroid, -1/float64(dim), n.vertices[dim]) 303 | floats.AddScaled(n.centroid, 1/float64(dim), x) 304 | } 305 | 306 | func (*NelderMead) Needs() struct { 307 | Gradient bool 308 | Hessian bool 309 | } { 310 | return struct { 311 | Gradient bool 312 | Hessian bool 313 | }{false, false} 314 | } 315 | -------------------------------------------------------------------------------- /newton.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2015 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "math" 9 | 10 | "github.com/gonum/matrix/mat64" 11 | ) 12 | 13 | const maxNewtonModifications = 20 14 | 15 | // Newton implements a modified Newton's method for Hessian-based unconstrained 16 | // minimization. It applies regularization when the Hessian is not positive 17 | // definite, and it can converge to a local minimum from any starting point. 18 | // 19 | // Newton iteratively forms a quadratic model to the objective function f and 20 | // tries to minimize this approximate model. It generates a sequence of 21 | // locations x_k by means of 22 | // solve H_k d_k = -∇f_k for d_k, 23 | // x_{k+1} = x_k + α_k d_k, 24 | // where H_k is the Hessian matrix of f at x_k and α_k is a step size found by 25 | // a line search. 26 | // 27 | // Away from a minimizer H_k may not be positive definite and d_k may not be a 28 | // descent direction. Newton implements a Hessian modification strategy that 29 | // adds successively larger multiples of identity to H_k until it becomes 30 | // positive definite. Note that the repeated trial factorization of the 31 | // modified Hessian involved in this process can be computationally expensive. 32 | // 33 | // If the Hessian matrix cannot be formed explicitly or if the computational 34 | // cost of its factorization is prohibitive, BFGS or L-BFGS quasi-Newton method 35 | // can be used instead. 36 | type Newton struct { 37 | // Linesearcher is used for selecting suitable steps along the descent 38 | // direction d. Accepted steps should satisfy at least one of the Wolfe, 39 | // Goldstein or Armijo conditions. 40 | // If Linesearcher == nil, an appropriate default is chosen. 41 | Linesearcher Linesearcher 42 | // Increase is the factor by which a scalar tau is successively increased 43 | // so that (H + tau*I) is positive definite. Larger values reduce the 44 | // number of trial Hessian factorizations, but also reduce the second-order 45 | // information in H. 46 | // Increase must be greater than 1. If Increase is 0, it is defaulted to 5. 47 | Increase float64 48 | 49 | ls *LinesearchMethod 50 | 51 | hess *mat64.SymDense // Storage for a copy of the Hessian matrix. 52 | chol mat64.Cholesky // Storage for the Cholesky factorization. 53 | tau float64 54 | } 55 | 56 | func (n *Newton) Init(loc *Location) (Operation, error) { 57 | if n.Increase == 0 { 58 | n.Increase = 5 59 | } 60 | if n.Increase <= 1 { 61 | panic("optimize: Newton.Increase must be greater than 1") 62 | } 63 | if n.Linesearcher == nil { 64 | n.Linesearcher = &Bisection{} 65 | } 66 | if n.ls == nil { 67 | n.ls = &LinesearchMethod{} 68 | } 69 | n.ls.Linesearcher = n.Linesearcher 70 | n.ls.NextDirectioner = n 71 | 72 | return n.ls.Init(loc) 73 | } 74 | 75 | func (n *Newton) Iterate(loc *Location) (Operation, error) { 76 | return n.ls.Iterate(loc) 77 | } 78 | 79 | func (n *Newton) InitDirection(loc *Location, dir []float64) (stepSize float64) { 80 | dim := len(loc.X) 81 | n.hess = resizeSymDense(n.hess, dim) 82 | n.tau = 0 83 | return n.NextDirection(loc, dir) 84 | } 85 | 86 | func (n *Newton) NextDirection(loc *Location, dir []float64) (stepSize float64) { 87 | // This method implements Algorithm 3.3 (Cholesky with Added Multiple of 88 | // the Identity) from Nocedal, Wright (2006), 2nd edition. 89 | 90 | dim := len(loc.X) 91 | d := mat64.NewVector(dim, dir) 92 | grad := mat64.NewVector(dim, loc.Gradient) 93 | n.hess.CopySym(loc.Hessian) 94 | 95 | // Find the smallest diagonal entry of the Hessian. 96 | minA := n.hess.At(0, 0) 97 | for i := 1; i < dim; i++ { 98 | a := n.hess.At(i, i) 99 | if a < minA { 100 | minA = a 101 | } 102 | } 103 | // If the smallest diagonal entry is positive, the Hessian may be positive 104 | // definite, and so first attempt to apply the Cholesky factorization to 105 | // the un-modified Hessian. If the smallest entry is negative, use the 106 | // final tau from the last iteration if regularization was needed, 107 | // otherwise guess an appropriate value for tau. 108 | if minA > 0 { 109 | n.tau = 0 110 | } else if n.tau == 0 { 111 | n.tau = -minA + 0.001 112 | } 113 | 114 | for k := 0; k < maxNewtonModifications; k++ { 115 | if n.tau != 0 { 116 | // Add a multiple of identity to the Hessian. 117 | for i := 0; i < dim; i++ { 118 | n.hess.SetSym(i, i, loc.Hessian.At(i, i)+n.tau) 119 | } 120 | } 121 | // Try to apply the Cholesky factorization. 122 | pd := n.chol.Factorize(n.hess) 123 | if pd { 124 | // Store the solution in d's backing array, dir. 125 | d.SolveCholeskyVec(&n.chol, grad) 126 | d.ScaleVec(-1, d) 127 | return 1 128 | } 129 | // Modified Hessian is not PD, so increase tau. 130 | n.tau = math.Max(n.Increase*n.tau, 0.001) 131 | } 132 | 133 | // Hessian modification failed to get a PD matrix. Return the negative 134 | // gradient as the descent direction. 135 | d.ScaleVec(-1, grad) 136 | return 1 137 | } 138 | 139 | func (n *Newton) Needs() struct { 140 | Gradient bool 141 | Hessian bool 142 | } { 143 | return struct { 144 | Gradient bool 145 | Hessian bool 146 | }{true, true} 147 | } 148 | -------------------------------------------------------------------------------- /printer.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "math" 11 | "os" 12 | "time" 13 | 14 | "github.com/gonum/floats" 15 | ) 16 | 17 | var printerHeadings = [...]string{ 18 | "Iter", 19 | "Runtime", 20 | "FuncEvals", 21 | "Func", 22 | "GradEvals", 23 | "|Gradient|∞", 24 | "HessEvals", 25 | } 26 | 27 | const ( 28 | printerBaseTmpl = "%9v %16v %9v %22v" // Base template for headings and values that are always printed. 29 | printerGradTmpl = " %9v %22v" // Appended to base template when loc.Gradient != nil. 30 | printerHessTmpl = " %9v" // Appended to base template when loc.Hessian != nil. 31 | ) 32 | 33 | // Printer writes column-format output to the specified writer as the optimization 34 | // progresses. By default, it writes to os.Stdout. 35 | type Printer struct { 36 | Writer io.Writer 37 | HeadingInterval int 38 | ValueInterval time.Duration 39 | 40 | lastHeading int 41 | lastValue time.Time 42 | } 43 | 44 | func NewPrinter() *Printer { 45 | return &Printer{ 46 | Writer: os.Stdout, 47 | HeadingInterval: 30, 48 | ValueInterval: 500 * time.Millisecond, 49 | } 50 | } 51 | 52 | func (p *Printer) Init() error { 53 | p.lastHeading = p.HeadingInterval // So the headings are printed the first time. 54 | p.lastValue = time.Now().Add(-p.ValueInterval) // So the values are printed the first time. 55 | return nil 56 | } 57 | 58 | func (p *Printer) Record(loc *Location, op Operation, stats *Stats) error { 59 | if op != MajorIteration && op != InitIteration && op != PostIteration { 60 | return nil 61 | } 62 | 63 | // Print values always on PostIteration or when ValueInterval has elapsed. 64 | printValues := time.Since(p.lastValue) > p.ValueInterval || op == PostIteration 65 | if !printValues { 66 | // Return early if not printing anything. 67 | return nil 68 | } 69 | 70 | // Print heading when HeadingInterval lines have been printed, but never on PostIteration. 71 | printHeading := p.lastHeading >= p.HeadingInterval && op != PostIteration 72 | if printHeading { 73 | p.lastHeading = 1 74 | } else { 75 | p.lastHeading++ 76 | } 77 | 78 | if printHeading { 79 | headings := "\n" + fmt.Sprintf(printerBaseTmpl, printerHeadings[0], printerHeadings[1], printerHeadings[2], printerHeadings[3]) 80 | if loc.Gradient != nil { 81 | headings += fmt.Sprintf(printerGradTmpl, printerHeadings[4], printerHeadings[5]) 82 | } 83 | if loc.Hessian != nil { 84 | headings += fmt.Sprintf(printerHessTmpl, printerHeadings[6]) 85 | } 86 | _, err := fmt.Fprintln(p.Writer, headings) 87 | if err != nil { 88 | return err 89 | } 90 | } 91 | 92 | values := fmt.Sprintf(printerBaseTmpl, stats.MajorIterations, stats.Runtime, stats.FuncEvaluations, loc.F) 93 | if loc.Gradient != nil { 94 | values += fmt.Sprintf(printerGradTmpl, stats.GradEvaluations, floats.Norm(loc.Gradient, math.Inf(1))) 95 | } 96 | if loc.Hessian != nil { 97 | values += fmt.Sprintf(printerHessTmpl, stats.HessEvaluations) 98 | } 99 | _, err := fmt.Fprintln(p.Writer, values) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | p.lastValue = time.Now() 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /stepsizers.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "math" 9 | 10 | "github.com/gonum/floats" 11 | ) 12 | 13 | const ( 14 | initialStepFactor = 1 15 | 16 | quadraticMinimumStepSize = 1e-3 17 | quadraticMaximumStepSize = 1 18 | quadraticThreshold = 1e-12 19 | 20 | firstOrderMinimumStepSize = quadraticMinimumStepSize 21 | firstOrderMaximumStepSize = quadraticMaximumStepSize 22 | ) 23 | 24 | // ConstantStepSize is a StepSizer that returns the same step size for 25 | // every iteration. 26 | type ConstantStepSize struct { 27 | Size float64 28 | } 29 | 30 | func (c ConstantStepSize) Init(_ *Location, _ []float64) float64 { 31 | return c.Size 32 | } 33 | 34 | func (c ConstantStepSize) StepSize(_ *Location, _ []float64) float64 { 35 | return c.Size 36 | } 37 | 38 | // QuadraticStepSize estimates the initial line search step size as the minimum 39 | // of a quadratic that interpolates f(x_{k-1}), f(x_k) and ∇f_k⋅p_k. 40 | // This is useful for line search methods that do not produce well-scaled 41 | // descent directions, such as gradient descent or conjugate gradient methods. 42 | // The step size is bounded away from zero. 43 | type QuadraticStepSize struct { 44 | // Threshold determines that the initial step size should be estimated by 45 | // quadratic interpolation when the relative change in the objective 46 | // function is larger than Threshold. Otherwise the initial step size is 47 | // set to 2*previous step size. 48 | // If Threshold is zero, it will be set to 1e-12. 49 | Threshold float64 50 | // InitialStepFactor sets the step size for the first iteration to be InitialStepFactor / |g|_∞. 51 | // If InitialStepFactor is zero, it will be set to one. 52 | InitialStepFactor float64 53 | // MinStepSize is the lower bound on the estimated step size. 54 | // MinStepSize times GradientAbsTol should always be greater than machine epsilon. 55 | // If MinStepSize is zero, it will be set to 1e-3. 56 | MinStepSize float64 57 | // MaxStepSize is the upper bound on the estimated step size. 58 | // If MaxStepSize is zero, it will be set to 1. 59 | MaxStepSize float64 60 | 61 | fPrev float64 62 | dirPrevNorm float64 63 | projGradPrev float64 64 | xPrev []float64 65 | } 66 | 67 | func (q *QuadraticStepSize) Init(loc *Location, dir []float64) (stepSize float64) { 68 | if q.Threshold == 0 { 69 | q.Threshold = quadraticThreshold 70 | } 71 | if q.InitialStepFactor == 0 { 72 | q.InitialStepFactor = initialStepFactor 73 | } 74 | if q.MinStepSize == 0 { 75 | q.MinStepSize = quadraticMinimumStepSize 76 | } 77 | if q.MaxStepSize == 0 { 78 | q.MaxStepSize = quadraticMaximumStepSize 79 | } 80 | if q.MaxStepSize <= q.MinStepSize { 81 | panic("optimize: MinStepSize not smaller than MaxStepSize") 82 | } 83 | 84 | gNorm := floats.Norm(loc.Gradient, math.Inf(1)) 85 | stepSize = math.Max(q.MinStepSize, math.Min(q.InitialStepFactor/gNorm, q.MaxStepSize)) 86 | 87 | q.fPrev = loc.F 88 | q.dirPrevNorm = floats.Norm(dir, 2) 89 | q.projGradPrev = floats.Dot(loc.Gradient, dir) 90 | q.xPrev = resize(q.xPrev, len(loc.X)) 91 | copy(q.xPrev, loc.X) 92 | return stepSize 93 | } 94 | 95 | func (q *QuadraticStepSize) StepSize(loc *Location, dir []float64) (stepSize float64) { 96 | stepSizePrev := floats.Distance(loc.X, q.xPrev, 2) / q.dirPrevNorm 97 | projGrad := floats.Dot(loc.Gradient, dir) 98 | 99 | stepSize = 2 * stepSizePrev 100 | if !floats.EqualWithinRel(q.fPrev, loc.F, q.Threshold) { 101 | // Two consecutive function values are not relatively equal, so 102 | // computing the minimum of a quadratic interpolant might make sense 103 | 104 | df := (loc.F - q.fPrev) / stepSizePrev 105 | quadTest := df - q.projGradPrev 106 | if quadTest > 0 { 107 | // There is a chance of approximating the function well by a 108 | // quadratic only if the finite difference (f_k-f_{k-1})/stepSizePrev 109 | // is larger than ∇f_{k-1}⋅p_{k-1} 110 | 111 | // Set the step size to the minimizer of the quadratic function that 112 | // interpolates f_{k-1}, ∇f_{k-1}⋅p_{k-1} and f_k 113 | stepSize = -q.projGradPrev * stepSizePrev / quadTest / 2 114 | } 115 | } 116 | // Bound the step size to lie in [MinStepSize, MaxStepSize] 117 | stepSize = math.Max(q.MinStepSize, math.Min(stepSize, q.MaxStepSize)) 118 | 119 | q.fPrev = loc.F 120 | q.dirPrevNorm = floats.Norm(dir, 2) 121 | q.projGradPrev = projGrad 122 | copy(q.xPrev, loc.X) 123 | return stepSize 124 | } 125 | 126 | // FirstOrderStepSize estimates the initial line search step size based on the 127 | // assumption that the first-order change in the function will be the same as 128 | // that obtained at the previous iteration. That is, the initial step size s^0_k 129 | // is chosen so that 130 | // s^0_k ∇f_k⋅p_k = s_{k-1} ∇f_{k-1}⋅p_{k-1} 131 | // This is useful for line search methods that do not produce well-scaled 132 | // descent directions, such as gradient descent or conjugate gradient methods. 133 | type FirstOrderStepSize struct { 134 | // InitialStepFactor sets the step size for the first iteration to be InitialStepFactor / |g|_∞. 135 | // If InitialStepFactor is zero, it will be set to one. 136 | InitialStepFactor float64 137 | // MinStepSize is the lower bound on the estimated step size. 138 | // MinStepSize times GradientAbsTol should always be greater than machine epsilon. 139 | // If MinStepSize is zero, it will be set to 1e-3. 140 | MinStepSize float64 141 | // MaxStepSize is the upper bound on the estimated step size. 142 | // If MaxStepSize is zero, it will be set to 1. 143 | MaxStepSize float64 144 | 145 | dirPrevNorm float64 146 | projGradPrev float64 147 | xPrev []float64 148 | } 149 | 150 | func (fo *FirstOrderStepSize) Init(loc *Location, dir []float64) (stepSize float64) { 151 | if fo.InitialStepFactor == 0 { 152 | fo.InitialStepFactor = initialStepFactor 153 | } 154 | if fo.MinStepSize == 0 { 155 | fo.MinStepSize = firstOrderMinimumStepSize 156 | } 157 | if fo.MaxStepSize == 0 { 158 | fo.MaxStepSize = firstOrderMaximumStepSize 159 | } 160 | if fo.MaxStepSize <= fo.MinStepSize { 161 | panic("optimize: MinStepSize not smaller than MaxStepSize") 162 | } 163 | 164 | gNorm := floats.Norm(loc.Gradient, math.Inf(1)) 165 | stepSize = math.Max(fo.MinStepSize, math.Min(fo.InitialStepFactor/gNorm, fo.MaxStepSize)) 166 | 167 | fo.dirPrevNorm = floats.Norm(dir, 2) 168 | fo.projGradPrev = floats.Dot(loc.Gradient, dir) 169 | fo.xPrev = resize(fo.xPrev, len(loc.X)) 170 | copy(fo.xPrev, loc.X) 171 | return stepSize 172 | } 173 | 174 | func (fo *FirstOrderStepSize) StepSize(loc *Location, dir []float64) (stepSize float64) { 175 | stepSizePrev := floats.Distance(loc.X, fo.xPrev, 2) / fo.dirPrevNorm 176 | projGrad := floats.Dot(loc.Gradient, dir) 177 | 178 | stepSize = stepSizePrev * fo.projGradPrev / projGrad 179 | stepSize = math.Max(fo.MinStepSize, math.Min(stepSize, fo.MaxStepSize)) 180 | 181 | fo.dirPrevNorm = floats.Norm(dir, 2) 182 | fo.projGradPrev = floats.Dot(loc.Gradient, dir) 183 | copy(fo.xPrev, loc.X) 184 | return stepSize 185 | } 186 | -------------------------------------------------------------------------------- /termination.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import "errors" 8 | 9 | // Status represents the status of the optimization. Programs 10 | // should not rely on the underlying numeric value of the Status being constant. 11 | type Status int 12 | 13 | const ( 14 | NotTerminated Status = iota 15 | Success 16 | FunctionThreshold 17 | FunctionConvergence 18 | GradientThreshold 19 | StepConvergence 20 | FunctionNegativeInfinity 21 | Failure 22 | IterationLimit 23 | RuntimeLimit 24 | FunctionEvaluationLimit 25 | GradientEvaluationLimit 26 | HessianEvaluationLimit 27 | ) 28 | 29 | func (s Status) String() string { 30 | return statuses[s].name 31 | } 32 | 33 | // Early returns true if the status indicates the optimization ended before a 34 | // minimum was found. As an example, if the maximum iterations was reached, a 35 | // minimum was not found, but if the gradient norm was reached then a minimum 36 | // was found. 37 | func (s Status) Early() bool { 38 | return statuses[s].early 39 | } 40 | 41 | // Err returns the error associated with an early ending to the minimization. If 42 | // Early returns false, Err will return nil. 43 | func (s Status) Err() error { 44 | return statuses[s].err 45 | } 46 | 47 | var statuses = []struct { 48 | name string 49 | early bool 50 | err error 51 | }{ 52 | { 53 | name: "NotTerminated", 54 | }, 55 | { 56 | name: "Success", 57 | }, 58 | { 59 | name: "FunctionThreshold", 60 | }, 61 | { 62 | name: "FunctionConvergence", 63 | }, 64 | { 65 | name: "GradientThreshold", 66 | }, 67 | { 68 | name: "StepConvergence", 69 | }, 70 | { 71 | name: "FunctionNegativeInfinity", 72 | }, 73 | { 74 | name: "Failure", 75 | early: true, 76 | err: errors.New("optimize: termination ended in failure"), 77 | }, 78 | { 79 | name: "IterationLimit", 80 | early: true, 81 | err: errors.New("optimize: maximum number of major iterations reached"), 82 | }, 83 | { 84 | name: "RuntimeLimit", 85 | early: true, 86 | err: errors.New("optimize: maximum runtime reached"), 87 | }, 88 | { 89 | name: "FunctionEvaluationLimit", 90 | early: true, 91 | err: errors.New("optimize: maximum number of function evaluations reached"), 92 | }, 93 | { 94 | name: "GradientEvaluationLimit", 95 | early: true, 96 | err: errors.New("optimize: maximum number of gradient evaluations reached"), 97 | }, 98 | { 99 | name: "HessianEvaluationLimit", 100 | early: true, 101 | err: errors.New("optimize: maximum number of Hessian evaluations reached"), 102 | }, 103 | } 104 | 105 | // NewStatus returns a unique Status variable to represent a custom status. 106 | // NewStatus is intended to be called only during package initialization, and 107 | // calls to NewStatus are not thread safe. 108 | // 109 | // NewStatus takes in three arguments, the string that should be output from 110 | // Status.String, a boolean if the status indicates early optimization conclusion, 111 | // and the error to return from Err (if any). 112 | func NewStatus(name string, early bool, err error) Status { 113 | statuses = append(statuses, struct { 114 | name string 115 | early bool 116 | err error 117 | }{name, early, err}) 118 | return Status(len(statuses) - 1) 119 | } 120 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | // Copyright ©2014 The gonum Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package optimize 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "math" 11 | "time" 12 | 13 | "github.com/gonum/matrix/mat64" 14 | ) 15 | 16 | const defaultGradientAbsTol = 1e-6 17 | 18 | // Operation represents the set of operations commanded by Method at each 19 | // iteration. It is a bitmap of various Iteration and Evaluation constants. 20 | // Individual constants must NOT be combined together by the binary OR operator 21 | // except for the Evaluation operations. 22 | type Operation uint64 23 | 24 | // Supported Operations. 25 | const ( 26 | // NoOperation specifies that no evaluation or convergence check should 27 | // take place. 28 | NoOperation Operation = 0 29 | // InitIteration is sent to Recorder to indicate the initial location. 30 | // All fields of the location to record must be valid. 31 | // Method must not return it. 32 | InitIteration Operation = 1 << (iota - 1) 33 | // PostIteration is sent to Recorder to indicate the final location 34 | // reached during an optimization run. 35 | // All fields of the location to record must be valid. 36 | // Method must not return it. 37 | PostIteration 38 | // MajorIteration indicates that the next candidate location for 39 | // an optimum has been found and convergence should be checked. 40 | MajorIteration 41 | // FuncEvaluation specifies that the objective function 42 | // should be evaluated. 43 | FuncEvaluation 44 | // GradEvaluation specifies that the gradient 45 | // of the objective function should be evaluated. 46 | GradEvaluation 47 | // HessEvaluation specifies that the Hessian 48 | // of the objective function should be evaluated. 49 | HessEvaluation 50 | 51 | // Mask for the evaluating operations. 52 | evalMask = FuncEvaluation | GradEvaluation | HessEvaluation 53 | ) 54 | 55 | func (op Operation) isEvaluation() bool { 56 | return op&evalMask != 0 && op&^evalMask == 0 57 | } 58 | 59 | func (op Operation) String() string { 60 | if op&evalMask != 0 { 61 | return fmt.Sprintf("Evaluation(Func: %t, Grad: %t, Hess: %t, Extra: 0b%b)", 62 | op&FuncEvaluation != 0, 63 | op&GradEvaluation != 0, 64 | op&HessEvaluation != 0, 65 | op&^(evalMask)) 66 | } 67 | s, ok := operationNames[op] 68 | if ok { 69 | return s 70 | } 71 | return fmt.Sprintf("Operation(%d)", op) 72 | } 73 | 74 | var operationNames = map[Operation]string{ 75 | NoOperation: "NoOperation", 76 | InitIteration: "InitIteration", 77 | MajorIteration: "MajorIteration", 78 | PostIteration: "PostIteration", 79 | } 80 | 81 | // Location represents a location in the optimization procedure. 82 | type Location struct { 83 | X []float64 84 | F float64 85 | Gradient []float64 86 | Hessian *mat64.SymDense 87 | } 88 | 89 | // Result represents the answer of an optimization run. It contains the optimum 90 | // location as well as the Status at convergence and Statistics taken during the 91 | // run. 92 | type Result struct { 93 | Location 94 | Stats 95 | Status Status 96 | } 97 | 98 | // Stats contains the statistics of the run. 99 | type Stats struct { 100 | MajorIterations int // Total number of major iterations 101 | FuncEvaluations int // Number of evaluations of Func 102 | GradEvaluations int // Number of evaluations of Grad 103 | HessEvaluations int // Number of evaluations of Hess 104 | Runtime time.Duration // Total runtime of the optimization 105 | } 106 | 107 | // complementEval returns an evaluating operation that evaluates fields of loc 108 | // not evaluated by eval. 109 | func complementEval(loc *Location, eval Operation) (complEval Operation) { 110 | if eval&FuncEvaluation == 0 { 111 | complEval = FuncEvaluation 112 | } 113 | if loc.Gradient != nil && eval&GradEvaluation == 0 { 114 | complEval |= GradEvaluation 115 | } 116 | if loc.Hessian != nil && eval&HessEvaluation == 0 { 117 | complEval |= HessEvaluation 118 | } 119 | return complEval 120 | } 121 | 122 | // Problem describes the optimization problem to be solved. 123 | type Problem struct { 124 | // Func evaluates the objective function at the given location. Func 125 | // must not modify x. 126 | Func func(x []float64) float64 127 | 128 | // Grad evaluates the gradient at x and stores the result in-place in grad. 129 | // Grad must not modify x. 130 | Grad func(grad []float64, x []float64) 131 | 132 | // Hess evaluates the Hessian at x and stores the result in-place in hess. 133 | // Hess must not modify x. 134 | Hess func(hess mat64.MutableSymmetric, x []float64) 135 | 136 | // Status reports the status of the objective function being optimized and any 137 | // error. This can be used to terminate early, for example when the function is 138 | // not able to evaluate itself. The user can use one of the pre-provided Status 139 | // constants, or may call NewStatus to create a custom Status value. 140 | Status func() (Status, error) 141 | } 142 | 143 | // TODO(btracey): Think about making this an exported function when the 144 | // constraint interface is designed. 145 | func (p Problem) satisfies(method Needser) error { 146 | if method.Needs().Gradient && p.Grad == nil { 147 | return errors.New("optimize: problem does not provide needed Grad function") 148 | } 149 | if method.Needs().Hessian && p.Hess == nil { 150 | return errors.New("optimize: problem does not provide needed Hess function") 151 | } 152 | return nil 153 | } 154 | 155 | // Settings represents settings of the optimization run. It contains initial 156 | // settings, convergence information, and Recorder information. In general, users 157 | // should use DefaultSettings rather than constructing a Settings literal. 158 | // 159 | // If UseInitData is true, InitialValue, InitialGradient and InitialHessian 160 | // specify function information at the initial location. 161 | // 162 | // If Recorder is nil, no information will be recorded. 163 | type Settings struct { 164 | UseInitialData bool // Use supplied information about the conditions at the initial x. 165 | InitialValue float64 // Function value at the initial x. 166 | InitialGradient []float64 // Gradient at the initial x. 167 | InitialHessian *mat64.SymDense // Hessian at the initial x. 168 | 169 | // FunctionThreshold is the threshold for acceptably small values of the 170 | // objective function. FunctionThreshold status is returned if 171 | // the objective function is less than this value. 172 | // The default value is -inf. 173 | FunctionThreshold float64 174 | 175 | // GradientThreshold determines the accuracy to which the minimum is found. 176 | // GradientThreshold status is returned if the infinity norm of 177 | // the gradient is less than this value. 178 | // Has no effect if gradient information is not used. 179 | // The default value is 1e-6. 180 | GradientThreshold float64 181 | 182 | // FunctionConverge tests that the function value decreases by a 183 | // significant amount over the specified number of iterations. 184 | // 185 | // If f < f_best and 186 | // f_best - f > FunctionConverge.Relative * maxabs(f, f_best) + FunctionConverge.Absolute 187 | // then a significant decrease has occured, and f_best is updated. 188 | // 189 | // If there is no significant decrease for FunctionConverge.Iterations 190 | // major iterations, FunctionConvergence status is returned. 191 | // 192 | // If this is nil or if FunctionConverge.Iterations == 0, it has no effect. 193 | FunctionConverge *FunctionConverge 194 | 195 | // MajorIterations is the maximum number of iterations allowed. 196 | // IterationLimit status is returned if the number of major iterations 197 | // equals or exceeds this value. 198 | // If it equals zero, this setting has no effect. 199 | // The default value is 0. 200 | MajorIterations int 201 | 202 | // Runtime is the maximum runtime allowed. RuntimeLimit status is returned 203 | // if the duration of the run is longer than this value. Runtime is only 204 | // checked at iterations of the Method. 205 | // If it equals zero, this setting has no effect. 206 | // The default value is 0. 207 | Runtime time.Duration 208 | 209 | // FuncEvaluations is the maximum allowed number of function evaluations. 210 | // FunctionEvaluationLimit status is returned if the total number of calls 211 | // to Func equals or exceeds this number. 212 | // If it equals zero, this setting has no effect. 213 | // The default value is 0. 214 | FuncEvaluations int 215 | 216 | // GradEvaluations is the maximum allowed number of gradient evaluations. 217 | // GradientEvaluationLimit status is returned if the total number of calls 218 | // to Grad equals or exceeds this number. 219 | // If it equals zero, this setting has no effect. 220 | // The default value is 0. 221 | GradEvaluations int 222 | 223 | // HessEvaluations is the maximum allowed number of Hessian evaluations. 224 | // HessianEvaluationLimit status is returned if the total number of calls 225 | // to Hess equals or exceeds this number. 226 | // If it equals zero, this setting has no effect. 227 | // The default value is 0. 228 | HessEvaluations int 229 | 230 | Recorder Recorder 231 | 232 | // Concurrent represents how many concurrent evaluations are possible. 233 | Concurrent int 234 | } 235 | 236 | // DefaultSettings returns a new Settings struct containing the default settings. 237 | func DefaultSettings() *Settings { 238 | return &Settings{ 239 | GradientThreshold: defaultGradientAbsTol, 240 | FunctionThreshold: math.Inf(-1), 241 | FunctionConverge: &FunctionConverge{ 242 | Absolute: 1e-10, 243 | Iterations: 20, 244 | }, 245 | } 246 | } 247 | 248 | // resize takes x and returns a slice of length dim. It returns a resliced x 249 | // if cap(x) >= dim, and a new slice otherwise. 250 | func resize(x []float64, dim int) []float64 { 251 | if dim > cap(x) { 252 | return make([]float64, dim) 253 | } 254 | return x[:dim] 255 | } 256 | 257 | func resizeSymDense(m *mat64.SymDense, dim int) *mat64.SymDense { 258 | if m == nil || cap(m.RawSymmetric().Data) < dim*dim { 259 | return mat64.NewSymDense(dim, nil) 260 | } 261 | return mat64.NewSymDense(dim, m.RawSymmetric().Data[:dim*dim]) 262 | } 263 | --------------------------------------------------------------------------------