├── LICENSE.md ├── Manifest.toml ├── README.md ├── REQUIRE ├── bracketing.ipynb ├── derivatives.ipynb ├── descent.ipynb ├── design-under-uncertainty.ipynb ├── direct.ipynb ├── discrete.ipynb ├── expr.ipynb ├── first-order.ipynb ├── gp.jl ├── introduction.ipynb ├── julia.ipynb ├── linear.ipynb ├── math-concepts.ipynb ├── mdo.ipynb ├── multiobjective.ipynb ├── penalty.ipynb ├── population.ipynb ├── prob-surrogate-models.ipynb ├── sampling-plans.ipynb ├── second-order.ipynb ├── stochastic.ipynb ├── support_code.jl ├── surrogate-models.ipynb ├── surrogate-optimization.ipynb ├── test-functions.ipynb └── uncertaintyprop.ipynb /LICENSE.md: -------------------------------------------------------------------------------- 1 | Permission is granted, free of charge, to use the code snippets associated with Algorithms for Optimization, 2 | subject to the condition that the source of the code is acknowledged. 3 | Algorithms for Optimization is Copyright (c) 2019 by the Massachusetts Institute of Technology. 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Algorithms for Optimization Jupyter Notebooks 2 | 3 | This repository contains supplemental Jupyter notebooks to accompany [Algorithms for Optimization](http://mitpress.mit.edu/books/algorithms-optimization) by Mykel Kochenderfer and Tim Wheeler. 4 | These notebooks were generated from the Algorithms for Optimization source code. 5 | We provide these notebooks to aid with the development of lectures and understanding the material, with the hope that you find it useful. 6 | 7 | ## Installation 8 | All notebooks have Julia 1.0.1 kernels. 9 | [Julia can be installed here.](https://julialang.org/downloads/) 10 | 11 | Rendering is managed by [PGFPlots.jl](https://github.com/JuliaTeX/PGFPlots.jl). 12 | Please see [their documentation](https://nbviewer.jupyter.org/github/JuliaTeX/PGFPlots.jl/blob/master/doc/PGFPlots.ipynb) for important installation instructions. 13 | 14 | Julia notebooks are supported by [IJulia](https://github.com/JuliaLang/IJulia.jl). 15 | 16 | Once the repo is cloned, one can set up the required packages from the terminal before launching the jupyter notebook: 17 | ``` 18 | export JULIA_PROJECT="@." 19 | julia -e 'using Pkg; pkg"instantiate"' 20 | jupyter notebook 21 | ``` 22 | -------------------------------------------------------------------------------- /REQUIRE: -------------------------------------------------------------------------------- 1 | julia 1.0.1 2 | 3 | Colors 4 | ColorSchemes 5 | DataStructures 6 | Discretizers 7 | Distributions 8 | ExprRules 9 | ForwardDiff 10 | IterTools 11 | LightGraphs 12 | LinearAlgebra 13 | NLopt 14 | ImageMagick 15 | OnlineStats 16 | Optim 17 | Parameters 18 | PGFPlots 19 | Polynomials 20 | Primes 21 | Printf 22 | QuadGK 23 | Random 24 | Sobol 25 | SpecialFunctions 26 | Statistics 27 | StatsBase 28 | SymEngine 29 | TikzPictures 30 | TreeView 31 | Vec 32 | Weave 33 | Zygote -------------------------------------------------------------------------------- /derivatives.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Derivatives\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "p = let\n", 32 | "\n", 33 | "f = x->sin(x)\n", 34 | "\n", 35 | "x₀ = 0.5\n", 36 | "ymin = -1.1\n", 37 | "lo = -2\n", 38 | "hi = 3\n", 39 | "\n", 40 | "function get_axis(h)\n", 41 | "a, b = x₀, x₀ + h\n", 42 | "fa, fb = f(a), f(b)\n", 43 | "m = (fb - fa) / (b-a)\n", 44 | "\n", 45 | "p = Plots.Plot[]\n", 46 | "push!(p, Plots.Linear(f, (lo,hi), style=\"thick, black\"))\n", 47 | "push!(p, Plots.Linear([a,a],[fa, ymin], style=\"solid, black, mark=none\"))\n", 48 | "push!(p, Plots.Linear([b,b],[fb, ymin], style=\"solid, black, mark=none\"))\n", 49 | "push!(p, Plots.Linear([lo,hi],[ymin, ymin], style=\"solid, black, mark=none\"))\n", 50 | "push!(p, Plots.Linear([lo, hi],[fa + (lo-a)*m, fa + (hi-a)*m], style=\"ultra thick, solid, pastelBlue, mark=none\"))\n", 51 | "push!(p, Plots.Scatter([a, b],[fa, fb], style=\"solid, mark=*, mark options={draw=black, fill=black}\"))\n", 52 | "\n", 53 | "Axis(p, style=\"axis lines=left, axis line style = {transparent}, xtick={$(x₀ + h/2)}, xticklabels={\\$h\\$}, ytick=\\\\empty, xtick style={draw=none}, ytick style={draw=none}\") #$\n", 54 | "end\n", 55 | "\n", 56 | "g = GroupPlot(3, 1, groupStyle=\"horizontal sep=1cm\", style=\"ymin=-1.1, ymax=1.1, width=6cm\")\n", 57 | "push!(g, get_axis(2.0))\n", 58 | "push!(g, get_axis(1.0))\n", 59 | "push!(g, get_axis(0.5))\n", 60 | "g\n", 61 | "end\n", 62 | "\n", 63 | "plot(p)" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": { 70 | "collapsed": true 71 | }, 72 | "outputs": [], 73 | "source": [ 74 | "% \tplot(Axis(\n", 75 | "% \t\t[Plots.Contour(x->exp(0-x[1]^2-x[2]^2)*(x[1]-2*x[2]), (-2,2), (-2,2), levels=[-0.1,-0.3, -0.5]),\n", 76 | "% \t\t Plots.Scatter([-0.75], [1.0], style=\"black, mark=*, mark options={draw=black}\"),\n", 77 | "% \t\t Plots.Command(\"\\\\draw [->, ultra thick] (axis cs:-0.75,1) -- (axis cs:-0.25,0.5);\"),\n", 78 | "% \t\t Plots.Command(\"\\\\node at (axis cs:-0.75,1) [anchor=north] {\\$\\\\vect x\\$};\"),\n", 79 | "% \t\t Plots.Command(\"\\\\node at (axis cs:-0.5,0.75) [anchor=south west] {\\$\\\\vect s\\$};\"),\n", 80 | "% \t\t],\n", 81 | "% \t style=\"width=9cm, axis lines=left, xtick=\\\\empty, ytick=\\\\empty, xmin=-2, xmax=1.75, ymin=-0.5, ymax=2, view={0}{90}, contour/labels=false, contour/draw color={black}\"))\n", 82 | "% \\end{jlcode}\n", 83 | "% \\begin{marginfigure}[-9cm]\n", 84 | "% \t\\centering\n", 85 | "% \t\\plot{fig_derivatives_2d_derivative}\n", 86 | "% \t\\caption{The two-dimensional directional derivative}\n", 87 | "% \\end{marginfigure}\n", 88 | "\n", 89 | "\\begin{example}\n", 90 | "\tWe wish to compute the directional derivative of $f(\\vect x) = x_1 x_2$ at $\\vect x = [1,0]$ in the direction $\\vect s = [-1, -1]$:\n", 91 | "\t\\begin{align*}\n", 92 | "\t\t\\nabla f(\\vect x) & = \\begin{bmatrix}\\frac{\\partial f}{\\partial x_1}, & \\frac{\\partial f}{\\partial x_2}\\end{bmatrix} = \\brock*{x_2, x_1} \\\\\n", 93 | "\t\t\\nabla_{\\vect s}f(\\vect x) & = \\nabla f(\\vect x)^\\transpose \\vect s = \\begin{bmatrix}0 & 1\\end{bmatrix} \\begin{bmatrix}-1 \\\\ -1\\end{bmatrix} = -1\n", 94 | "\t\\end{align*}\n", 95 | "\tWe can also compute the directional derivative as follows:\n", 96 | "\t\\begin{align*}\n", 97 | "\t\tg(\\alpha) &= f(\\vect x + \\alpha \\vect s) = (1 - \\alpha)(-\\alpha) = \\alpha^2 - \\alpha \\\\\n", 98 | "\t\tg'(\\alpha) &= 2\\alpha - 1 \\\\\n", 99 | "\t\tg'(0) &= -1\n", 100 | "\t\\end{align*}\n", 101 | "\t\\caption{Computing a directional derivative.\\label{ex:directional-deriv}}\n", 102 | "\\end{example}\n", 103 | "\\begin{marginfigure}[-8cm]\n", 104 | "\t\\centering\n", 105 | "\t\\begin{tikzpicture}\n", 106 | "\t\\begin{axis}[\n", 107 | "\twidth=\\marginparwidth,\n", 108 | "\txlabel={$\\alpha$},\n", 109 | "\tylabel={$f(\\alpha)$},\n", 110 | "\taxis lines=middle, xtick={1}, ytick=\\empty,\n", 111 | "\t]\n", 112 | "\t\\addplot[mark=none, samples=100, domain=-0.25:1.25]{(1-x)*x};\n", 113 | "\t\\addplot[mark=none, samples=2, domain=-0.25:0.5]{x};\n", 114 | "\t\\end{axis}\n", 115 | "\t\\end{tikzpicture}\n", 116 | "\\end{marginfigure}\n", 117 | "\n", 118 | "\n", 119 | "\n", 120 | "\\begin{ignore}\n", 121 | "\tFor multi-input, multi-output functions\\sidenote{$\\vect f: \\mathbb{R}^n \\rightarrow \\mathbb{R}^m$}, we can compute the \\vocab{Jacobian}:\n", 122 | "\n", 123 | "\t\\begin{equation}\n", 124 | "\t\\nabla \\vect f(\\vect x) = \\begin{bmatrix}\n", 125 | "\t\\frac{\\partial f_1(\\vect x)}{\\partial x_1} & \\frac{\\partial f_1(\\vect x)}{\\partial x_2} & \\cdots & \\frac{\\partial f_1(\\vect x)}{\\partial x_n} \\\\\n", 126 | "\t& \\vdots & & \\\\\n", 127 | "\t\\frac{\\partial f_m(\\vect x)}{\\partial x_1} & \\frac{\\partial f_m(\\vect x)}{\\partial x_2} & \\cdots & \\frac{\\partial f_m(\\vect x)}{\\partial x_n} \\\\\n", 128 | "\t\\end{bmatrix}\n", 129 | "\t\\end{equation}\n", 130 | "\n", 131 | "\tThe Jacobian forms the best linear approximation for $\\vect f$ at $\\vect x$, because it contains all of the gradient components.\n", 132 | "\t\\tim{I don't think we use the Jacobian in the text. Should we leave it out?} \\mykel{Let's comment it out. Let's try to make the book as simple as possible (but no simpler). If a topic is not connected with anything, we should strongly consider dropping it. Also, the Jacobian gets weird when we have the convention where the gradient is a column vector.}\n", 133 | "\\end{ignore}\n", 134 | "\n", 135 | "\\section{Numerical Differentiation}\n", 136 | "\n", 137 | "The process of estimating derivatives numerically is referred to as \\vocab{numerical differentiation}.\n", 138 | "Estimates can be derived in different ways from function evaluations. This section discusses finite difference methods and the complex step method.\\sidenote{For a more comprehensive treatment of the topics discussed in the remainder of this chapter, see \\fullcite{Griewank2008}.}\n", 139 | "\n", 140 | "\n", 141 | "% The gradient $\\nabla f$ or the derivative along a search direction ${\\frac{\\partial}{\\partial \\alpha} f(\\vect x + \\alpha \\vect d)}$ are often not readily avaiable for direct use, but they are critical to many optimization methods.\n", 142 | "% Given a method for derivative estimation one also automatically has a method for gradient estimation - one need merely estimate the partial derivative along each component direction.\n", 143 | "\n", 144 | "\\subsection{Finite Difference Methods}\n", 145 | "\n", 146 | "As the name implies, \\vocab{finite difference methods} compute the difference between two values that differ by a finite step size.\n", 147 | "They approximate the derivative definitions in \\cref{eq:derivative_defintions} using small differences:\n", 148 | "\\begin{fullwidth}\n", 149 | "\\begin{equation}\n", 150 | "\tf'(x) \\approx \\underbrace{\\frac{f(x + h) - f(x)}{h}}_\\text{forward difference} \\approx \\underbrace{\\frac{f(x + h/2) - f(x - h/2)}{h}}_\\text{central difference} \\approx \\underbrace{\\frac{f(x) - f(x - h)}{h}}_\\text{backward difference}\n", 151 | "\\end{equation}\n", 152 | "\\end{fullwidth}\n", 153 | "\n", 154 | "Mathematically, the smaller the step size $h$, the better the derivative estimate.\n", 155 | "Practically, values for $h$ that are too small can result in numerical cancellation errors.\n", 156 | "This effect is shown later in \\cref{fig:derivatives_derivative_error}. \\Cref{alg:finite-diff} provides implementations for these methods.\n", 157 | "\n", 158 | "\\begin{algorithm}\n", 159 | "\\begin{juliaverbatim}\n", 160 | "diff_forward(f, x; h=sqrt(eps(Float64))) = (f(x+h) - f(x))/h\n", 161 | "diff_central(f, x; h=cbrt(eps(Float64))) = (f(x+h/2) - f(x-h/2))/h\n", 162 | "diff_backward(f, x; h=sqrt(eps(Float64))) = (f(x) - f(x-h))/h\n", 163 | "\\end{juliaverbatim}\n", 164 | "\t\\caption{\n", 165 | "\t\t\\label{alg:finite-diff}\n", 166 | "\t\tFinite difference methods for estimating the derivative of a function \\jlv{f} at \\jlv{x} with finite difference \\jlv{h}.\n", 167 | "\t\tThe default step sizes are the square root or cube root of the machine precision for floating point values.\n", 168 | "\t\tThese step sizes balance machine round-off error with step size error.\n", 169 | "\n", 170 | "\t\tThe \\jlv{eps} function provides the step size between \\jlv{1.0} and the next larger representable floating-point value.\n", 171 | "\t}\n", 172 | "\\end{algorithm}\n", 173 | "\n", 174 | "\n", 175 | "The finite difference methods can be derived using the Taylor expansion.\n", 176 | "We will derive the forward difference derivative estimate, beginning with the Taylor expansion of $f$ about $x$:\n", 177 | "\\begin{equation}\n", 178 | "\tf(x + h) = f(x) + \\frac{f'(x)}{1!} h + \\frac{f''(x)}{2!} h^2 + \\frac{f'''(x)}{3!} h^3 + \\cdots\n", 179 | "\\end{equation}\n", 180 | "\n", 181 | "We can rearrange and solve for the first derivative:\n", 182 | "\\begin{align}\n", 183 | "\t\tf'(x) h & = f(x+h) - f(x) - \\frac{f''(x)}{2!} h^2 - \\frac{f'''(x)}{3!} h^3 - \\cdots \\\\\n", 184 | "\t\tf'(x) & = \\frac{f(x+h) - f(x)}{h} - \\frac{f''(x)}{2!} h - \\frac{f'''(x)}{3!} h^2 - \\cdots \\\\\n", 185 | "\t\tf'(x) & \\approx \\frac{f(x+h) - f(x)}{h}\n", 186 | "\\end{align}\n", 187 | "\n", 188 | "The forward difference approximates the true derivative for small $h$ with error dependent on $\\frac{f''(x)}{2!} h + \\frac{f'''(x)}{3!} h^2 + \\cdots$.\n", 189 | "This error term is $O(h)$, meaning the forward difference has linear error as $h$ approaches zero.\\sidenote[][]{\n", 190 | "\tAsymptotic notation is covered in \\chref{the appendix}{ch:math_concepts}.\n", 191 | "}\n", 192 | "\n", 193 | "The central difference method has an error term of $O(h^2)$.\\cite{Mathews2004}\n", 194 | "We can derive this error term using the Taylor expansion.\n", 195 | "The Taylor expansions about $x$ for $f(x+h/2)$ and $f(x-h/2)$ are:\n", 196 | "\\begin{align}\n", 197 | "\t\tf(x+h/2) & = f(x) + f'(x) \\frac{h}{2} + \\frac{f''(x)}{2!} \\paren*{\\frac{h}{2}}^2 + \\frac{f'''(x)}{3!}\\paren*{\\frac{h}{2}}^3 + \\cdots \\\\\n", 198 | "\t\tf(x-h/2) & = f(x) - f'(x) \\frac{h}{2} + \\frac{f''(x)}{2!} \\paren*{\\frac{h}{2}}^2 - \\frac{f'''(x)}{3!}\\paren*{\\frac{h}{2}}^3 + \\cdots\n", 199 | "\\end{align}\n", 200 | "\n", 201 | "Subtracting these expansions produces:\n", 202 | "\\begin{equation}\n", 203 | "\tf(x+h/2) - f(x-h/2) \\approx 2f'(x)\\frac{h}{2} + \\frac{2}{3!} f'''(x) \\paren*{\\frac{h}{2}}^3\n", 204 | "\\end{equation}\n", 205 | "\n", 206 | "We rearrange to obtain:\n", 207 | "\\begin{equation}\n", 208 | "\tf'(x) \\approx \\frac{f(x+h/2) - f(x-h/2)}{h} - \\frac{f'''(x)h^2}{24}\n", 209 | "\\end{equation}\n", 210 | "which shows that the approximation has quadratic error.\n", 211 | "\n", 212 | "\n", 213 | "\\subsection{Complex Step Method}\n", 214 | "\n", 215 | "We often run into the problem of needing to choose a step size $h$ small enough to provide a good approximation but not too small so as to lead to numerical subtractive cancellation issues.\n", 216 | "The \\vocab{complex step method} bypasses the effect of subtractive cancellation by using a single function evaluation. We evaluate the function once after taking a step in the imaginary direction.\\sidenote{\\fullcite{Martins2003}. Special care must be taken to ensure that the implementation of $f$ properly supports complex numbers as input.}\n", 217 | "\n", 218 | "The Taylor expansion for an imaginary step is:\n", 219 | "\\begin{equation}\n", 220 | "\tf(x + i h) = f(x) + i h f'(x) - h^2 \\frac{f''(x)}{2!} - i h^3 \\frac{f'''(x)}{3!} + \\cdots\n", 221 | "\\end{equation}\n", 222 | "Taking only the imaginary component of each side produces a derivative approximation:\n", 223 | "\\begin{align}\n", 224 | "\t\t\\Imag(f(x + i h)) & = h f'(x) - h^3 \\frac{f'''(x)}{3!} + \\cdots \\\\\n", 225 | "\t\t\\Rightarrow f'(x) & = \\frac{\\Imag (f(x + i h))}{h} + h^2 \\frac{f'''(x)}{3!} - \\cdots \\\\\n", 226 | "\t\t\t\t\t\t & = \\frac{\\Imag (f(x + i h))}{h} + O(h^2) \\text{ as } h \\rightarrow 0\n", 227 | "\\end{align}\n", 228 | "An implementation is provided by \\cref{alg:complex-step}. The real part approximates $f(x)$ to within $O(h^2)$ as $h \\rightarrow 0$:\n", 229 | "\\begin{align}\n", 230 | "\t\t\\Real(f(x + i h)) = f(x) - h^2 \\frac{f''(x)}{2!} + \\hdots \\\\\n", 231 | "\t\t\\Rightarrow f(x) = \\Real(f(x + i h)) + h^2 \\frac{f''(x)}{2!} - \\cdots\n", 232 | "\\end{align}\n", 233 | "\n", 234 | "Thus, we can evaluate both $f(x)$ and $f'(x)$ using a single evaluation of $f$ with complex arguments. \\Cref{ex:complex-step} shows the calculations involved for estimating the derivative of a function at a particular point.\n", 235 | "\\Cref{alg:complex-step} implements the complex step method.\n", 236 | "\\Cref{fig:derivatives_derivative_error} compares the numerical error of the complex step method to the forward and central difference methods as the step size is varied.\n", 237 | "\n", 238 | "\n", 239 | "\\begin{algorithm}\n", 240 | "\\begin{juliaverbatim}\n", 241 | "diff_complex(f, x; h=1e-20) = imag(f(x + h*im)) / h\n", 242 | "\\end{juliaverbatim}\n", 243 | "\\caption{\n", 244 | "\tThe complex step method for estimating the derivative of a function \\jlv{f} at \\jlv{x} with finite difference \\jlv{h}.\n", 245 | "\t\\label{alg:complex-step}\n", 246 | "}\n", 247 | "\\end{algorithm}\n", 248 | "\n", 249 | "%Use of the complex step method requires that the function be able to accept complex values as input.\n", 250 | "%Existing packages sometimes require modifying the code that implements $f$, which can limit the method's direct application.\n", 251 | "\n", 252 | "\\begin{example}\n", 253 | "Consider $f(x) = \\sin(x^2)$.\n", 254 | "The function value at $x = \\pi/2$ is approximately \\num{0.624266} and the derivative is $\\pi \\cos(\\pi^2/4) \\approx -2.45425$.\n", 255 | "We can arrive at this using the complex step method:\n", 256 | "\\begin{juliaconsole}\n", 257 | "f = x -> sin(x^2);\n", 258 | "v = f(π/2 + 0.001im);\n", 259 | "real(v) # f(x)\n", 260 | "imag(v)/0.001 # f'(x)\n", 261 | "\\end{juliaconsole}\n", 262 | "\\caption{The complex step method for estimating derivatives.\\label{ex:complex-step}}\n", 263 | "\\end{example}\n", 264 | "\n", 265 | "\\begin{jlcode}\n", 266 | "\tdiff_forward(f, x; h=sqrt(eps(Float64))) = (f(x+h) - f(x))/h\n", 267 | "\tdiff_central(f, x; h=cbrt(eps(Float64))) = (f(x+h/2) - f(x-h/2))/h\n", 268 | "\tdiff_backward(f, x; h=sqrt(eps(Float64))) = (f(x) - f(x-h))/h\n", 269 | "\tdiff_complex(f, x; h=1e-20) = imag(f(x + h*im)) / h\n", 270 | "\n", 271 | "\tp = let\n", 272 | "\t\tx = 0.5\n", 273 | "\t\td_true = 1.0\n", 274 | "\n", 275 | "\t\tarr_h = collect(10 .^ range(-17, stop=1, length=101))\n", 276 | "\t\tarr_forward = [abs(d_true - diff_forward(sin, x, h=h)/cos(x))\n", 277 | "\t\t\t\t\t\tfor h in arr_h]\n", 278 | "\t\tarr_central = [abs(d_true - diff_central(sin, x, h=h)/cos(x))\n", 279 | "\t\t\t\t\t\tfor h in arr_h]\n", 280 | "\t\tarr_complex = [abs(d_true - diff_complex(sin, x, h=h)/cos(x))\n", 281 | "\t\t\t\t\t\tfor h in arr_h]\n", 282 | "\n", 283 | "\t\tAxis([\n", 284 | "\t\t\tPlots.Linear(arr_h, arr_complex, style=\"thick, solid,\n", 285 | "\t\t\t\tpastelGreen, mark=none\", legendentry=\"complex\"),\n", 286 | "\t\t\tPlots.Linear(arr_h, arr_forward, style=\"thick, solid,\n", 287 | "\t\t\t\tpastelBlue, mark=none\", legendentry=\"forward\"),\n", 288 | "\t\t\tPlots.Linear(arr_h, arr_central, style=\"thick, solid,\n", 289 | "\t\t\t\tpastelRed, mark=none\", legendentry=\"central\")],\n", 290 | "\t\t\txmode=\"log\", ymode=\"log\",\n", 291 | "\t\t\txlabel=\"step size \\$h\\$\",\n", 292 | "\t\t\tylabel=\"absolute relative error\",\n", 293 | "\t\t\twidth=\"10cm\", height=\"8cm\",\n", 294 | "\t\t\tstyle=\"legend pos=outer north east\"\n", 295 | "\t\t)\n", 296 | "\tend\n", 297 | "\tplot(p)" 298 | ] 299 | } 300 | ], 301 | "metadata": { 302 | "kernelspec": { 303 | "display_name": "Julia 1.5.1", 304 | "language": "julia", 305 | "name": "julia-1.5" 306 | }, 307 | "language_info": { 308 | "file_extension": ".jl", 309 | "mimetype": "application/julia", 310 | "name": "julia", 311 | "version": "1.5.1" 312 | } 313 | }, 314 | "nbformat": 4, 315 | "nbformat_minor": 2 316 | } 317 | -------------------------------------------------------------------------------- /design-under-uncertainty.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Design Under Uncertainty\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "\tusing Distributions\n", 32 | "\tusing Optim\n", 33 | "\timport QuadGK: quadgk\n", 34 | "\n", 35 | "\tp = let\n", 36 | "\n", 37 | "\t\ta = 1\n", 38 | "\t\tb = 2\n", 39 | "\t\tc = 4.5\n", 40 | "\t\tf = x -> 2.5 - 2exp(-100(x-a)^2) - exp(-(x-b)^2) - 1.4exp(-0.15(x-c)^2)\n", 41 | "\n", 42 | "\t\tp = Plots.Plot[]\n", 43 | "\t\tx_arr = append!(collect(range(0.0,stop=1.5,length=300)), collect(range(1.5,stop=9,length=200)[2:end]))\n", 44 | "\t\tpush!(p, Plots.Linear(x_arr, f.(x_arr), style=\"solid, thick, black, mark=none\", legendentry=\"noise-free\"))\n", 45 | "\t\tlabels=[\"very low\", \"low\", \"high\", \"very high\"]\n", 46 | "\t\tfor (i,σ) in enumerate([0.1,0.5,1.0,2.0])\n", 47 | "\t\t P = Normal(0,σ)\n", 48 | "\t\t g = x -> quadgk(z->f(x+z)*pdf(P, z), -5, 5)[1]\n", 49 | "\t\t push!(p, Plots.Linear(g, (0, 9), xbins=401, style=\"solid, pastelBlue!$(100-25(i-1)), mark=none\", legendentry=labels[i]*\" noise\"))\n", 50 | "\t\tend\n", 51 | "\n", 52 | "\t\tmin_a = optimize(f, a-0.5, a+0.5).minimizer\n", 53 | "\t\tmin_b = optimize(f, b-0.5, b+0.5).minimizer\n", 54 | "\t\tmin_c = optimize(f, c-1.0, c+1.0).minimizer\n", 55 | "\n", 56 | "\t\tAxis(p, width=\"12cm\", height=\"6cm\", style=\"axis lines=left, xtick={$(min_a), $(min_b), $(min_c)}, xticklabels={\\$a\\$, \\$b\\$, \\$c\\$}, xticklabel style={text height=2ex}, ytick=\\\\empty, legend style={draw=none, at={(0.5,-0.25)}, anchor=north, legend columns=-1, /tikz/every even column/.append style={column sep=0.5cm}}\",\n", 57 | "\t\t xlabel=L\"x\", ylabel=\"expected value\", xmin=-0.5, ymin=-0.25)\n", 58 | " end\n", 59 | " plot(p)" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": null, 65 | "metadata": { 66 | "collapsed": true 67 | }, 68 | "outputs": [], 69 | "source": [ 70 | "using Optim\n", 71 | "p = let\n", 72 | "\tf = x -> -x*(x < 0) + x^2*(x > 0)\n", 73 | "\tdom = (-1,1)\n", 74 | "\tp = Plots.Plot[]\n", 75 | "\tpush!(p, Plots.Linear(f, dom, style=\"solid, black, mark=none\", legendentry=\"true\"))\n", 76 | "\n", 77 | "\tF_lin(x,ϵ) = -x + ϵ # valid for x - ϵ ≤ 0\n", 78 | "\tF_qua(x,ϵ) = (x + ϵ)^2 # valid for x + ϵ > 0\n", 79 | "\n", 80 | "\tfor (i,ϵ) in enumerate([0.2,0.4,0.6,0.8,1.0,1.2,1.4,1.6,1.8,2.0])\n", 81 | "\t\tx_arr = Float64[dom[1]]\n", 82 | "\t\ty_arr = Float64[F_lin(x_arr[1], ϵ)]\n", 83 | "\t\tx_vertex = -(2ϵ+1)/2 + 0.5*sqrt((2ϵ+1)^2 - 4*(ϵ^2-ϵ))\n", 84 | "\t\tfor x in range(x_vertex, stop=dom[2], length=101)\n", 85 | "\t\t\tpush!(x_arr, x)\n", 86 | "\t\t\tpush!(y_arr, F_qua(x, ϵ))\n", 87 | "\t\tend\n", 88 | "\t\tpush!(p, Plots.Linear(x_arr, y_arr, style=\"solid, pastelBlue!$(100-9(i-1)), mark=none\", legendentry=\"\\$\\\\epsilon = $ϵ\\$\"))\n", 89 | "\tend\n", 90 | "\tAxis(p, xmin=-1, xmax=1, ymax=4, xlabel=L\"x\", ylabel=L\"y\", style=\"width=8cm, legend pos=outer north east\")\n", 91 | "end\n", 92 | "plot(p)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": { 99 | "collapsed": true 100 | }, 101 | "outputs": [], 102 | "source": [ 103 | "p = let\n", 104 | "\tZ = ϵ -> [-ϵ, ϵ]\n", 105 | "\tf = (x, z) -> (x+z)^2 + 6exp(-(x+z)^2)\n", 106 | "\n", 107 | "\tg = (x,ϵ) -> maximum(f(x,z) for z in range(Z(ϵ)[1], stop=Z(ϵ)[2], length=51))\n", 108 | "\tdom = (-3,3)\n", 109 | "\n", 110 | "\tG = GroupPlot(1,2, groupStyle=\"vertical sep=0.5cm, xlabels at=edge bottom, xticklabels at=edge bottom\", style=\"xlabel=\\$x\\$, ylabel=\\$y\\$, xmin=$(dom[1]), xmax=$(dom[2]), ymin=2, ymax=10, width=9.5cm, height=4cm\")\n", 111 | "\n", 112 | "\tp = Plots.Plot[]\n", 113 | "\tfor (i,ϵ) in enumerate(range(0.0,stop=3.0,length=11))\n", 114 | "\t\tif i == 1\n", 115 | "\t\t\tpush!(p, Plots.Linear(x->g(x,ϵ), dom, style=\"solid, black, mark=none\", legendentry=\"noise-free\"))\n", 116 | "\t\telse\n", 117 | "\t\t\tpush!(p, Plots.Linear(x->g(x,ϵ), dom, xbins=101, style=\"solid, pastelBlue!$(100-9(i-2)), mark=none\", legendentry=\"\\$\\\\epsilon = $(ϵ)\\$\"))\n", 118 | "\t\tend\n", 119 | "\tend\n", 120 | "\tpush!(p, Plots.Linear([ 2, 2],[2,10], style=\"solid, gray, mark=none, forget plot\"))\n", 121 | "\tpush!(p, Plots.Linear([-2, -2],[2,10], style=\"solid, gray, mark=none, forget plot\"))\n", 122 | "\tpush!(p, Plots.Node(L\"\\mathcal{Z}\",0,8.1))\n", 123 | "\tpush!(p, Plots.Node(L\"x^*\",0,5))\n", 124 | "\tpush!(p, Plots.Linear([-2, 2],[7.5,7.5], style=\"solid, thick, red, mark=none, forget plot\"))\n", 125 | "\tpush!(p, Plots.Linear([-2,-2],[7.25,7.75], style=\"solid, thick, red, mark=none, forget plot\"))\n", 126 | "\tpush!(p, Plots.Linear([ 2, 2],[7.25,7.75], style=\"solid, thick, red, mark=none, forget plot\"))\n", 127 | "\tpush!(p, Plots.Scatter([0],[f(0,0)], style=\"solid, black, mark=*, mark size=1.5, mark options={solid, draw=black, fill=black}, forget plot\"))\n", 128 | "\tpush!(G, Axis(p, style=\"legend pos=outer north east, legend style={draw=none, fill=none}\"))\n", 129 | "\n", 130 | "\tp = Plots.Plot[]\n", 131 | "\tfor (i,ϵ) in enumerate(range(0.0,stop=3.0,length=11))\n", 132 | "\t\tif i == 1\n", 133 | "\t\t\tpush!(p, Plots.Linear(x->g(x,ϵ), dom, style=\"solid, black, mark=none\"))\n", 134 | "\t\telse\n", 135 | "\t\t\tpush!(p, Plots.Linear(x->g(x,ϵ), dom, xbins=201, style=\"solid, pastelBlue!$(100-9(i-2)), mark=none\"))\n", 136 | "\t\tend\n", 137 | "\tend\n", 138 | "\tpush!(p, Plots.Linear([ 2, 2],[2,10], style=\"solid, gray, mark=none\"))\n", 139 | "\tpush!(p, Plots.Linear([-2, -2],[2,10], style=\"solid, gray, mark=none\"))\n", 140 | "\tpush!(p, Plots.Linear([dom[1], dom[2]],[5,5], style=\"solid, gray, mark=none\"))\n", 141 | "\n", 142 | "\tz_hi = -0.478808\n", 143 | "\tz_lo = -2\n", 144 | "\txstar = (z_hi + z_lo)/2\n", 145 | "\n", 146 | "\tpush!(p, Plots.Node(L\"\\mathcal{Z}\",-1.25,8))\n", 147 | "\tpush!(p, Plots.Node(L\"x^*\",xstar+0.5,f(xstar,0)+0.2))\n", 148 | "\tpush!(p, Plots.Linear([z_lo,z_hi],[7.5,7.5], style=\"solid, thick, red, mark=none\"))\n", 149 | "\tpush!(p, Plots.Linear([z_lo,z_lo],[7.25,7.75], style=\"solid, thick, red, mark=none\"))\n", 150 | "\tpush!(p, Plots.Linear([z_hi,z_hi],[7.25,7.75], style=\"solid, thick, red, mark=none\"))\n", 151 | "\tpush!(p, Plots.Scatter([xstar],[f(xstar,0)], style=\"solid, black, mark=*, mark size=1.5, mark options={solid, draw=black, fill=black}\"))\n", 152 | "\n", 153 | "\tpush!(G, Axis(p))\n", 154 | "end\n", 155 | "plot(p)" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": { 162 | "collapsed": true 163 | }, 164 | "outputs": [], 165 | "source": [ 166 | "using Distributions\n", 167 | "p = let\n", 168 | "\tf = x -> sin(2x)/x\n", 169 | "\tdom = (-10,10)\n", 170 | "\tp = Plots.Plot[]\n", 171 | "\tpush!(p, Plots.Linear(f, dom, xbins=151, style=\"solid, black, mark=none\", legendentry=\"\\$\\\\nu = 0\\$\"))\n", 172 | "\tfor (i,ϵ) in enumerate([0.5,1,1.5,2])\n", 173 | "\t P = Normal(0, ϵ)\n", 174 | "\t F = x -> quadgk(z->pdf(P, z)*f(x+z), -10, 10)[1]\n", 175 | "\t push!(p, Plots.Linear(F, dom, style=\"smooth,solid, pastelBlue!$(100-10(i-1)), mark=none\", legendentry=\"\\$\\\\nu = $ϵ\\$\"))\n", 176 | "\tend\n", 177 | "\tAxis(p, xmin=dom[1], xmax=dom[2], xlabel=L\"x\", ylabel=L\"y\", style=\"width=9cm, height=4cm, legend cell align=left, legend pos=outer north east\")\n", 178 | "end\n", 179 | "plot(p)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": null, 185 | "metadata": { 186 | "collapsed": true 187 | }, 188 | "outputs": [], 189 | "source": [ 190 | "using Distributions\n", 191 | "p = let\n", 192 | "\tf = x -> isapprox(x, 0.0) ? 2.0 : sin(2x)/x\n", 193 | "\tdom = (-10,10)\n", 194 | "\tP = Normal(0, 0.1)\n", 195 | "\tp = Plots.Plot[]\n", 196 | "\tpush!(p, Plots.Linear(f, dom, xbins=151, style=\"solid, thick, black, mark=none\", legendentry=\"noise-free\"))\n", 197 | "\t# for (i,k) in enumerate([0.1, 0.5, 1.0, 1.5, 2.0])\n", 198 | "\tfor (i,k) in enumerate([1.0, 1.25, 1.5, 1.75, 2.0])\n", 199 | "\t F = x -> quadgk(z->pdf(P, z)*sign(f(x+z))*abs(f(x+z))^k, -Inf, Inf)[1]\n", 200 | "\t push!(p, Plots.Linear(F, dom, xbins=150, style=\"solid, pastelBlue!$(100-15(i-1)), mark=none\", legendentry=\"\\$k = $k\\$\"))\n", 201 | "\tend\n", 202 | "\tAxis(p, xmin=dom[1], xmax=dom[2], xlabel=L\"x\", ylabel=L\"y\", style=\"width=11cm, height=6cm, legend cell align=left, legend style={draw=none, at={(0.5,-0.15)}, anchor=north, legend columns=1},\")\n", 203 | "end\n", 204 | "plot(p)" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": null, 210 | "metadata": { 211 | "collapsed": true 212 | }, 213 | "outputs": [], 214 | "source": [ 215 | "using Distributions\n", 216 | "using Optim\n", 217 | "\n", 218 | "p = let\n", 219 | "\n", 220 | "\tf = x -> x^3 + 1\n", 221 | "\tp = Plots.Plot[]\n", 222 | "\tpush!(p, Plots.Linear(f, (0, 5), style=\"solid, black, mark=none\"))\n", 223 | "\n", 224 | "\tymin = -3\n", 225 | "\n", 226 | "\tδ = 0.49\n", 227 | "\tP = Normal(1.5,0.25)\n", 228 | "\tdom = quantile.(P, [0.5-δ, 0.5+δ])\n", 229 | "\ta, b = dom[1], dom[2]\n", 230 | "\tfa, fb = f(a), f(b)\n", 231 | "\tpush!(p, Plots.Linear(x->2pdf(P, x)+ymin, (a, b), style=\"solid, pastelBlue, mark=none\"))\n", 232 | "\tpush!(p, Plots.Linear([-0.5, a, a], [fa, fa, ymin], style=\"draw=none, mark=none, name path=A, forget plot\"))\n", 233 | "\tpush!(p, Plots.Linear([-0.5, b, b], [fb, fb, ymin], style=\"draw=none, mark=none, name path=B, forget plot\"))\n", 234 | "\tpush!(p, Plots.Command(\"\\\\addplot[pastelBlue!40] fill between[of=A and B];\"))\n", 235 | "\t_μ = (fa+fb)/2\n", 236 | "\tσ = optimize(σ->(quantile(Normal(_μ, σ), 0.5+δ) - fb)^2, 0.1, 10.0).minimizer\n", 237 | "\tQ = Normal(_μ, σ)\n", 238 | "\ty_arr = range(fa, stop=fb, length=101)\n", 239 | "\tx_arr = [3*(pdf(Q, y)-pdf(Q,fa))-0.5 for y in y_arr]\n", 240 | "\tpush!(p, Plots.Linear(x_arr, y_arr, style=\"solid, pastelBlue, mark=none\"))\n", 241 | "\tpush!(p, Plots.Node(\"robust\",0,fb, style=\"above right\"))\n", 242 | "\n", 243 | "\tP = Normal(4,0.25)\n", 244 | "\tdom = quantile.(P, [0.5-δ, 0.5+δ])\n", 245 | "\ta, b = dom[1], dom[2]\n", 246 | "\tfa, fb = f(a), f(b)\n", 247 | "\tpush!(p, Plots.Linear(x->2pdf(P, x)+ymin, (a, b), style=\"solid, pastelBlue, mark=none\"))\n", 248 | "\tpush!(p, Plots.Linear([-0.5, a, a], [fa, fa, -3], style=\"draw=none, mark=none, name path=A, forget plot\"))\n", 249 | "\tpush!(p, Plots.Linear([-0.5, b, b], [fb, fb, -3], style=\"draw=none, mark=none, name path=B, forget plot\"))\n", 250 | "\tpush!(p, Plots.Command(\"\\\\addplot[pastelBlue!40] fill between[of=A and B];\"))\n", 251 | "\t_μ = (fa+fb)/2\n", 252 | "\tσ = optimize(σ->(quantile(Normal(_μ, σ), 0.5+δ) - fb)^2, 0.1, 10.0).minimizer\n", 253 | "\tQ = Normal(_μ, σ)\n", 254 | "\ty_arr = range(fa, stop=fb, length=101)\n", 255 | "\tx_arr = [3*(pdf(Q, y)-pdf(Q,fa))-0.5 for y in y_arr]\n", 256 | "\tpush!(p, Plots.Linear(x_arr, y_arr, style=\"solid, pastelBlue, mark=none\"))\n", 257 | "\tpush!(p, Plots.Node(\"sensitive\",0,fb, style=\"above right\"))\n", 258 | "\n", 259 | "\tAxis(p, width=\"9cm\", xmin=-0.5, ymin=ymin, xlabel=L\"x\", ylabel=L\"y\", style=\"axis lines=left, axis on top, xtick=\\\\empty, ytick=\\\\empty\")\n", 260 | "end\n", 261 | "plot(p)" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": null, 267 | "metadata": { 268 | "collapsed": true 269 | }, 270 | "outputs": [], 271 | "source": [ 272 | "p = let\n", 273 | "\t_μ = x -> x^2 + 4/(1+abs(x))\n", 274 | "\t_σ = x -> sqrt(8/(1+abs(x)))\n", 275 | "\tAxis([\n", 276 | "\t Plots.Linear(x -> _μ(x) + _σ(x), (-3,3), xbins=51, style=\"draw=none, mark=none, name path=A, forget plot\"),\n", 277 | "\t Plots.Linear(x -> _μ(x) - _σ(x), (-3,3), xbins=51, style=\"draw=none, mark=none, name path=B, forget plot\"),\n", 278 | "\t Plots.Command(\"\\\\addplot[pastelBlue!40] fill between[of=A and B];\\\\addlegendentry{\\$\\\\pm \\\\sigma\\$}\"),\n", 279 | "\t Plots.Linear(x -> _μ(x), (-3,3), xbins=51, style=\"solid, black, mark=none\", legendentry=\"expected value\"),\n", 280 | "\t], xmin=-3, xmax=3, width=\"10cm\", height=\"5cm\", xlabel=L\"x\", ylabel=L\"y\", style=\"legend style={draw=none, at={(0.5,-0.25)}, anchor=north, legend columns=-1, /tikz/every even column/.append style={column sep=0.5cm}}\")\n", 281 | "end\n", 282 | "plot(p)" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": null, 288 | "metadata": { 289 | "collapsed": true 290 | }, 291 | "outputs": [], 292 | "source": [ 293 | "p = let\n", 294 | "\t_μ = x -> x^2 + 4/(1+abs(x))\n", 295 | "\t_σ = x -> sqrt(8/(1+abs(x)))\n", 296 | "\tp = Plots.Plot[]\n", 297 | "\tp_scatter = Plots.Plot[]\n", 298 | "\tfor α in range(0.0,stop=1.0,length=11)\n", 299 | "\t f = x -> α*_μ(x) + (1-α)*_σ(x)\n", 300 | "\t xopt = 0.0\n", 301 | "\t for x in range(-3,stop=0,length=101)\n", 302 | "\t if f(x) < f(xopt)\n", 303 | "\t xopt = x\n", 304 | "\t end\n", 305 | "\t end\n", 306 | "\t push!(p, Plots.Linear(f, (-3,3), xbins=151, style=\"solid, color=pastelBlue!$(round(Int, 80α+20))\"))\n", 307 | "\t push!(p_scatter, Plots.Scatter([xopt, -xopt], f.([xopt, -xopt]), style=\"solid, mark=*, mark size=0.75, mark options={draw=black, fill=black}\"))\n", 308 | "\tend\n", 309 | "\tappend!(p, p_scatter)\n", 310 | "\tAxis(p, xmin=-3, xmax=3, width=\"10cm\", height=\"5cm\", xlabel=L\"x\", ylabel=L\"\\alpha \\mu + (1-\\alpha)\\sigma\", style=\"axis on top, clip marker paths=true, legend pos = outer north east\")\n", 311 | "end\n", 312 | "plot(p)" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": null, 318 | "metadata": { 319 | "collapsed": true 320 | }, 321 | "outputs": [], 322 | "source": [ 323 | "using Distributions\n", 324 | "p = let\n", 325 | "\txmax = 7.0\n", 326 | "\tymax = 0.4\n", 327 | "\n", 328 | "\tG = Truncated(Gamma(2.0,1.0), 0.0, xmax)\n", 329 | "\n", 330 | "\tμ_ = quadgk(x -> x*pdf(G, x), 0.0, xmax)[1]\n", 331 | "\n", 332 | "\tα = 0.1\n", 333 | "\tvalue_at_risk(α) = minimum(1-cdf(G, t) <= α ? t : Inf for t in range(0.0, stop=7.0, length=201))\n", 334 | "\tVaR = value_at_risk(α)\n", 335 | "\tcond_value_at_risk(α) = (1/α)*quadgk(t -> value_at_risk(1-t), 1-α, 1-0.001)[1]\n", 336 | "\tCVaR = cond_value_at_risk(α)\n", 337 | "\n", 338 | "\txq = quantile(G, 1-α)\n", 339 | "\n", 340 | "\tAxis([\n", 341 | "\t Plots.Linear(z->pdf(G, z), (0, xmax), style=\"solid, pastelBlue, mark=none\"),\n", 342 | "\t Plots.Linear(z->pdf(G, z), (xq, xmax), style=\"solid, pastelBlue, ultra thick, mark=none\"),\n", 343 | "\t Plots.Linear([μ_,μ_], [0, ymax], style=\"solid, gray, mark=none\"),\n", 344 | "\t Plots.Linear([VaR,VaR], [0, ymax], style=\"solid, gray, mark=none\"),\n", 345 | "\t Plots.Linear([CVaR,CVaR], [0, ymax], style=\"solid, gray, mark=none\"),\n", 346 | "\t Plots.Node(\"top \\$1-\\\\alpha\\$ quantile\", (xq+xmax)/2+0.35, 0.08),\n", 347 | "\t], xlabel=L\"y\", ylabel=L\"p(y)\", style=\"xtick={$μ_,$VaR, $CVaR}, xticklabels={expected value, VaR, CVaR}\", ymin=0, ymax=ymax, xmin=0, xmax=xmax, width=\"12cm\", height=\"4cm\"\n", 348 | "\t)\n", 349 | "end\n", 350 | "plot(p)" 351 | ] 352 | } 353 | ], 354 | "metadata": { 355 | "kernelspec": { 356 | "display_name": "Julia 1.5.1", 357 | "language": "julia", 358 | "name": "julia-1.5" 359 | }, 360 | "language_info": { 361 | "file_extension": ".jl", 362 | "mimetype": "application/julia", 363 | "name": "julia", 364 | "version": "1.5.1" 365 | } 366 | }, 367 | "nbformat": 4, 368 | "nbformat_minor": 2 369 | } 370 | -------------------------------------------------------------------------------- /discrete.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Discrete\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | " using LinearAlgebra\n", 32 | " p = let\n", 33 | " f_ = (x,y) -> x + y\n", 34 | " dom = (-3.25,3.25)\n", 35 | "\n", 36 | " Xgood = Vector{Float64}[]\n", 37 | " Xbad = Vector{Float64}[]\n", 38 | " for x in -3 : 3\n", 39 | " for y in -3 : 3\n", 40 | " p = [x,y]\n", 41 | " if norm(p) ≤ 2\n", 42 | " push!(Xgood, p)\n", 43 | " else\n", 44 | " push!(Xbad, p)\n", 45 | " end\n", 46 | " end\n", 47 | " end\n", 48 | "\n", 49 | " y_arr = (x->f_(x[1],x[2])).(Xgood)\n", 50 | " y_min = minimum(y_arr)\n", 51 | " Xopt = Xgood[findall(y_arr .== y_min)]\n", 52 | "\n", 53 | " Axis([\n", 54 | " Plots.Image(f_, dom, dom, xbins=600, ybins=600, colormap=pasteljet, colorbar = false),\n", 55 | " Plots.Contour((x,y)->norm([x,y]), dom, dom, levels=[2], contour_style=\"draw color=white\", style=\"thick, white\", xbins=101, ybins=101),\n", 56 | " Plots.Scatter([x[1] for x in Xgood], [x[2] for x in Xgood], style=\"solid, white, mark=*, mark size=1.25, mark options={draw=white, fill=white}\"),\n", 57 | " Plots.Scatter([x[1] for x in Xbad], [x[2] for x in Xbad], style=\"solid, white, mark=o, mark size=1.25, mark options={draw=white}\"),\n", 58 | " Plots.Scatter([x[1] for x in Xopt], [x[2] for x in Xopt], style=\"solid, pastelRed, mark=*, mark size=1.5, mark options={draw=pastelRed, fill=pastelRed}\"),\n", 59 | " Plots.Scatter([-sqrt(2)], [-sqrt(2)], style=\"solid, pastelRed, mark=+, mark size=5, mark options={draw=pastelRed, fill=pastelRed}\"),\n", 60 | " ], xlabel=L\"x_1\", ylabel=L\"x_2\", width=\"4.75cm\", height=\"4.75cm\", style=\"contour/labels=false, view={0}{90}\")\n", 61 | " end\n", 62 | " plot(p)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": { 69 | "collapsed": true 70 | }, 71 | "outputs": [], 72 | "source": [ 73 | "p = let\n", 74 | "f_ = x->1 - exp(-2(x-2)^2) - 1.25exp(-2(x-4)^2)\n", 75 | "dom = (0,5)\n", 76 | "m = 11\n", 77 | "x_pts = collect(range(dom[1], stop=dom[2], length=m))\n", 78 | "\n", 79 | "a = 0.0\n", 80 | "b = 3.75\n", 81 | "lo = -0.5\n", 82 | "hi = 1.25\n", 83 | "\n", 84 | "Axis([\n", 85 | "Plots.Linear(f_, dom, style=\"solid, black, mark=none\"),\n", 86 | "Plots.Linear([a,a], [lo,hi], style=\"name path=A, draw=none, mark=none\"),\n", 87 | "Plots.Linear([b,b], [lo,hi], style=\"name path=B, draw=none, mark=none\"),\n", 88 | "Plots.Command(\"\\\\addplot[pastelBlue!40] fill between[of=A and B]\"),\n", 89 | "Plots.Scatter(x_pts[1:8], f_.(x_pts[1:8]), style=\"solid, black, mark=*, mark size=1, mark options={draw=black, fill=black}\"),\n", 90 | "Plots.Scatter(x_pts[9:end], f_.(x_pts[9:end]), style=\"solid, black, mark=*, mark size=1.25, mark options={draw=black, fill=white}\"),\n", 91 | "Plots.Scatter([b], [f_(b)], style=\"solid, pastelRed, mark=*, mark size=1.25, mark options={draw=pastelRed, fill=pastelRed}\"),\n", 92 | "# Plots.Scatter([2], [f_(2)], style=\"solid, white, mark=*, mark size=1.25, mark options={draw=white, fill=white}\"),\n", 93 | "Plots.Node(L\"x^*\", 2, f_(2), style=\"below\"),\n", 94 | "Plots.Command(\"\\\\draw[->, pastelRed] (axis cs:$b,$(f_(b))) -- (axis cs:3.5,$(f_(3.5)));\"),\n", 95 | "], style=\"enlarge x limits=0, xtick=\\\\empty, ytick=\\\\empty, xlabel=\\$x\\$, ylabel=\\$y\\$\", ymin=lo, ymax=hi, width=\"9cm\"\n", 96 | ")\n", 97 | "end\n", 98 | "plot(p)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": { 105 | "collapsed": true 106 | }, 107 | "outputs": [], 108 | "source": [ 109 | " p = let\n", 110 | " f_ = x->sin(x)*sin(2x+0.1)*sin(4x+0.2)*sin(8x+0.3)\n", 111 | " dom = (0,3)\n", 112 | " m = 20\n", 113 | " x_pts = collect(range(dom[1],stop=dom[2],length=m))\n", 114 | "\n", 115 | " G = GroupPlot(2,1, groupStyle=\"horizontal sep=0.5cm, ylabels at=edge left\",\n", 116 | " style=\"width=7cm, xtick=\\\\empty, ytick=\\\\empty, xlabel=\\$x\\$, ylabel=\\$y\\$\")\n", 117 | " push!(G, Axis(\n", 118 | " Plots.Linear(f_, dom, style=\"solid, black, mark=none\", xbins=201),\n", 119 | " style=\"enlarge x limits=0\"\n", 120 | " ))\n", 121 | " push!(G, Axis(\n", 122 | " Plots.Scatter(x_pts, f_.(x_pts), style=\"solid, black, mark=*, mark size=1, mark options={draw=black, fill=black}\")\n", 123 | " ))\n", 124 | " G\n", 125 | " end\n", 126 | " plot(p)" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": { 133 | "collapsed": true 134 | }, 135 | "outputs": [], 136 | "source": [ 137 | " using LightGraphs\n", 138 | " using StatsBase\n", 139 | " using Vec\n", 140 | " using Random\n", 141 | "\n", 142 | " function execute_ant(graph, lengths, A, τ, best_tour, best_tour_length)\n", 143 | " path = [1]\n", 144 | " while length(path) < nv(graph)\n", 145 | " i = path[end]\n", 146 | " neighbors = outneighbors(graph, i)\n", 147 | " valid_neighbors = setdiff(neighbors, path)\n", 148 | " if isempty(valid_neighbors) # ant got stuck\n", 149 | " return (best_tour, best_tour_length)\n", 150 | " end\n", 151 | " as = [A[(i,j)] for j in valid_neighbors]\n", 152 | " push!(path, valid_neighbors[sample(Weights(as))])\n", 153 | " end\n", 154 | "\n", 155 | " l = sum(lengths[(path[i-1],path[i])] for i in 2 : length(path))\n", 156 | " for i in 2 : length(path)\n", 157 | " τ[(path[i-1],path[i])] += 1/l\n", 158 | " end\n", 159 | " if l < best_tour_length\n", 160 | " return (path, l)\n", 161 | " else\n", 162 | " return (best_tour, best_tour_length)\n", 163 | " end\n", 164 | " end\n", 165 | "\n", 166 | " function edge_attractiveness(graph, τ, η, α, β)\n", 167 | " A = Dict{Tuple{Int,Int},Float64}()\n", 168 | " for i in 1 : nv(graph)\n", 169 | " neighbors = outneighbors(graph, i)\n", 170 | " for j in neighbors\n", 171 | " v = τ[(i,j)]^α * η[(i,j)]^β\n", 172 | " A[(i,j)] = v\n", 173 | " end\n", 174 | " end\n", 175 | " return A\n", 176 | " end\n", 177 | "\n", 178 | " let\n", 179 | " dist_threshold = 0.8\n", 180 | "\n", 181 | " function closest_point(P::VecE2, vertices::Vector{VecE2})\n", 182 | " d_min = Inf\n", 183 | " retval = 0\n", 184 | " for (i,Q) in enumerate(vertices)\n", 185 | " d = abs(Q-P)\n", 186 | " if 0 < d < d_min\n", 187 | " d_min = d\n", 188 | " retval = i\n", 189 | " end\n", 190 | " end\n", 191 | " return retval\n", 192 | " end\n", 193 | " function edge_collides_with_other_edges(i::Int, j::Int, vertices::Vector{VecE2}, graph::DiGraph)\n", 194 | " seg = LineSegment(vertices[i], vertices[j])\n", 195 | " for e in edges(graph)\n", 196 | " A = vertices[e.src]\n", 197 | " B = vertices[e.dst]\n", 198 | " r = polar(1.0, atan(B - A))\n", 199 | " seg2 = LineSegment(A + r*0.05, B - r*0.05)\n", 200 | " if intersects(seg, seg2)\n", 201 | " return true\n", 202 | " end\n", 203 | " end\n", 204 | " return false\n", 205 | " end\n", 206 | "\n", 207 | " Random.seed!(1)\n", 208 | " vertices = VecE2[]\n", 209 | " for θ in range(0, stop=2π, length=21)[1:20]\n", 210 | " push!(vertices, polar(1.25 + sin(4θ),θ))\n", 211 | " end\n", 212 | "\n", 213 | " graph = DiGraph(length(vertices))\n", 214 | " for i in 1 : length(vertices)-1\n", 215 | " add_edge!(graph, i, i+1)\n", 216 | " end\n", 217 | " add_edge!(graph, length(vertices), 1)\n", 218 | "\n", 219 | " for k in 1 : 100\n", 220 | " i = rand(1:nv(graph))\n", 221 | " j = sample([j for j in 1 : nv(graph) if norm(vertices[i] - vertices[j]) < 2])\n", 222 | " if i != j && !has_edge(graph, i, j) && !has_edge(graph, j, i) &&\n", 223 | " !edge_collides_with_other_edges(i, j, vertices, graph)\n", 224 | "\n", 225 | " add_edge!(graph,i,j)\n", 226 | " end\n", 227 | " end\n", 228 | " for k in 1 : 14\n", 229 | " i = rand(1:nv(graph))\n", 230 | " j = sample([j for j in 1 : nv(graph)])\n", 231 | " if i != j && !has_edge(graph, i, j) && !has_edge(graph, j, i) &&\n", 232 | " !edge_collides_with_other_edges(i, j, vertices, graph)\n", 233 | "\n", 234 | " add_edge!(graph,i,j)\n", 235 | " end\n", 236 | " end\n", 237 | "\n", 238 | " lengths = Dict{Tuple{Int,Int}, Float64}()\n", 239 | " for e in edges(graph)\n", 240 | " lengths[(e.src, e.dst)] = norm(vertices[e.src] - vertices[e.dst])\n", 241 | " end\n", 242 | "\n", 243 | " Random.seed!(0)\n", 244 | "\n", 245 | " k_max = 1*4\n", 246 | " m = 50\n", 247 | " α = 1.0\n", 248 | " β = 5.0\n", 249 | " ρ = 0.5\n", 250 | " η = Dict((e.src,e.dst)=>1/lengths[(e.src,e.dst)] for e in edges(graph))\n", 251 | " τ = Dict((e.src,e.dst)=>1.0 for e in edges(graph))\n", 252 | "\n", 253 | " # rendering params\n", 254 | " r = 0.03\n", 255 | " scale = 0.7\n", 256 | "\n", 257 | " best_tour = Int[]\n", 258 | " best_tour_length = Inf\n", 259 | "\n", 260 | " Δx = 4.5\n", 261 | " Δy = 4.5\n", 262 | " max_node_x = 3\n", 263 | " nodex = 0\n", 264 | " nodey = 0\n", 265 | "\n", 266 | " o = stdout # IOBuffer()\n", 267 | " println(o, \"\\\\begin{tikzpicture}[x=$(scale)cm, y=$(scale)cm]\")\n", 268 | "\n", 269 | " for k in 1 : k_max\n", 270 | "\n", 271 | " println(o, \"\\\\node at ($(Δx*nodex),$(Δy*nodey)) { \\\\begin{tikzpicture}\")\n", 272 | " τmax = maximum(values(τ))\n", 273 | " for e in edges(graph)\n", 274 | " A = vertices[e.src]\n", 275 | " B = vertices[e.dst]\n", 276 | " α = τ[(e.src, e.dst)] / τmax\n", 277 | " println(o, \"\\\\draw[-{Latex[length=0.7mm, width=0.5mm]}, shorten >=3pt, pastelBlue!$(round(90*α + 10))] ($(A.x), $(A.y)) -- ($(B.x), $(B.y));\")\n", 278 | " end\n", 279 | " for A in vertices\n", 280 | " println(o, \"\\\\fill ($(A.x), $(A.y)) circle ($r);\")\n", 281 | " end\n", 282 | " println(o, \"\\\\end{tikzpicture} };\")\n", 283 | " nodex += 1\n", 284 | " if nodex > max_node_x\n", 285 | " nodex = 0\n", 286 | " nodey -= 1\n", 287 | " end\n", 288 | "\n", 289 | " A = edge_attractiveness(graph, τ, η, α, β)\n", 290 | " for (e,v) in τ\n", 291 | " τ[e] = (1-ρ)*v\n", 292 | " end\n", 293 | " for ant in 1 : m\n", 294 | " (best_tour, best_tour_length) = execute_ant(graph, lengths, A, τ, best_tour, best_tour_length)\n", 295 | " end\n", 296 | " end\n", 297 | "\n", 298 | " println(o, \"\\\\end{tikzpicture}\")\n", 299 | " end" 300 | ] 301 | } 302 | ], 303 | "metadata": { 304 | "kernelspec": { 305 | "display_name": "Julia 1.5.1", 306 | "language": "julia", 307 | "name": "julia-1.5" 308 | }, 309 | "language_info": { 310 | "file_extension": ".jl", 311 | "mimetype": "application/julia", 312 | "name": "julia", 313 | "version": "1.5.1" 314 | } 315 | }, 316 | "nbformat": 4, 317 | "nbformat_minor": 2 318 | } 319 | -------------------------------------------------------------------------------- /expr.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Expr\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | " abstract type SelectionMethod end\n", 32 | " abstract type CrossoverMethod end\n", 33 | " abstract type MutationMethod end\n", 34 | "\n", 35 | " struct TruncationSelection <: SelectionMethod\n", 36 | " k::Int\n", 37 | " end" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": { 44 | "collapsed": true 45 | }, 46 | "outputs": [], 47 | "source": [ 48 | " struct RouletteWheelSelection <: SelectionMethod end" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": { 55 | "collapsed": true 56 | }, 57 | "outputs": [], 58 | "source": [ 59 | " using ExprRules\n", 60 | " using TreeView\n", 61 | " using Distributions\n", 62 | " using Random\n", 63 | "\n", 64 | " struct TreeCrossover <: CrossoverMethod\n", 65 | " grammar::Grammar\n", 66 | " max_depth::Int\n", 67 | " end\n", 68 | " struct TreeMutation <: MutationMethod\n", 69 | " grammar::Grammar\n", 70 | " p::Float64\n", 71 | " end\n", 72 | "\n", 73 | " g = let\n", 74 | "\n", 75 | " grammar = @grammar begin\n", 76 | " R = |(1:9)\n", 77 | " R = R + R\n", 78 | " R = R - R\n", 79 | " R = R / R\n", 80 | " R = R * R\n", 81 | " end\n", 82 | "\n", 83 | " function select(T::TruncationSelection, y)\n", 84 | " p = sortperm(y)\n", 85 | " return [p[rand(1:T.k, 2)] for i in y]\n", 86 | " end\n", 87 | " function select(::RouletteWheelSelection, y)\n", 88 | " y = maximum(y) - y\n", 89 | " cat = Categorical(normalize(y, 1))\n", 90 | " return [rand(cat, 2) for i in y]\n", 91 | " end\n", 92 | "\n", 93 | " function crossover(C::TreeCrossover, a, b)\n", 94 | " child = deepcopy(a)\n", 95 | " crosspoint = sample(b)\n", 96 | " typ = return_type(C.grammar, crosspoint.ind)\n", 97 | " d_subtree = depth(crosspoint)\n", 98 | " d_max = C.max_depth + 1 - d_subtree\n", 99 | " if d_max > 0 && contains_returntype(child, C.grammar, typ, d_max)\n", 100 | " loc = sample(NodeLoc, child, typ, C.grammar, d_max)\n", 101 | " insert!(child, loc, deepcopy(crosspoint))\n", 102 | " end\n", 103 | " child\n", 104 | " end\n", 105 | "\n", 106 | " function mutate(M::TreeMutation, a)\n", 107 | " child = deepcopy(a)\n", 108 | " if rand() < M.p\n", 109 | " # println(\"mutate!\")\n", 110 | " loc = sample(NodeLoc, child)\n", 111 | " # @show loc\n", 112 | " typ = return_type(M.grammar, get(child, loc).ind)\n", 113 | " # @show loc\n", 114 | " subtree = rand(RuleNode, M.grammar, typ)\n", 115 | " # @show subtree\n", 116 | " child = insert!(child, loc, subtree)\n", 117 | " # @show child\n", 118 | " end\n", 119 | " return child\n", 120 | " end\n", 121 | "\n", 122 | " f = (node) -> begin\n", 123 | " value = Core.eval(node, grammar)\n", 124 | " if isinf(value) || isnan(value)\n", 125 | " return Inf\n", 126 | " end\n", 127 | " Δ = abs(value - π)\n", 128 | " return log(Δ) + length(node)/1e3\n", 129 | " end\n", 130 | "\n", 131 | " function genetic_algorithm(f, population, k_max, S, C, M)\n", 132 | " for k in 1 : k_max\n", 133 | " parents = select(S, f.(population))\n", 134 | " children = [crossover(C,population[p[1]],population[p[2]]) for p in parents]\n", 135 | " population = [mutate(M, c) for c in children]\n", 136 | " end\n", 137 | " population[argmin(f.(population))]\n", 138 | " end\n", 139 | "\n", 140 | " Random.seed!(1)\n", 141 | " m = 1000\n", 142 | " population = [rand(RuleNode, grammar, :R) for i in 1:m]\n", 143 | " k_max = 20\n", 144 | " best_tree = genetic_algorithm(f, population, k_max,\n", 145 | " TruncationSelection(50),\n", 146 | " TreeCrossover(grammar, 10),\n", 147 | " TreeMutation(grammar, 0.25))\n", 148 | " expr = get_executable(best_tree, grammar)\n", 149 | " tree = walk_tree(expr)\n", 150 | " picture = TreeView.tikz_representation(tree)\n", 151 | " picture.options = \"scale = 0.6\"\n", 152 | " picture\n", 153 | " end\n", 154 | "\n", 155 | " global cur_plot = g" 156 | ] 157 | } 158 | ], 159 | "metadata": { 160 | "kernelspec": { 161 | "display_name": "Julia 1.5.1", 162 | "language": "julia", 163 | "name": "julia-1.5" 164 | }, 165 | "language_info": { 166 | "file_extension": ".jl", 167 | "mimetype": "application/julia", 168 | "name": "julia", 169 | "version": "1.5.1" 170 | } 171 | }, 172 | "nbformat": 4, 173 | "nbformat_minor": 2 174 | } 175 | -------------------------------------------------------------------------------- /gp.jl: -------------------------------------------------------------------------------- 1 | using PGFPlots 2 | using Distributions 3 | using Optim 4 | using Parameters 5 | 6 | import LinearAlgebra: norm, I, diag 7 | 8 | const Dset = Vector{Vector{Float64}} 9 | 10 | function myopt(f; a=0.0, b=8.0, Δx=0.1) 11 | best_x = a 12 | best_y = Inf 13 | m = ceil(Int, (b-a)/Δx) 14 | pts = range(a, stop=b, length=m+1) 15 | for i in 1 : m 16 | res = optimize(f, pts[i], pts[i+1]) 17 | if res.minimum < best_y 18 | best_y = res.minimum 19 | best_x = res.minimizer 20 | end 21 | end 22 | return best_x 23 | end 24 | 25 | μ(X::Dset, m::Function) = [m(x) for x in X] 26 | Σ(X::Dset, k::Function) = [k(x,x′) for x in X, x′ in X] 27 | K(X::Dset, X′::Dset, k::Function) = [k(x,x′) for x in X, x′ in X′] 28 | 29 | @with_kw struct GaussianProcess 30 | m::Function = x -> 0.0 31 | k::Function = (x,x′)->exp(-(norm(x-x′))^2) 32 | X::Vector{Vector{Float64}} = Vector{Float64}[] 33 | y::Vector{Float64} = Float64[] 34 | ν::Float64 = 0.0 # variance when sampling f 35 | end 36 | 37 | mvnrand(μ::Vector{Float64}, Σ::Matrix{Float64}, inflation=1e-6) = rand(MvNormal(μ, Σ + inflation*I)); 38 | Base.rand(GP::GaussianProcess, X::Dset) = mvnrand(μ(X, GP.m), Σ(X, GP.k)) 39 | function Base.push!(GP::GaussianProcess, x::Vector{Float64}, y::Real) 40 | push!(GP.X, x) 41 | push!(GP.y, y) 42 | return GP 43 | end 44 | function Base.pop!(GP::GaussianProcess) 45 | pop!(GP.X) 46 | pop!(GP.y) 47 | return GP 48 | end 49 | 50 | function predict(GP::GaussianProcess, X_pred::Dset) 51 | m, k, ν = GP.m, GP.k, GP.ν 52 | tmp = K(X_pred, GP.X, k) / (K(GP.X, GP.X, k) + ν*I) 53 | μₚ = μ(X_pred, m) + tmp*(GP.y - μ(GP.X, m)) 54 | S = K(X_pred, X_pred, k) - tmp*K(GP.X, X_pred, k) 55 | νₚ = diag(S) .+ eps() # eps prevents numerical issues 56 | return (μₚ, νₚ) 57 | end 58 | function predict(GP::GaussianProcess, x_pred::Vector{Float64}) 59 | (μₚ, νₚ) = predict(GP, [x_pred]) 60 | return Normal(μₚ[1], sqrt(νₚ[1])) 61 | end 62 | 63 | prob_of_improvement(N::Normal{Float64}, y_max::Real) = isapprox(N.σ, 0, atol=1e-4) ? 0.0 : cdf(N, y_max) 64 | prob_is_safe(N::Normal{Float64}, y_max::Real) = cdf(N, y_max) 65 | 66 | function upperbound(GP::GaussianProcess, x_pred::Vector{Float64}, β::Real) 67 | (μₚ, νₚ) = predict(GP, [x_pred]) 68 | return μₚ[1] + sqrt(β*νₚ[1]) 69 | end 70 | function lowerbound(GP::GaussianProcess, x_pred::Vector{Float64}, β::Real) 71 | (μₚ, νₚ) = predict(GP, [x_pred]) 72 | return μₚ[1] - sqrt(β*νₚ[1]) 73 | end 74 | function width(GP::GaussianProcess, x_pred::Vector{Float64}, β::Real) 75 | (μₚ, νₚ) = predict(GP, [x_pred]) 76 | return 2sqrt(β*νₚ[1]) 77 | end 78 | 79 | is_safe(GP::GaussianProcess, x::Vector{Float64}, β::Real, y_max::Real) = upperbound(GP, x, β) <= y_max 80 | function get_safe_regions(GP::GaussianProcess, β::Real, a::Real, b::Real, y_max::Real; Δx=(b-a)/201) 81 | 82 | a = 1.0*a 83 | b = 1.0*b 84 | 85 | x = Float64[a] 86 | safe_regions = Tuple{Float64,Float64}[] 87 | while x[1] ≤ b 88 | while !is_safe(GP, x, β, y_max) && x[1] < b 89 | x[1] += Δx 90 | end 91 | lo = x[1] 92 | if is_safe(GP, x, β, y_max) 93 | while is_safe(GP, x, β, y_max) && x[1] < b 94 | x[1] += Δx 95 | end 96 | hi = x[1] 97 | push!(safe_regions, (lo,hi)) 98 | x[1] += Δx 99 | end 100 | end 101 | return safe_regions 102 | end 103 | 104 | function get_best_upperbound(GP::GaussianProcess, β::Real, safe_regions::Vector{Tuple{Float64,Float64}}) 105 | best_hi = Inf 106 | for (a,b) in safe_regions 107 | best_hi = min(best_hi, myopt(x->upperbound(GP, [x], β), a=a, b=b)) 108 | end 109 | return upperbound(GP, [best_hi], β) 110 | end 111 | 112 | function get_potential_maximizers(GP::GaussianProcess, β::Real, safe_regions::Vector{Tuple{Float64,Float64}}; Δx=0.01) 113 | 114 | best_hi = get_best_upperbound(GP, β, safe_regions) 115 | 116 | M_regions = Tuple{Float64,Float64}[] 117 | for (a,b) in safe_regions 118 | x = [a*1.0] 119 | while x[1] ≤ b 120 | while lowerbound(GP, x, β) > best_hi && x[1] < b 121 | x[1] += Δx 122 | end 123 | lo = x[1] 124 | if lowerbound(GP, x, β) ≤ best_hi 125 | while lowerbound(GP, x, β) ≤ best_hi && x[1] < b 126 | x[1] += Δx 127 | end 128 | hi = x[1] 129 | push!(M_regions, (lo,hi)) 130 | x .+= Δx 131 | else 132 | x .+= Δx 133 | end 134 | end 135 | end 136 | return M_regions 137 | end 138 | 139 | function is_expander( 140 | GP::GaussianProcess, 141 | x::Vector{Float64}, 142 | β::Float64, 143 | a::Float64, 144 | b::Float64, 145 | L::Float64, 146 | d::Function, 147 | y_max::Float64, 148 | ) 149 | 150 | ℓ = lowerbound(GP, x, β) 151 | return ℓ + L*min(d(x, [a]), d(x, [b])) ≤ y_max 152 | end 153 | function get_potential_expanders( 154 | GP::GaussianProcess, 155 | β::Real, 156 | safe_regions::Vector{Tuple{Float64,Float64}}, 157 | L::Float64, 158 | d::Function, 159 | y_max::Float64, 160 | ; 161 | Δx = 0.01, 162 | ) 163 | 164 | E_regions = Tuple{Float64,Float64}[] 165 | for (a,b) in safe_regions 166 | x = [a*1.0] 167 | while is_expander(GP, x, β, a, b, L, d, y_max) && x[1] < b 168 | x[1] += Δx 169 | end 170 | if !is_expander(GP, x, β, a, b, L, d, y_max) && x[1] > a 171 | push!(E_regions, (a, x[1])) 172 | end 173 | 174 | x = [b*1.0] 175 | while is_expander(GP, x, β, a, b, L, d, y_max) && x[1] > a 176 | x[1] -= Δx 177 | end 178 | if !is_expander(GP, x, β, a, b, L, d, y_max) && x[1] > a 179 | push!(E_regions, (x[1], b)) 180 | end 181 | 182 | end 183 | 184 | return E_regions 185 | end 186 | 187 | function plot_GP_data(GP::GaussianProcess; legendentry::String="") 188 | p = Plots.Scatter([x[1] for x in GP.X], GP.y, style="only marks, mark=*, mark size=1, mark options={draw=black, fill=black}") 189 | if !isempty(legendentry) 190 | p.legendentry = legendentry 191 | end 192 | return p 193 | end 194 | 195 | function plot_transparent_interval(xdom::Tuple{A,B}, ydom::Tuple{C,D}; style::String="") where {A<:Real, B<:Real, C<:Real, D<:Real} 196 | a,b = xdom 197 | lo,hi = ydom 198 | 199 | p = Plots.Plot[] 200 | push!(p, Plots.Linear([a,a], [lo,hi], style="name path=A, draw=none, mark=none")) 201 | push!(p, Plots.Linear([b,b], [lo,hi], style="name path=B, draw=none, mark=none")) 202 | push!(p, Plots.Command("\\addplot[$style] fill between[of=A and B]")) 203 | return p 204 | end 205 | 206 | function plot_transparent_intervals(regions::Vector{Tuple{Float64,Float64}}, ydom::Tuple{Float64,Float64}, color::String, opacity::Real, legendentry::String="") 207 | p = Plots.Plot[] 208 | lo, hi = ydom 209 | for (i,tup) in enumerate(regions) 210 | a, b = tup 211 | 212 | push!(p, Plots.Linear([a,a], [lo,hi], style="name path=A, draw=none, mark=none, forget plot")) 213 | push!(p, Plots.Linear([b,b], [lo,hi], style="name path=B, draw=none, mark=none, forget plot")) 214 | 215 | if i != length(regions) || isempty(legendentry) 216 | push!(p, Plots.Command("\\addplot[$color, forget plot, opacity=$(string(opacity))] fill between[of=A and B]")) 217 | else 218 | push!(p, Plots.Command("\\addplot[$color, opacity=$(string(opacity))] fill between[of=A and B]")) 219 | push!(p, Plots.Command("\\addlegendentry{$legendentry}")) 220 | end 221 | end 222 | return p 223 | end 224 | 225 | function update_confidence_intervals!(GP, X, u, ℓ, β) 226 | μₚ, νₚ = predict(GP, X) 227 | u[:] = μₚ + sqrt.(β*νₚ) 228 | ℓ[:] = μₚ - sqrt.(β*νₚ) 229 | return (u, ℓ) 230 | end 231 | function compute_sets!(GP, S, M, E, X, u, l, y_max, β) 232 | fill!(M, false) 233 | fill!(E, false) 234 | 235 | # safe set 236 | S[:] = u .≤ y_max 237 | 238 | if any(S) 239 | 240 | # potential minimizers 241 | M[S] = l[S] .< minimum(u[S]) 242 | 243 | # maximum width (in M) 244 | w_max = maximum(u[M] - l[M]) 245 | 246 | # expanders - skip values in M or those with w ≤ w_max 247 | E[:] = S .& .~M # skip points in M 248 | if any(E) 249 | E[E] .= maximum(u[E] - l[E]) .> w_max 250 | for (i,e) in enumerate(E) 251 | if e && u[i] - l[i] > w_max 252 | push!(GP, X[i], l[i]) 253 | μₚ, νₚ = predict(GP, X[.~S]) 254 | pop!(GP) 255 | E[i] = any(μₚ + sqrt.(β*νₚ) .≥ y_max) 256 | if E[i]; w_max = u[i] - l[i]; end 257 | end 258 | end 259 | end 260 | end 261 | 262 | return (S,M,E) 263 | end 264 | function compute_sets!(S, M, E, X, u, ℓ, y_max) 265 | # update the safe set based on current confidence bounds 266 | S[:] = u .≤ y_max 267 | 268 | # maximizers 269 | fill!(M, false) 270 | M[S] = u[S] .≥ maximum(ℓ[S]) 271 | 272 | # maximum width (in M) 273 | w_max = maximum(u[M] - ℓ[M]) 274 | 275 | # expanders 276 | #= 277 | For the run of the algorithm we do not need to calculate the 278 | full set of potential expanders: 279 | We can skip the ones already in M and ones that have lower 280 | variance than the maximum variance in M, w_max or the threshold. 281 | Amongst the remaining ones we only need to find the 282 | potential expander with maximum variance 283 | =# 284 | E[:] = S .& .~M # skip points in M 285 | if any(E) 286 | E[E] = maximum(u[E] - ℓ[E]) .> w_max # skip points with low width 287 | for (i,e) in enumerate(E) 288 | if e && u[i] - ℓ[i] > w_max # is potentially an expander and higher width 289 | push!(GP, X[i], ℓ[i]) # Add safe point with its lowest possible value to the GP 290 | μₚ, νₚ = predict(GP, X[.~S]) # Prediction of previously unsafe points based on that 291 | pop!(GP) # Remove the fake data point from the GP again 292 | E[i] = any(μₚ + sqrt.(β*νₚ) ≥ y_max) # If any unsafe upperr bound is suddenly below fmax then the point is an expander 293 | if E[i]; w_max = u[i] - ℓ[i]; end # so we don't consider other expanders with lower width 294 | end 295 | end 296 | end 297 | 298 | return (S,M,E) 299 | end 300 | function get_new_query_point(M, E, u, ℓ) 301 | ME = M .| E 302 | any(ME) || error("There are no points to evaluate") 303 | return something(findfirst(isequal(argmax(u[ME] - ℓ[ME])), cumsum(ME)), 0) 304 | end 305 | 306 | function safe_opt(GP, X, xi, f, y_max; β=3.0, K=10) 307 | push!(GP, X[i], f(X[i])) # make first observation 308 | 309 | m = length(X) 310 | u, ℓ = fill(Inf, m), fill(-Inf, m) 311 | S, M, E = falses(m), falses(m), falses(m) 312 | 313 | for k in 1 : K 314 | update_confidence_intervals!(GP, X, u, ℓ, β) 315 | compute_sets!(S, M, E, X, u, ℓ, y_max) 316 | i = get_new_query_point(M, E, u, ℓ) 317 | push!(GP, X[i], f(X[i])) 318 | end 319 | 320 | # return the best point 321 | update_confidence_intervals!(GP, X, u, ℓ, β) 322 | S[:] = u .≤ y_max 323 | if any(S) 324 | u_best, i_best = findmin(u[S]) 325 | i_best = something(findfirst(isequal(i_best), cumsum(S)), 0) 326 | return (u_best, i_best) 327 | else 328 | return (NaN,0) 329 | end 330 | end -------------------------------------------------------------------------------- /introduction.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Introduction\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "\n", 32 | "p = let\n", 33 | "\tf = (x,y) -> (1-x)^2 + 5*(y - x^2)^2\n", 34 | "\tAxis([Plots.Contour(f, (-2,2), (-2,2), levels=[1,2,3,5,10,20,50,100]),\n", 35 | "\t\t Plots.Scatter([1], [1], style=\"black, solid, mark=*, mark size = 1, mark options={draw=black}\"),\n", 36 | "\t\t ], width=\"9cm\", height=\"9cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"view={0}{90}, contour/labels=false\"\n", 37 | "\t)\n", 38 | "end\n", 39 | "\n", 40 | "plot(p)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": { 47 | "collapsed": true 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "\tp = let\n", 52 | "\t\txdom = (-2.5,2.5)\n", 53 | "\t\tydom = (-2.5,2.5)\n", 54 | "\n", 55 | "\t\tg = GroupPlot(2,1,groupStyle=\"horizontal sep=1.5cm, ylabels at=edge left\",\n", 56 | " \t style=\"width=5.25cm, xlabel=\\$x_1\\$, ylabel=\\$x_2\\$, xmin=$(xdom[1]), xmax=$(xdom[2]), ymin=$(ydom[1]), ymax=$(ydom[2])\")\n", 57 | "\n", 58 | "\t\tf = (x,y) -> x^2 - y^2\n", 59 | "\n", 60 | "\t\tc1 = x -> -sqrt(2 + x^2)\n", 61 | "\t\tc2 = x -> -sqrt(4 + x^2)\n", 62 | "\t\tc3 = y -> sqrt(2 + y^2)\n", 63 | "\t\tc4 = y -> sqrt(4 + y^2)\n", 64 | "\n", 65 | "\t\txs = range(xdom[1],stop=xdom[2],length=51)\n", 66 | "\t\txs3 = range(-2, stop=0.5, length=41)\n", 67 | "\t\txs4 = range(-1.5, stop=0, length=41)\n", 68 | "\n", 69 | "\t\tpush!(g,\n", 70 | "\t\t\tAxis(\n", 71 | "\t\t\t\t[\n", 72 | "\t\t\t\t\tPlots.Command(\"\"\"\n", 73 | "\t\t\t\t\t \\\\addplot3[patch,patch refines=3,\n", 74 | "\t\t\t\t\t\t\tshader=faceted interp,\n", 75 | "\t\t\t\t\t\t\tpatch type=biquadratic]\n", 76 | "\t\t\t\t\t \ttable[z expr=x^2-y^2]\n", 77 | "\t\t\t\t\t {\n", 78 | "\t\t\t\t\t x y\n", 79 | "\t\t\t\t\t -2.5 -2.5\n", 80 | "\t\t\t\t\t 2.5 -2.5\n", 81 | "\t\t\t\t\t 2.5 2.5\n", 82 | "\t\t\t\t\t -2.5 2.5\n", 83 | "\t\t\t\t\t 0 -2.5\n", 84 | "\t\t\t\t\t 2.5 0\n", 85 | "\t\t\t\t\t 0 2.5\n", 86 | "\t\t\t\t\t -2.5 0\n", 87 | "\t\t\t\t\t 0 0\n", 88 | "\t\t\t\t\t };\n", 89 | "\t\t\t\t\t\"\"\"),\n", 90 | "\t\t\t\t\tPlots.Linear3([-2.5,0.7],[-2.5,0.7],[0,0], style=\"solid, white, mark=none\"),\n", 91 | "\t\t\t\t\tPlots.Linear3([-1.5,2.5],[1.5,-2.5],[0,0], style=\"solid, white, mark=none\"),\n", 92 | "\t\t\t\t\tPlots.Linear3(xs,[c1(x) for x in xs],fill(-2, length(xs)), style=\"solid, white, mark=none\"),\n", 93 | "\t\t\t\t\tPlots.Linear3(xs,[c2(x) for x in xs],fill(-4, length(xs)), style=\"solid, white, mark=none\"),\n", 94 | "\t\t\t\t\tPlots.Linear3([ c3(x) for x in xs3], xs3, fill( 2, length(xs3)), style=\"solid, white, mark=none\"),\n", 95 | "\t\t\t\t\tPlots.Linear3([ c4(x) for x in xs4], xs4, fill( 4, length(xs4)), style=\"solid, white, mark=none\"),\n", 96 | "\t\t\t\t\tPlots.Linear3([-c3(x) for x in xs], xs, fill( 2, length(xs)), style=\"solid, white, mark=none\"),\n", 97 | "\t\t\t\t\tPlots.Linear3([-c4(x) for x in xs], xs, fill( 4, length(xs)), style=\"solid, white, mark=none\"),\n", 98 | "\t\t\t\t]\n", 99 | "\t\t\t)\n", 100 | "\t\t)\n", 101 | "\t\t# push!(g,\n", 102 | "\t\t# \tAxis(Plots.Image(f, xdom, ydom, xbins=600, ybins=600, colormap = pasteljet, colorbar = false),\n", 103 | "\t\t# \t xmin=xdom[1], xmax=xdom[2], ymin=ydom[1], ymax=ydom[2], width=\"5.25cm\", height=\"5.25cm\", style=\"view={0}{90}\", xlabel=L\"x_1\", ylabel=L\"x_2\")\n", 104 | "\t\t# )\n", 105 | "\t\tpush!(g,\n", 106 | "\t\t\tAxis(Plots.Contour(f, xdom, ydom, xbins=100, ybins=100, levels=[-4,-2,0,2,4]),\n", 107 | "\t\t\t style=\"axis equal, height=5.25cm, view={0}{90}\")\n", 108 | "\t\t)\n", 109 | "\n", 110 | "\t\tg\n", 111 | "\tend\n", 112 | "\n", 113 | "\tplot(p)" 114 | ] 115 | } 116 | ], 117 | "metadata": { 118 | "kernelspec": { 119 | "display_name": "Julia 1.5.1", 120 | "language": "julia", 121 | "name": "julia-1.5" 122 | }, 123 | "language_info": { 124 | "file_extension": ".jl", 125 | "mimetype": "application/julia", 126 | "name": "julia", 127 | "version": "1.5.1" 128 | } 129 | }, 130 | "nbformat": 4, 131 | "nbformat_minor": 2 132 | } 133 | -------------------------------------------------------------------------------- /julia.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Julia\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | " \t\tprintln(\"\\\\begin{tabular}{ll}\")\n", 32 | " \t\tprintln(\"\\\\toprule\")\n", 33 | " \t\tprintln(\"Package & Version \\\\\\\\\")\n", 34 | " \t\tprintln(\"\\\\midrule\")\n", 35 | " \t\topen(\"REQUIRE\") do f\n", 36 | " \t\tfor l in readlines(f)\n", 37 | " \t\tif startswith(l, \"julia\") || startswith(l, \"Vec\") || startswith(l, \"Weave\") || startswith(l, \"ColorSchemes\") || startswith(l, \"Optim\") || startswith(l, \"PGFPlots\") || startswith(l, \"SymEngine\") || startswith(l, \"TikzGraphs\") || length(l) < 1\n", 38 | " \t\tcontinue\n", 39 | " \t\tend\n", 40 | " \t\tprintln(\"$l & $(Pkg.installed(l))\\\\\\\\\")\n", 41 | " \t\tend\n", 42 | " \t\tend\n", 43 | " \t\tprintln(\"\\\\bottomrule\")\n", 44 | " \t\tprintln(\"\\\\end{tabular}\")" 45 | ] 46 | } 47 | ], 48 | "metadata": { 49 | "kernelspec": { 50 | "display_name": "Julia 1.5.1", 51 | "language": "julia", 52 | "name": "julia-1.5" 53 | }, 54 | "language_info": { 55 | "file_extension": ".jl", 56 | "mimetype": "application/julia", 57 | "name": "julia", 58 | "version": "1.5.1" 59 | } 60 | }, 61 | "nbformat": 4, 62 | "nbformat_minor": 2 63 | } 64 | -------------------------------------------------------------------------------- /linear.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Linear\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "using Vec\n", 32 | "using LinearAlgebra\n", 33 | "\n", 34 | "p = let\n", 35 | "\n", 36 | " g = GroupPlot(4,1,groupStyle=\"horizontal sep=0.5cm, ylabels at=edge left\",\n", 37 | " \tstyle = \"width=5cm, height=5cm, xmin=0, xmax=1, ymin=0, ymax=1, ytick=\\\\empty, xtick=\\\\empty, xlabel = \\$x_1\\$, ylabel= \\$x_2\\$, contour/labels=false, axis equal, view={0}{90}\"\n", 38 | " )\n", 39 | "\n", 40 | "\n", 41 | "\n", 42 | " c = Float64[2, 1]\n", 43 | " f = x->c⋅x\n", 44 | "\n", 45 | " A = VecE2(0.20,0.20)\n", 46 | " B = A + polar(2.0, deg2rad(10))\n", 47 | " C = A + polar(2.0, deg2rad(75))\n", 48 | "\n", 49 | " arrow_lo = [0.7,0.9]\n", 50 | " arrow_hi = arrow_lo - 0.6*normalize(c)\n", 51 | " push!(g, Axis([Plots.Contour(f, (0,1), (0,1), contour_style=\"draw color=gray\"),\n", 52 | " \t\t\t Plots.Linear([A.x, B.x], [A.y, B.y], style=\"solid, black, mark=none, name path=A\"),\n", 53 | " \t\t\t Plots.Linear([A.x, C.x], [A.y, C.y], style=\"solid, black, mark=none, name path=B\"),\n", 54 | " \t\t\t Plots.Command(\"\\\\addplot[pastelBlue!50] fill between[of=A and B]\"),\n", 55 | " \t\t\t Plots.Scatter([A.x], [A.y], style=\"mark=*, mark size=1, mark options={draw=pastelBlue, fill=pastelBlue}\"),\n", 56 | " \t\t\t Plots.Node(L\"\\vect x^*\",A.x, A.y, style=\"below right\"),\n", 57 | " \t\t\t Plots.Command(\"\\\\draw[myarrow] (axis cs:$(arrow_lo[1]), $(arrow_lo[2])) -- (axis cs:$(arrow_hi[1]), $(arrow_hi[2]));\"),\n", 58 | " \t\t\t Plots.Node(L\"-\\vect c\", (arrow_lo[1] + arrow_hi[1])/2, (arrow_lo[2] + arrow_hi[2])/2, style=\"below right\", axis=\"axis description cs\"),\n", 59 | " \t\t], title=\"One Solution\"\n", 60 | " ))\n", 61 | "\n", 62 | " r = VecE2(1,-2)\n", 63 | " q = VecE2(2, 1)\n", 64 | "\n", 65 | " A = VecE2(0.20,0.20)\n", 66 | " B = A + polar(2.0, deg2rad(10))\n", 67 | " C = A + polar(2.0, deg2rad(75))\n", 68 | " push!(g, Axis([Plots.Contour(x->-f(x), (0,1), (0,1), contour_style=\"draw color=gray\"),\n", 69 | " \t\t\t Plots.Linear([A.x, B.x], [A.y, B.y], style=\"solid, black, mark=none, name path=A\"),\n", 70 | " \t\t\t Plots.Linear([A.x, C.x], [A.y, C.y], style=\"solid, black, mark=none, name path=B\"),\n", 71 | " \t\t\t Plots.Command(\"\\\\addplot[pastelBlue!50] fill between[of=A and B]\"),\n", 72 | " \t\t\t Plots.Command(\"\\\\draw[myarrow] (axis cs:$(arrow_hi[1]), $(arrow_hi[2])) -- (axis cs:$(arrow_lo[1]), $(arrow_lo[2]));\"),\n", 73 | " \t\t\t Plots.Node(L\"-\\vect c\", (arrow_lo[1] + arrow_hi[1])/2, (arrow_lo[2] + arrow_hi[2])/2, style=\"below right\", axis=\"axis description cs\"),\n", 74 | " \t\t], title=\"Unbounded Solution\"\n", 75 | " ))\n", 76 | "\n", 77 | " A = 0.1*q - 0.15*r\n", 78 | " B = 0.1*q + 0.00*r\n", 79 | " C = B + polar(0.75, deg2rad(10))\n", 80 | " D = A + polar(0.6, deg2rad(75))\n", 81 | " push!(g, Axis([Plots.Contour(f, (0,1), (0,1), contour_style=\"draw color=gray\"),\n", 82 | " Plots.Linear([A.x, B.x, C.x, D.x, A.x],[A.y, B.y, C.y, D.y, A.y], style=\"solid, black, mark=none, name path=A\"),\n", 83 | " Plots.Linear([A.x, B.x],[A.y, B.y], style=\"solid, ultra thick, pastelBlue, mark=none, name path=B\"),\n", 84 | " Plots.Command(\"\\\\addplot[pastelBlue!50] fill between[of=A and B]\"),\n", 85 | " \t\t\t Plots.Node(L\"\\vect x^*\",(A.x+B.x)/2, (A.y+B.y)/2, style=\"above right\"),\n", 86 | " \t\t\t Plots.Command(\"\\\\draw[myarrow] (axis cs:$(arrow_lo[1]), $(arrow_lo[2])) -- (axis cs:$(arrow_hi[1]), $(arrow_hi[2]));\"),\n", 87 | " \t\t\t Plots.Node(L\"-\\vect c\", (arrow_lo[1] + arrow_hi[1])/2, (arrow_lo[2] + arrow_hi[2])/2 + 0.05, style=\"below right\", axis=\"axis description cs\"),\n", 88 | " \t\t], title=\"Infinite Solutions\"\n", 89 | " ))\n", 90 | "\n", 91 | " push!(g, Axis([Plots.Contour(f, (0,1), (0,1), contour_style=\"draw color=gray\"),\n", 92 | " \t\t\t Plots.Command(\"\\\\draw[myarrow] (axis cs:$(arrow_lo[1]), $(arrow_lo[2])) -- (axis cs:$(arrow_hi[1]), $(arrow_hi[2]));\"),\n", 93 | " \t\t\t Plots.Node(L\"-\\vect c\", (arrow_lo[1] + arrow_hi[1])/2, (arrow_lo[2] + arrow_hi[2])/2, style=\"below right\", axis=\"axis description cs\"),\n", 94 | " \t\t], title=\"No Solution\"\n", 95 | " ))\n", 96 | "\n", 97 | " g\n", 98 | "end\n", 99 | "\n", 100 | "plot(p)" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": { 107 | "collapsed": true 108 | }, 109 | "outputs": [], 110 | "source": [ 111 | "p = let\n", 112 | "\tAxis(\n", 113 | "\t\t[Plots.Linear([0,1], [-1,0], style=\"solid, black, mark=none\"),\n", 114 | "\t\t Plots.Linear([1,4], [0,3], style=\"solid, thick, pastelBlue, mark=none\"),\n", 115 | "\t\t ],\n", 116 | "\t\txlabel=L\"x\", ylabel=L\"s\",\n", 117 | "\t\txmin=0, xmax=4.5, ymin=-1.25, ymax=3.25, width=\"1.2*9cm\",\n", 118 | "\t\tstyle=\"axis lines=center\",\n", 119 | "\t)\n", 120 | "end\n", 121 | "plot(p)" 122 | ] 123 | } 124 | ], 125 | "metadata": { 126 | "kernelspec": { 127 | "display_name": "Julia 1.5.1", 128 | "language": "julia", 129 | "name": "julia-1.5" 130 | }, 131 | "language_info": { 132 | "file_extension": ".jl", 133 | "mimetype": "application/julia", 134 | "name": "julia", 135 | "version": "1.5.1" 136 | } 137 | }, 138 | "nbformat": 4, 139 | "nbformat_minor": 2 140 | } 141 | -------------------------------------------------------------------------------- /math-concepts.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Math Concepts\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "\n", 32 | "p = let\n", 33 | "\tf = x -> cos(x) + x + 10x^1.5 + 3x^2\n", 34 | "\tf0 = x -> 110abs(cos(x))\n", 35 | "\tf1 = x -> 16x^1.5\n", 36 | "\tf2 = x -> 12x^2\n", 37 | "\tf3 = x -> 6x^3\n", 38 | "\n", 39 | "\tg = GroupPlot(2, 1, groupStyle=\"horizontal sep=1cm\")\n", 40 | "\ta, b = 0, 2\n", 41 | "\tpush!(g,\n", 42 | "\t\tAxis([\n", 43 | "\t \tPlots.Linear(f, (a, b), style=\"solid, thick, black, mark=none\"),\n", 44 | "\t \t# Plots.Linear(f0, (a, b), style=\"solid, mark=none, pastelGreen\"),\n", 45 | " \tPlots.Linear(f1, (a, b), style=\"solid, mark=none, pastelBlue\"),\n", 46 | " \tPlots.Linear(f2, (a, b), style=\"solid, mark=none, pastelPurple\"),\n", 47 | " \tPlots.Linear(f3, (a, b), style=\"solid, mark=none, pastelSeaGreen\"),\n", 48 | " ], xmin=a, xmax=b, xlabel=L\"x\", width=\"6cm\")\n", 49 | "\t)\n", 50 | "\n", 51 | "\ta,b = 1,1000\n", 52 | "\tpush!(g,\n", 53 | "\t\tAxis([\n", 54 | "\t \tPlots.Linear(f, (a, b), style=\"solid, thick, black, mark=none\", legendentry=L\"f(x)\"),\n", 55 | " \t# Plots.Linear(f0, (a, b), style=\"solid, mark=none, pastelGreen\", legendentry=L\"110|\\cos(x)|\"),\n", 56 | " \tPlots.Linear(f1, (a, b), style=\"solid, mark=none, pastelBlue\", legendentry=L\"16x^{3/2}\"),\n", 57 | " \tPlots.Linear(f2, (a, b), style=\"solid, mark=none, pastelPurple\", legendentry=L\"12x^2\"),\n", 58 | " \tPlots.Linear(f3, (a, b), style=\"solid, mark=none, pastelSeaGreen\", legendentry=L\"6x^3\"),\n", 59 | " ], xmin=a, xmax=b, xlabel=L\"x\", width=\"6cm\", style=\"xmode=log, ymode=log, legend style={at={(-0.11,-0.27)},anchor=north,legend columns=5}\")\n", 60 | "\t)\n", 61 | "\tg\n", 62 | "end\n", 63 | "\n", 64 | "plot(p)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "metadata": { 71 | "collapsed": true 72 | }, 73 | "outputs": [], 74 | "source": [ 75 | "\n", 76 | "p = let\n", 77 | "\tf = x -> cos(x)\n", 78 | "\tf′ = x -> -sin(x)\n", 79 | "\tf′′ = x ->-cos(x)\n", 80 | "\tf′′′ = x ->sin(x)\n", 81 | "\tf′′′′ = x ->cos(x)\n", 82 | "\tf′′′′′ = x ->-sin(x)\n", 83 | "\n", 84 | "\tp = 1.0\n", 85 | "\ta = p-5.0\n", 86 | "\tb = p+5.0\n", 87 | "\n", 88 | "\ttaylor0 = x -> f(p)\n", 89 | "\ttaylor1 = x -> f(p) + f′(p)*(x-p)\n", 90 | "\ttaylor2 = x -> f(p) + f′(p)*(x-p) + f′′(p)*(x-p)^2/2\n", 91 | "\ttaylor3 = x -> f(p) + f′(p)*(x-p) + f′′(p)*(x-p)^2/2 + f′′′(p)*(x-p)^3/6\n", 92 | "\ttaylor4 = x -> f(p) + f′(p)*(x-p) + f′′(p)*(x-p)^2/2 + f′′′(p)*(x-p)^3/6 + f′′′′(p)*(x-p)^4/24\n", 93 | "\ttaylor5 = x -> f(p) + f′(p)*(x-p) + f′′(p)*(x-p)^2/2 + f′′′(p)*(x-p)^3/6 + f′′′′(p)*(x-p)^4/24 + f′′′′′(p)*(x-p)^5/120\n", 94 | "\n", 95 | "\tx_arr = collect(range(a, stop=b, length=151))\n", 96 | "\n", 97 | "\tAxis([\n", 98 | "\t Plots.Linear(x_arr, f.(x_arr), style=\"solid, thick, black, mark=none\", legendentry=L\"\\cos(x)\"),\n", 99 | " Plots.Linear(x_arr, taylor0.(x_arr), style=\"solid, mark=none\", legendentry=\"0th degree\"),\n", 100 | " Plots.Linear(x_arr, taylor1.(x_arr), style=\"solid, mark=none\", legendentry=\"1st degree\"),\n", 101 | " Plots.Linear(x_arr, taylor2.(x_arr), style=\"solid, mark=none\", legendentry=\"2nd degree\"),\n", 102 | " Plots.Linear(x_arr, taylor3.(x_arr), style=\"solid, mark=none\", legendentry=\"3rd degree\"),\n", 103 | " Plots.Linear(x_arr, taylor4.(x_arr), style=\"solid, mark=none\", legendentry=\"4th degree\"),\n", 104 | " Plots.Linear(x_arr, taylor5.(x_arr), style=\"solid, mark=none\", legendentry=\"5th degree\"),\n", 105 | " Plots.Scatter([p], [f(p)], style=\"solid, mark=*, mark size=1, mark options={draw=none, fill=black}\"),\n", 106 | " ], xmin=a, xmax=b, ymin=-3, ymax=3, xlabel=L\"x\", width=\"10cm\", height=\"8cm\", style=\"legend columns=1, legend pos=outer north east, cycle list name = pastelcolors\")\n", 107 | "end\n", 108 | "\n", 109 | "plot(p)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "metadata": { 116 | "collapsed": true 117 | }, 118 | "outputs": [], 119 | "source": [ 120 | "using Distributions\n", 121 | "\n", 122 | "p = let\n", 123 | "\n", 124 | "\tN = Normal(0.0,1.0)\n", 125 | "\n", 126 | "\tAxis(Plots.Linear(x->pdf(N, x), (-3,3), style=\"black,solid,mark=none\"),\n", 127 | "\t\t xmin=-3, xmax=3, ymin=0, ymax=0.6, width=\"9cm\",\n", 128 | "\t\t xlabel=L\"x\", ylabel=L\"p(x\\mid \\mu, \\sigma)\",\n", 129 | "\t\t style=\"xtick={-1,0,1}, ytick=\\\\empty, xticklabels={\\$\\\\mu-\\\\sigma\\$,\\$\\\\mu\\$,\\$\\\\mu+\\\\sigma\\$}, xticklabel style={text height=2ex}\",\n", 130 | "\t\t )\n", 131 | "end\n", 132 | "\n", 133 | "plot(p)" 134 | ] 135 | } 136 | ], 137 | "metadata": { 138 | "kernelspec": { 139 | "display_name": "Julia 1.5.1", 140 | "language": "julia", 141 | "name": "julia-1.5" 142 | }, 143 | "language_info": { 144 | "file_extension": ".jl", 145 | "mimetype": "application/julia", 146 | "name": "julia", 147 | "version": "1.5.1" 148 | } 149 | }, 150 | "nbformat": 4, 151 | "nbformat_minor": 2 152 | } 153 | -------------------------------------------------------------------------------- /mdo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Mdo\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "p = let\n", 32 | "\n", 33 | "\tg = GroupPlot(1, 2, groupStyle=\"vertical sep=1em, xlabels at=edge bottom, ylabels at=edge left, xticklabels at=edge bottom, yticklabels at=edge bottom\", style=\"width=10cm, height=3.75cm, xlabel=iteration\")\n", 34 | "\n", 35 | "\tfunction F1(A)\n", 36 | "\t\tA[\"y1\"] = A[\"y2\"] - A[\"x\"]\n", 37 | "\t\treturn A\n", 38 | "\tend\n", 39 | "\tfunction F2(A)\n", 40 | "\t\tA[\"y2\"] = sin(A[\"y1\"] + A[\"y3\"])\n", 41 | "\t\treturn A\n", 42 | "\tend\n", 43 | "\tfunction F3(A)\n", 44 | "\t A[\"y3\"] = cos(A[\"x\"] + A[\"y2\"] + A[\"y1\"])\n", 45 | "\t return A\n", 46 | "\tend\n", 47 | "\n", 48 | "\t_K = 20\n", 49 | "\tarr_t = collect(0:_K)\n", 50 | "\n", 51 | "\tfunction get_plot(Fs, add_legend)\n", 52 | "\n", 53 | "\t\tA = Dict{String,Float64}(\"x\"=>1.0, \"y1\"=>1.0, \"y2\"=>1.0, \"y3\"=>1.0)\n", 54 | "\n", 55 | "\t\tarr_y1 = Array{Float64}(undef, _K+1)\n", 56 | "\t\tarr_y2 = Array{Float64}(undef, _K+1)\n", 57 | "\t\tarr_y3 = Array{Float64}(undef, _K+1)\n", 58 | "\t\tarr_y1[1] = A[\"y1\"]\n", 59 | "\t\tarr_y2[1] = A[\"y2\"]\n", 60 | "\t\tarr_y3[1] = A[\"y3\"]\n", 61 | "\n", 62 | "\t\tfor i in 1 : _K\n", 63 | "\t\t\tfor F in Fs\n", 64 | "\t\t\t\tF(A)\n", 65 | "\t\t\tend\n", 66 | "\t\t\tarr_y1[i+1] = A[\"y1\"]\n", 67 | "\t\t\tarr_y2[i+1] = A[\"y2\"]\n", 68 | "\t\t\tarr_y3[i+1] = A[\"y3\"]\n", 69 | "\t\tend\n", 70 | "\t\tp1 = Plots.Linear(arr_t, arr_y1, style=\"pastelPurple, solid, mark=*, mark size=1, mark options={draw=pastelPurple, fill=pastelPurple}\")\n", 71 | "\t\tp2 = Plots.Linear(arr_t, arr_y2, style=\"pastelBlue, solid, mark=*, mark size=1, mark options={draw=pastelBlue, fill=pastelBlue}\")\n", 72 | "\t\tp3 = Plots.Linear(arr_t, arr_y3, style=\"pastelSeaGreen, solid, mark=*, mark size=1, mark options={draw=pastelSeaGreen, fill=pastelSeaGreen}\")\n", 73 | "\t\tif add_legend\n", 74 | "\t\t\tp1.legendentry = L\"y^{(1)}\"\n", 75 | "\t\t\tp2.legendentry = L\"y^{(2)}\"\n", 76 | "\t\t\tp3.legendentry = L\"y^{(3)}\"\n", 77 | "\t\tend\n", 78 | "\n", 79 | "\n", 80 | "\t\treturn Axis([p1, p2, p3], style= add_legend ? \"legend pos=outer north east\" : nothing)\n", 81 | "\tend\n", 82 | "\n", 83 | "\tpush!(g, get_plot([F1, F2, F3], true))\n", 84 | "\tpush!(g, get_plot([F1, F3, F2], false))\n", 85 | "\tg\n", 86 | "end\n", 87 | "plot(p)" 88 | ] 89 | } 90 | ], 91 | "metadata": { 92 | "kernelspec": { 93 | "display_name": "Julia 1.5.1", 94 | "language": "julia", 95 | "name": "julia-1.5" 96 | }, 97 | "language_info": { 98 | "file_extension": ".jl", 99 | "mimetype": "application/julia", 100 | "name": "julia", 101 | "version": "1.5.1" 102 | } 103 | }, 104 | "nbformat": 4, 105 | "nbformat_minor": 2 106 | } 107 | -------------------------------------------------------------------------------- /multiobjective.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Multiobjective\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "using Vec\n", 32 | "using Random\n", 33 | "\n", 34 | "p = let\n", 35 | "\n", 36 | "\tRandom.seed!(0)\n", 37 | "\tx_arr = Float64[]\n", 38 | "\ty_arr = Float64[]\n", 39 | "\tA = VecE2(1,1)\n", 40 | "\tfor θ in range(180, stop=270, length=90)\n", 41 | "\t B = A + polar(0.9 - 0.3rand(), deg2rad(θ))\n", 42 | "\t push!(x_arr, B.x)\n", 43 | "\t push!(y_arr, B.y)\n", 44 | "\tend\n", 45 | "\n", 46 | " Axis([Plots.Command(\"\\\\draw[pastelBlue, line width=5mm, line cap=round] (axis cs:0.12,1) [out=270, in=180] to (axis cs:1,0.12);\"),\n", 47 | " \t Plots.Linear(x_arr,y_arr, style=\"only marks, mark=*, mark size=1, mark options={draw=black, fill=black}\"),\n", 48 | " \t Plots.Node(\"Ideal\",0,0, style=\"anchor=south west\"),\n", 49 | " \t Plots.Node(\"Pareto Frontier\",0.35,0.1, style=\"anchor=west\"),\n", 50 | " \t Plots.Node(\"Pareto dominated points (suboptimal)\",0.45,0.85, style=\"anchor=west, text width=4cm\"),\n", 51 | " \t Plots.Node(\"criterion space \\\\\\\\ (denoted \\$\\\\mathcal{Y}\\$) \",0.65,0.6, style=\"anchor=west, text width=4cm\"),\n", 52 | " \t], width=\"10cm\", xlabel=\"alert rate\", ylabel=\"collision rate\",\n", 53 | "\t xmin=0, xmax=1.1, ymin=0, ymax=1.1,\n", 54 | "\t style=\"xtick=\\\\empty, ytick=\\\\empty\",\n", 55 | " )\n", 56 | "end\n", 57 | "\n", 58 | "plot(p)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": { 65 | "collapsed": true 66 | }, 67 | "outputs": [], 68 | "source": [ 69 | "using Vec\n", 70 | "using Random\n", 71 | "\n", 72 | "function dominates(y, y′)\n", 73 | "\tlo, hi = extrema(y′ - y)\n", 74 | "\treturn hi > 0 && lo ≥ 0\n", 75 | "end\n", 76 | "function naive_pareto(xs, ys)\n", 77 | " pareto_xs, pareto_ys = [], []\n", 78 | " for (x,y) in zip(xs,ys)\n", 79 | " if !any(dominates(y′,y) for y′ in ys)\n", 80 | " push!(pareto_xs, x)\n", 81 | " push!(pareto_ys, y)\n", 82 | " end\n", 83 | " end\n", 84 | " return (pareto_xs, pareto_ys)\n", 85 | "end\n", 86 | "\n", 87 | "p = let\n", 88 | "\n", 89 | "\tRandom.seed!(1)\n", 90 | "\txs = Vector{Float64}[]\n", 91 | "\tys = Vector{Float64}[]\n", 92 | "\tA = VecE2(1,1)\n", 93 | "\tfor θ in range(180, stop=270, length=90)\n", 94 | "\t B = A + polar(0.9 - 0.3rand(), deg2rad(θ))\n", 95 | "\t push!(xs, [B.x, B.y])\n", 96 | "\t push!(ys, [B.x, B.y])\n", 97 | "\tend\n", 98 | "\n", 99 | "\tpareto, _ = naive_pareto(xs, ys)\n", 100 | "\tx_arr = [x[1] for x in pareto]\n", 101 | "\ty_arr = [x[2] for x in pareto]\n", 102 | "\tp = sortperm(x_arr)\n", 103 | "\tx_arr[:] = x_arr[p]\n", 104 | "\ty_arr[:] = y_arr[p]\n", 105 | "\n", 106 | "\tfilter!(x->x ∉ pareto, xs)\n", 107 | "\n", 108 | "\n", 109 | " Axis([Plots.Linear([x[1] for x in xs], [x[2] for x in xs], style=\"only marks, mark=*, mark size=0.5, mark options={draw=black, fill=black}\"),\n", 110 | " \t Plots.Linear(x_arr, y_arr, style=\"pastelBlue, mark=*, mark size=0.5, mark options={draw=none, fill=pastelBlue}\"),\n", 111 | " \t], width=\"9cm\", xlabel=L\"y_1\", ylabel=L\"y_2\",\n", 112 | "\t xmin=0, xmax=1.1, ymin=0, ymax=1.1,\n", 113 | "\t style=\"xtick=\\\\empty, ytick=\\\\empty, axis lines=middle\",\n", 114 | " )\n", 115 | "end\n", 116 | "\n", 117 | "plot(p)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": { 124 | "collapsed": true 125 | }, 126 | "outputs": [], 127 | "source": [ 128 | "abstract type SelectionMethod end\n", 129 | "struct TruncationSelection <: SelectionMethod\n", 130 | "\tk # top k to keep\n", 131 | "end\n", 132 | "abstract type MutationMethod end\n", 133 | "struct GaussianMutation <: MutationMethod\n", 134 | "\tσ\n", 135 | "end" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": { 142 | "collapsed": true 143 | }, 144 | "outputs": [], 145 | "source": [ 146 | "using Random\n", 147 | "\n", 148 | "function Base.partialsort(t::TruncationSelection, y)\n", 149 | " p = sortperm(y)\n", 150 | " return [p[rand(1:t.k, 2)] for i in y]\n", 151 | "end\n", 152 | "\n", 153 | "function rand_population_uniform(M, a, b)\n", 154 | " d = length(a)\n", 155 | " return [a+rand(d).*(b-a) for i in 1:M]\n", 156 | "end\n", 157 | "\n", 158 | "abstract type CrossoverMethod end\n", 159 | "struct InterpolationCrossover <: CrossoverMethod\n", 160 | " λ\n", 161 | "end\n", 162 | "crossover(C::InterpolationCrossover, a, b) = (1-C.λ)*a + C.λ*b\n", 163 | "\n", 164 | "function mutate(M::GaussianMutation, child)\n", 165 | " return child + randn(length(child))*M.σ\n", 166 | "end\n", 167 | "\n", 168 | "p = let\n", 169 | "\n", 170 | "\tf = x -> begin\n", 171 | "\t r = 0.5 + 0.5*(2x[1]/(1+x[1]^2))\n", 172 | "\t θ = x[2]\n", 173 | "\t y1 = 1 - r*cos(θ)\n", 174 | "\t y2 = 1 - r*sin(θ)\n", 175 | "\t return [y1, y2]\n", 176 | "\tend\n", 177 | "\tm_pop = 40\n", 178 | "\tRandom.seed!(0)\n", 179 | "\tpopulation = rand_population_uniform(m_pop, [-5,-5], [5,5])\n", 180 | "\n", 181 | "\tS = TruncationSelection(10)\n", 182 | "\tC = InterpolationCrossover(0.1)\n", 183 | "\tM = GaussianMutation(0.1)\n", 184 | "\n", 185 | "\tG = GroupPlot(4,1,groupStyle=\"xlabels at=edge bottom, ylabels at =edge left, xticklabels at=edge bottom, yticklabels at=edge left, horizontal sep=0.5cm, vertical sep=0.5cm\", style=\"xlabel=\\$y_1\\$, ylabel=\\$y_2\\$, width=5cm, height=5cm, xmin=0, xmax=2, ymin=0, ymax=2\")\n", 186 | "\n", 187 | "\n", 188 | "\tm = length(f(population[1]))\n", 189 | "\tm_pop = length(population)\n", 190 | "\tm_subpop = m_pop ÷ m\n", 191 | "\tfor k in 1 : 4\n", 192 | "\t fitnesses = f.(population)\n", 193 | "\t parents = partialsort(S, [y[1] for y in fitnesses])[1:m_subpop]\n", 194 | "\t for i in 2 : m\n", 195 | "\t subpop=partialsort(S,[y[i] for y in fitnesses])[1:m_subpop]\n", 196 | "\t append!(parents, subpop)\n", 197 | "\t end\n", 198 | "\n", 199 | "\t plots = Plots.Plot[]\n", 200 | "\t push!(plots, Plots.Linear(y1->1-sqrt(1-(1-y1)^2), (0,1), style=\"solid, pastelBlue, ultra thick\"))\n", 201 | "\t push!(plots, Plots.Scatter([y[1] for y in fitnesses],\n", 202 | "\t [y[2] for y in fitnesses], style=\"clip marker paths, mark=*, mark size=1, mark options={draw=gray, fill=gray}\"))\n", 203 | "\t push!(G, Axis(plots, title=\"Gen $k\")) #$\n", 204 | "\n", 205 | "\t p = randperm(2m_pop)\n", 206 | "\t p_ind=i->parents[mod(p[i]-1,m_pop)+1][(p[i]-1)÷m_pop + 1]\n", 207 | "\t parents = [[p_ind(i), p_ind(i+1)] for i in 1 : 2 : 2m_pop]\n", 208 | "\t children = [crossover(C,population[p[1]],population[p[2]])\n", 209 | "\t for p in parents]\n", 210 | "\t population = [mutate(M, c) for c in children]\n", 211 | "\tend\n", 212 | "\n", 213 | "\tG\n", 214 | "end\n", 215 | "plot(p)" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": null, 221 | "metadata": { 222 | "collapsed": true 223 | }, 224 | "outputs": [], 225 | "source": [ 226 | "using Vec\n", 227 | "using Random\n", 228 | "\n", 229 | "p = let\n", 230 | "\n", 231 | "\tRandom.seed!(1)\n", 232 | "\tys = Vector{Float64}[]\n", 233 | "\tA = VecE2(1,1)\n", 234 | "\tfor θ in range(180, stop=270, length=101)\n", 235 | "\t B = A + polar(0.9 - 0.3rand(), deg2rad(θ))\n", 236 | "\t push!(ys, [B.x, B.y])\n", 237 | "\tend\n", 238 | "\n", 239 | "\tfunction dominates2(y, y′)\n", 240 | "\t\tlo, hi = extrema(y′ - y)\n", 241 | "\t\treturn hi > 0 && lo ≥ 0\n", 242 | "\tend\n", 243 | "\tfunction get_non_domination_levels(ys)\n", 244 | "\t L, m = 0, length(ys)\n", 245 | "\t levels = zeros(Int, m)\n", 246 | "\t while minimum(levels) == 0\n", 247 | "\t L += 1\n", 248 | "\t for (i,y) in enumerate(ys)\n", 249 | "\t if levels[i] == 0 &&\n", 250 | "\t !any((levels[i] == 0 || levels[i] == L) && dominates2(ys[i],y) for i in 1 : m)\n", 251 | "\t levels[i] = L\n", 252 | "\t end\n", 253 | "\t end\n", 254 | "\t end\n", 255 | "\t return levels\n", 256 | "\tend\n", 257 | "\n", 258 | "\tp = Plots.Plot[]\n", 259 | "\tlevels = get_non_domination_levels(ys)\n", 260 | "\tfor lvl in 1 : maximum(levels)\n", 261 | "\t x_arr = [x[1] for x in ys[levels .== lvl]]\n", 262 | "\t y_arr = [x[2] for x in ys[levels .== lvl]]\n", 263 | "\t color = \"pastelBlue!$(round(Int, 100*(maximum(levels) - lvl + 1) / maximum(levels)))\" # $\n", 264 | "\t push!(p, Plots.Linear(x_arr, y_arr, style=\"solid, $color, mark=*, mark size=1, mark options={draw=$color, fill=$color}\"))\n", 265 | "\tend\n", 266 | "\tpush!(p, Plots.Node(\"\\\\small Level \\$1\\$\", 0.72, 0.20, style=\"anchor=center\"))\n", 267 | "\tpush!(p, Plots.Node(\"\\\\small Level \\$2\\$\", 0.99, 0.15, style=\"anchor=west\"))\n", 268 | "\tpush!(p, Plots.Node(\"\\\\small Level \\$10\\$\", 0.40, 0.94, style=\"anchor=west\"))\n", 269 | "\tAxis(p, width=\"10cm\", xlabel=L\"y_1\", ylabel=L\"y_2\",\n", 270 | "\t xmin=0, xmax=1.2, ymin=0, ymax=1.1,\n", 271 | "\t style=\"xtick=\\\\empty, ytick=\\\\empty, axis lines=middle\",\n", 272 | "\t)\n", 273 | "end\n", 274 | "\n", 275 | "plot(p)" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": null, 281 | "metadata": { 282 | "collapsed": true 283 | }, 284 | "outputs": [], 285 | "source": [ 286 | "using Random\n", 287 | "using LinearAlgebra\n", 288 | "\n", 289 | "p = let\n", 290 | "\n", 291 | "\trng = MersenneTwister(0)\n", 292 | "\n", 293 | "\tfunction discard_closest_pair!(xs, ys)\n", 294 | "\t\tindex, min_dist = 0, Inf\n", 295 | "\t\tfor (i,y) in enumerate(ys)\n", 296 | "\t\t\tfor (j, y′) in enumerate(ys[i+1:end])\n", 297 | "\t\t\t\tdist = norm(y - y′)\n", 298 | "\t\t\t\tif dist < min_dist\n", 299 | "\t\t\t\t\tindex, min_dist = rand(rng, [i,j]), dist\n", 300 | "\t\t\t\tend\n", 301 | "\t\t\tend\n", 302 | "\t\tend\n", 303 | "\t\tdeleteat!(xs, index)\n", 304 | "\t\tdeleteat!(ys, index)\n", 305 | "\t\treturn (xs, ys)\n", 306 | "\tend\n", 307 | "\tfunction update_pareto_filter!(filter_xs, filter_ys, xs, ys;\n", 308 | "\t\tcapacity=length(xs),\n", 309 | "\t\t)\n", 310 | "\t for (x,y) in zip(xs, ys)\n", 311 | "\t \tif !any(dominates(y′,y) for y′ in filter_ys)\n", 312 | "\t push!(filter_xs, x)\n", 313 | "\t push!(filter_ys, y)\n", 314 | "\t end\n", 315 | "\t end\n", 316 | "\t filter_xs, filter_ys = naive_pareto(filter_xs, filter_ys)\n", 317 | "\t while length(filter_xs) > capacity\n", 318 | "\t \tdiscard_closest_pair!(filter_xs, filter_ys)\n", 319 | "\t end\n", 320 | "\t return (filter_xs, filter_ys)\n", 321 | "\tend\n", 322 | "\n", 323 | "\tf = x -> begin\n", 324 | "\t r = 0.5 + 0.5*(2x[1]/(1+x[1]^2))\n", 325 | "\t θ = x[2]\n", 326 | "\t y1 = 1 - r*cos(θ)\n", 327 | "\t y2 = 1 - r*sin(θ)\n", 328 | "\t return [y1, y2]\n", 329 | "\tend\n", 330 | "\tm_pop = 40\n", 331 | "\tRandom.seed!(0)\n", 332 | "\tpopulation = rand_population_uniform(m_pop, [-5,-5], [5,5])\n", 333 | "\n", 334 | "\tfilter_xs = Vector{Float64}[]\n", 335 | "\tfilter_ys = Vector{Float64}[]\n", 336 | "\n", 337 | "\tS = TruncationSelection(10)\n", 338 | "\tC = InterpolationCrossover(0.1)\n", 339 | "\tM = GaussianMutation(0.1)\n", 340 | "\n", 341 | "\tG = GroupPlot(1,1,groupStyle=\"xlabels at=edge bottom, ylabels at =edge left, xticklabels at=edge bottom, yticklabels at=edge left, horizontal sep=0.5cm, vertical sep=0.5cm\", style=\"xlabel=\\$y_1\\$, ylabel=\\$y_2\\$, width=7cm, height=7cm, xmin=0, xmax=1.5, ymin=0, ymax=1.5\")\n", 342 | "\n", 343 | "\n", 344 | "\tK = 10\n", 345 | "\tm = length(f(population[1]))\n", 346 | "\tm_pop = length(population)\n", 347 | "\tm_subpop = m_pop ÷ m\n", 348 | "\tfor k in 1 : K\n", 349 | "\n", 350 | "\t\tfitnesses = f.(population)\n", 351 | "\n", 352 | "\t\tfilter_xs, filter_ys = update_pareto_filter!(filter_xs, filter_ys, population, fitnesses)\n", 353 | "\n", 354 | "\n", 355 | "\t parents = partialsort(S, [y[1] for y in fitnesses])[1:m_subpop]\n", 356 | "\t for i in 2 : m\n", 357 | "\t subpop=partialsort(S,[y[i] for y in fitnesses])[1:m_subpop]\n", 358 | "\t append!(parents, subpop)\n", 359 | "\t end\n", 360 | "\n", 361 | "\t if k == K\n", 362 | "\t\t plots = Plots.Plot[]\n", 363 | "\t\t push!(plots, Plots.Linear(y1->1-sqrt(1-(1-y1)^2), (0,1), style=\"solid, pastelBlue, ultra thick\"))\n", 364 | "\t\t push!(plots, Plots.Scatter([y[1] for y in filter_ys],\n", 365 | "\t\t [y[2] for y in filter_ys], style=\"clip marker paths, mark=*, mark size=1, mark options={draw=gray, fill=gray}\"))\n", 366 | "\t\t push!(G, Axis(plots))\n", 367 | "\t\tend\n", 368 | "\n", 369 | "\t p = randperm(2m_pop)\n", 370 | "\t p_ind=i->parents[mod(p[i]-1,m_pop)+1][(p[i]-1)÷m_pop + 1]\n", 371 | "\t parents = [[p_ind(i), p_ind(i+1)] for i in 1 : 2 : 2m_pop]\n", 372 | "\t children = [crossover(C,population[p[1]],population[p[2]])\n", 373 | "\t for p in parents]\n", 374 | "\t population = [mutate(M, c) for c in children]\n", 375 | "\tend\n", 376 | "\n", 377 | "\tG\n", 378 | "end\n", 379 | "plot(p)" 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": null, 385 | "metadata": { 386 | "collapsed": true 387 | }, 388 | "outputs": [], 389 | "source": [ 390 | "using Random\n", 391 | "\n", 392 | "p = let\n", 393 | "\n", 394 | "\trng = MersenneTwister(0)\n", 395 | "\n", 396 | "\tfunction discard_closest_pair!(xs, ys)\n", 397 | "\t\tindex, min_dist = 0, Inf\n", 398 | "\t\tfor (i,y) in enumerate(ys)\n", 399 | "\t\t\tfor (j, y′) in enumerate(ys[i+1:end])\n", 400 | "\t\t\t\tdist = norm(y - y′)\n", 401 | "\t\t\t\tif dist < min_dist\n", 402 | "\t\t\t\t\tindex, min_dist = rand(rng, [i,j]), dist\n", 403 | "\t\t\t\tend\n", 404 | "\t\t\tend\n", 405 | "\t\tend\n", 406 | "\t\tdeleteat!(xs, index)\n", 407 | "\t\tdeleteat!(ys, index)\n", 408 | "\t\treturn (xs, ys)\n", 409 | "\tend\n", 410 | "\tfunction update_pareto_filter!(filter_xs, filter_ys, xs, ys;\n", 411 | "\t\tcapacity=length(xs),\n", 412 | "\t\t)\n", 413 | "\t for (x,y) in zip(xs, ys)\n", 414 | "\t \tif !any(dominates(y′,y) for y′ in filter_ys)\n", 415 | "\t push!(filter_xs, x)\n", 416 | "\t push!(filter_ys, y)\n", 417 | "\t end\n", 418 | "\t end\n", 419 | "\t filter_xs, filter_ys = naive_pareto(filter_xs, filter_ys)\n", 420 | "\t while length(filter_xs) > capacity\n", 421 | "\t \tdiscard_closest_pair!(filter_xs, filter_ys)\n", 422 | "\t end\n", 423 | "\t return (filter_xs, filter_ys)\n", 424 | "\tend\n", 425 | "\n", 426 | "\tf = x -> begin\n", 427 | "\t r = 0.5 + 0.5*(2x[1]/(1+x[1]^2))\n", 428 | "\t θ = x[2]\n", 429 | "\t y1 = 1 - r*cos(θ)\n", 430 | "\t y2 = 1 - r*sin(θ)\n", 431 | "\t return [y1, y2]\n", 432 | "\tend\n", 433 | "\tm_pop = 40\n", 434 | "\tRandom.seed!(0)\n", 435 | "\tpopulation = rand_population_uniform(m_pop, [-5,-5], [5,5])\n", 436 | "\n", 437 | "\tfilter_xs = Vector{Float64}[]\n", 438 | "\tfilter_ys = Vector{Float64}[]\n", 439 | "\n", 440 | "\tS = TruncationSelection(10)\n", 441 | "\tC = InterpolationCrossover(0.1)\n", 442 | "\tM = GaussianMutation(0.1)\n", 443 | "\n", 444 | "\tG = GroupPlot(1,1,groupStyle=\"xlabels at=edge bottom, ylabels at =edge left, xticklabels at=edge bottom, yticklabels at=edge left, horizontal sep=0.5cm, vertical sep=0.5cm\", style=\"xlabel=\\$y_1\\$, ylabel=\\$y_2\\$, width=7cm, height=7cm, xmin=0, xmax=1.5, ymin=0, ymax=1.5\")\n", 445 | "\n", 446 | "\n", 447 | "\tdist = 0.15\n", 448 | "\n", 449 | "\tK = 10\n", 450 | "\tm = length(f(population[1]))\n", 451 | "\tm_pop = length(population)\n", 452 | "\tm_subpop = m_pop ÷ m\n", 453 | "\tfor k in 1 : K\n", 454 | "\n", 455 | "\t\tfitnesses = f.(population)\n", 456 | "\t\tneighbors = zeros(m_pop)\n", 457 | "\t\tfor (i,y) in enumerate(fitnesses)\n", 458 | "\t\t\tfor j in i+1 : m_pop\n", 459 | "\t\t\t\ty′ = fitnesses[j]\n", 460 | "\t\t\t\tif norm(y - y′) < dist\n", 461 | "\t\t\t\t\tneighbors[i] += 1\n", 462 | "\t\t\t\t\tneighbors[j] += 1\n", 463 | "\t\t\t\tend\n", 464 | "\t\t\tend\n", 465 | "\t\tend\n", 466 | "\n", 467 | "\t\tfilter_xs, filter_ys = update_pareto_filter!(filter_xs, filter_ys, population, fitnesses)\n", 468 | "\n", 469 | "\t\tfor (i,y) in enumerate(fitnesses)\n", 470 | "\t\t\tfitnesses[i] = y*neighbors[i]\n", 471 | "\t\tend\n", 472 | "\n", 473 | "\n", 474 | "\n", 475 | "\n", 476 | "\t parents = partialsort(S, [y[1] for y in fitnesses])[1:m_subpop]\n", 477 | "\t for i in 2 : m\n", 478 | "\t subpop=partialsort(S,[y[i] for y in fitnesses])[1:m_subpop]\n", 479 | "\t append!(parents, subpop)\n", 480 | "\t end\n", 481 | "\n", 482 | "\t if k == K\n", 483 | "\t\t plots = Plots.Plot[]\n", 484 | "\t\t push!(plots, Plots.Linear(y1->1-sqrt(1-(1-y1)^2), (0,1), style=\"solid, pastelBlue, ultra thick\"))\n", 485 | "\t\t push!(plots, Plots.Scatter([y[1] for y in filter_ys],\n", 486 | "\t\t [y[2] for y in filter_ys], style=\"clip marker paths, mark=*, mark size=1, mark options={draw=gray, fill=gray}\"))\n", 487 | "\t\t push!(G, Axis(plots))\n", 488 | "\t\tend\n", 489 | "\n", 490 | "\t p = randperm(2m_pop)\n", 491 | "\t p_ind=i->parents[mod(p[i]-1,m_pop)+1][(p[i]-1)÷m_pop + 1]\n", 492 | "\t parents = [[p_ind(i), p_ind(i+1)] for i in 1 : 2 : 2m_pop]\n", 493 | "\t children = [crossover(C,population[p[1]],population[p[2]])\n", 494 | "\t for p in parents]\n", 495 | "\t population = [mutate(M, c) for c in children]\n", 496 | "\tend\n", 497 | "\n", 498 | "\tG\n", 499 | "end\n", 500 | "plot(p)" 501 | ] 502 | } 503 | ], 504 | "metadata": { 505 | "kernelspec": { 506 | "display_name": "Julia 1.5.1", 507 | "language": "julia", 508 | "name": "julia-1.5" 509 | }, 510 | "language_info": { 511 | "file_extension": ".jl", 512 | "mimetype": "application/julia", 513 | "name": "julia", 514 | "version": "1.5.1" 515 | } 516 | }, 517 | "nbformat": 4, 518 | "nbformat_minor": 2 519 | } 520 | -------------------------------------------------------------------------------- /penalty.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Penalty\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "p = let\n", 32 | "\n", 33 | "\tf = x -> x[1]^2 - 0.9*x[1]*x[2] + x[2]^2\n", 34 | "\n", 35 | "\txdomain = (-2, 1)\n", 36 | "\tydomain = (-2, 1)\n", 37 | "\n", 38 | "\n", 39 | "\tplots = Plots.Plot[]\n", 40 | "\tpush!(plots, Plots.Contour(f, xdomain, ydomain, levels=[0.1,0.2,0.5,1,2,3,4], style=\"width=\\\\columnwidth\"))\n", 41 | "\tpush!(plots, Plots.Linear3([-1.5,-1.5], [-2.0, 1.0], [5,5], style=\"black, dotted, mark=none\"))\n", 42 | "\tpush!(plots, Plots.Linear3([-1.5,-1.5], [-1.5, 0.5], [6,6], style=\"black, solid, mark=none\"))\n", 43 | "\tpush!(plots, Plots.Linear3([ 0.5, 0.5], [-2.0, 1.0], [5,5], style=\"black, dotted, mark=none\"))\n", 44 | "\tpush!(plots, Plots.Linear3([ 0.5, 0.5], [-1.5, 0.5], [6,6], style=\"black, solid, mark=none\"))\n", 45 | "\tpush!(plots, Plots.Linear3([-2.0, 1.0], [-1.5,-1.5], [5,5], style=\"black, dotted, mark=none\"))\n", 46 | "\tpush!(plots, Plots.Linear3([-1.5, 0.5], [-1.5,-1.5], [6,6], style=\"black, solid, mark=none\"))\n", 47 | "\tpush!(plots, Plots.Linear3([-2.0, 1.0], [ 0.5, 0.5], [5,5], style=\"black, dotted, mark=none\"))\n", 48 | "\tpush!(plots, Plots.Linear3([-1.5, 0.5], [ 0.5, 0.5], [6,6], style=\"black, solid, mark=none\"))\n", 49 | "\n", 50 | "\tAxis(plots, width=\"1.1*9cm\", height=\"1.1*9cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"xtick=\\\\empty, ytick=\\\\empty, contour/labels=false, axis equal, view={0}{90}\")\n", 51 | "end\n", 52 | "\n", 53 | "plot(p)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": { 60 | "collapsed": true 61 | }, 62 | "outputs": [], 63 | "source": [ 64 | "p = let\n", 65 | "\n", 66 | "\tf = x -> begin\n", 67 | "\t\tr = norm(x)\n", 68 | "\t\tθ = atan(x[2], x[1])\n", 69 | "\t\t(sin(3θ) + r - 1)\n", 70 | "\tend\n", 71 | "\n", 72 | "\txdomain = (-2,2)\n", 73 | "\tydomain = (-2,2)\n", 74 | "\n", 75 | "\tplots = Plots.Plot[]\n", 76 | "\tpush!(plots, Plots.Contour(f, xdomain, ydomain, levels=[-1, -0.5, 0.5, 1, 1.5, 2], labels=false))\n", 77 | "\tpush!(plots, Plots.Contour(f, xdomain, ydomain, levels=[0], contour_style=\"draw color=black\"))\n", 78 | "\n", 79 | "\tAxis(plots, width=\"1.1*9cm\", height=\"1.1*9cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"xtick=\\\\empty, ytick=\\\\empty, axis equal, view={0}{90}\")\n", 80 | "end\n", 81 | "\n", 82 | "plot(p)" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": { 89 | "collapsed": true 90 | }, 91 | "outputs": [], 92 | "source": [ 93 | "using Optim\n", 94 | "\n", 95 | "p = let\n", 96 | "\n", 97 | "\tf = x -> -exp(-(x[1]*x[2] - 1.5)^2 -(x[2]-1.5)^2)\n", 98 | "\th = x -> x[1] - x[2]^2\n", 99 | "\n", 100 | "\txdomain = (0, 3)\n", 101 | "\tydomain = (0, 3)\n", 102 | "\n", 103 | "\tx = optimize(x->f(x) + 10000*abs(h(x))^2, [1.0,1.0]).minimizer\n", 104 | "\n", 105 | "\tplots = Plots.Plot[]\n", 106 | "\tpush!(plots, Plots.Contour(f, xdomain, ydomain, xbins=101, ybins=101, levels=[-0.1,-0.2,-0.3,-0.4,-0.5,-0.7,f(x),-0.97]))\n", 107 | "\tpush!(plots, Plots.Contour(h, xdomain, ydomain, xbins=101, ybins=101, levels=[0], contour_style=\"draw color=black\"))\n", 108 | "\tpush!(plots, Plots.Scatter([x[1]], [x[2]], style=\"black, mark size=1, mark options={draw=black, fill=black}\"))\n", 109 | "\tpush!(plots, Plots.Node(L\"h(\\vect x) = 0\", 2.75, sqrt(2.75), axis=\"axis cs\", style=\"anchor=south east\"))\n", 110 | "\tpush!(plots, Plots.Node(L\"\\vect x^*\",x[1],x[2], axis=\"axis cs\", style=\"anchor=north west\"))\n", 111 | "\n", 112 | "\tAxis(plots, xmin=xdomain[1], xmax=xdomain[2], ymin=ydomain[1], ymax=ydomain[2],\n", 113 | "\t width=\"9cm\", height=\"9cm\", xlabel=L\"x_1\", ylabel=L\"x_2\",\n", 114 | "\t style=\"xtick=\\\\empty, ytick=\\\\empty, contour/labels=false, axis equal, view={0}{90}\")\n", 115 | "end\n", 116 | "\n", 117 | "plot(p)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": { 124 | "collapsed": true 125 | }, 126 | "outputs": [], 127 | "source": [ 128 | "using Optim\n", 129 | "import Printf: @sprintf\n", 130 | "p = let\n", 131 | "\tf = x -> sin(x)\n", 132 | "\t# x^2 <= 1\n", 133 | "\tL = (x,μ) -> f(x) + μ*(x^2 - 1)\n", 134 | "\tdom = (-4,4)\n", 135 | "\n", 136 | "\tx_star = -1\n", 137 | "\n", 138 | "\tplots = Plots.Plot[]\n", 139 | "\tfor (i,μ) in enumerate(collect(0:8)*0.25)\n", 140 | "\t g = x->L(x,μ)\n", 141 | "\t color = \"pastelPurple!$(109-9i)\"\n", 142 | "\t push!(plots, Plots.Linear(g, dom, style=\"solid, thick, $color, mark=none\", legendentry=@sprintf(\"\\$\\\\mu = %.2f\\$\", μ)))\n", 143 | "\t x = optimize(g, -4, 4).minimizer\n", 144 | "\t push!(plots, Plots.Scatter([x], [g(x)], style=\"mark=*, mark size=1, mark options={draw=$color, fill=$color}, forget plot\"))\n", 145 | "\tend\n", 146 | "\tpush!(plots, Plots.Linear(f, dom, style=\"solid, black, mark=none\"))\n", 147 | "\tpush!(plots, Plots.Linear(f, (-1,1), style=\"solid, thick, pastelBlue, mark=none\"))\n", 148 | "\tpush!(plots, Plots.Scatter([x_star], [f(x_star)], style=\"mark=*, mark size=1, mark options={draw=pastelBlue, fill=pastelBlue}\"))\n", 149 | "\tpush!(plots, Plots.Node(L\"x^*\", x_star, f(x_star), style=\"anchor=south west\"))\n", 150 | "\tAxis(plots, style=\"xlabel=\\$x\\$, ylabel=\\$y\\$, enlarge x limits=0, ymax=10, legend cell align=left, legend style={draw=black, fill=white}, legend pos=outer north east, axis on top\", width=\"7cm\")\n", 151 | "end\n", 152 | "\n", 153 | "plot(p)" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": { 160 | "collapsed": true 161 | }, 162 | "outputs": [], 163 | "source": [ 164 | "using Optim\n", 165 | "\n", 166 | "p = let\n", 167 | "\n", 168 | "\tG = GroupPlot(1,2,groupStyle=\"vertical sep=1cm\")\n", 169 | "\n", 170 | "\t# PLOT 1\n", 171 | "\n", 172 | "\tf = x -> x[1] + x[2] + x[1]*x[2]\n", 173 | "\th = x -> x[1]^2 + x[2]^2 - 1\n", 174 | "\n", 175 | "\txdomain = (-2, 2)\n", 176 | "\tydomain = (-2, 2)\n", 177 | "\n", 178 | "\ts = (√2-1)/(√2-2)\n", 179 | "\tplots = Plots.Plot[]\n", 180 | "\tpush!(plots, Plots.Contour(f, xdomain, ydomain, levels=[f([-1,0]),0,f([(1+√2)/(2+√2),(1+√2)/(2+√2)]),f([s,s]), -2, 1, 3]))\n", 181 | "\tpush!(plots, Plots.Contour(h, xdomain, ydomain, levels=[0], contour_style=\"draw color=black, labels=false\"))\n", 182 | "\tpush!(plots, Plots.Scatter([-1,0,(1+√2)/(2+√2),s], [0,-1,(1+√2)/(2+√2),s], style=\"black, mark size=1.5, mark options={draw=black, fill=black}\"))\n", 183 | "\tpush!(G, Axis(plots, width=\"9cm\", height=\"9cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"contour/labels=false, axis equal, view={0}{90}\"))\n", 184 | "\n", 185 | "\t# PLOT 2\n", 186 | "\n", 187 | "\tλ_opt = 1/2\n", 188 | "\tpush!(G,\n", 189 | "\t\tAxis(\n", 190 | "\t\t\t[Plots.Linear(λ -> -λ - 1/(2*λ+1), (1/2,5), style=\"solid, black, mark=none\"),\n", 191 | "\t\t\t Plots.Linear([λ_opt,λ_opt],[-20,5], style=\"solid, black!40, mark=none\"),\n", 192 | " \t\t\t\t Plots.Node(L\"\\lambda^*\", λ_opt + 0.45, -5, axis=\"axis cs\"),\n", 193 | "\t\t\t],\n", 194 | "\t\t\txlabel = L\"\\lambda\",\n", 195 | "\t\t\tylabel = L\"\\mathcal{D}(\\lambda)\",\n", 196 | "\t\t\txmin=0, xmax=5,\n", 197 | "\t\t\tymin=-7, ymax=0,\n", 198 | "\t\t\twidth = \"9cm\",\n", 199 | "\t\t)\n", 200 | "\t)\n", 201 | "end\n", 202 | "\n", 203 | "plot(p)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "metadata": { 210 | "collapsed": true 211 | }, 212 | "outputs": [], 213 | "source": [ 214 | "using Optim\n", 215 | "import LinearAlgebra: norm\n", 216 | "\n", 217 | "p = let\n", 218 | "\n", 219 | "\tG = GroupPlot(4,1,groupStyle=\"xlabels at=edge bottom, ylabels at =edge left,\n", 220 | "\t xticklabels at=edge bottom, yticklabels at=edge left,\n", 221 | "\t horizontal sep=0.25cm, vertical sep=0.25cm\",\n", 222 | "\t style=\"xlabel=\\$x_1\\$, ylabel=\\$x_2\\$\")\n", 223 | "\n", 224 | "\tf = x -> norm(x) + sin(4atan(x[2], x[1]))\n", 225 | "\tg = x -> 2 - x[1]^2 - x[2]^2\n", 226 | "\n", 227 | "\tP = x -> max(g(x), 0)^2\n", 228 | "\n", 229 | "\txdomain = (-3, 3)\n", 230 | "\tydomain = (-3, 3)\n", 231 | "\n", 232 | "\tx_arr = Vector{Float64}[]\n", 233 | "\tx = [2.5, 2.5]\n", 234 | "\tpush!(x_arr, x)\n", 235 | "\n", 236 | "\tfor r in [0.0, 1.0, 2.0, 3.0]\n", 237 | "\t obj = x -> f(x) + r*P(x)\n", 238 | "\t x = optimize(obj, x+[0.25,0.25]).minimizer\n", 239 | "\t push!(x_arr, x)\n", 240 | "\n", 241 | "\t push!(G,\n", 242 | "\t Axis([\n", 243 | "\t Plots.Contour(obj, xdomain, ydomain, labels=false, xbins=151, ybins=151),\n", 244 | "\t Plots.Contour(g, xdomain, ydomain, contour_style=\"draw color=black,labels=false\", levels=[0.0]),\n", 245 | "\t Plots.Scatter([x[1]], [x[2]], style=\"mark=*, mark size=1, mark options={draw=black, fill=black}\"),\n", 246 | "\t Plots.Node(L\"x\", x[1]+0.2, x[2]+0.2),\n", 247 | "\t ], width=\"5.25cm\", height=\"5.25cm\", style=\"view={0}{90}\", title=\"\\$\\\\rho=$(round(Int, r))\\$\",\n", 248 | "\t )\n", 249 | "\t )\n", 250 | "\tend\n", 251 | "\n", 252 | "\tG\n", 253 | "end\n", 254 | "\n", 255 | "plot(p)" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "metadata": { 262 | "collapsed": true 263 | }, 264 | "outputs": [], 265 | "source": [ 266 | "using Optim\n", 267 | "\n", 268 | "p = let\n", 269 | "\n", 270 | "\tf = x -> norm(x) + sin(4atan(x[2], x[1]))\n", 271 | "\th = x -> 2 - x[1]^2 - x[2]^2\n", 272 | "\n", 273 | "\tP = x -> h(x)^2\n", 274 | "\n", 275 | "\txs_P = Vector{Float64}[]\n", 276 | "\tx = x_start = [3, 2.5]\n", 277 | "\tc = c_start = 1.0\n", 278 | "\tγ = 1.5\n", 279 | "\n", 280 | "\tis = collect(1:10)\n", 281 | "\n", 282 | "\tfor i in is\n", 283 | "\t obj = x -> f(x) + c*P(x)\n", 284 | "\t x = optimize(obj, x+randn(2)/5).minimizer\n", 285 | "\t push!(xs_P, x)\n", 286 | "\t c *= γ\n", 287 | "\tend\n", 288 | "\n", 289 | "\ths = [h]\n", 290 | "\tc = c_start\n", 291 | "\txs_ALM = Vector{Float64}[]\n", 292 | "\tλs = zeros(length(hs))\n", 293 | "\tx = x_start\n", 294 | "\n", 295 | "\tfor i in is\n", 296 | "\t P = x -> f(x) + c/2*sum(h(x)^2 for h in hs) -\n", 297 | "\t sum(λ*h(x) for (λ,h) in zip(λs, hs))\n", 298 | "\t x = optimize(x -> f(x) + P(x), x+randn(2)/10).minimizer\n", 299 | "\t λs -= c*[h(x) for h in hs]\n", 300 | "\t c *= γ\n", 301 | "\t push!(xs_ALM, x)\n", 302 | "\tend\n", 303 | "\n", 304 | "\tax = Axis([\n", 305 | "\t Plots.Linear(is, [h(x) for x in xs_P], style=\"black, solid, mark=none\", legendentry=\"quadratic penalty method\"),\n", 306 | "\t Plots.Linear(is, [h(x) for x in xs_ALM], style=\"pastelBlue, solid, mark=none\", legendentry=\"augmented Lagrangian\"),\n", 307 | "\t ], ymode=\"log\", width=\"8cm\", height=\"6cm\", xlabel=\"number of iterations\", ylabel=L\"h(x)\", style=\"legend cell align=left\")\n", 308 | "\tax.legendStyle = \"{at={(1.05,1.0)},anchor=north west}\"\n", 309 | "\tax\n", 310 | "end\n", 311 | "\n", 312 | "plot(p)" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": null, 318 | "metadata": { 319 | "collapsed": true 320 | }, 321 | "outputs": [], 322 | "source": [ 323 | "using Optim\n", 324 | "\n", 325 | "p = let\n", 326 | "\n", 327 | "\tG = GroupPlot(4,1,groupStyle=\"xlabels at=edge bottom, ylabels at =edge left,\n", 328 | "\t xticklabels at=edge bottom, yticklabels at=edge left,\n", 329 | "\t horizontal sep=0.25cm, vertical sep=0.25cm\",\n", 330 | "\t style=\"xlabel=\\$x_1\\$, ylabel=\\$x_2\\$\")\n", 331 | "\n", 332 | "\tf = x -> norm(x) + sin(4atan(x[2], x[1]))\n", 333 | "\tg = x -> 2 - x[1]^2 - x[2]^2\n", 334 | "\n", 335 | "\tp = x -> g(x) <= 0 ? -1/g(x) : Inf\n", 336 | "\n", 337 | "\txdomain = (-2.5, 2.5)\n", 338 | "\tydomain = (-2.5, 2.5)\n", 339 | "\n", 340 | "\tx_arr = Vector{Float64}[]\n", 341 | "\tx = [3, 2.5]\n", 342 | "\tpush!(x_arr, x)\n", 343 | "\n", 344 | "\tfor ρ in [0.5, 1.0, 10.0, 100.0]\n", 345 | "\t obj = x -> f(x) + p(x)/ρ\n", 346 | "\t x = optimize(obj, x).minimizer\n", 347 | "\t push!(x_arr, x)\n", 348 | "\n", 349 | "\t push!(G,\n", 350 | "\t Axis([\n", 351 | "\t Plots.Contour(x->min(obj(x), 10.0), xdomain, ydomain, labels=false),\n", 352 | "\t Plots.Contour(g, xdomain, ydomain, contour_style=\"draw color=black,labels=false\", levels=[0.0]),\n", 353 | "\t Plots.Scatter([x[1]], [x[2]], style=\"mark=*, mark size=1, mark options={draw=black, fill=black}\"),\n", 354 | "\t Plots.Node(L\"x\", x[1]+0.2, x[2]+0.2),\n", 355 | "\t ], width=\"5.25cm\", height=\"5.25cm\", style=\"view={0}{90}\", title=\"\\$\\\\rho=$ρ\\$\",\n", 356 | "\t )\n", 357 | "\t )\n", 358 | "\tend\n", 359 | "\n", 360 | "\tG\n", 361 | "end\n", 362 | "\n", 363 | "plot(p)" 364 | ] 365 | } 366 | ], 367 | "metadata": { 368 | "kernelspec": { 369 | "display_name": "Julia 1.5.1", 370 | "language": "julia", 371 | "name": "julia-1.5" 372 | }, 373 | "language_info": { 374 | "file_extension": ".jl", 375 | "mimetype": "application/julia", 376 | "name": "julia", 377 | "version": "1.5.1" 378 | } 379 | }, 380 | "nbformat": 4, 381 | "nbformat_minor": 2 382 | } 383 | -------------------------------------------------------------------------------- /population.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Population\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | " using Distributions\n", 32 | " using Random\n", 33 | " using LinearAlgebra\n", 34 | "\n", 35 | " p = let\n", 36 | " function rand_population_uniform(M, a, b)\n", 37 | " d = length(a)\n", 38 | " return [a+rand(d).*(b-a) for i in 1:M]\n", 39 | " end\n", 40 | "\n", 41 | " using Distributions\n", 42 | " function rand_population_normal(M, μ, Σ)\n", 43 | " D = MvNormal(μ,Σ)\n", 44 | " return [rand(D) for i in 1:M]\n", 45 | " end\n", 46 | "\n", 47 | " using Distributions\n", 48 | " function rand_population_cauchy(M, μ, σ)\n", 49 | " d = length(μ)\n", 50 | " return [[rand(Cauchy(μ[j],σ[j])) for j in 1:d] for i in 1:M]\n", 51 | " end\n", 52 | "\n", 53 | " Random.seed!(0)\n", 54 | " m = 1000\n", 55 | " pop1 = rand_population_uniform(m, [-2.0, -2.0], [2.0,2.0])\n", 56 | " pop2 = rand_population_normal(m, [0.0, 0.0], [1.0,1.0])\n", 57 | " pop3 = rand_population_cauchy(m, [0.0, 0.0], [1.0,1.0])\n", 58 | " filter!(x->norm(x) < 10, pop3)\n", 59 | "\n", 60 | " scatter_style = \"clip marker paths, mark=*, mark size=0.75, mark options={draw=pastelBlue, fill=pastelBlue, opacity=0.5}\"\n", 61 | "\n", 62 | " G = GroupPlot(3,1,groupStyle=\"xlabels at=edge bottom, ylabels at =edge left, xticklabels at=edge bottom, yticklabels at=edge left, horizontal sep=0.5cm, vertical sep=0.5cm\", style=\"xlabel=\\$x_1\\$, ylabel=\\$x_2\\$, width=4.75cm, height=4.75cm, xmin=-4, xmax=4, ymin=-4, ymax=4, title style={text height=2ex}\")\n", 63 | " push!(G, Axis(\n", 64 | " Plots.Scatter([x[1] for x in pop1], [x[2] for x in pop1], style=scatter_style),\n", 65 | " title = \"Uniform\",\n", 66 | " ))\n", 67 | " push!(G, Axis(\n", 68 | " Plots.Scatter([x[1] for x in pop2], [x[2] for x in pop2], style=scatter_style),\n", 69 | " title = \"Normal\",\n", 70 | " ))\n", 71 | " push!(G, Axis(\n", 72 | " Plots.Scatter([x[1] for x in pop3], [x[2] for x in pop3], style=scatter_style),\n", 73 | " title = \"Cauchy\",\n", 74 | " ))\n", 75 | " G\n", 76 | " end\n", 77 | "\n", 78 | " plot(p)" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "metadata": { 85 | "collapsed": true 86 | }, 87 | "outputs": [], 88 | "source": [ 89 | " using Random\n", 90 | " using Distributions\n", 91 | "\n", 92 | " function rand_population_uniform(M, a, b)\n", 93 | " d = length(a)\n", 94 | " return [a+rand(d).*(b-a) for i in 1:M]\n", 95 | " end\n", 96 | "\n", 97 | " abstract type SelectionMethod end\n", 98 | " struct TruncationSelection <: SelectionMethod\n", 99 | " k # top k to keep\n", 100 | " end\n", 101 | " function select(t::TruncationSelection, y)\n", 102 | " p = sortperm(y)\n", 103 | " return [p[rand(1:t.k, 2)] for i in y]\n", 104 | " end\n", 105 | "\n", 106 | " abstract type CrossoverMethod end\n", 107 | " struct SinglePointCrossover <: CrossoverMethod end\n", 108 | " function crossover(::SinglePointCrossover, a, b)\n", 109 | " i = rand(1:length(a))\n", 110 | " return vcat(a[1:i], b[i+1:end])\n", 111 | " end\n", 112 | "\n", 113 | " abstract type MutationMethod end\n", 114 | " struct GaussianMutation <: MutationMethod\n", 115 | " σ\n", 116 | " end\n", 117 | " mutate(U::GaussianMutation, child) = child + randn(length(child))*U.σ\n", 118 | "\n", 119 | " p = let\n", 120 | " K = 4\n", 121 | " G = GroupPlot(K,1,groupStyle=\"horizontal sep=0.25cm, vertical sep=0.25cm, ylabels at=edge left, xlabels at=edge bottom\",\n", 122 | " style=\"width=5cm, height=5cm, xlabel=\\$x_1\\$, ylabel=\\$x_2\\$, xtick=\\\\empty, ytick=\\\\empty, contour/labels=false, view={0}{90}\")\n", 123 | "\n", 124 | " function michalewicz(x; m=10)\n", 125 | " return -sum(sin(v)*sin(i*v^2/π)^(2m) for\n", 126 | " (i,v) in enumerate(x))\n", 127 | " end\n", 128 | " f = x -> michalewicz(x)\n", 129 | " xdomain = (0,4)\n", 130 | " ydomain = (0,4)\n", 131 | "\n", 132 | " S = TruncationSelection(10)\n", 133 | " C = SinglePointCrossover()\n", 134 | " M = GaussianMutation(0.1)\n", 135 | "\n", 136 | " m = 40\n", 137 | " Random.seed!(0)\n", 138 | " population = rand_population_uniform(m, [0.0, 0.0], [4.0,4.0])\n", 139 | "\n", 140 | " for i in 1 : K\n", 141 | "\n", 142 | " push!(G, Axis([\n", 143 | " Plots.Image((x,y)->f([x,y]), xdomain, ydomain, xbins=600, ybins=600, colormap = pasteljet, colorbar = false),\n", 144 | " Plots.Scatter([P[1] for P in population], [P[2] for P in population], style=\"mark=*, mark size=1, mark options={draw=black, fill=black}\"),\n", 145 | " ]))\n", 146 | "\n", 147 | " parents = select(S, f.(population))\n", 148 | " children = [crossover(C,population[p[1]],population[p[2]])\n", 149 | " for p in parents]\n", 150 | " population = [mutate(M, c) for c in children]\n", 151 | " end\n", 152 | "\n", 153 | " G\n", 154 | " end\n", 155 | "\n", 156 | " plot(p)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": { 163 | "collapsed": true 164 | }, 165 | "outputs": [], 166 | "source": [ 167 | " using Random\n", 168 | " using StatsBase\n", 169 | "\n", 170 | " p = let\n", 171 | " N = 4\n", 172 | " g = GroupPlot(N,2,groupStyle=\"horizontal sep=0.25cm, vertical sep=0.25cm, ylabels at=edge left, xlabels at=edge bottom\",\n", 173 | " style=\"width=5cm, height=5cm, xlabel=\\$x_1\\$, ylabel=\\$x_2\\$, xtick=\\\\empty, ytick=\\\\empty, contour/labels=false, view={0}{90}\")\n", 174 | "\n", 175 | " f = x -> -20*exp(-0.2*sqrt(0.5)*norm(x)) - exp(0.5*(cos(2π*x[1]) + cos(2π*x[2]))) + ℯ + 20\n", 176 | " f2 = (x,y) -> f([x,y])\n", 177 | " xdomain = (-5,5)\n", 178 | " ydomain = (-5,5)\n", 179 | " P = 0.5\n", 180 | " w = 0.2\n", 181 | "\n", 182 | " function tick_differential_evolution!(f, population, P, w)\n", 183 | " n, m = length(population[1]), length(population)\n", 184 | " for (k,x) in enumerate(population)\n", 185 | " weights = Weights([j!=k for j in 1 : m])\n", 186 | " a, b, c = sample(population, weights, 3, replace=false)\n", 187 | " z = a + w*(b-c)\n", 188 | " j = rand(1:n)\n", 189 | " x′ = [i == j || rand() < P ? z[i] : x[i] for i in 1:n]\n", 190 | " if f(x′) < f(x)\n", 191 | " x[:] = x′\n", 192 | " end\n", 193 | " end\n", 194 | " return population\n", 195 | " end\n", 196 | "\n", 197 | " Random.seed!(0)\n", 198 | " population = [(rand(2) .- 0.5).*9 for i in 1 : 20]\n", 199 | "\n", 200 | " for i in 1 : 2N\n", 201 | "\n", 202 | " ax = Axis([\n", 203 | " Plots.Image(f2, xdomain, ydomain, xbins=600, ybins=600, colormap = pasteljet, colorbar = false),\n", 204 | " Plots.Scatter([P[1] for P in population], [P[2] for P in population], style=\"mark=*, mark size=1, mark options={draw=black, fill=black}\"),\n", 205 | " ])\n", 206 | " push!(g, ax)\n", 207 | "\n", 208 | " tick_differential_evolution!(f, population, P, w)\n", 209 | " end\n", 210 | "\n", 211 | " g\n", 212 | " end\n", 213 | "\n", 214 | " plot(p)" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": null, 220 | "metadata": { 221 | "collapsed": true 222 | }, 223 | "outputs": [], 224 | "source": [ 225 | " using Random\n", 226 | " using Vec\n", 227 | " struct Particle\n", 228 | " x\n", 229 | " v\n", 230 | " x_best\n", 231 | " end\n", 232 | " function initialize_particles_uniform(f, N, a_low, a_high)\n", 233 | " m = length(a_low)\n", 234 | " Δ = a_high - a_low\n", 235 | " spawn = () -> begin\n", 236 | " x = a_low + rand(m).*Δ\n", 237 | " v = (rand(m) .- 0.5) .* 2Δ\n", 238 | " Particle(x, v, x)\n", 239 | " end\n", 240 | " [spawn() for i in 1 : N]\n", 241 | " end\n", 242 | "\n", 243 | " p = let\n", 244 | "\n", 245 | " g = GroupPlot(4,2,groupStyle=\"xlabels at=edge bottom, ylabels at =edge left, horizontal sep=0.5cm, vertical sep=0.5cm\")\n", 246 | "\n", 247 | " f = x -> -exp(-(x[1]*x[2] - 1.5)^2 -(x[2]-1.5)^2)\n", 248 | "\n", 249 | " xdomain = (0, 3)\n", 250 | " ydomain = (0, 3)\n", 251 | "\n", 252 | " p_contour = Plots.Contour(f, xdomain, ydomain, style=\"width=\\\\columnwidth\")\n", 253 | "\n", 254 | " function add_particles!(particles)\n", 255 | "\n", 256 | " plots = Plots.Plot[]\n", 257 | " push!(plots, p_contour)\n", 258 | " push!(plots, Plots.Scatter([P.x[1] for P in particles], [P.x[2] for P in particles], style=\"mark=*, mark size=1, mark options={draw=black, fill=black}\"))\n", 259 | " for P in particles\n", 260 | " A = VecE2(P.x[1], P.x[2])\n", 261 | " B = A + VecE2(P.v[1], P.v[2])/8\n", 262 | " push!(plots, Plots.Linear([A.x, B.x], [A.y, B.y], style=\"solid, black, mark=none\"))\n", 263 | " end\n", 264 | "\n", 265 | " ax = Axis(plots, ymin=0, xmin=0, ymax=3, xmax=3, width=\"5cm\", height=\"5cm\", style=\"xtick=\\\\empty, ytick=\\\\empty, contour/labels=false, axis equal, view={0}{90}\")\n", 266 | " push!(g, ax)\n", 267 | " end\n", 268 | "\n", 269 | " Random.seed!(0)\n", 270 | " N = 40\n", 271 | " population = initialize_particles_uniform(f, N, [xdomain[1],ydomain[1]], [xdomain[2],ydomain[2]])\n", 272 | " add_particles!(population)\n", 273 | "\n", 274 | " w, c1, c2, m = 0.1, 0.25, 2.0, 2\n", 275 | " x_best, y_best = copy(population[1].x_best), Inf\n", 276 | " for P in population\n", 277 | " y = f(P.x)\n", 278 | " if y < y_best; x_best[:], y_best = P.x, y; end\n", 279 | " end\n", 280 | " for iter in 1 : g.dimensions[1]*g.dimensions[2]-1\n", 281 | "\n", 282 | " for P in population\n", 283 | " r1, r2 = rand(m), rand(m)\n", 284 | " P.x[:] = P.x + P.v\n", 285 | " P.v[:] = w*P.v + c1*r1.*(P.x_best - P.x) + c2*r2.*(x_best - P.x)\n", 286 | " y = f(P.x)\n", 287 | " if y < y_best; x_best[:], y_best = P.x, y; end\n", 288 | " if y < f(P.x_best); P.x_best[:] = P.x; end\n", 289 | " end\n", 290 | " add_particles!(population)\n", 291 | " end\n", 292 | "\n", 293 | " g\n", 294 | " end\n", 295 | "\n", 296 | " plot(p)" 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": null, 302 | "metadata": { 303 | "collapsed": true 304 | }, 305 | "outputs": [], 306 | "source": [ 307 | " using Distributions\n", 308 | " using Random\n", 309 | "\n", 310 | " p = let\n", 311 | " G = GroupPlot(4,1,groupStyle=\"xlabels at=edge bottom, ylabels at =edge left, horizontal sep=0.5cm, vertical sep=0.5cm\")\n", 312 | "\n", 313 | " function branin(x; a=1, b=5.1/(4π^2), c=5/π, r=6, s=10, t=1/(8π))\n", 314 | " return a*(x[2]-b*x[1]^2+c*x[1]-r)^2 + s*(1-t)*cos(x[1]) + s\n", 315 | " end\n", 316 | " f = x -> branin(x)\n", 317 | "\n", 318 | " xdomain = (-5, 10)\n", 319 | " ydomain = ( 0, 15)\n", 320 | "\n", 321 | " p_contour = Plots.Contour(f, xdomain, ydomain, levels=[1,2,3,5,10,20,50,100], xbins=101, ybins=101, style=\"width=\\\\columnwidth\")\n", 322 | "\n", 323 | " function add_axis!(population)\n", 324 | " plots = Plots.Plot[]\n", 325 | " push!(plots, p_contour)\n", 326 | " push!(plots, Plots.Scatter([x[1] for x in population], [x[2] for x in population], style=\"mark=*, mark size=1, mark options={draw=black, fill=black}\"))\n", 327 | " ax = Axis(plots, ymin=ydomain[1], xmin=xdomain[1], ymax=ydomain[2], xmax=xdomain[2], width=\"5cm\", height=\"5cm\", style=\"xtick=\\\\empty, ytick=\\\\empty, contour/labels=false, axis equal, view={0}{90}\")\n", 328 | " push!(G, ax)\n", 329 | " end\n", 330 | "\n", 331 | " Random.seed!(0)\n", 332 | " population = [[rand(Uniform(xdomain[1], xdomain[2])),\n", 333 | " rand(Uniform(ydomain[1], ydomain[2]))] for i in 1 : 40]\n", 334 | " add_axis!(population)\n", 335 | "\n", 336 | "\n", 337 | " α = 0.5\n", 338 | " β = 1.0\n", 339 | " I = r->exp(-0.1r^2)\n", 340 | "\n", 341 | " N = MvNormal(diagm(0=>ones(length(population[1]))))\n", 342 | " for K in 1 : 3\n", 343 | " for a in population, b in population\n", 344 | " if f(b) < f(a)\n", 345 | " r = norm(b-a)\n", 346 | " a[:] += β*I(r)*(b-a) + α*rand(N)\n", 347 | " end\n", 348 | " end\n", 349 | " add_axis!(population)\n", 350 | " end\n", 351 | "\n", 352 | " G\n", 353 | " end\n", 354 | "\n", 355 | " plot(p)" 356 | ] 357 | }, 358 | { 359 | "cell_type": "code", 360 | "execution_count": null, 361 | "metadata": { 362 | "collapsed": true 363 | }, 364 | "outputs": [], 365 | "source": [ 366 | " using Distributions\n", 367 | " using Random\n", 368 | "\n", 369 | " mutable struct Nest\n", 370 | " x # position\n", 371 | " y # value, f(x)\n", 372 | " end\n", 373 | "\n", 374 | " p = let\n", 375 | " G = GroupPlot(4,1,groupStyle=\"xlabels at=edge bottom, ylabels at =edge left, horizontal sep=0.5cm, vertical sep=0.5cm\")\n", 376 | "\n", 377 | " function branin(x; a=1, b=5.1/(4π^2), c=5/π, r=6, s=10, t=1/(8π))\n", 378 | " return a*(x[2]-b*x[1]^2+c*x[1]-r)^2 + s*(1-t)*cos(x[1]) + s\n", 379 | " end\n", 380 | " f = x -> branin(x)\n", 381 | "\n", 382 | " xdomain = (-5, 10)\n", 383 | " ydomain = ( 0, 15)\n", 384 | "\n", 385 | " p_contour = Plots.Contour(f, xdomain, ydomain, levels=[1,2,3,5,10,20,50,100], xbins=101, ybins=101, style=\"width=\\\\columnwidth\")\n", 386 | "\n", 387 | " function add_axis!(nests)\n", 388 | " plots = Plots.Plot[]\n", 389 | " push!(plots, p_contour)\n", 390 | " push!(plots, Plots.Scatter([P.x[1] for P in nests], [P.x[2] for P in nests], style=\"mark=*, mark size=1, mark options={draw=black, fill=black}\"))\n", 391 | " ax = Axis(plots, ymin=ydomain[1], xmin=xdomain[1], ymax=ydomain[2], xmax=xdomain[2], width=\"5cm\", height=\"5cm\", style=\"xtick=\\\\empty, ytick=\\\\empty, contour/labels=false, axis equal, view={0}{90}\")\n", 392 | " push!(G, ax)\n", 393 | " end\n", 394 | "\n", 395 | " Random.seed!(0)\n", 396 | " N = 40\n", 397 | " population = Array{Nest}(undef, N)\n", 398 | " for i in 1 : N\n", 399 | " x = [xdomain[1] + rand()*(xdomain[2] - xdomain[1]),\n", 400 | " ydomain[1] + rand()*(ydomain[2] - ydomain[1])]\n", 401 | " population[i] = Nest(x, f(x))\n", 402 | " end\n", 403 | " add_axis!(population)\n", 404 | "\n", 405 | " a = 30\n", 406 | " s = 0.5\n", 407 | " C = Distributions.Cauchy(0,1.0)\n", 408 | "\n", 409 | " N, m = length(population), length(population[1].x)\n", 410 | " for iter in 1 : 3\n", 411 | " # get a cuckoo randomly and generate a new nest by Levy flight\n", 412 | " # replace another random cuckoo with that new position if it is better\n", 413 | " i, j = rand(1:N), rand(1:N)\n", 414 | " x = population[j].x + s*[rand(C) for k in 1 : m]\n", 415 | " y = f(x)\n", 416 | " if y < population[i].y\n", 417 | " population[i].x[:] = x\n", 418 | " population[i].y = y\n", 419 | " end\n", 420 | "\n", 421 | " p = sortperm(population, by=nest->nest.y, rev=true)\n", 422 | " for i in 1 : a\n", 423 | " j = rand(1:N-a)+a\n", 424 | " population[p[i]].x = population[p[j]].x +\n", 425 | " s*[rand(C) for k in 1 : m]\n", 426 | " population[p[i]].y = f(population[p[i]].x)\n", 427 | " end\n", 428 | " add_axis!(population)\n", 429 | " end\n", 430 | "\n", 431 | " G\n", 432 | " end\n", 433 | "\n", 434 | " plot(p)" 435 | ] 436 | }, 437 | { 438 | "cell_type": "code", 439 | "execution_count": null, 440 | "metadata": { 441 | "collapsed": true 442 | }, 443 | "outputs": [], 444 | "source": [ 445 | " p = let\n", 446 | "\n", 447 | " G = GroupPlot(2,1,groupStyle=\"horizontal sep=1cm\")\n", 448 | "\n", 449 | " f = x -> -exp(-x^2) + -2exp(-(x-3)^2)\n", 450 | " x = [-1.8, -1, -0.4, 0.6, 1.0, 1.2]\n", 451 | " x2 = x .* 0.3\n", 452 | "\n", 453 | " p = Plots.Plot[]\n", 454 | " push!(p, Plots.Linear(f, (-2,5), style=\"solid, black, mark=none\"))\n", 455 | " push!(p, Plots.Scatter(x2, f.(x2), style=\"black, mark=*, mark size=1.5, mark options={draw=black, fill=black}\"))\n", 456 | " push!(G, Axis(p, xlabel=L\"x\", ylabel=L\"y\", style=\"enlarge x limits=0\", width=\"6cm\", title=\"Lamarckian\"))\n", 457 | "\n", 458 | " p = Plots.Plot[]\n", 459 | " push!(p, Plots.Linear(f, (-2,5), style=\"solid, black, mark=none\"))\n", 460 | " for i in 1 : length(x)\n", 461 | " push!(p, Plots.Linear([x[i], x[i]], [f(x[i]), f(x2[i])], style=\"solid, pastelBlue, mark=*, mark size=1.45, mark options={draw=pastelBlue, fill=white}\"))\n", 462 | " end\n", 463 | " push!(p, Plots.Scatter(x, f.(x), style=\"black, mark=*, mark size=1.5, mark options={draw=black, fill=black}\"))\n", 464 | "\n", 465 | " push!(G, Axis(p, xlabel=L\"x\", style=\"enlarge x limits=0\", width=\"6cm\", title=\"Baldwinian\"))\n", 466 | " end\n", 467 | " plot(p)" 468 | ] 469 | } 470 | ], 471 | "metadata": { 472 | "kernelspec": { 473 | "display_name": "Julia 1.5.1", 474 | "language": "julia", 475 | "name": "julia-1.5" 476 | }, 477 | "language_info": { 478 | "file_extension": ".jl", 479 | "mimetype": "application/julia", 480 | "name": "julia", 481 | "version": "1.5.1" 482 | } 483 | }, 484 | "nbformat": 4, 485 | "nbformat_minor": 2 486 | } 487 | -------------------------------------------------------------------------------- /sampling-plans.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Sampling Plans\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "using Random\n", 32 | "p = let\n", 33 | "\tL = 6π\n", 34 | "\tf = x -> (cos(x) - 1)*min(abs(L - x), abs(-x - L))\n", 35 | "\tX = collect(-L:2π:L)\n", 36 | "\tRandom.seed!(4)\n", 37 | "\tX2 = X + rand(length(X))*2π .- π\n", 38 | "\tplots = Plots.Plot[]\n", 39 | "\tpush!(plots, Plots.Linear(f, (-L, L), xbins=151, style=\"solid, black, mark=none\", legendentry=L\"f(x)\"))\n", 40 | "\tpush!(plots, Plots.Linear(X, f.(X), style=\"only marks, mark=*, mark size=1, mark options={draw=pastelBlue, fill=pastelBlue}\", legendentry=\"sampling on grid\"))\n", 41 | "\tpush!(plots, Plots.Linear(X2, f.(X2), style=\"only marks, mark=*, mark size=1, mark options={draw=pastelRed, fill=pastelRed}\", legendentry=\"stratified sampling\"))\n", 42 | "\tAxis(plots, width=\"9cm\", xlabel=L\"x\", ylabel=L\"y\", style=\"xtick=\\\\empty, ytick=\\\\empty, legend cell align=left, legend style={draw=none, at={(0.5,-0.15)}, anchor=north, legend columns=1}\")\n", 43 | "end\n", 44 | "plot(p)" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": { 51 | "collapsed": true 52 | }, 53 | "outputs": [], 54 | "source": [ 55 | "using Random\n", 56 | "using LinearAlgebra\n", 57 | "\n", 58 | "function mutate!(Xs)\n", 59 | " m, n = length(Xs), length(Xs[1])\n", 60 | " j = rand(1:n)\n", 61 | " i = randperm(m)[1:2]\n", 62 | " Xs[i[1]][j], Xs[i[2]][j] = Xs[i[2]][j], Xs[i[1]][j]\n", 63 | " return Xs\n", 64 | "end\n", 65 | "function uniform_projection_plan(n)\n", 66 | "\tp = randperm(n)\n", 67 | "\t[[u,v] for (u,v) in zip(p,1:n)]\n", 68 | "end\n", 69 | "function pairwise_distances(Xs, p=2)\n", 70 | " n = length(Xs)\n", 71 | " [norm(Xs[i]-Xs[j], p) for i in 1:(n-1) for j in (i+1):n]\n", 72 | "end\n", 73 | "phiq(Xs, q, p) = sum(d^(-q) for d in pairwise_distances(Xs, p))^(1/q)\n", 74 | "\n", 75 | "p = let\n", 76 | "\n", 77 | "\tN = 20\n", 78 | "\tcapacity = 6*3\n", 79 | "\n", 80 | "\tRandom.seed!(0)\n", 81 | "\tX = Vector{Vector{Float64}}[]\n", 82 | "\tpush!(X, [Float64[i,i] for i in 1:N])\n", 83 | "\tfor i in 2 : capacity\n", 84 | "\t\tpush!(X, mutate!(mutate!(deepcopy(X[end]))))\n", 85 | "\tend\n", 86 | "\tΦ = [phiq(x, 1, 2) for x in X]\n", 87 | "\n", 88 | "\tfor i in 1 : 50\n", 89 | "\t\t# remove the index with minimum distance to neighbors\n", 90 | "\t\ti_min = 1\n", 91 | "\t\td_min = Inf\n", 92 | "\t\tp = sortperm(Φ)\n", 93 | "\t\tX[:] = X[p]\n", 94 | "\t\tΦ[:] = Φ[p]\n", 95 | "\t\tfor i in 2 : length(X)-1\n", 96 | "\t\t\td = Φ[i+1] - Φ[i-1]\n", 97 | "\t\t\t@assert d >= 0\n", 98 | "\t\t\tif d < d_min\n", 99 | "\t\t\t\td_min = d\n", 100 | "\t\t\t\ti_min = 1\n", 101 | "\t\t\tend\n", 102 | "\t\tend\n", 103 | "\t\tX[i_min] = mutate!(X[i_min])\n", 104 | "\t\tΦ[i_min] = phiq(X[i_min], 1, 2)\n", 105 | "\tend\n", 106 | "\n", 107 | "\tsort!(X, by=x->phiq(x, 1, 2))\n", 108 | "\n", 109 | " g = GroupPlot(6,3,groupStyle=\"horizontal sep=0.5cm, vertical sep=0.75cm\")\n", 110 | "\n", 111 | " \tfor Y in X\n", 112 | " \t\tpush!(g, Axis(\n", 113 | " \t\t\tPlots.Scatter([x[1] for x in Y], [x[2] for x in Y], style=\"mark=*, mark size=0.7, mark options={draw=black, fill=black}\"),\n", 114 | " \t\t\twidth=\"4cm\", height=\"4cm\", style=\"ytick=\\\\empty, xtick=\\\\empty, title style={yshift=-0.5em}\",\n", 115 | " \t\t\ttitle = \"\\$\\\\Phi_1 = $(round(phiq(Y, 1, 2), digits=1))\\$\", # $\n", 116 | " \t\t\t)\n", 117 | "\t\t)\n", 118 | " \tend\n", 119 | "\n", 120 | " g\n", 121 | "end\n", 122 | "\n", 123 | "plot(p)" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": { 130 | "collapsed": true 131 | }, 132 | "outputs": [], 133 | "source": [ 134 | "using Random\n", 135 | "\n", 136 | "min_dist(a, B, p) = minimum(norm(a-b, p) for b in B)\n", 137 | "d_max(A, B, p=2) = maximum(min_dist(a, B, p) for a in A)\n", 138 | "function greedy_local_search(X, m, d=d_max)\n", 139 | "\tS = [X[rand(1:m)]]\n", 140 | "\tfor i in 2 : m\n", 141 | "\t\tj = argmin([x ∈ S ? Inf : d(X, push!(copy(S), x))\n", 142 | "\t\t for x in X])\n", 143 | "\t\tpush!(S, X[j])\n", 144 | "\tend\n", 145 | "\treturn S\n", 146 | "end\n", 147 | "function exchange_algorithm(X, m, d=d_max)\n", 148 | "\tS = X[randperm(length(X))[1:m]]\n", 149 | "\tδ, done = d(X, S), false\n", 150 | "\twhile !done\n", 151 | "\t\tbest_pair = (0,0)\n", 152 | "\t\tfor i in 1 : m\n", 153 | "\t\t\ts = S[i]\n", 154 | "\t\t\tfor (j,x) in enumerate(X)\n", 155 | "\t\t\t\tif !in(x, S)\n", 156 | "\t\t\t\t\tS[i] = x\n", 157 | "\t\t\t\t\tδ′ = d(X, S)\n", 158 | "\t\t\t\t\tif δ′ < δ\n", 159 | "\t\t\t\t\t\tδ = δ′\n", 160 | "\t\t\t\t\t\tbest_pair = (i,j)\n", 161 | "\t\t\t\t\tend\n", 162 | "\t\t\t\tend\n", 163 | "\t\t\tend\n", 164 | "\t\t\tS[i] = s\n", 165 | "\t\tend\n", 166 | "\t\tdone = best_pair == (0,0)\n", 167 | "\t\tif !done\n", 168 | "\t\t\ti,j = best_pair\n", 169 | "\t\t\tS[i] = X[j]\n", 170 | "\t\tend\n", 171 | "\tend\n", 172 | "\treturn S\n", 173 | "end\n", 174 | "function multistart_local_search(X, m, alg, k_max, d=d_max)\n", 175 | "\tsets = [alg(X, m, d) for i in 1 : k_max]\n", 176 | "\treturn sets[argmin([d(X, S) for S in sets])]\n", 177 | "end\n", 178 | "\n", 179 | "p = let\n", 180 | "\n", 181 | "\tRandom.seed!(0)\n", 182 | "\tX = Vector{Float64}[]\n", 183 | "\tfor i in 1 : 100\n", 184 | "\t\tpush!(X, rand(2))\n", 185 | "\tend\n", 186 | "\n", 187 | "\tXs_gls = multistart_local_search(X, 8, greedy_local_search, 10)\n", 188 | "\tXs_ea = multistart_local_search(X, 8, exchange_algorithm, 10)\n", 189 | "\n", 190 | "\tpX = Plots.Scatter([x[1] for x in X], [x[2] for x in X], style=\"mark=*, mark size=1, mark options={draw=black, fill=black}\")\n", 191 | "\n", 192 | " g = GroupPlot(2,1,groupStyle=\"horizontal sep=0.5cm\")\n", 193 | "\n", 194 | " push!(g, Axis(\n", 195 | " \t[pX,\n", 196 | " \t Plots.Scatter([x[1] for x in Xs_gls], [x[2] for x in Xs_gls], style=\"mark=*, mark size=1.5, mark options={draw=pastelRed, fill=pastelRed}\")\n", 197 | " \t],\n", 198 | " \twidth=\"6.25cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", title=\"greedy local search\",\n", 199 | " \tstyle=\"ytick=\\\\empty, xtick=\\\\empty\",\n", 200 | " ))\n", 201 | "\n", 202 | " push!(g, Axis(\n", 203 | " \t[pX,\n", 204 | " \t Plots.Scatter([x[1] for x in Xs_ea], [x[2] for x in Xs_ea], style=\"mark=*, mark size=1.5, mark options={draw=pastelRed, fill=pastelRed}\")\n", 205 | " \t],\n", 206 | " \twidth=\"6.25cm\", xlabel=L\"x_1\", title=\"exchange algorithm\",\n", 207 | " \tstyle=\"ytick=\\\\empty, xtick=\\\\empty\",\n", 208 | " ))\n", 209 | "\n", 210 | "\n", 211 | " g\n", 212 | "end\n", 213 | "\n", 214 | "plot(p)" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": null, 220 | "metadata": { 221 | "collapsed": true 222 | }, 223 | "outputs": [], 224 | "source": [ 225 | "using Sobol\n", 226 | "import OnlineStats: Mean, Series, value\n", 227 | "using Random\n", 228 | "\n", 229 | "p = let\n", 230 | "\n", 231 | "\tf = x -> sin(10x)\n", 232 | "\n", 233 | "\n", 234 | "\tRandom.seed!(0)\n", 235 | "\tn_tries_per = 4\n", 236 | "\tf_true = sin(5)^2 / 5\n", 237 | "\n", 238 | "\tarr_n = [round(Int, n) for n in 10 .^ range(0, stop=4, length=101)]\n", 239 | "\tarr_err_rand = map(arr_n) do n\n", 240 | "\t o = Series(Mean())\n", 241 | "\t for i in 1 : n_tries_per\n", 242 | "\t v = Series(Mean())\n", 243 | "\t for j in 1 : n\n", 244 | "\t fit!(v, f(rand()))\n", 245 | "\t end\n", 246 | "\t fit!(o, value(v)[1])\n", 247 | "\t end\n", 248 | "\t abs(value(o)[1] - f_true) / f_true\n", 249 | "\tend\n", 250 | "\tarr_err_sobol = map(arr_n) do n\n", 251 | "\t o = Series(Mean())\n", 252 | "\t for x in Iterators.take(SobolSeq(1), n)\n", 253 | "\t fit!(o, f(x[1]))\n", 254 | "\t end\n", 255 | "\t abs(value(o)[1] - f_true) / f_true\n", 256 | "\tend\n", 257 | "\n", 258 | "\tAxis([\n", 259 | "\t Plots.Linear(arr_n, arr_err_rand, style=\"mark=none, pastelRed\", legendentry=\"rand\"),\n", 260 | "\t Plots.Linear(arr_n, arr_err_sobol, style=\"mark=none, pastelBlue\", legendentry=\"Sobol\"),\n", 261 | "\t ], xlabel=\"number of samples\", ylabel=\"relative error\", xmode=\"log\", ymode=\"log\", width=\"9cm\",\n", 262 | "\t style=\"trim axis left, trim axis right, legend cell align=left, legend style={draw=none, at={(0.5,-0.4)}, anchor=north, legend columns=2}\")\n", 263 | "end\n", 264 | "\n", 265 | "plot(p)" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": null, 271 | "metadata": { 272 | "collapsed": true 273 | }, 274 | "outputs": [], 275 | "source": [ 276 | "using Primes\n", 277 | "function halton(i, b)\n", 278 | " result, f = 0.0, 1.0\n", 279 | " while i > 0\n", 280 | " f = f / b;\n", 281 | " result = result + f * mod(i, b)\n", 282 | " i = floor(Int, i / b)\n", 283 | " end\n", 284 | " return result\n", 285 | "end\n", 286 | "get_filling_set_halton(m; b=2) = [halton(i,b) for i in 1: m]\n", 287 | "function get_filling_set_halton(m, n)\n", 288 | " bs = primes(max(ceil(Int, n*(log(n) + log(log(n)))), 6))\n", 289 | " seqs = [get_filling_set_halton(m, b=b) for b in bs[1:n]]\n", 290 | " return [collect(x) for x in zip(seqs...)]\n", 291 | "end\n", 292 | "\n", 293 | "p = let\n", 294 | "\n", 295 | "\tn = 19\n", 296 | "\tseqs = [get_filling_set_halton(n, b=b) for b in [19,23]]\n", 297 | "\tX = [collect(x) for x in zip(seqs...)]\n", 298 | "\n", 299 | "\tAxis(\n", 300 | "\t Plots.Scatter([x[1] for x in X], [x[2] for x in X],\n", 301 | "\t style=\"mark=*, mark size=1, mark options={draw=black, fill=black}\"),\n", 302 | "\t width=\"5cm\", height=\"5cm\",\n", 303 | "\tstyle=\"xtick=\\\\empty, ytick=\\\\empty, clip marker paths=true, axis on top=true\"\n", 304 | "\t )\n", 305 | "end\n", 306 | "\n", 307 | "plot(p)" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": null, 313 | "metadata": { 314 | "collapsed": true 315 | }, 316 | "outputs": [], 317 | "source": [ 318 | "using Primes\n", 319 | "using Random\n", 320 | "import Printf: @sprintf\n", 321 | "\n", 322 | "function get_filling_set_random_walk(n)\n", 323 | " Xs = [rand()]\n", 324 | " for i in 2 : n\n", 325 | " push!(Xs, mod(Xs[end] + 0.5 + rand(), 1))\n", 326 | " end\n", 327 | " return Xs\n", 328 | "end\n", 329 | "\n", 330 | "function get_filling_set_additive_recurrence(n; c=φ-1)\n", 331 | " Xs = [rand()]\n", 332 | " for i in 2 : n\n", 333 | " push!(Xs, mod(Xs[end] + c, 1))\n", 334 | " end\n", 335 | " return Xs\n", 336 | "end\n", 337 | "function get_filling_set_additive_recurrence(n, k)\n", 338 | " ps = primes(max(ceil(Int, k*(log(k) + log(log(k)))), 6))\n", 339 | " seqs = [get_filling_set_additive_recurrence(n, c=sqrt(p))\n", 340 | " for p in ps[1:k]]\n", 341 | " return [collect(x) for x in zip(seqs...)]\n", 342 | "end\n", 343 | "\n", 344 | "p = let\n", 345 | "\n", 346 | "\tg = GroupPlot(5,3, groupStyle=\"horizontal sep=0.25cm, vertical sep=0.25cm\")\n", 347 | "\n", 348 | "\tns = [10,100,1000]\n", 349 | "\n", 350 | "\tfunction add_plot(Xs,mark_size)\n", 351 | "\t\tpush!(g, Axis(\n", 352 | "\t\t Plots.Scatter([x[1] for x in Xs], [x[2] for x in Xs], [i/length(Xs) for i in 1 : length(Xs)],\n", 353 | "\t\t style=\"mark=*, mark size=$(mark_size), scatter/use mapped color={draw=mapped color, fill=mapped color}, colormap name=pasteljet\"),\n", 354 | "\t\t width=\"4.5cm\", height=\"4.5cm\", enlargelimits=0,\n", 355 | "\t\tstyle=\"xtick=\\\\empty, ytick=\\\\empty, clip marker paths=true, axis on top=true\"\n", 356 | "\t\t ))\n", 357 | "\tend\n", 358 | "\n", 359 | "\tRandom.seed!(0)\n", 360 | "\tmark_sizes = [1,0.75,0.5]\n", 361 | "\tfor (j,n) in enumerate(ns)\n", 362 | "\t\tmark_size = mark_sizes[j]\n", 363 | "\t add_plot([rand(2) for i in 1 : n], mark_size)\n", 364 | "\t g.axes[end].ylabel=@sprintf(\"\\\\small \\$m = %d\\$\", n)\n", 365 | "\n", 366 | "\t add_plot(get_filling_set_additive_recurrence(n, 2), mark_size)\n", 367 | "\t add_plot(get_filling_set_halton(n, 2), mark_size)\n", 368 | "\n", 369 | "\t Xs = Array{Vector{Float64}}(undef, n)\n", 370 | "\t for (i,x) in enumerate(Iterators.take(SobolSeq(2), n))\n", 371 | "\t Xs[i] = x\n", 372 | "\t end\n", 373 | "\t add_plot(Xs, mark_size)\n", 374 | "\n", 375 | "\t add_plot(uniform_projection_plan(n)./n, mark_size)\n", 376 | "\tend\n", 377 | "\n", 378 | "\tg.axes[end-4].xlabel=\"\\\\small Random\"\n", 379 | "\tg.axes[end-3].xlabel=\"\\\\small Additive Recurrence\"\n", 380 | "\tg.axes[end-2].xlabel=\"\\\\small Halton\"\n", 381 | "\tg.axes[end-1].xlabel=\"\\\\small Sobol\"\n", 382 | "\tg.axes[end].xlabel=\"\\\\small Uniform Projection\"\n", 383 | "\n", 384 | "\tg\n", 385 | "end\n", 386 | "\n", 387 | "plot(p)" 388 | ] 389 | } 390 | ], 391 | "metadata": { 392 | "kernelspec": { 393 | "display_name": "Julia 1.5.1", 394 | "language": "julia", 395 | "name": "julia-1.5" 396 | }, 397 | "language_info": { 398 | "file_extension": ".jl", 399 | "mimetype": "application/julia", 400 | "name": "julia", 401 | "version": "1.5.1" 402 | } 403 | }, 404 | "nbformat": 4, 405 | "nbformat_minor": 2 406 | } 407 | -------------------------------------------------------------------------------- /second-order.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Second Order\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "using Vec\n", 32 | "\n", 33 | "p = let\n", 34 | "\td = 0.2\n", 35 | "\te = 0.02\n", 36 | "\tf = x -> x^2 + d*x^3 - e*x^4\n", 37 | "\tf′ = x -> 2x + 3d*x^2 - 4e*x^3\n", 38 | "\tf″ = x -> 2 + 6d*x - 12e*x^2\n", 39 | "\txdomain = (-2,2)\n", 40 | "\n", 41 | "\tx = 1.25\n", 42 | "\n", 43 | "\tG = GroupPlot(2, 1, groupStyle=\"horizontal sep=1.25cm\", style=\"ymin=-1, xmin=-2.25, width=6cm, ymax=$(f(2)), axis lines=left, xtick=\\\\empty, ytick=\\\\empty, xlabel=\\$x\\$, ylabel=\\$f\\$\") # $\n", 44 | "\n", 45 | "\tg = α -> f(x) + (α-x)*f′(x)\n", 46 | "\tplots = Plots.Plot[]\n", 47 | "\tpush!(plots, Plots.Linear(f, xdomain, style=\"solid, thick, black, mark=none\"))\n", 48 | "\tpush!(plots, Plots.Scatter([x], [f(x)], style=\"mark=*, mark size = 1, mark options={draw=black, fill=black}\"))\n", 49 | "\tpush!(plots, Plots.Linear(g, xdomain, style=\"solid, thick, pastelBlue, mark=none\"))\n", 50 | "\tpush!(G, Axis(plots))\n", 51 | "\n", 52 | "\tg = α -> f(x) + (α-x)*f′(x) + (α-x)^2*f″(x)/2\n", 53 | "\tplots = Plots.Plot[]\n", 54 | "\tpush!(plots, Plots.Linear(f, xdomain, style=\"solid, thick, black, mark=none\"))\n", 55 | "\tpush!(plots, Plots.Scatter([x], [f(x)], style=\"mark=*, mark size = 1, mark options={draw=black, fill=black}\"))\n", 56 | "\tpush!(plots, Plots.Linear(g, xdomain, style=\"solid, thick, pastelBlue, mark=none\"))\n", 57 | "\tpush!(G, Axis(plots))\n", 58 | "\n", 59 | "\tG\n", 60 | "end\n", 61 | "\n", 62 | "plot(p)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": { 69 | "collapsed": true 70 | }, 71 | "outputs": [], 72 | "source": [ 73 | "using Vec\n", 74 | "\n", 75 | "p = let\n", 76 | "\td = 0.2\n", 77 | "\te = 0.02\n", 78 | "\tf = x -> x^2 + d*x^3 - e*x^4\n", 79 | "\tf′ = x -> 2x + 3d*x^2 - 4e*x^3\n", 80 | "\tf″ = x -> 2 + 6d*x - 12e*x^2\n", 81 | "\txdomain = (-2,2)\n", 82 | "\n", 83 | "\tx = 1.25\n", 84 | "\tx2 = x - f′(x)/f″(x)\n", 85 | "\n", 86 | "\tG = GroupPlot(2, 1, groupStyle=\"horizontal sep=1.25cm\", style=\"ymin=-1, xmin=-2.25, width=6cm, ymax=$(f(2)), axis lines=left, xtick={$x, $x2}, xticklabels={\\$x^{(k)}\\$, \\$x^{(k+1)}\\$}, xlabel=\\$x\\$, xticklabel style={text height=2ex}\") # $\n", 87 | "\n", 88 | "\tg = α -> f(x) + (α-x)*f′(x) + (α-x)^2*f″(x)/2\n", 89 | "\tplots = Plots.Plot[]\n", 90 | "\tpush!(plots, Plots.Linear(f, xdomain, style=\"solid, thick, black, mark=none\"))\n", 91 | "\tpush!(plots, Plots.Scatter([x, x2], [f(x), f(x2)], style=\"mark=*, mark size = 1, mark options={draw=black, fill=black}\"))\n", 92 | "\tpush!(plots, Plots.Linear(g, xdomain, style=\"solid, thick, pastelBlue, mark=none\"))\n", 93 | "\tpush!(G, Axis(plots, ylabel=L\"f\", style=\"ytick=\\\\empty\"))\n", 94 | "\n", 95 | "\tg = α -> f′(x) + (α-x)*f″(x)\n", 96 | "\tplots = Plots.Plot[]\n", 97 | "\tpush!(plots, Plots.Linear([xdomain[1], xdomain[2]], [0,0], style=\"solid, gray, mark=none\"))\n", 98 | "\tpush!(plots, Plots.Linear([x2,x2], [-2,6], style=\"solid, gray, mark=none\"))\n", 99 | "\tpush!(plots, Plots.Linear(f′, xdomain, style=\"solid, thick, black, mark=none\"))\n", 100 | "\tpush!(plots, Plots.Scatter([x, x2], [f′(x), f′(x2)], style=\"mark=*, mark size = 1, mark options={draw=black, fill=black}\"))\n", 101 | "\tpush!(plots, Plots.Linear(g, xdomain, style=\"solid, thick, pastelBlue, mark=none\"))\n", 102 | "\tpush!(G, Axis(plots, ylabel=L\"f'\", xmin=-2, ymin=-2, ymax=6, style=\"ytick={0}\"))\n", 103 | "\n", 104 | "\tG\n", 105 | "end\n", 106 | "\n", 107 | "plot(p)" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "metadata": { 114 | "collapsed": true 115 | }, 116 | "outputs": [], 117 | "source": [ 118 | "p = let\n", 119 | "\n", 120 | "\tG = GroupPlot(3, 1, groupStyle=\"horizontal sep=1.25cm, xlabels at=edge bottom, ylabels at=edge left\", style=\"xlabel=\\$x\\$, ylabel=\\$f\\$, width=6cm, axis lines=left, xticklabel style={text height=2ex}\")\n", 121 | "\n", 122 | "\tf = x -> log(exp(x)+1) - x/2 + 1\n", 123 | "\tf′ = x -> 1/(1+exp(-x)) - 0.5\n", 124 | "\tf″ = x -> exp(x)/(exp(x)+1)^2\n", 125 | "\n", 126 | "\tx = 2.17732\n", 127 | "\tx2 = x - f′(x)/f″(x)\n", 128 | "\txdomain = (-4,4)\n", 129 | "\tydomain = (1.15,3.25)\n", 130 | "\n", 131 | "\tg = α -> f(x) + (α-x)*f′(x) + (α-x)^2*f″(x)/2\n", 132 | "\tg2 = α -> f(x2) + (α-x2)*f′(x2) + (α-x2)^2*f″(x2)/2\n", 133 | "\tplots = Plots.Plot[]\n", 134 | "\tpush!(plots, Plots.Linear([x,x], [ydomain[1], ydomain[2]], style=\"solid, gray, mark=none\"))\n", 135 | "\tpush!(plots, Plots.Linear([x2,x2], [ydomain[1], ydomain[2]], style=\"solid, gray, mark=none\"))\n", 136 | "\tpush!(plots, Plots.Linear(f, xdomain, style=\"solid, thick, black, mark=none\"))\n", 137 | "\tpush!(plots, Plots.Scatter([x,x2], f.([x,x2]), style=\"mark=*, mark size = 1, mark options={draw=black, fill=black}\"))\n", 138 | "\tpush!(plots, Plots.Linear(g, xdomain, style=\"solid, thick, pastelBlue, mark=none\"))\n", 139 | "\tpush!(plots, Plots.Linear(g2, xdomain, style=\"solid, thick, pastelPurple, mark=none\"))\n", 140 | "\tpush!(G, Axis(plots, title=\"Oscillation\", xmin=xdomain[1], xmax=xdomain[2], ymin=ydomain[1], ymax=ydomain[2], style=\"xtick=\\\\empty, ytick=\\\\empty\"))\n", 141 | "\n", 142 | "\tf = x -> x < -π/2 ? sin(x) : 2sin((x-π/2)/2) + 1\n", 143 | "\tf′ = x -> cos(x)\n", 144 | "\tf″ = x -> -sin(x)\n", 145 | "\n", 146 | "\tx = -π + 0.2\n", 147 | "\tx2 = x - f′(x)/f″(x)\n", 148 | "\txdomain = (-4.25,3)\n", 149 | "\tydomain = (-3,2)\n", 150 | "\n", 151 | "\tg = α -> f(x) + (α-x)*f′(x) + (α-x)^2*f″(x)/2\n", 152 | "\tplots = Plots.Plot[]\n", 153 | "\tpush!(plots, Plots.Linear([x,x], [ydomain[1], ydomain[2]], style=\"solid, gray, mark=none\"))\n", 154 | "\tpush!(plots, Plots.Linear([x2,x2], [ydomain[1], ydomain[2]], style=\"solid, gray, mark=none\"))\n", 155 | "\tpush!(plots, Plots.Linear(f, xdomain, style=\"solid, thick, black, mark=none\"))\n", 156 | "\tpush!(plots, Plots.Scatter([x,x2], f.([x,x2]), style=\"mark=*, mark size = 1, mark options={draw=black, fill=black}\"))\n", 157 | "\tpush!(plots, Plots.Linear(g, xdomain, style=\"solid, thick, pastelBlue, mark=none\"))\n", 158 | "\tax = Axis(plots, title=\"Overshoot\", xmin=xdomain[1], xmax=xdomain[2], ymin=ydomain[1], ymax=ydomain[2],\n", 159 | "\t style=\"xtick={$x, $(x2)}, xticklabels={\\$x^{(k)}\\$, \\$x^{(k+1)}\\$}, ytick=\\\\empty, \")\n", 160 | "\tpush!(G, ax)\n", 161 | "\n", 162 | "\tc = -π - 0.4\n", 163 | "\tf = x -> x > c ? sin(x) : cos(c)*(x-c) + sin(c) - 0.1*abs(x-c)^2\n", 164 | "\tf′ = x -> cos(x)\n", 165 | "\tf″ = x -> -sin(x)\n", 166 | "\n", 167 | "\tx = -π - 0.3\n", 168 | "\tx2 = x - f′(x)/f″(x)\n", 169 | "\txdomain = (-8,-1)\n", 170 | "\tydomain = (-2,4)\n", 171 | "\n", 172 | "\tg = α -> f(x) + (α-x)*f′(x) + (α-x)^2*f″(x)/2\n", 173 | "\tplots = Plots.Plot[]\n", 174 | "\tpush!(plots, Plots.Linear([x,x], [ydomain[1], ydomain[2]], style=\"solid, gray, mark=none\"))\n", 175 | "\tpush!(plots, Plots.Linear([x2,x2], [ydomain[1], ydomain[2]], style=\"solid, gray, mark=none\"))\n", 176 | "\tpush!(plots, Plots.Linear(f, xdomain, style=\"solid, thick, black, mark=none\"))\n", 177 | "\tpush!(plots, Plots.Scatter([x,x2], f.([x,x2]), style=\"mark=*, mark size = 1, mark options={draw=black, fill=black}\"))\n", 178 | "\tpush!(plots, Plots.Linear(g, xdomain, style=\"solid, thick, pastelBlue, mark=none\"))\n", 179 | "\tax = Axis(plots, title=\"Negative \\$f''\\$\", xmin=xdomain[1], xmax=xdomain[2], ymin=ydomain[1], ymax=ydomain[2],\n", 180 | "\t style=\"xtick={$x, $(x2)}, xticklabels={\\$x^{(k)}\\$, \\$x^{(k+1)}\\$}, ytick=\\\\empty, \")\n", 181 | "\tpush!(G, ax)\n", 182 | "\n", 183 | "\tG\n", 184 | "end\n", 185 | "\n", 186 | "plot(p)" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": null, 192 | "metadata": { 193 | "collapsed": true 194 | }, 195 | "outputs": [], 196 | "source": [ 197 | "using Vec\n", 198 | "using LinearAlgebra\n", 199 | "\n", 200 | "function _line_search(f, x, d)\n", 201 | " d = normalize(d)\n", 202 | " objective = α -> f(x + α*d)\n", 203 | " v, α = f(x), 1e-6\n", 204 | " while f(x + α*d) < v\n", 205 | " v = f(x + α*d)\n", 206 | " α += 1e-6\n", 207 | " end\n", 208 | " return x + α*d\n", 209 | "end\n", 210 | "\n", 211 | "abstract type DescentMethod end\n", 212 | "mutable struct DFP <: DescentMethod\n", 213 | "\tQ\n", 214 | "end\n", 215 | "function init!(M::DFP, f, ∇f, x)\n", 216 | "\tM.Q = Matrix(1.0I, length(x), length(x))\n", 217 | "\treturn M\n", 218 | "end\n", 219 | "function step!(M::DFP, f, ∇f, x)\n", 220 | "\tQ, g = M.Q, ∇f(x)\n", 221 | "\tx′ = _line_search(f, x, -Q*g)\n", 222 | "\tg′ = ∇f(x′)\n", 223 | "\tδ = x′ - x\n", 224 | " γ = g′ - g\n", 225 | " Q[:] = Q - Q*γ*γ'*Q/(γ'*Q*γ) + δ*δ'/(δ'*γ)\n", 226 | " return x′\n", 227 | "end\n", 228 | "\n", 229 | "mutable struct myBFGS <: DescentMethod\n", 230 | "\tQ\n", 231 | "end\n", 232 | "function init!(M::myBFGS, f, ∇f, x)\n", 233 | "\tM.Q = Matrix(1.0I, length(x), length(x))\n", 234 | "\treturn M\n", 235 | "end\n", 236 | "function step!(M::myBFGS, f, ∇f, x)\n", 237 | "\tQ, g = M.Q, ∇f(x)\n", 238 | "\tx′ = _line_search(f, x, -Q*g)\n", 239 | "\tg′ = ∇f(x′)\n", 240 | "\tδ = x′ - x\n", 241 | " γ = g′ - g\n", 242 | " Q[:] = Q - (δ*γ'*Q + Q*γ*δ')/(δ'*γ) + (1 + (γ'*Q*γ)/(δ'*γ))[1]*(δ*δ')/(δ'*γ)\n", 243 | " return x′\n", 244 | "end\n", 245 | "\n", 246 | "mutable struct LimitedMemoryBFGS <: DescentMethod\n", 247 | "\tm\n", 248 | "\tδs\n", 249 | "\tγs\n", 250 | "\tqs\n", 251 | "end\n", 252 | "function init!(M::LimitedMemoryBFGS, f, ∇f, x)\n", 253 | "\tM.δs = []\n", 254 | "\tM.γs = []\n", 255 | " M.qs = []\n", 256 | "\treturn M\n", 257 | "end\n", 258 | "function step!(M::LimitedMemoryBFGS, f, ∇f, x)\n", 259 | " δs, γs, qs, g = M.δs, M.γs, M.qs, ∇f(x)\n", 260 | " m = length(δs)\n", 261 | " if m > 0\n", 262 | " q = g\n", 263 | " for i in m : -1 : 1\n", 264 | " qs[i] = copy(q)\n", 265 | " q -= (δs[i]⋅q)/(γs[i]⋅δs[i])*γs[i]\n", 266 | " end\n", 267 | " z = (γs[m] .* δs[m] .* q) / (γs[m]⋅γs[m])\n", 268 | " for i in 1 : m\n", 269 | " z += δs[i]*(δs[i]⋅qs[i] - γs[i]⋅z)/(γs[i]⋅δs[i])\n", 270 | " end\n", 271 | " x′ = _line_search(f, x, -z)\n", 272 | " else\n", 273 | " x′ = _line_search(f, x, -g)\n", 274 | " end\n", 275 | " g′ = ∇f(x′)\n", 276 | " push!(δs, x′ - x); push!(γs, g′ - g)\n", 277 | " push!(qs, zeros(length(x)))\n", 278 | " while length(δs) > M.m\n", 279 | " popfirst!(δs); popfirst!(γs); popfirst!(qs)\n", 280 | " end\n", 281 | " return x′\n", 282 | "end\n", 283 | "\n", 284 | "p = let\n", 285 | "\tf = x -> (1-x[1])^2 + 5*(4x[2] - x[1]^2)^2\n", 286 | "\t∇f = x -> [2*(10x[1]^3 - 40x[1]*x[2] + x[1] - 1), -40*(x[1]^2 - 4x[2])]\n", 287 | "\n", 288 | "\txdomain = (-3, 2)\n", 289 | "\tydomain = (-0.5, 2)\n", 290 | "\n", 291 | "\tfunction this_step!(M::DescentMethod, v::VecE2{Float64})\n", 292 | "\t x = convert(Vector{Float64}, v)\n", 293 | "\t return VecE2{Float64}(step!(M, f, ∇f, x))\n", 294 | "\tend\n", 295 | "\tfunction run_descent_method(M::DescentMethod, x₀::VecE2{Float64}, N::Int)\n", 296 | "\t pts = [x₀]\n", 297 | "\t init!(M, f, ∇f, convert(Vector{Float64}, x₀))\n", 298 | "\t for i in 1 : N\n", 299 | "\t push!(pts, this_step!(M, pts[end]))\n", 300 | "\t end\n", 301 | "\t return pts\n", 302 | "\tend\n", 303 | "\tfunction get_descent_plot(pts::Vector{VecE2{Float64}}, name, color::String=\"black\")\n", 304 | "\t Plots.Linear([P.x for P in pts], [P.y for P in pts], style=\"thick, $color, solid, mark=none, line join=round\", legendentry=name) #$\n", 305 | "\tend\n", 306 | "\n", 307 | "\tx₀ = VecE2{Float64}(-1,1.75)\n", 308 | "\tN = 15\n", 309 | "\tplots = Plots.Plot[]\n", 310 | "\tpush!(plots, Plots.Contour(f, xdomain, ydomain, levels=[2,10,50,200,500], style=\"forget plot, width=\\\\columnwidth\", xbins=150, ybins=150))\n", 311 | "\n", 312 | "\tstuff = Tuple{DescentMethod, String, String}[]\n", 313 | "\tpush!(stuff, (DFP(NaN), \"DFP\", \"black\"))\n", 314 | "\tpush!(stuff, (myBFGS(NaN), \"BFGS\", \"gray\"))\n", 315 | "\tpush!(stuff, (LimitedMemoryBFGS(3,NaN,NaN,NaN), \"L-BFGS (\\$m=3\\$)\", \"pastelGreen\"))\n", 316 | "\tpush!(stuff, (LimitedMemoryBFGS(2,NaN,NaN,NaN), \"L-BFGS (\\$m=2\\$)\", \"pastelSeaGreen\"))\n", 317 | "\tpush!(stuff, (LimitedMemoryBFGS(1,NaN,NaN,NaN), \"L-BFGS (\\$m=1\\$)\", \"pastelBlue\"))\n", 318 | "\tfor (M, name, color) in stuff\n", 319 | "\t pts = run_descent_method(M, x₀, N)\n", 320 | "\t push!(plots, get_descent_plot(pts, name, color))\n", 321 | "\tend\n", 322 | "\tpush!(plots, Plots.Scatter([1], [1/4], style=\"mark=*, mark size = 1, mark options={draw=black, fill=black}\"))\n", 323 | "\tAxis(plots, width=\"8cm\", height=\"8cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"xtick=\\\\empty, ytick=\\\\empty, contour/labels=false, view={0}{90}, legend pos=outer north east\")\n", 324 | "end\n", 325 | "plot(p)" 326 | ] 327 | } 328 | ], 329 | "metadata": { 330 | "kernelspec": { 331 | "display_name": "Julia 1.5.1", 332 | "language": "julia", 333 | "name": "julia-1.5" 334 | }, 335 | "language_info": { 336 | "file_extension": ".jl", 337 | "mimetype": "application/julia", 338 | "name": "julia", 339 | "version": "1.5.1" 340 | } 341 | }, 342 | "nbformat": 4, 343 | "nbformat_minor": 2 344 | } 345 | -------------------------------------------------------------------------------- /support_code.jl: -------------------------------------------------------------------------------- 1 | using PGFPlots 2 | using Colors 3 | using ColorSchemes 4 | using LinearAlgebra 5 | pasteljet = ColorMaps.RGBArrayMap(ColorSchemes.viridis, interpolation_levels=500, invert=true); 6 | pastelRedBlue = ColorMaps.RGBArrayMap([RGB(246/255, 21/255, 92/255), 7 | RGB(1.0,1.0,1.0), 8 | RGB( 27/255,161/255,234/255)], interpolation_levels=500); 9 | pushPGFPlotsPreamble("\\pgfplotscreateplotcyclelist{pastelcolors}{%") 10 | pushPGFPlotsPreamble(" solid, pastelPurple, mark=none\\\\%") 11 | pushPGFPlotsPreamble(" solid, pastelBlue, mark=none\\\\%") 12 | pushPGFPlotsPreamble(" solid, pastelGreen, mark=none\\\\%") 13 | pushPGFPlotsPreamble(" solid, pastelRed, mark=none\\\\%") 14 | pushPGFPlotsPreamble(" solid, pastelMagenta, mark=none\\\\%") 15 | pushPGFPlotsPreamble(" solid, pastelOrange, mark=none\\\\%") 16 | pushPGFPlotsPreamble(" solid, pastelSeaGreen, mark=none\\\\%") 17 | pushPGFPlotsPreamble("}") 18 | pushPGFPlotsPreamble("\\usepackage{amsmath}") 19 | pushPGFPlotsPreamble("\\usepackage{xfrac}") 20 | pushPGFPlotsPreamble("\\usepgfplotslibrary{fillbetween}") 21 | pushPGFPlotsPreamble("\\usepgfplotslibrary{statistics}") 22 | pushPGFPlotsPreamble("\\definecolor{pastelMagenta}{HTML}{FF48CF}") 23 | pushPGFPlotsPreamble("\\definecolor{pastelPurple}{HTML}{8770FE}") 24 | pushPGFPlotsPreamble("\\definecolor{pastelBlue}{HTML}{1BA1EA}") 25 | pushPGFPlotsPreamble("\\definecolor{pastelSeaGreen}{HTML}{14B57F}") 26 | pushPGFPlotsPreamble("\\definecolor{pastelGreen}{HTML}{3EAA0D}") 27 | pushPGFPlotsPreamble("\\definecolor{pastelOrange}{HTML}{C38D09}") 28 | pushPGFPlotsPreamble("\\definecolor{pastelRed}{HTML}{F5615C}") 29 | pushPGFPlotsPreamble("\\tikzset{myarrow/.style={line width = 0.05cm, ->, rounded corners=5mm}}") 30 | pushPGFPlotsPreamble("\\newcommand{\\vect}[1]{\\boldsymbol{\\mathbf{#1}}}") 31 | pushPGFPlotsPreamble("\\newcommand{\\mat}[1]{\\vect{#1}}") 32 | pushPGFPlotsPreamble("\\pgfplotsset{") 33 | pushPGFPlotsPreamble(" colormap={pasteljet}{") 34 | pushPGFPlotsPreamble(" rgb=(0.99325,0.90616,0.14394)") 35 | pushPGFPlotsPreamble(" rgb=(0.98387,0.90487,0.13690)") 36 | pushPGFPlotsPreamble(" rgb=(0.97442,0.90359,0.13021)") 37 | pushPGFPlotsPreamble(" rgb=(0.96489,0.90232,0.12394)") 38 | pushPGFPlotsPreamble(" rgb=(0.95530,0.90107,0.11813)") 39 | pushPGFPlotsPreamble(" rgb=(0.94564,0.89982,0.11284)") 40 | pushPGFPlotsPreamble(" rgb=(0.93590,0.89857,0.10813)") 41 | pushPGFPlotsPreamble(" rgb=(0.92611,0.89733,0.10407)") 42 | pushPGFPlotsPreamble(" rgb=(0.91624,0.89609,0.10072)") 43 | pushPGFPlotsPreamble(" rgb=(0.90631,0.89485,0.09813)") 44 | pushPGFPlotsPreamble(" rgb=(0.89632,0.89362,0.09634)") 45 | pushPGFPlotsPreamble(" rgb=(0.88627,0.89237,0.09537)") 46 | pushPGFPlotsPreamble(" rgb=(0.87617,0.89112,0.09525)") 47 | pushPGFPlotsPreamble(" rgb=(0.86601,0.88987,0.09595)") 48 | pushPGFPlotsPreamble(" rgb=(0.85581,0.88860,0.09745)") 49 | pushPGFPlotsPreamble(" rgb=(0.84556,0.88732,0.09970)") 50 | pushPGFPlotsPreamble(" rgb=(0.83527,0.88603,0.10265)") 51 | pushPGFPlotsPreamble(" rgb=(0.82494,0.88472,0.10622)") 52 | pushPGFPlotsPreamble(" rgb=(0.81458,0.88339,0.11035)") 53 | pushPGFPlotsPreamble(" rgb=(0.80418,0.88205,0.11496)") 54 | pushPGFPlotsPreamble(" rgb=(0.79376,0.88068,0.12001)") 55 | pushPGFPlotsPreamble(" rgb=(0.78331,0.87928,0.12540)") 56 | pushPGFPlotsPreamble(" rgb=(0.77285,0.87787,0.13111)") 57 | pushPGFPlotsPreamble(" rgb=(0.76237,0.87642,0.13706)") 58 | pushPGFPlotsPreamble(" rgb=(0.75188,0.87495,0.14323)") 59 | pushPGFPlotsPreamble(" rgb=(0.74139,0.87345,0.14956)") 60 | pushPGFPlotsPreamble(" rgb=(0.73089,0.87192,0.15603)") 61 | pushPGFPlotsPreamble(" rgb=(0.72039,0.87035,0.16260)") 62 | pushPGFPlotsPreamble(" rgb=(0.70990,0.86875,0.16926)") 63 | pushPGFPlotsPreamble(" rgb=(0.69942,0.86712,0.17597)") 64 | pushPGFPlotsPreamble(" rgb=(0.68894,0.86545,0.18272)") 65 | pushPGFPlotsPreamble(" rgb=(0.67849,0.86374,0.18950)") 66 | pushPGFPlotsPreamble(" rgb=(0.66805,0.86200,0.19629)") 67 | pushPGFPlotsPreamble(" rgb=(0.65764,0.86022,0.20308)") 68 | pushPGFPlotsPreamble(" rgb=(0.64726,0.85840,0.20986)") 69 | pushPGFPlotsPreamble(" rgb=(0.63690,0.85654,0.21662)") 70 | pushPGFPlotsPreamble(" rgb=(0.62658,0.85464,0.22335)") 71 | pushPGFPlotsPreamble(" rgb=(0.61629,0.85271,0.23005)") 72 | pushPGFPlotsPreamble(" rgb=(0.60604,0.85073,0.23671)") 73 | pushPGFPlotsPreamble(" rgb=(0.59584,0.84872,0.24333)") 74 | pushPGFPlotsPreamble(" rgb=(0.58568,0.84666,0.24990)") 75 | pushPGFPlotsPreamble(" rgb=(0.57556,0.84457,0.25642)") 76 | pushPGFPlotsPreamble(" rgb=(0.56550,0.84243,0.26288)") 77 | pushPGFPlotsPreamble(" rgb=(0.55548,0.84025,0.26928)") 78 | pushPGFPlotsPreamble(" rgb=(0.54552,0.83804,0.27563)") 79 | pushPGFPlotsPreamble(" rgb=(0.53562,0.83579,0.28191)") 80 | pushPGFPlotsPreamble(" rgb=(0.52578,0.83349,0.28813)") 81 | pushPGFPlotsPreamble(" rgb=(0.51599,0.83116,0.29428)") 82 | pushPGFPlotsPreamble(" rgb=(0.50627,0.82879,0.30036)") 83 | pushPGFPlotsPreamble(" rgb=(0.49661,0.82638,0.30638)") 84 | pushPGFPlotsPreamble(" rgb=(0.48703,0.82393,0.31232)") 85 | pushPGFPlotsPreamble(" rgb=(0.47750,0.82144,0.31820)") 86 | pushPGFPlotsPreamble(" rgb=(0.46805,0.81892,0.32400)") 87 | pushPGFPlotsPreamble(" rgb=(0.45867,0.81636,0.32973)") 88 | pushPGFPlotsPreamble(" rgb=(0.44937,0.81377,0.33538)") 89 | pushPGFPlotsPreamble(" rgb=(0.44014,0.81114,0.34097)") 90 | pushPGFPlotsPreamble(" rgb=(0.43098,0.80847,0.34648)") 91 | pushPGFPlotsPreamble(" rgb=(0.42191,0.80577,0.35191)") 92 | pushPGFPlotsPreamble(" rgb=(0.41291,0.80304,0.35727)") 93 | pushPGFPlotsPreamble(" rgb=(0.40400,0.80027,0.36255)") 94 | pushPGFPlotsPreamble(" rgb=(0.39517,0.79748,0.36776)") 95 | pushPGFPlotsPreamble(" rgb=(0.38643,0.79464,0.37289)") 96 | pushPGFPlotsPreamble(" rgb=(0.37778,0.79178,0.37794)") 97 | pushPGFPlotsPreamble(" rgb=(0.36921,0.78889,0.38291)") 98 | pushPGFPlotsPreamble(" rgb=(0.36074,0.78596,0.38781)") 99 | pushPGFPlotsPreamble(" rgb=(0.35236,0.78301,0.39264)") 100 | pushPGFPlotsPreamble(" rgb=(0.34407,0.78003,0.39738)") 101 | pushPGFPlotsPreamble(" rgb=(0.33588,0.77702,0.40205)") 102 | pushPGFPlotsPreamble(" rgb=(0.32780,0.77398,0.40664)") 103 | pushPGFPlotsPreamble(" rgb=(0.31981,0.77091,0.41115)") 104 | pushPGFPlotsPreamble(" rgb=(0.31193,0.76782,0.41559)") 105 | pushPGFPlotsPreamble(" rgb=(0.30415,0.76470,0.41994)") 106 | pushPGFPlotsPreamble(" rgb=(0.29648,0.76156,0.42422)") 107 | pushPGFPlotsPreamble(" rgb=(0.28892,0.75839,0.42843)") 108 | pushPGFPlotsPreamble(" rgb=(0.28148,0.75520,0.43255)") 109 | pushPGFPlotsPreamble(" rgb=(0.27415,0.75199,0.43660)") 110 | pushPGFPlotsPreamble(" rgb=(0.26694,0.74875,0.44057)") 111 | pushPGFPlotsPreamble(" rgb=(0.25986,0.74549,0.44447)") 112 | pushPGFPlotsPreamble(" rgb=(0.25290,0.74221,0.44828)") 113 | pushPGFPlotsPreamble(" rgb=(0.24607,0.73891,0.45202)") 114 | pushPGFPlotsPreamble(" rgb=(0.23937,0.73559,0.45569)") 115 | pushPGFPlotsPreamble(" rgb=(0.23281,0.73225,0.45928)") 116 | pushPGFPlotsPreamble(" rgb=(0.22640,0.72889,0.46279)") 117 | pushPGFPlotsPreamble(" rgb=(0.22012,0.72551,0.46623)") 118 | pushPGFPlotsPreamble(" rgb=(0.21400,0.72211,0.46959)") 119 | pushPGFPlotsPreamble(" rgb=(0.20803,0.71870,0.47287)") 120 | pushPGFPlotsPreamble(" rgb=(0.20222,0.71527,0.47608)") 121 | pushPGFPlotsPreamble(" rgb=(0.19657,0.71183,0.47922)") 122 | pushPGFPlotsPreamble(" rgb=(0.19109,0.70837,0.48228)") 123 | pushPGFPlotsPreamble(" rgb=(0.18578,0.70489,0.48527)") 124 | pushPGFPlotsPreamble(" rgb=(0.18065,0.70140,0.48819)") 125 | pushPGFPlotsPreamble(" rgb=(0.17571,0.69790,0.49103)") 126 | pushPGFPlotsPreamble(" rgb=(0.17095,0.69438,0.49380)") 127 | pushPGFPlotsPreamble(" rgb=(0.16638,0.69086,0.49650)") 128 | pushPGFPlotsPreamble(" rgb=(0.16202,0.68732,0.49913)") 129 | pushPGFPlotsPreamble(" rgb=(0.15785,0.68376,0.50169)") 130 | pushPGFPlotsPreamble(" rgb=(0.15389,0.68020,0.50417)") 131 | pushPGFPlotsPreamble(" rgb=(0.15015,0.67663,0.50659)") 132 | pushPGFPlotsPreamble(" rgb=(0.14662,0.67305,0.50894)") 133 | pushPGFPlotsPreamble(" rgb=(0.14330,0.66946,0.51121)") 134 | pushPGFPlotsPreamble(" rgb=(0.14021,0.66586,0.51343)") 135 | pushPGFPlotsPreamble(" rgb=(0.13734,0.66225,0.51557)") 136 | pushPGFPlotsPreamble(" rgb=(0.13469,0.65864,0.51765)") 137 | pushPGFPlotsPreamble(" rgb=(0.13227,0.65501,0.51966)") 138 | pushPGFPlotsPreamble(" rgb=(0.13007,0.65138,0.52161)") 139 | pushPGFPlotsPreamble(" rgb=(0.12809,0.64775,0.52349)") 140 | pushPGFPlotsPreamble(" rgb=(0.12633,0.64411,0.52531)") 141 | pushPGFPlotsPreamble(" rgb=(0.12478,0.64046,0.52707)") 142 | pushPGFPlotsPreamble(" rgb=(0.12344,0.63681,0.52876)") 143 | pushPGFPlotsPreamble(" rgb=(0.12231,0.63315,0.53040)") 144 | pushPGFPlotsPreamble(" rgb=(0.12138,0.62949,0.53197)") 145 | pushPGFPlotsPreamble(" rgb=(0.12064,0.62583,0.53349)") 146 | pushPGFPlotsPreamble(" rgb=(0.12008,0.62216,0.53495)") 147 | pushPGFPlotsPreamble(" rgb=(0.11970,0.61849,0.53635)") 148 | pushPGFPlotsPreamble(" rgb=(0.11948,0.61482,0.53769)") 149 | pushPGFPlotsPreamble(" rgb=(0.11942,0.61114,0.53898)") 150 | pushPGFPlotsPreamble(" rgb=(0.11951,0.60746,0.54022)") 151 | pushPGFPlotsPreamble(" rgb=(0.11974,0.60379,0.54140)") 152 | pushPGFPlotsPreamble(" rgb=(0.12009,0.60010,0.54253)") 153 | pushPGFPlotsPreamble(" rgb=(0.12057,0.59642,0.54361)") 154 | pushPGFPlotsPreamble(" rgb=(0.12115,0.59274,0.54464)") 155 | pushPGFPlotsPreamble(" rgb=(0.12183,0.58905,0.54562)") 156 | pushPGFPlotsPreamble(" rgb=(0.12261,0.58537,0.54656)") 157 | pushPGFPlotsPreamble(" rgb=(0.12346,0.58169,0.54744)") 158 | pushPGFPlotsPreamble(" rgb=(0.12440,0.57800,0.54829)") 159 | pushPGFPlotsPreamble(" rgb=(0.12539,0.57432,0.54909)") 160 | pushPGFPlotsPreamble(" rgb=(0.12645,0.57063,0.54984)") 161 | pushPGFPlotsPreamble(" rgb=(0.12757,0.56695,0.55056)") 162 | pushPGFPlotsPreamble(" rgb=(0.12873,0.56327,0.55123)") 163 | pushPGFPlotsPreamble(" rgb=(0.12993,0.55958,0.55186)") 164 | pushPGFPlotsPreamble(" rgb=(0.13117,0.55590,0.55246)") 165 | pushPGFPlotsPreamble(" rgb=(0.13244,0.55222,0.55302)") 166 | pushPGFPlotsPreamble(" rgb=(0.13374,0.54853,0.55354)") 167 | pushPGFPlotsPreamble(" rgb=(0.13507,0.54485,0.55403)") 168 | pushPGFPlotsPreamble(" rgb=(0.13641,0.54117,0.55448)") 169 | pushPGFPlotsPreamble(" rgb=(0.13777,0.53749,0.55491)") 170 | pushPGFPlotsPreamble(" rgb=(0.13915,0.53381,0.55530)") 171 | pushPGFPlotsPreamble(" rgb=(0.14054,0.53013,0.55566)") 172 | pushPGFPlotsPreamble(" rgb=(0.14194,0.52645,0.55599)") 173 | pushPGFPlotsPreamble(" rgb=(0.14334,0.52277,0.55629)") 174 | pushPGFPlotsPreamble(" rgb=(0.14476,0.51909,0.55657)") 175 | pushPGFPlotsPreamble(" rgb=(0.14618,0.51541,0.55682)") 176 | pushPGFPlotsPreamble(" rgb=(0.14761,0.51173,0.55705)") 177 | pushPGFPlotsPreamble(" rgb=(0.14904,0.50805,0.55725)") 178 | pushPGFPlotsPreamble(" rgb=(0.15048,0.50437,0.55743)") 179 | pushPGFPlotsPreamble(" rgb=(0.15192,0.50069,0.55759)") 180 | pushPGFPlotsPreamble(" rgb=(0.15336,0.49700,0.55772)") 181 | pushPGFPlotsPreamble(" rgb=(0.15482,0.49331,0.55784)") 182 | pushPGFPlotsPreamble(" rgb=(0.15627,0.48962,0.55794)") 183 | pushPGFPlotsPreamble(" rgb=(0.15773,0.48593,0.55801)") 184 | pushPGFPlotsPreamble(" rgb=(0.15919,0.48224,0.55807)") 185 | pushPGFPlotsPreamble(" rgb=(0.16067,0.47854,0.55812)") 186 | pushPGFPlotsPreamble(" rgb=(0.16214,0.47484,0.55814)") 187 | pushPGFPlotsPreamble(" rgb=(0.16362,0.47113,0.55815)") 188 | pushPGFPlotsPreamble(" rgb=(0.16512,0.46742,0.55814)") 189 | pushPGFPlotsPreamble(" rgb=(0.16662,0.46371,0.55812)") 190 | pushPGFPlotsPreamble(" rgb=(0.16813,0.45999,0.55808)") 191 | pushPGFPlotsPreamble(" rgb=(0.16965,0.45626,0.55803)") 192 | pushPGFPlotsPreamble(" rgb=(0.17118,0.45253,0.55797)") 193 | pushPGFPlotsPreamble(" rgb=(0.17272,0.44879,0.55788)") 194 | pushPGFPlotsPreamble(" rgb=(0.17427,0.44504,0.55779)") 195 | pushPGFPlotsPreamble(" rgb=(0.17584,0.44129,0.55768)") 196 | pushPGFPlotsPreamble(" rgb=(0.17742,0.43753,0.55756)") 197 | pushPGFPlotsPreamble(" rgb=(0.17902,0.43376,0.55743)") 198 | pushPGFPlotsPreamble(" rgb=(0.18063,0.42997,0.55728)") 199 | pushPGFPlotsPreamble(" rgb=(0.18226,0.42618,0.55712)") 200 | pushPGFPlotsPreamble(" rgb=(0.18390,0.42238,0.55694)") 201 | pushPGFPlotsPreamble(" rgb=(0.18556,0.41857,0.55675)") 202 | pushPGFPlotsPreamble(" rgb=(0.18723,0.41475,0.55655)") 203 | pushPGFPlotsPreamble(" rgb=(0.18892,0.41091,0.55633)") 204 | pushPGFPlotsPreamble(" rgb=(0.19063,0.40706,0.55609)") 205 | pushPGFPlotsPreamble(" rgb=(0.19236,0.40320,0.55584)") 206 | pushPGFPlotsPreamble(" rgb=(0.19410,0.39932,0.55556)") 207 | pushPGFPlotsPreamble(" rgb=(0.19586,0.39543,0.55528)") 208 | pushPGFPlotsPreamble(" rgb=(0.19764,0.39153,0.55497)") 209 | pushPGFPlotsPreamble(" rgb=(0.19943,0.38761,0.55464)") 210 | pushPGFPlotsPreamble(" rgb=(0.20124,0.38367,0.55429)") 211 | pushPGFPlotsPreamble(" rgb=(0.20306,0.37972,0.55393)") 212 | pushPGFPlotsPreamble(" rgb=(0.20490,0.37575,0.55353)") 213 | pushPGFPlotsPreamble(" rgb=(0.20676,0.37176,0.55312)") 214 | pushPGFPlotsPreamble(" rgb=(0.20862,0.36775,0.55268)") 215 | pushPGFPlotsPreamble(" rgb=(0.21050,0.36373,0.55221)") 216 | pushPGFPlotsPreamble(" rgb=(0.21240,0.35968,0.55171)") 217 | pushPGFPlotsPreamble(" rgb=(0.21430,0.35562,0.55118)") 218 | pushPGFPlotsPreamble(" rgb=(0.21621,0.35153,0.55063)") 219 | pushPGFPlotsPreamble(" rgb=(0.21813,0.34743,0.55004)") 220 | pushPGFPlotsPreamble(" rgb=(0.22006,0.34331,0.54941)") 221 | pushPGFPlotsPreamble(" rgb=(0.22199,0.33916,0.54875)") 222 | pushPGFPlotsPreamble(" rgb=(0.22393,0.33499,0.54805)") 223 | pushPGFPlotsPreamble(" rgb=(0.22586,0.33081,0.54731)") 224 | pushPGFPlotsPreamble(" rgb=(0.22780,0.32659,0.54653)") 225 | pushPGFPlotsPreamble(" rgb=(0.22974,0.32236,0.54571)") 226 | pushPGFPlotsPreamble(" rgb=(0.23167,0.31811,0.54483)") 227 | pushPGFPlotsPreamble(" rgb=(0.23360,0.31383,0.54391)") 228 | pushPGFPlotsPreamble(" rgb=(0.23553,0.30953,0.54294)") 229 | pushPGFPlotsPreamble(" rgb=(0.23744,0.30520,0.54192)") 230 | pushPGFPlotsPreamble(" rgb=(0.23935,0.30085,0.54084)") 231 | pushPGFPlotsPreamble(" rgb=(0.24124,0.29648,0.53971)") 232 | pushPGFPlotsPreamble(" rgb=(0.24311,0.29209,0.53852)") 233 | pushPGFPlotsPreamble(" rgb=(0.24497,0.28768,0.53726)") 234 | pushPGFPlotsPreamble(" rgb=(0.24681,0.28324,0.53594)") 235 | pushPGFPlotsPreamble(" rgb=(0.24863,0.27877,0.53456)") 236 | pushPGFPlotsPreamble(" rgb=(0.25043,0.27429,0.53310)") 237 | pushPGFPlotsPreamble(" rgb=(0.25219,0.26978,0.53158)") 238 | pushPGFPlotsPreamble(" rgb=(0.25394,0.26525,0.52998)") 239 | pushPGFPlotsPreamble(" rgb=(0.25565,0.26070,0.52831)") 240 | pushPGFPlotsPreamble(" rgb=(0.25732,0.25613,0.52656)") 241 | pushPGFPlotsPreamble(" rgb=(0.25897,0.25154,0.52474)") 242 | pushPGFPlotsPreamble(" rgb=(0.26057,0.24692,0.52283)") 243 | pushPGFPlotsPreamble(" rgb=(0.26214,0.24229,0.52084)") 244 | pushPGFPlotsPreamble(" rgb=(0.26366,0.23763,0.51876)") 245 | pushPGFPlotsPreamble(" rgb=(0.26515,0.23296,0.51660)") 246 | pushPGFPlotsPreamble(" rgb=(0.26658,0.22826,0.51435)") 247 | pushPGFPlotsPreamble(" rgb=(0.26797,0.22355,0.51201)") 248 | pushPGFPlotsPreamble(" rgb=(0.26931,0.21882,0.50958)") 249 | pushPGFPlotsPreamble(" rgb=(0.27059,0.21407,0.50705)") 250 | pushPGFPlotsPreamble(" rgb=(0.27183,0.20930,0.50443)") 251 | pushPGFPlotsPreamble(" rgb=(0.27301,0.20452,0.50172)") 252 | pushPGFPlotsPreamble(" rgb=(0.27413,0.19972,0.49891)") 253 | pushPGFPlotsPreamble(" rgb=(0.27519,0.19490,0.49600)") 254 | pushPGFPlotsPreamble(" rgb=(0.27619,0.19007,0.49300)") 255 | pushPGFPlotsPreamble(" rgb=(0.27713,0.18523,0.48990)") 256 | pushPGFPlotsPreamble(" rgb=(0.27801,0.18037,0.48670)") 257 | pushPGFPlotsPreamble(" rgb=(0.27883,0.17549,0.48340)") 258 | pushPGFPlotsPreamble(" rgb=(0.27957,0.17060,0.48000)") 259 | pushPGFPlotsPreamble(" rgb=(0.28025,0.16569,0.47650)") 260 | pushPGFPlotsPreamble(" rgb=(0.28087,0.16077,0.47290)") 261 | pushPGFPlotsPreamble(" rgb=(0.28141,0.15583,0.46920)") 262 | pushPGFPlotsPreamble(" rgb=(0.28189,0.15088,0.46541)") 263 | pushPGFPlotsPreamble(" rgb=(0.28229,0.14591,0.46151)") 264 | pushPGFPlotsPreamble(" rgb=(0.28262,0.14093,0.45752)") 265 | pushPGFPlotsPreamble(" rgb=(0.28288,0.13592,0.45343)") 266 | pushPGFPlotsPreamble(" rgb=(0.28307,0.13090,0.44924)") 267 | pushPGFPlotsPreamble(" rgb=(0.28319,0.12585,0.44496)") 268 | pushPGFPlotsPreamble(" rgb=(0.28323,0.12078,0.44058)") 269 | pushPGFPlotsPreamble(" rgb=(0.28320,0.11568,0.43611)") 270 | pushPGFPlotsPreamble(" rgb=(0.28309,0.11055,0.43155)") 271 | pushPGFPlotsPreamble(" rgb=(0.28291,0.10539,0.42690)") 272 | pushPGFPlotsPreamble(" rgb=(0.28266,0.10020,0.42216)") 273 | pushPGFPlotsPreamble(" rgb=(0.28233,0.09495,0.41733)") 274 | pushPGFPlotsPreamble(" rgb=(0.28192,0.08967,0.41241)") 275 | pushPGFPlotsPreamble(" rgb=(0.28145,0.08432,0.40741)") 276 | pushPGFPlotsPreamble(" rgb=(0.28089,0.07891,0.40233)") 277 | pushPGFPlotsPreamble(" rgb=(0.28027,0.07342,0.39716)") 278 | pushPGFPlotsPreamble(" rgb=(0.27957,0.06784,0.39192)") 279 | pushPGFPlotsPreamble(" rgb=(0.27879,0.06214,0.38659)") 280 | pushPGFPlotsPreamble(" rgb=(0.27794,0.05632,0.38119)") 281 | pushPGFPlotsPreamble(" rgb=(0.27702,0.05034,0.37572)") 282 | pushPGFPlotsPreamble(" rgb=(0.27602,0.04417,0.37016)") 283 | pushPGFPlotsPreamble(" rgb=(0.27495,0.03775,0.36454)") 284 | pushPGFPlotsPreamble(" rgb=(0.27381,0.03150,0.35885)") 285 | pushPGFPlotsPreamble(" rgb=(0.27259,0.02556,0.35309)") 286 | pushPGFPlotsPreamble(" rgb=(0.27131,0.01994,0.34727)") 287 | pushPGFPlotsPreamble(" rgb=(0.26994,0.01463,0.34138)") 288 | pushPGFPlotsPreamble(" rgb=(0.26851,0.00961,0.33543)") 289 | pushPGFPlotsPreamble(" rgb=(0.26700,0.00487,0.32942)") 290 | pushPGFPlotsPreamble(" }") 291 | pushPGFPlotsPreamble("}"); 292 | -------------------------------------------------------------------------------- /test-functions.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Test Functions\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "p = let\n", 32 | "\tfunction ackley(x, a=20, b=0.2, c=2π)\n", 33 | "\t\td = length(x)\n", 34 | "\t\treturn -a*exp(-b*sqrt(sum(x.^2)/d)) -\n", 35 | "\t\t exp(sum(cos.(c*xi) for xi in x)/d) + a + ℯ\n", 36 | "\tend\n", 37 | "\n", 38 | "\txdomain = (-30, 30)\n", 39 | "\tydomain = (-30, 30)\n", 40 | "\n", 41 | "\tplots = Plots.Plot[]\n", 42 | "\tpush!(plots, Plots.Image((x,y)->ackley([x,y]), xdomain, ydomain, xbins=600, ybins=600, colormap = pasteljet, colorbar = false))\n", 43 | "\tAxis(plots, width=\"8cm\", height=\"8cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"axis equal, view={0}{90}\")\n", 44 | "end\n", 45 | "\n", 46 | "plot(p)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "metadata": { 53 | "collapsed": true 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "p = let\n", 58 | "\tbooth(x) = (x[1]+2x[2]-7)^2 + (2x[1]+x[2]-5)^2\n", 59 | "\n", 60 | "\txdomain = (-10, 10)\n", 61 | "\tydomain = (-10, 10)\n", 62 | "\n", 63 | "\tplots = Plots.Plot[]\n", 64 | "\tpush!(plots, Plots.Contour(booth, xdomain, ydomain, style=\"width=\\\\columnwidth\", xbins=100, ybins=100, levels=[0.5,2,5,10,20,50,100,500,1000]))\n", 65 | "\tAxis(plots, width=\"8cm\", height=\"8cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"contour/labels=false, axis equal, view={0}{90}\")\n", 66 | "end\n", 67 | "\n", 68 | "plot(p)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": { 75 | "collapsed": true 76 | }, 77 | "outputs": [], 78 | "source": [ 79 | "\n", 80 | "p = let\n", 81 | "\tbranin(x; a=1, b=5.1/(4π^2), c=5/π, r=6, s=10, t=1/(8π)) = a*(x[2] - b*x[1]^2 + c*x[1] - r)^2 + s*(1-t)*cos(x[1]) + s\n", 82 | "\n", 83 | "\txdomain = (-2π, 6π)\n", 84 | "\tydomain = ( -π, 7π)\n", 85 | "\n", 86 | "\tplots = Plots.Plot[]\n", 87 | "\tpush!(plots, Plots.Contour(branin, xdomain, ydomain, levels=[1,2,3,5,10,20,50,100], style=\"width=\\\\columnwidth\", xbins=100, ybins=100))\n", 88 | "\tAxis(plots, width=\"8cm\", height=\"8cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"contour/labels=false, axis equal, view={0}{90}\")\n", 89 | "end\n", 90 | "\n", 91 | "plot(p)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": { 98 | "collapsed": true 99 | }, 100 | "outputs": [], 101 | "source": [ 102 | "using LinearAlgebra\n", 103 | "p = let\n", 104 | "\tkochenderfer(x; a=1, b=1, c=4) = a*norm(x) + b*sin(c*atan(x[2], x[1]))\n", 105 | "\n", 106 | "\txdomain = ( -3, 3)\n", 107 | "\tydomain = ( -3, 3)\n", 108 | "\n", 109 | "\tplots = Plots.Plot[]\n", 110 | "\tpush!(plots, Plots.Contour(kochenderfer, xdomain, ydomain, levels=collect(-4:4), style=\"width=\\\\columnwidth\", xbins=101, ybins=101))\n", 111 | "\tAxis(plots, width=\"8cm\", height=\"8cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"contour/labels=false, axis equal, view={0}{90}\")\n", 112 | "end\n", 113 | "\n", 114 | "plot(p)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "metadata": { 121 | "collapsed": true 122 | }, 123 | "outputs": [], 124 | "source": [ 125 | "p = let\n", 126 | "michalewicz(x; m=10) = -sum(sin(v)*sin(i*v^2/π)^(2m) for (i,v) in enumerate(x))\n", 127 | "\n", 128 | "xdomain = (0, 4)\n", 129 | "ydomain = (0, 4)\n", 130 | "\n", 131 | "plots = Plots.Plot[]\n", 132 | "# push!(plots, Plots.Contour(michalewicz, xdomain, ydomain, levels=[-1.8, -1.6, -1.4, -1.2, -1.0, -0.8, -0.6, -0.4, -0.2], style=\"width=\\\\columnwidth\", xbins=101, ybins=101))\n", 133 | "# Axis(plots, width=\"8cm\", height=\"8cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"contour/labels=false, axis equal, view={0}{90}\")\n", 134 | "Axis(Plots.Image((x,y)->michalewicz([x,y]), xdomain, ydomain, xbins=600, ybins=600, colormap = pasteljet, colorbar = false),\n", 135 | "xmin=xdomain[1], xmax=xdomain[2], ymin=ydomain[1], ymax=ydomain[2], width=\"8cm\", height=\"8cm\", style=\"view={0}{90}\", xlabel=L\"x_1\", ylabel=L\"x_2\")\n", 136 | "end\n", 137 | "\n", 138 | "plot(p)" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": { 145 | "collapsed": true 146 | }, 147 | "outputs": [], 148 | "source": [ 149 | "p = let\n", 150 | "\trosenbrock(x; a=1, b=5) = (a-x[1])^2 + b*(x[2] - x[1]^2)^2\n", 151 | "\n", 152 | "\txdomain = (-2, 2)\n", 153 | "\tydomain = (-2, 2)\n", 154 | "\n", 155 | "\tplots = Plots.Plot[]\n", 156 | "\tpush!(plots, Plots.Contour(rosenbrock, xdomain, ydomain, levels=[1,2,3,5,10,20,50,100], style=\"width=\\\\columnwidth\", xbins=100, ybins=100))\n", 157 | "\tAxis(plots, width=\"8cm\", height=\"8cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"contour/labels=false, axis equal, view={0}{90}\")\n", 158 | "end\n", 159 | "\n", 160 | "plot(p)" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": null, 166 | "metadata": { 167 | "collapsed": true 168 | }, 169 | "outputs": [], 170 | "source": [ 171 | "p = let\n", 172 | "\twheeler(x, a=1.5) = -exp(-(x[1]*x[2] - a)^2 -(x[2]-a)^2)\n", 173 | "\n", 174 | "\txdom = (-8,25)\n", 175 | "\tydom = (-3,6)\n", 176 | "\n", 177 | "\tAxis(Plots.Image((x,y)->wheeler([x,y]), xdom, ydom, xbins=1200, ybins=480, colormap = pasteljet, colorbar = false),\n", 178 | "\t xmin=xdom[1], xmax=xdom[2], ymin=ydom[1], ymax=ydom[2], width=\"12cm\", height=\"4.8cm\", style=\"view={0}{90}\", xlabel=L\"x_1\", ylabel=L\"x_2\")\n", 179 | "end\n", 180 | "\n", 181 | "plot(p)" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": { 188 | "collapsed": true 189 | }, 190 | "outputs": [], 191 | "source": [ 192 | "p = let\n", 193 | "\n", 194 | "\tf = x -> -exp(-(x[1]*x[2] - 1.5)^2 -(x[2]-1.5)^2)\n", 195 | "\n", 196 | "\txdomain = (0, 3)\n", 197 | "\tydomain = (0, 3)\n", 198 | "\n", 199 | "\tp_contour = Plots.Contour(f, xdomain, ydomain, style=\"width=\\\\columnwidth\", xbins=151, ybins=151) # , levels=[-15,-10,-5,-1,0,0.5,1,1.5,2,2.5,3]\n", 200 | "\n", 201 | "\tplots = Plots.Plot[]\n", 202 | "\tpush!(plots, p_contour)\n", 203 | "\tax = Axis(plots, ymin=0, xmin=0, ymax=3, xmax=3, width=\"9cm\", height=\"9cm\", xlabel=L\"x_1\", ylabel=L\"x_2\", style=\"contour/labels=false, axis equal, view={0}{90}\")\n", 204 | "end\n", 205 | "\n", 206 | "plot(p)" 207 | ] 208 | } 209 | ], 210 | "metadata": { 211 | "kernelspec": { 212 | "display_name": "Julia 1.5.1", 213 | "language": "julia", 214 | "name": "julia-1.5" 215 | }, 216 | "language_info": { 217 | "file_extension": ".jl", 218 | "mimetype": "application/julia", 219 | "name": "julia", 220 | "version": "1.5.1" 221 | } 222 | }, 223 | "nbformat": 4, 224 | "nbformat_minor": 2 225 | } 226 | -------------------------------------------------------------------------------- /uncertaintyprop.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Uncertaintyprop\n", 8 | "\n", 9 | "This notebook was automatically generated from the Algorithms for Optimization source code. Each cell generates a figure from the original text. While this code is not optimized for use in lectures, we provide it here to be adapted for such projects. We hope you find it useful." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "include(\"support_code.jl\");" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "using LinearAlgebra\n", 32 | "using Distributions\n", 33 | "using Optim\n", 34 | "using ForwardDiff\n", 35 | "import QuadGK: quadgk\n", 36 | "p = let\n", 37 | "\tf = (x,z1,z2) -> sin(x+z1)*cos(x+z2)\n", 38 | "\tdom = (-π,π)\n", 39 | "\n", 40 | "\tfunction taylor_approx(f, μ, ν, secondorder=false)\n", 41 | "\t\tμhat = f(μ)\n", 42 | "\t\t∇ = (z -> ForwardDiff.gradient(f, z))(μ)\n", 43 | "\t\tνhat = ∇.^2⋅ν\n", 44 | "\t\tif secondorder\n", 45 | "\t\t\tH = (z -> ForwardDiff.hessian(f, z))(μ)\n", 46 | "\t\t\tμhat += (diag(H)⋅ν)/2\n", 47 | "\t\t\tνhat += ν⋅(H.^2*ν)/2\n", 48 | "\t\tend\n", 49 | "\t\treturn (μhat, νhat)\n", 50 | "\tend\n", 51 | "\n", 52 | "\tμ_ = [0.0,0.0]\n", 53 | " ν = [0.1,0.2]\n", 54 | " P1 = Normal(μ_[1],sqrt(ν[1]))\n", 55 | " P2 = Normal(μ_[2],sqrt(ν[2]))\n", 56 | " P = MvNormal(μ_, sqrt.(ν))\n", 57 | "\n", 58 | " ϵ = 1e-6\n", 59 | " E = x -> quadgk(z1->quadgk(z2->f(x,z1,z2) *pdf(P2,z2), -Inf, Inf, atol=ϵ)[1]*pdf(P1,z1), -Inf,Inf, atol=ϵ)[1]\n", 60 | " V = x -> quadgk(z1->quadgk(z2->f(x,z1,z2)^2*pdf(P2,z2), -Inf, Inf, atol=ϵ)[1]*pdf(P1,z1), -Inf,Inf, atol=ϵ)[1] - E(x)^2\n", 61 | "\n", 62 | " G = GroupPlot(1,2, groupStyle=\"vertical sep=0.25cm, xlabels at=edge bottom, xticklabels at=edge bottom\", style=\"width=10cm, height=3cm, xmin=$(convert(Float64, dom[1])), xmax=$(convert(Float64, dom[2])), xlabel=\\$x\\$\")\n", 63 | " push!(G,\n", 64 | " Axis([\n", 65 | " Plots.Linear(E, dom, style=\"solid, thick, black, mark=none\"),\n", 66 | " Plots.Linear(x->taylor_approx(z->f(x,z[1],z[2]), μ_, ν)[1], dom, style=\"solid, pastelBlue, mark=none\"),\n", 67 | " Plots.Linear(x->taylor_approx(z->f(x,z[1],z[2]), μ_, ν, true)[1], dom, style=\"solid, pastelRed, mark=none\"),\n", 68 | " ], ylabel=\"mean\"))\n", 69 | " push!(G,\n", 70 | " Axis([\n", 71 | " Plots.Linear(V, dom, style=\"solid, thick, black, mark=none\", legendentry=\"true\"),\n", 72 | " Plots.Linear(x->taylor_approx(z->f(x,z[1],z[2]), μ_, ν)[2], dom, style=\"solid, pastelBlue, mark=none\", legendentry=\"first-order Taylor approx.\"),\n", 73 | " Plots.Linear(x->taylor_approx(z->f(x,z[1],z[2]), μ_, ν, true)[2], dom, style=\"solid, pastelRed, mark=none\", legendentry=\"second-order Taylor approx.\"),\n", 74 | " ], ylabel=\"variance\", ymin=0, style=\"scaled ticks=false, tick label style={/pgf/number format/fixed},legend style={draw=none, at={(0.5,-0.6)}, anchor=north, legend columns=-1, /tikz/every even column/.append style={column sep=0.5cm}}\"))\n", 75 | "end\n", 76 | "plot(p)" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "metadata": { 83 | "collapsed": true 84 | }, 85 | "outputs": [], 86 | "source": [ 87 | "using Optim\n", 88 | "using Distributions\n", 89 | "p = let\n", 90 | "\n", 91 | "\tf = (x,z1,z2) -> sin(x+z1)*cos(x+z2) + 2\n", 92 | "\tdom = (-0.9π,0.9π)\n", 93 | "\n", 94 | "\tν1 = 1.0\n", 95 | "\tν2 = 0.5\n", 96 | "\tP1 = Normal(0,sqrt(ν1))\n", 97 | "\tP2 = Normal(0,sqrt(ν2))\n", 98 | "\n", 99 | "\tn = 2\n", 100 | "\tk = 1\n", 101 | "\tϕ = (x,z1,z2) -> -k*log(f(x,z1,z2)) - logpdf(P1, z1) - logpdf(P2,z2)\n", 102 | "\n", 103 | "\tget_z_star = x -> optimize(z->ϕ(x,z[1],z[2]), zeros(n), Optim.BFGS()).minimizer\n", 104 | "\n", 105 | "\t∂1 = (x,z1,z2) -> -k*cos(x+z1)*cos(x+z2) / f(x,z1,z2) + z1\n", 106 | "\t∂2 = (x,z1,z2) -> k*sin(x+z1)*sin(x+z2) / f(x,z1,z2) + 2z2\n", 107 | "\t∂11 = (x,z1,z2) -> k*cos(x+z1)^2*cos(x+z2)^2 / f(x,z1,z2)^2 + k*sin(x+z1)*cos(x+z2)/f(x,z1,z2) + 1\n", 108 | "\t∂22 = (x,z1,z2) -> k*sin(x+z1)^2*sin(x+z2)^2 / f(x,z1,z2)^2 + k*sin(x+z1)*cos(x+z2)/f(x,z1,z2) + 2\n", 109 | "\t∂12 = (x,z1,z2) -> k*cos(x+z1)*sin(x+z2)/f(x,z1,z2) - k*sin(x+z1)*cos(x+z1)*sin(x+z2)*cos(x+z2)/f(x,z1,z2)^2\n", 110 | "\n", 111 | "\tH = (x,z1,z2) -> [∂11(x,z1,z2) ∂12(x,z1,z2); ∂12(x,z1,z2) ∂22(x,z1,z2)]\n", 112 | "\n", 113 | "\tϵ = 1e-5\n", 114 | "\tE = x -> quadgk(z1->quadgk(z2->f(x,z1,z2) *pdf(P2,z2), -Inf, Inf, atol=ϵ)[1]*pdf(P1,z1), -Inf,Inf, atol=ϵ)[1]\n", 115 | "\t_E = x -> begin\n", 116 | "\t zstar = get_z_star(x)\n", 117 | "\t Hstar = H(x, zstar[1], zstar[2])\n", 118 | "\t dHstar = det(Hstar)\n", 119 | "\t if dHstar > 0.0\n", 120 | "\t return clamp(exp(-ϕ(x, zstar[1], zstar[2]))*(2π)^(n/2)*dHstar^(-0.5), 1.0, 10.0)\n", 121 | "\t else\n", 122 | "\t return 0.0\n", 123 | "\t end\n", 124 | "\tend\n", 125 | "\n", 126 | "\tAxis(\n", 127 | "\t [\n", 128 | "\t Plots.Linear(E, dom, style=\"solid, black, mark=none\", xbins=101, legendentry=\"true\")\n", 129 | "\t Plots.Linear(_E, dom, style=\"solid, pastelBlue, mark=none\", xbins=251, legendentry=\"Laplace approx.\")\n", 130 | "\t ], xlabel=L\"x\", ylabel=L\"\\Expectation[x]\", width=\"8cm\", height=\"6cm\", style=\"enlarge x limits=false, legend cell align=left, legend style={draw=none}, legend pos=outer north east\",\n", 131 | "\t)\n", 132 | "end\n", 133 | "plot(p)" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": null, 139 | "metadata": { 140 | "collapsed": true 141 | }, 142 | "outputs": [], 143 | "source": [ 144 | "using Distributions\n", 145 | "using Polynomials\n", 146 | "using Random\n", 147 | "import QuadGK: quadgk\n", 148 | "\n", 149 | "p = let\n", 150 | "\tfunction hermite(i)\n", 151 | "\t p = Polynomial([1])\n", 152 | "\t x = Polynomial([0,1])\n", 153 | "\t for j in 2 : i\n", 154 | "\t p = x*p - derivative(p)\n", 155 | "\t end\n", 156 | "\t return p\n", 157 | "\tend\n", 158 | "\n", 159 | "\tbs = [hermite(i) for i in 1 : 4]\n", 160 | "\n", 161 | "\tZ = Normal(0.0, 1.0)\n", 162 | "\n", 163 | "\tfunction fobj(x::Float64, z::Float64=0.0)\n", 164 | "\t return 1 - exp(-(x+z-1)^2) - 2exp(-(x+z-3)^2)\n", 165 | "\tend\n", 166 | "\tE = x -> quadgk(z->fobj(x,z)*pdf(Z,z), -Inf,Inf)[1]\n", 167 | "\n", 168 | "\tfunction polyE(x::Float64; m::Int=7)\n", 169 | "\t zs = rand(Z,m)\n", 170 | "\t ys = [fobj(x,z) for z in zs]\n", 171 | "\t Bmat = [b(z) for z in zs, b in bs]\n", 172 | "\t θ = pinv(Bmat)*ys\n", 173 | "\t # return the mean\n", 174 | "\t return θ[1]\n", 175 | "\tend\n", 176 | "\n", 177 | "\tfunction polyEBounds(x::Float64, m::Int=7, m2::Int=1000)\n", 178 | "\t N = fit_mle(Normal, [polyE(x, m=m) for i in 1 : m2])\n", 179 | "\t return (quantile(N, 0.05), mean(N), quantile(N, 0.95))\n", 180 | "\tend\n", 181 | "\n", 182 | "\tdom = (-2,6)\n", 183 | "\n", 184 | "\tsample_counts = [10,30,50]\n", 185 | "\tG = GroupPlot(1, length(sample_counts), groupStyle=\"vertical sep=0.5cm, xlabels at=edge bottom, xticklabels at=edge bottom\",\n", 186 | "\t\t\t\t\t\tstyle=\"width=9.5cm, height=3cm, xlabel=\\$x\\$, ylabel={\\$\\\\Expectation[f \\\\mid x]\\$}, enlarge x limits=0, ymax=1.5, ymin=-1.5\")\n", 187 | "\n", 188 | "\tx_arr = range(dom[1], stop=dom[2], length=200)\n", 189 | "\tfor (i,m) in enumerate(sample_counts)\n", 190 | "\t\tRandom.seed!(0)\n", 191 | "\t\tp = Plots.Plot[]\n", 192 | "\t\tpush!(p, Plots.Linear(fobj, dom, xbins=100, style=\"solid, pastelRed, mark=none\", legendentry= i==1 ? \"noise-free\" : nothing))\n", 193 | "\t\tpush!(p, Plots.Linear(E, dom, xbins=100, style=\"solid, black, mark=none\", legendentry= i==1 ? \"exact\" : nothing))\n", 194 | "\n", 195 | "\t\tbounds = [polyEBounds(x, m, 1000) for x in x_arr]\n", 196 | "\t\tpush!(p, Plots.Linear(x_arr, [b[1] for b in bounds], style=\"name path=A, draw=none, mark=none, forget plot\"))\n", 197 | "\t\tpush!(p, Plots.Linear(x_arr, [b[3] for b in bounds], style=\"name path=B, draw=none, mark=none, forget plot\"))\n", 198 | "\t\tpush!(p, Plots.Command(\"\\\\addplot[pastelBlue!40] fill between[of=A and B];\"))\n", 199 | "\t\tif i == 1\n", 200 | "\t\t\tpush!(p, Plots.Command(\"\\\\addlegendentry{\\\\SI{95}{\\\\percent} interval}\"))\n", 201 | "\t\tend\n", 202 | "\t\tpush!(p, Plots.Linear(x_arr, [b[2] for b in bounds], style=\"solid, pastelBlue, mark=none\", legendentry= i==1 ? \"mean\" : nothing))\n", 203 | "\t\tpush!(p, Plots.Node(\"\\\\scriptsize \\$$m\\$ samples\", 0.02, 0.25, style=\"right\", axis=\"axis description cs\"))\n", 204 | "\t\tpush!(G, Axis(p))\n", 205 | "\tend\n", 206 | "\n", 207 | "\tG.axes[1].style = \"legend pos=outer north east\"\n", 208 | "\n", 209 | "\tG\n", 210 | "end\n", 211 | "plot(p)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": null, 217 | "metadata": { 218 | "collapsed": true 219 | }, 220 | "outputs": [], 221 | "source": [ 222 | "using Polynomials\n", 223 | "function legendre(i)\n", 224 | "\tn = i-1\n", 225 | " p = Polynomial([-1,0,1])^n\n", 226 | " for i in 1 : n\n", 227 | " p = derivative(p)\n", 228 | " end\n", 229 | " return p / (2^n * factorial(n))\n", 230 | "end\n", 231 | "function laguerre(i)\n", 232 | " p = Polynomial([1])\n", 233 | " for j in 2 : i\n", 234 | " p = integrate(derivative(p) - p)\n", 235 | " end\n", 236 | " return p\n", 237 | "end\n", 238 | "function hermite(i)\n", 239 | " p = Polynomial([1])\n", 240 | " x = Polynomial([0,1])\n", 241 | " for j in 2 : i\n", 242 | " p = x*p - derivative(p)\n", 243 | " end\n", 244 | " return p\n", 245 | "end\n", 246 | "\n", 247 | "p = begin\n", 248 | "\tG = GroupPlot(3,1,groupStyle=\"horizontal sep=0.75cm, vertical sep=1.5cm, xlabels at=edge bottom, xticklabels at=edge bottom\",\n", 249 | "\t style=\"cycle list name = pastelcolors, width=6cm, every axis title/.style={font=\\\\footnotesize, at={(0.5,1.1)}, align=center}\")\n", 250 | "\n", 251 | "\tp = Plots.Plot[]\n", 252 | "\tfor i in 1 : 6\n", 253 | "\t\tL = legendre(i)\n", 254 | "\t push!(p, Plots.Linear(x->L(x), (-1, 1), xbins=201, style=\"solid, mark=none\"))\n", 255 | "\tend\n", 256 | "\tpush!(G, Axis(p, xlabel=L\"z\", title=\"Legendre\", xmin=-1, xmax=1))\n", 257 | "\n", 258 | "\tp = Plots.Plot[]\n", 259 | "\tfor i in 1 : 6\n", 260 | "\t\tL = laguerre(i)\n", 261 | "\t push!(p, Plots.Linear(x->L(x), (0, 15), xbins=201, style=\"solid, mark=none\"))\n", 262 | "\tend\n", 263 | "\tpush!(G, Axis(p, xlabel=L\"z\", title=\"Laguerre\", ymin=-20, ymax=20, xmin=0, xmax=15))\n", 264 | "\n", 265 | "\tp = Plots.Plot[]\n", 266 | "\tfor i in 1 : 6\n", 267 | "\t H = hermite(i)\n", 268 | "\t push!(p, Plots.Linear(z->clamp(H(z), -17, 17), (-5, 5), xbins=201, style=\"solid, mark=none\", legendentry=\"\\$b_$i\\$\"))\n", 269 | "\tend\n", 270 | "\tpush!(G, Axis(p, xlabel=L\"z\", style=\"legend pos=outer north east\", title=\"Hermite\", xmin=-5, xmax=5, ymin=-15, ymax=15))\n", 271 | "\n", 272 | "\tG\n", 273 | "end\n", 274 | "\n", 275 | "plot(p)" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": null, 281 | "metadata": { 282 | "collapsed": true 283 | }, 284 | "outputs": [], 285 | "source": [ 286 | "using QuadGK\n", 287 | "import QuadGK: quadgk\n", 288 | "import Printf: @sprintf\n", 289 | "\n", 290 | "p = let\n", 291 | "\tf = z -> sin(z*π)\n", 292 | "\tX = [[-1], [-0.2], [0.3], [0.7], [0.9]]\n", 293 | "\ty = [f(x[1]) for x in X]\n", 294 | "\n", 295 | "\tvar_term(b, ρ, dom) = quadgk(x->b(x)^2*ρ(x), dom...)[1]\n", 296 | "\tvar_term_legendre(i) = var_term(legendre(i), x->1/2, [-1,1])\n", 297 | "\tfunction get_var(θs, var_terms)\n", 298 | "\t μ_ = θs[1]\n", 299 | "\t return θs.^2⋅var_terms - μ_^2\n", 300 | "\tend\n", 301 | "\n", 302 | "\n", 303 | "\tfunction get_plot(i)\n", 304 | "\t bases = [z->legendre(j)(z[1]) for j in 1 : i]\n", 305 | "\t var_terms = [var_term_legendre(j) for j in 1 : i]\n", 306 | "\t B = [b(x) for x in X, b in bases]\n", 307 | "\t θ = pinv(B)*y\n", 308 | "\t fhat = z -> sum(θ[j] * bases[j](z) for j in 1 : i)\n", 309 | "\t ν = get_var(θ, var_terms)\n", 310 | "\t return Plots.Linear(z->fhat([z]), (-1,1), style=\"solid, mark=none\", legendentry=\"\\$i = $i \\\\quad \\\\mu = $(@sprintf(\"%+6.3f\", θ[1])) \\\\quad \\\\nu = $(@sprintf(\"%6.3f\", ν))\\$\")\n", 311 | "\tend\n", 312 | "\n", 313 | "\tp = Plots.Plot[]\n", 314 | "\tpush!(p, Plots.Linear(f, (-1,1), style=\"solid, black, mark=none\", legendentry=\"true\"))\n", 315 | "\tfor i in 1 : 5\n", 316 | "\t push!(p, get_plot(i))\n", 317 | "\tend\n", 318 | "\tpush!(p, Plots.Scatter([x[1] for x in X], y, style=\"solid, mark=*, mark size=1, mark options={draw=black, fill=black}\"))\n", 319 | "\tAxis(p, style=\"legend pos=outer north east, cycle list name = pastelcolors\", xlabel=L\"z\", width=\"7.5cm\", xmin=-1, xmax=1)\n", 320 | "end\n", 321 | "plot(p)" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": null, 327 | "metadata": { 328 | "collapsed": true 329 | }, 330 | "outputs": [], 331 | "source": [ 332 | "using Distributions\n", 333 | "using Polynomials\n", 334 | "\n", 335 | "p = let\n", 336 | "\tf = z -> sin(z*π)\n", 337 | "\tX = [[2.1], [2.5], [3.3], [4.7], [3.9]]\n", 338 | "\ty = [f(x[1]) for x in X]\n", 339 | "\n", 340 | "\tdom = (2,5)\n", 341 | "\tP = TruncatedNormal(3, 1, dom...)\n", 342 | "\tρ = z -> pdf(P, z)\n", 343 | "\n", 344 | "\tvar_term(b, ρ, dom) = begin\n", 345 | "\t\tb2 = b*b\n", 346 | "\t\tquadgk(x->b2(x)*ρ(x), dom...)[1]\n", 347 | "\tend\n", 348 | "\tfunction get_var(θs, var_terms)\n", 349 | "\t μ_ = θs[1]\n", 350 | "\t return θs.^2⋅var_terms - μ_^2\n", 351 | "\tend\n", 352 | "\tfunction orthogonal_recurrence(bs, p, dom, ϵ=1e-6)\n", 353 | "\t i = length(bs)\n", 354 | "\t c1 = quadgk(z->z*bs[i](z)^2*p(z), dom..., atol=ϵ)[1]\n", 355 | "\t c2 = quadgk(z-> bs[i](z)^2*p(z), dom..., atol=ϵ)[1]\n", 356 | "\t α = c1 / c2\n", 357 | "\t if i > 1\n", 358 | "\t \tc3 = quadgk(z->bs[i-1](z)^2*p(z), dom..., atol=ϵ)[1]\n", 359 | "\t \tβ = c2 / c3\n", 360 | "\t \treturn Polynomial([-α, 1])*bs[i] - β*bs[i-1]\n", 361 | "\t else\n", 362 | "\t \treturn Polynomial([-α, 1])*bs[i]\n", 363 | "\t\tend\n", 364 | "\tend\n", 365 | "\n", 366 | "\tbs = [Polynomial([1.0])]\n", 367 | "\tvar_terms = [var_term(bs[end], ρ, dom)]\n", 368 | "\n", 369 | "\tp = Plots.Plot[]\n", 370 | "\tpush!(p, Plots.Linear(f, dom, style=\"solid, black, mark=none\", legendentry=\"true\"))\n", 371 | "\tfor i in 1 : 5\n", 372 | "\n", 373 | "\t B = [b(x[1]) for x in X, b in bs]\n", 374 | "\t θ = pinv(B)*y\n", 375 | "\t fhat = z -> sum(θ[j] * bs[j](z) for j in 1 : i)\n", 376 | "\t ν = get_var(θ, var_terms)\n", 377 | "\n", 378 | "\t push!(p, Plots.Linear(z->fhat(z), dom, style=\"solid, mark=none\", legendentry=\"\\$i = $i \\\\quad \\\\mu = $(@sprintf(\"%+6.3f\", θ[1])) \\\\quad \\\\nu = $(@sprintf(\"%6.3f\", ν))\\$\"))\n", 379 | "\n", 380 | "\t push!(bs, orthogonal_recurrence(bs, ρ, dom))\n", 381 | "\t push!(var_terms, var_term(bs[end], ρ, dom))\n", 382 | "\tend\n", 383 | "\tpush!(p, Plots.Scatter([x[1] for x in X], y, style=\"solid, mark=*, mark size=1, mark options={draw=black, fill=black}\"))\n", 384 | "\tAxis(p, style=\"legend pos=outer north east, cycle list name = pastelcolors\", xlabel=L\"z\", width=\"7.5cm\", xmin=dom[1], xmax=dom[2])\n", 385 | "end\n", 386 | "plot(p)" 387 | ] 388 | }, 389 | { 390 | "cell_type": "code", 391 | "execution_count": null, 392 | "metadata": { 393 | "collapsed": true 394 | }, 395 | "outputs": [], 396 | "source": [ 397 | "if !@isdefined(GaussianProcess)\n", 398 | "\tinclude(\"../chapter/gp.jl\")\n", 399 | "end\n", 400 | "\n", 401 | "p = let\n", 402 | "\tf = (x,z) -> sin(x+z[1])*cos(x+z[2])\n", 403 | "\tw = [1.0,1.0]\n", 404 | "\tμz = [0.0,0.0]\n", 405 | "\tΣz = Matrix(Diagonal([1.0,0.5]))\n", 406 | "\tD = MvNormal(μz, Σz)\n", 407 | "\n", 408 | "\tfunction bayesian_monte_carlo(GP, w, μz, Σz)\n", 409 | "\t\tW = Matrix(Diagonal((w.^2)))\n", 410 | "\t\tinvK = inv(K(GP.X, GP.X, GP.k))\n", 411 | "\t\tq = [exp(-0.5*((z-μz)'inv(W+Σz)*(z-μz))[1]) for z in GP.X]\n", 412 | "\t\tq .*= (det(W\\Σz + I))^(-0.5)\n", 413 | "\t\tμ = q'*invK*GP.y\n", 414 | "\t\tν = (det(2W\\Σz + I))^(-0.5) - (q'*invK*q)[1]\n", 415 | "\t\treturn (μ, ν)\n", 416 | "\tend\n", 417 | "\n", 418 | "\tdom = (-0.9π,0.9π)\n", 419 | "\tϵ = 1e-5\n", 420 | "\tP1 = Normal(0,sqrt(1.0))\n", 421 | "\tP2 = Normal(0,sqrt(1/2))\n", 422 | "\tE = x -> QuadGK.quadgk(z1->QuadGK.quadgk(z2->f(x,[z1,z2]) *pdf(P2,z2), -Inf, Inf, atol=ϵ)[1]*pdf(P1,z1), -Inf,Inf, atol=ϵ)[1]\n", 423 | "\n", 424 | "\tx_arr = collect(range(dom[1], stop=dom[2], length=251))\n", 425 | "\tsamples = [[rand(D) for j in 1 : 10] for i in 1 : length(x_arr)]\n", 426 | "\n", 427 | "\ty_bmc = Array{Float64}(undef, length(x_arr))\n", 428 | "\tfor i in 1 : length(x_arr)\n", 429 | "\t\tx = x_arr[i]\n", 430 | "\t\tGP = GaussianProcess(k = (x,x′) ->\n", 431 | "\t\t exp(-0.5*sum((x[i]-x′[i])^2/w[i]^2 for i in 1 : length(x)))\n", 432 | "\t\t)\n", 433 | "\t\tfor z in samples[i]\n", 434 | "\t\t push!(GP, z, f(x,z))\n", 435 | "\t\tend\n", 436 | "\t\ty_bmc[i] = bayesian_monte_carlo(GP, w, μz, Σz)[1]\n", 437 | "\tend\n", 438 | "\n", 439 | "\ty_mean = [mean(f(x,z) for z in zs) for (x,zs) in zip(x_arr, samples)]\n", 440 | "\n", 441 | "\tAxis(\n", 442 | "\t [\n", 443 | "\t Plots.Linear(E, dom, style=\"solid, black, mark=none\", xbins=101, legendentry=\"true\")\n", 444 | "\t Plots.Linear(x_arr, y_mean, style=\"solid, pastelRed, mark=none\", legendentry=\"sample mean\")\n", 445 | "\t Plots.Linear(x_arr, y_bmc, style=\"solid, pastelBlue, mark=none\", legendentry=\"Bayesian MC\")\n", 446 | "\t ], xlabel=L\"x\", ylabel=L\"\\Expectation[f \\mid x]\", width=\"8cm\", height=\"6cm\", style=\"enlarge x limits=false, legend cell align=left, legend style={draw=none}, legend pos=outer north east\",\n", 447 | "\t)\n", 448 | "end\n", 449 | "plot(p)" 450 | ] 451 | } 452 | ], 453 | "metadata": { 454 | "kernelspec": { 455 | "display_name": "Julia 1.5.1", 456 | "language": "julia", 457 | "name": "julia-1.5" 458 | }, 459 | "language_info": { 460 | "file_extension": ".jl", 461 | "mimetype": "application/julia", 462 | "name": "julia", 463 | "version": "1.5.1" 464 | } 465 | }, 466 | "nbformat": 4, 467 | "nbformat_minor": 2 468 | } 469 | --------------------------------------------------------------------------------