├── .gitattributes
├── .gitignore
├── 02-Unconstrained_Optimization.Rmd
├── 03-One_Dimensional_Optimization.Rmd
├── 04-Line_Search_Descent_Methods.Rmd
├── 05-Conjugate_Gradient_Methods.Rmd
├── 06-Quasi_Newton_Methods.Rmd
├── CODE_OF_CONDUCT.md
├── README.md
├── _bookdown.yml
├── _bookdown_files
└── bookdownproj_files
│ └── figure-html
│ ├── unnamed-chunk-1-1.png
│ ├── unnamed-chunk-1-3.png
│ ├── unnamed-chunk-10-1.png
│ ├── unnamed-chunk-10-2.png
│ ├── unnamed-chunk-10-3.png
│ ├── unnamed-chunk-10-5.png
│ ├── unnamed-chunk-2-1.png
│ ├── unnamed-chunk-2-11.png
│ ├── unnamed-chunk-2-13.png
│ ├── unnamed-chunk-2-15.png
│ ├── unnamed-chunk-2-17.png
│ ├── unnamed-chunk-2-19.png
│ ├── unnamed-chunk-2-21.png
│ ├── unnamed-chunk-2-23.png
│ ├── unnamed-chunk-2-25.png
│ ├── unnamed-chunk-2-27.png
│ ├── unnamed-chunk-2-29.png
│ ├── unnamed-chunk-2-3.png
│ ├── unnamed-chunk-2-31.png
│ ├── unnamed-chunk-2-33.png
│ ├── unnamed-chunk-2-35.png
│ ├── unnamed-chunk-2-5.png
│ ├── unnamed-chunk-2-7.png
│ ├── unnamed-chunk-2-9.png
│ ├── unnamed-chunk-6-1.png
│ ├── unnamed-chunk-6-11.png
│ ├── unnamed-chunk-6-13.png
│ ├── unnamed-chunk-6-15.png
│ ├── unnamed-chunk-6-17.png
│ ├── unnamed-chunk-6-19.png
│ ├── unnamed-chunk-6-21.png
│ ├── unnamed-chunk-6-23.png
│ ├── unnamed-chunk-6-25.png
│ ├── unnamed-chunk-6-27.png
│ ├── unnamed-chunk-6-29.png
│ ├── unnamed-chunk-6-3.png
│ ├── unnamed-chunk-6-31.png
│ ├── unnamed-chunk-6-33.png
│ ├── unnamed-chunk-6-35.png
│ ├── unnamed-chunk-6-37.png
│ ├── unnamed-chunk-6-39.png
│ ├── unnamed-chunk-6-41.png
│ ├── unnamed-chunk-6-43.png
│ ├── unnamed-chunk-6-45.png
│ ├── unnamed-chunk-6-47.png
│ ├── unnamed-chunk-6-48.png
│ ├── unnamed-chunk-6-49.png
│ ├── unnamed-chunk-6-5.png
│ ├── unnamed-chunk-6-51.png
│ ├── unnamed-chunk-6-53.png
│ ├── unnamed-chunk-6-55.png
│ ├── unnamed-chunk-6-57.png
│ ├── unnamed-chunk-6-59.png
│ ├── unnamed-chunk-6-61.png
│ ├── unnamed-chunk-6-63.png
│ ├── unnamed-chunk-6-65.png
│ ├── unnamed-chunk-6-7.png
│ └── unnamed-chunk-6-9.png
├── _output.yml
├── book-description.html
├── docs
├── a-caucus-race-and-a-long-tale.html
├── bookdownproj_files
│ └── figure-html
│ │ ├── unnamed-chunk-1-1.png
│ │ ├── unnamed-chunk-1-3.png
│ │ ├── unnamed-chunk-10-1.png
│ │ ├── unnamed-chunk-10-2.png
│ │ ├── unnamed-chunk-10-3.png
│ │ ├── unnamed-chunk-10-5.png
│ │ ├── unnamed-chunk-2-1.png
│ │ ├── unnamed-chunk-2-11.png
│ │ ├── unnamed-chunk-2-13.png
│ │ ├── unnamed-chunk-2-15.png
│ │ ├── unnamed-chunk-2-17.png
│ │ ├── unnamed-chunk-2-19.png
│ │ ├── unnamed-chunk-2-21.png
│ │ ├── unnamed-chunk-2-23.png
│ │ ├── unnamed-chunk-2-25.png
│ │ ├── unnamed-chunk-2-27.png
│ │ ├── unnamed-chunk-2-29.png
│ │ ├── unnamed-chunk-2-3.png
│ │ ├── unnamed-chunk-2-31.png
│ │ ├── unnamed-chunk-2-33.png
│ │ ├── unnamed-chunk-2-35.png
│ │ ├── unnamed-chunk-2-5.png
│ │ ├── unnamed-chunk-2-7.png
│ │ ├── unnamed-chunk-2-9.png
│ │ ├── unnamed-chunk-6-1.png
│ │ ├── unnamed-chunk-6-11.png
│ │ ├── unnamed-chunk-6-13.png
│ │ ├── unnamed-chunk-6-15.png
│ │ ├── unnamed-chunk-6-17.png
│ │ ├── unnamed-chunk-6-19.png
│ │ ├── unnamed-chunk-6-21.png
│ │ ├── unnamed-chunk-6-23.png
│ │ ├── unnamed-chunk-6-25.png
│ │ ├── unnamed-chunk-6-27.png
│ │ ├── unnamed-chunk-6-29.png
│ │ ├── unnamed-chunk-6-3.png
│ │ ├── unnamed-chunk-6-31.png
│ │ ├── unnamed-chunk-6-33.png
│ │ ├── unnamed-chunk-6-35.png
│ │ ├── unnamed-chunk-6-37.png
│ │ ├── unnamed-chunk-6-39.png
│ │ ├── unnamed-chunk-6-41.png
│ │ ├── unnamed-chunk-6-43.png
│ │ ├── unnamed-chunk-6-45.png
│ │ ├── unnamed-chunk-6-47.png
│ │ ├── unnamed-chunk-6-49.png
│ │ ├── unnamed-chunk-6-5.png
│ │ ├── unnamed-chunk-6-51.png
│ │ ├── unnamed-chunk-6-53.png
│ │ ├── unnamed-chunk-6-55.png
│ │ ├── unnamed-chunk-6-57.png
│ │ ├── unnamed-chunk-6-59.png
│ │ ├── unnamed-chunk-6-61.png
│ │ ├── unnamed-chunk-6-63.png
│ │ ├── unnamed-chunk-6-65.png
│ │ ├── unnamed-chunk-6-7.png
│ │ └── unnamed-chunk-6-9.png
├── conjugate-gradient-methods-1.html
├── contributor-covenant-code-of-conduct.html
├── img 1.png
├── img 10.png
├── img 11.png
├── img 12.png
├── img 13.png
├── img 14.png
├── img 15.png
├── img 16.png
├── img 17.png
├── img 18.png
├── img 19.png
├── img 2.png
├── img 20.png
├── img 21.png
├── img 22.png
├── img 23.png
├── img 24.png
├── img 25.png
├── img 26.png
├── img 27.png
├── img 3.png
├── img 4.png
├── img 5.png
├── img 6.png
├── img 7.png
├── img 8.png
├── img 9.png
├── index.html
├── introduction-to-unconstrained-optimization.html
├── libs
│ ├── accessible-code-block-0.0.1
│ │ └── empty-anchor.js
│ ├── anchor-sections-1.0.1
│ │ ├── anchor-sections.css
│ │ └── anchor-sections.js
│ ├── gitbook-2.6.7
│ │ ├── css
│ │ │ ├── fontawesome
│ │ │ │ └── fontawesome-webfont.ttf
│ │ │ ├── plugin-bookdown.css
│ │ │ ├── plugin-clipboard.css
│ │ │ ├── plugin-fontsettings.css
│ │ │ ├── plugin-highlight.css
│ │ │ ├── plugin-search.css
│ │ │ ├── plugin-table.css
│ │ │ └── style.css
│ │ └── js
│ │ │ ├── app.min.js
│ │ │ ├── clipboard.min.js
│ │ │ ├── jquery.highlight.js
│ │ │ ├── lunr.js
│ │ │ ├── plugin-bookdown.js
│ │ │ ├── plugin-clipboard.js
│ │ │ ├── plugin-fontsettings.js
│ │ │ ├── plugin-search.js
│ │ │ └── plugin-sharing.js
│ └── jquery-2.2.3
│ │ └── jquery.min.js
├── licence.html
├── line-search-descent-methods.html
├── quasi-newton-methods.html
├── reference-keys.txt
├── search_index.json
├── solving-one-dimensional-optimization-problems.html
├── style.css
├── the-pool-of-tears.html
├── what-is-numerical-optimization-1.html
└── what-is-numerical-optimization.html
├── fibonaccisearch.csv
├── img 1.png
├── img 10.png
├── img 11.png
├── img 12.png
├── img 13.png
├── img 14.png
├── img 15.png
├── img 16.png
├── img 17.png
├── img 18.png
├── img 19.png
├── img 2.png
├── img 20.png
├── img 21.png
├── img 22.png
├── img 23.png
├── img 24.png
├── img 25.png
├── img 26.png
├── img 27.png
├── img 3.png
├── img 4.png
├── img 5.png
├── img 6.png
├── img 7.png
├── img 8.png
├── img 9.png
├── index.Rmd
└── style.css
/.gitattributes:
--------------------------------------------------------------------------------
1 | *.css linguist-detectable=false
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .Rhistory
2 | .RData
3 | .Rproj.user
4 | *.Rproj
5 |
--------------------------------------------------------------------------------
/02-Unconstrained_Optimization.Rmd:
--------------------------------------------------------------------------------
1 | # Introduction to Unconstrained Optimization
2 |
3 | This chapter introduces what exactly an unconstrained optimization problem is. A detailed discussion of Taylor's Theorem is provided and has been use to study the first order and second order necessary and sufficient conditions for local minimizer in an unconstrained optimization tasks. Examples have been supplied too in view of understanding the necessary and sufficient conditions better. The Python package `scipy.optimize`, which will form an integral part in solving many optimization problems in the later chapters of this book, is introduced too. The chapter ends with an overview of how an algorithm to solve unconstrained minimization problem works, covering briefly two procedures: **line search descent method** and **trust region method**.
4 |
5 | ---
6 |
7 | ## The Unconstrained Optimization Problem
8 |
9 | As we have discussed in the first chapter, an unconstrained optimization problem deals with finding the local minimizer $\mathbf{x}^*$ of a real valued and smooth objective function $f(\mathbf{x})$ of $n$ variables, given by $f: \mathbb{R}^n \rightarrow \mathbb{R}$, formulated as,
10 |
11 | \begin{equation}
12 | \underset{\mathbf{x} \in \mathbb{R}^n}{\min f(\mathbf{x})} (\#eq:1)
13 | \end{equation}
14 | with no restrictions on the decision variables $\mathbf{x}$. We work towards computing $\mathbf{x}^*$, such that $\forall\ \mathbf{x}$ near $\mathbf{x}^*$, the following inequality is satisfied:
15 | \begin{equation}
16 | f(\mathbf{x}^*) \leq f(\mathbf{x}) (\#eq:2)
17 | \end{equation}
18 |
19 | ## Smooth Functions
20 |
21 | In terms of analysis, the measure of the number of continuous derivative a function has, characterizes the *smoothness* of a function.
22 |
23 | ```{definition}
24 | A function $f$ is *smooth* if it can be differentiated everywhere, i.e, the function has continuous derivatives up to some desired order over particular domain [Weisstein, Eric W. "Smooth Function." https://mathworld.wolfram.com/SmoothFunction.html].
25 | ```
26 |
27 | Some examples of smooth functions are , $f(x) = x$, $f(x)=e^x$, $f(x)=\sin(x)$, etc. To study the local minima $\mathbf{x}^*$ of a smooth objective function $f(\mathbf{x})$, we emphasize on *Taylor's theorem for a multivariate function*, thus focusing on the computations of the gradient vector $\nabla f(\mathbf{x})$ and the Hessian matrix $\mathbf{H} f(\mathbf{x})$.
28 |
29 | ## Taylor's Theorem
30 |
31 | ```{theorem}
32 | For a smooth function of a single variable given by $f: \mathbb{R} \rightarrow \mathbb{R}$, $m(\geq 1)$ times differentiable at the point $p \in \mathbb{R}$, there exists a function $j_m: \mathbb{R} \rightarrow \mathbb{R}$, such that the following equations are satisfied:
33 | \begin{align}
34 | f(x) &= f(p) + (x - p)f^{'}(p) + \frac{(x-p)^2}{2!}f^{''}(p) + \ldots \\ &+ \frac{(x-p)^m}{m!}f^m(p) + (x-p)^m j_m(x) (\#eq:3)
35 | \end{align}
36 | and
37 | \begin{equation}
38 | \lim_{x \to p}j_m(x)=0 (\#eq:4)
39 | \end{equation}
40 | ```
41 |
42 | The $m-$th order Taylor polynomial of the function $f$ around the point $p$ is given by:
43 | \begin{align}
44 | P_m(x)&=f(p) + (x - p)f^{'}(p) + \frac{(x-p)^2}{2!}f^{''}(p) + \ldots \\ &+ \frac{(x-p)^m}{m!}f^m(p) (\#eq:5)
45 | \end{align}
46 |
47 | Now let $f$ be a smooth, continuously differentiable function that takes in multiple variables, i.e, $f: \mathbb{R}^n \rightarrow \mathbb{R}$ and $\mathbf{x}, \mathbf{p}, \mathbf{\delta} \in \mathbb{R}^n$, where $\mathbf{\delta}$ is the direction in which the line $\mathbf{x} = \mathbf{p}+\alpha \mathbf{\delta}$ passes through the point $\mathbf{p}$ [*Snyman, Jan A. Practical mathematical optimization. Springer Science+ Business Media, Incorporated, 2005.*]. Here, $\alpha \in [0,1]$. We have,
48 | \begin{equation}
49 | f(\mathbf{x}) = f(\mathbf{p} + \alpha \mathbf{\delta}) (\#eq:6)
50 | \end{equation}
51 |
52 | From the definition of the *directional derivative*, we get,
53 | \begin{equation}
54 | \frac{df(\mathbf{x})}{d\alpha}|\mathbf{\delta} = \nabla^T f(\mathbf{x})\mathbf{\delta}=\hat{f}(\mathbf{x}) (\#eq:7)
55 | \end{equation}
56 | Again, differentiating $\hat{f}(\mathbf{x})$ with respect to $\alpha$.
57 |
58 | \begin{equation}
59 | \frac{d \hat{f}(\mathbf{x})}{d \alpha}|\mathbf{\delta} = \frac{d^2 f(\mathbf{x})}{d \alpha^2}=\nabla^T\hat{f}(\mathbf{x})\mathbf{\delta}=\mathbf{\delta}^T\mathbf{H}f(\mathbf{x})\mathbf{\delta} (\#eq:8)
60 | \end{equation}
61 |
62 | So, using equations \@ref(eq:5) and \@ref(eq:6) we can generate the Taylor expansion for a multivariable function at a point $\mathbf{p}$. So, around $\alpha = 0$, we get,
63 | \begin{equation}
64 | f(\mathbf{x}) = f(\mathbf{p}+\alpha \mathbf{\delta}) = f(\mathbf{p}) + \nabla^Tf(\mathbf{p})\alpha \mathbf{\delta} + \frac{1}{2}\alpha \mathbf{\delta}^T\mathbf{H} f(\mathbf{p})\alpha \mathbf{\delta} + \ldots (\#eq:9)
65 | \end{equation}
66 |
67 | The truncated Taylor expansion of the multivariable function, where the higher order terms are ignored, can be written as,
68 | \begin{equation}
69 | f(\mathbf{x}) = f(\mathbf{p}+\alpha \mathbf{\delta}) = f(\mathbf{p}) + \nabla^Tf(\mathbf{p})\alpha \mathbf{\delta} + \frac{1}{2}\alpha \mathbf{\delta}^T\mathbf{H} f(\mathbf{p}+\beta\mathbf{\delta})\alpha \mathbf{\delta} (\#eq:10)
70 | \end{equation}
71 | where, $\beta \in [0,1]$.
72 |
73 | ## Necessary and Sufficient Conditions for Local Minimizer in Unconstrained Optimization
74 |
75 | ### First-Order Necessary Condition
76 |
77 | If there exists a local minimizer $\mathbf{x}^*$ for a real-valued smooth function $f(\mathbf{x}): \mathbb{R}^n \rightarrow \mathbb{R}$, in an open neighborhood $\subset \mathbb{R}^n$ of $\mathbf{x}^*$ along the direction $\mathbf{\delta}$, then the *first order necessary condition* for the minimizer is given by:
78 | \begin{equation}
79 | \nabla^Tf(\mathbf{x}^*)\mathbf{\delta}=0\ \forall\ \mathbf{\delta} \neq 0 (\#eq:11)
80 | \end{equation}
81 | i.e, the *directional derivative* is $0$, which ultimately reduces to the equation:
82 | \begin{equation}
83 | \nabla f(\mathbf{x}^*)=0 (\#eq:12)
84 | \end{equation}
85 |
86 | ```{proof}
87 | Let a real-valued smooth function $f(\mathbf{x})$ be differentiable at the point $\mathbf{x}^* \in \mathbb{R}^n$. Using the *Taylor expansion*, we can write:
88 |
89 | \begin{equation}
90 | f(\mathbf{x})=f(\mathbf{x}^*) + \nabla^T f(\mathbf{x}^*) (\mathbf{x} - \mathbf{x}^*)+\sum_{|\gamma|\leq m}\frac{\mathfrak{D}^{\gamma}f(\mathbf{x}^*)}{\gamma!}(\mathbf{x}-\mathbf{x}^*)^{\gamma} + \sum_{|\gamma|=m}j_{\gamma}(\mathbf{x})(\mathbf{x} - \mathbf{x}^*)^{\gamma} (\#eq:13)
91 | \end{equation}
92 |
93 | where $\mathfrak{D}$ represents the differential and $m$ is the smoothness of the objective function $f$. Also $\lim_{\mathbf{x} \to \mathbf{x}^*}j_{\gamma(\mathbf{X})}=0$. Clubbing together the higher order terms, we can write Eq.\@ref(eq:13) as,
94 | \begin{equation}
95 | \label{eq:2.14}
96 | f(\mathbf{x})=f(\mathbf{x}^*) + \nabla^T f(\mathbf{x}^*) (\mathbf{x} - \mathbf{x}^*)+ \mathcal{O}(\|\mathbf{x} - \mathbf{x}^*\|) (\#eq:14)
97 | \end{equation}
98 |
99 | In this case,
100 | \begin{equation}
101 | \lim_{\mathbf{x} \to \mathbf{x}^*}\frac{\mathcal{O}(\|\mathbf{x} - \mathbf{x}^*\|)}{\|\mathbf{x} - \mathbf{x}^*\|}=0 (\#eq:15)
102 | \end{equation}
103 | Let us consider, $\mathbf{x} = \mathbf{x}^*-\beta \nabla f(\mathbf{x}^*)$, where $\beta \in [0,1]$. From Eq.\@ref(eq:14) we can write,
104 | \begin{equation}
105 | f(\mathbf{x}^*-\beta \nabla f(\mathbf{x}^*))=f(\mathbf{x}^*)-\beta\|\nabla f(\mathbf{x}^*)\|^2+\mathcal{O}(\beta\|\nabla f(\mathbf{x}^*)\|) (\#eq:16)
106 | \end{equation}
107 |
108 | Now, dividing Eq.\@ref(eq:16)-$f(\mathbf{x}^*)$ by $\beta$, we get,
109 |
110 | \begin{equation}
111 | \frac{f(\mathbf{x}^*-\beta \nabla f(\mathbf{x}^*))-f(\mathbf{x}^*)}{\beta} = -\|\nabla f(\mathbf{x}^*)\|^2 + \frac{\mathcal{O}(\beta \|\nabla f(\mathbf{x}^*)\|)}{\beta} \geq 0 (\#eq:17)
112 | \end{equation}
113 |
114 | Now, considering the limit $\beta \to 0^{+}$, we get,
115 | \begin{equation}
116 | -\|\nabla f(\mathbf{x}^*)\|^2 \leq 0 (\#eq:18)
117 | \end{equation}
118 |
119 | Combining which along with Eq.\@ref(eq:18), we get,
120 | \begin{equation}
121 | 0 \leq -\|\nabla f(\mathbf{x}^*)\|^2 \leq 0(\#eq:19)
122 | \end{equation}
123 | This ultimately gives $\nabla f(\mathbf{x}^*)=0$, proving the first-order necessary condition.
124 | ```
125 |
126 | ```{example}
127 | The **Rosenbrock function** of $n$-variables is given by:
128 | \begin{equation}
129 | f(\mathbf{x}) = \sum_{i=1}^{n-1}(100(x_{i+1}-x_i^2)^2 + (1-x_i)^2) (\#eq:20)
130 | \end{equation}
131 |
132 | where, $\mathbf{x} \in \mathbb{R}^n$. For this example let us consider the *Rosenbrock function* for two variables, given by:
133 | \begin{equation}
134 | f(\mathbf{x}) = 100(x_2-x_1^2)^2+(1-x_1)^2 (\#eq:21)
135 | \end{equation}
136 |
137 | We will show that the first order necessary condition is satisfied for the local minimizer $\mathbf{x^*}=\begin{bmatrix} 1 \\ 1 \end{bmatrix}$. We first check whether $\mathbf{x}^*$ is a minimizer or not. Putting $x_1=x_2=1$ in $f(\mathbf{x})$, we get $f(\mathbf{x})=0$. Now, we check whether the $\mathbf{x^*}$ satisfies the first order necessary condition. For that we calculate $\nabla f(\mathbf{x}^*)$.
138 | \begin{equation}
139 | \nabla f(\mathbf{x}^*) = \begin{bmatrix} -400x_1(x_2-x_1)^2-2(1-x_1) \\ 200(x_2-x_1^2)\end{bmatrix}_{\mathbf{x}^*} = \begin{bmatrix} 0 \\ 0 \end{bmatrix} (\#eq:22)
140 | \end{equation}
141 |
142 | So, we see that the first order necessary condition is satisfied. We can do similar analysis using the `scipy.optimize` package in Python. The Scipy official reference states that the `scipy.optimize` package provides the user with many commonly used optimization algorithms and test functions. It packages the following functionalities and aspects:
143 |
144 | * Minimization of multivariate scalar objective functions covering both the unconstrained and constrained domains, using a range of optimization algorithms,
145 | * Algorithms for minimization of scalar univariate functions,
146 | * A variety of brute-force optimization algorithms, also called global optimization algorithms,
147 | * Algorithms like minimization of least-squares and curve-fitting,
148 | * Root finding algorithms, and
149 | * Algorithms for solving multivariate equation systems.
150 |
151 | We will build upon the basic concepts of optimization using this package and cover most of the concepts one by one as we advance in the book. Note that, this is not the only package that we are going to use throughout the book. As we advance, we will be required to look into other Python resources according to our needs. But to keep in mind, `scipy.optimize` is the most important package that we will be using. Now, going back to the example, `scipy.optimize` already provides with the \mathbfit{Rosenbrock} test function, its gradient and its Hessian. We will import them first and check whether the given point $\mathbf{x}^*$ is a minimizer or not:
152 | ```
153 |
154 | ```{python}
155 | import numpy as np
156 | import scipy
157 | # Import the Rosenbrock function, its gradient and Hessian respectively
158 | from scipy.optimize import rosen, rosen_der, rosen_hess
159 | x_m = np.array([1, 1]) #given local minimizer
160 | rosen(x_m) # check whether x_m is a minimizer
161 | ```
162 |
163 | the result is $0.0$. So $\mathbf{x}^*$ is a minimizer. We then check for the first order necessary condition, using the gradient:
164 |
165 | ```{python}
166 | rosen_der(x_m) # calculates the gradient at the point x_m
167 | ```
168 |
169 | This matches with our calculations and also satisfies the first-order necessary condition.
170 |
171 | ### Second-Order Necessary Conditions
172 |
173 | If there exists a local minimizer $\mathbf{x}^*$ for a real-valued smooth function $f(\mathbf{x}): \mathbb{R}^n \rightarrow \mathbb{R}$, in an open neighborhood $\subset \mathbb{R}^n$ of $\mathbf{x}^*$ along the feasible direction $\mathbf{\delta}$, and $\mathbf{H} f(\mathbf{x})$ exists and is continuous in the open neighborhood, then the second order necessary conditions for the minimizer are given by:
174 | \begin{equation}
175 | \nabla^T f(\mathbf{x}^*)\mathbf{\delta} = 0, \forall\ \mathbf{\delta} \neq 0 (\#eq:23)
176 | \end{equation}
177 | and
178 | \begin{equation}
179 | \mathbf{\delta}^T\mathbf{H}f(\mathbf{x}^*)\mathbf{\delta} \geq 0, \forall\ \mathbf{\delta} \neq 0 (\#eq:24)
180 | \end{equation}
181 | which reduces to the following:
182 | \begin{equation}
183 | \nabla f(\mathbf{x}^*) = 0 (\#eq:25)
184 | \end{equation}
185 | and
186 | \begin{equation}
187 | \mathbf{\delta}\mathbf{H}f(\mathbf{x}^*) \geq 0 (\#eq:26)
188 | \end{equation}
189 | where equation Eq.\@ref(eq:26) means that the Hessian matrix should be positive semi-definite.
190 |
191 | ```{proof}
192 | From the proof of first order necessary condition, we know that $\nabla f(\mathbf{x}^*) = 0$, which satisfies equation Eq.\@ref(eq:25). Now for proving equation Eq.\@ref(eq:26), we use the Taylor series expansion again. We have,
193 | \begin{equation}
194 | f(\mathbf{x}) = f(\mathbf{x}^*)+\nabla^T f(\mathbf{x}^*)(\mathbf{x} - \mathbf{x}^*) + \frac{1}{2}\mathbf{H} f(\mathbf{x}^*)(\mathbf{x} - \mathbf{x}^*) + \mathcal{O}(\|\mathbf{x} - \mathbf{x}^*\|^2) (\#eq:27)
195 | \end{equation}
196 |
197 | Given, the feasible direction $\mathbf{\delta}$, we take $\beta \in [0,1]$, such that $\mathbf{x} = \mathbf{x}^* + \beta \mathbf{\delta}$. Putting this in Eq.\@ref(eq:27) and rearranging, we get,
198 | \begin{equation}
199 | f(\mathbf{x}^* + \beta \mathbf{\delta}) - f(\mathbf{x}^*) = \beta \nabla^T f(\mathbf{x}^*)\mathbf{\delta} + \frac{\beta^2}{2}\mathbf{\delta}^T \mathbf{H} f(\mathbf{x}^*)\mathbf{\delta} + \mathcal{O}(\beta^2) (\#eq:28)
200 | \end{equation}
201 |
202 | As, $\nabla f(\mathbf{x}^*)=0$, we have
203 | \begin{equation}
204 | f(\mathbf{x}^* + \beta \mathbf{\delta}) - f(\mathbf{x}^*) = \frac{\beta^2}{2}\mathbf{\delta}^T \mathbf{H} f(\mathbf{x}^*)\mathbf{\delta} + \mathcal{O}(\beta^2) (\#eq:29)
205 | \end{equation}
206 |
207 | Now, dividing Eq.\@ref(eq:29) by $\beta^2$, we have,
208 | \begin{equation}
209 | \frac{f(\mathbf{x}^* + \beta \mathbf{\delta}) - f(\mathbf{x}^*)}{\beta^2} = \frac{1}{2}\mathbf{\delta}^T \mathbf{H} f(\mathbf{x}^*)\mathbf{\delta} + \frac{\mathcal{O}(\beta^2)}{\beta^2} \geq 0 (\#eq:30)
210 | \end{equation}
211 |
212 | Taking the limit $\beta \to 0$, we have,
213 | \begin{equation}
214 | \mathbf{\delta}^T \mathbf{H} f(\mathbf{x}^*) \mathbf{\delta} \geq 0 (\#eq:31)
215 | \end{equation}
216 | Which ultimately reduces to,
217 | \begin{equation}
218 | \mathbf{H} f(\mathbf{x}^*) \geq 0 (\#eq:32)
219 | \end{equation}
220 | We see that the Hessian matrix is positive semi-definite. This completes the proof of the second order necessary conditions. [refer to chapter 1, optimality conditions]
221 | ```
222 |
223 | ### Second-Order Sufficient Conditions
224 |
225 | For a real-valued smooth objective function $f(\mathbf{x})$, if $\mathbf{x}^*$ is its local minimizer and $\mathbf{H} f(\mathbf{x})$ exists and is continuous in an open neighborhood $\subset \mathbb{R}^n$ of $\mathbf{x}^*$ along the feasible direction $\mathbf{\delta}$, then the conditions:
226 | \begin{equation}
227 | \nabla^T f(\mathbf{x}^*)\delta = 0 (\#eq:33)
228 | \end{equation}
229 | i.e,
230 | \begin{equation}
231 | \nabla f(\mathbf{x}^*) = 0 (\#eq:34)
232 | \end{equation}
233 | and
234 |
235 | \begin{equation}
236 | \mathbf{\delta}^T \mathbf{H} f(\mathbf{x}^*) \mathbf{\delta} > 0 (\#eq:35)
237 | \end{equation}
238 |
239 | i.e, a positive definite Hessian matrix imply that $\mathbf{x}^*$ is a strong local minimizer of $f(\mathbf{x})$.
240 |
241 | ```{proof}
242 | Let $\mathfrak{r} >0$ be a radius such that the Hessian of the objective function, $\mathbf{H} f(\mathbf{x})$ is positive definite $\forall\ \mathbf{x}$ in the open ball defined by $\mathfrak{B} = \{\mathbf{y} \mid \|\mathbf{y} - \mathbf{x}^*\| \leq \mathfrak{r}\}$. This comes from the fact that $\mathbf{H} f(\mathbf{x})$ is positive definite at the local minimizer $\mathbf{x}^*$. Now, considering a nonzero vector $\mathbf{\eta}$, such that $\|\mathbf{\eta}\| < \mathfrak{r}$ and $c \in [0,1]$, we will have, $\mathbf{x}=\mathbf{x}^*+c\mathbf{\eta} \in \mathfrak{B}$. Now, using the Taylor's expansion, i.e, Eq.\@ref(eq:27), we will have,
243 |
244 | \begin{equation}
245 | f(\mathbf{x}^* + \mathbf{\eta}) = f(\mathbf{x}^*) + c\nabla^Tf(\mathbf{x}^*)\mathbf{\eta} + \frac{c^2}{2}\mathbf{\eta}^T\mathbf{H} f(\mathbf{y})\mathbf{\eta} + \mathcal{O}(c^2) (\#eq:36)
246 | \end{equation}
247 |
248 | Since, $\nabla f(\mathbf{x}^*) = 0$, we have,
249 | \begin{equation}
250 | \frac{f(\mathbf{x}^* + \mathbf{\eta}) - f(\mathbf{x}^*)}{c^2} = \frac{1}{2}\mathbf{\eta}^T\mathbf{H}f(\mathbf{y})\mathbf{\eta}+\frac{\mathcal{O}(c^2)}{c^2} (\#eq:37)
251 | \end{equation}
252 |
253 | Here, $\mathbf{y} = \mathbf{x}^* + \beta \mathbf{\eta}$ with $\beta \in [0,1]$. As, $\mathbf{y} \in \mathfrak{B}$, we have,
254 | \begin{equation}
255 | \mathbf{\eta}^T\mathbf{H}f(\mathbf{y})\mathbf{\eta} > 0 (\#eq:38)
256 | \end{equation}
257 |
258 | So, taking the limit, $c \to 0$, we have,
259 | \begin{equation}
260 | f(\mathbf{x}^* + \mathbf{\eta}) - f(\mathbf{x}^*) > 0 (\#eq:39)
261 | \end{equation}
262 |
263 | this proves our claim that,
264 | \begin{equation}
265 | f(\mathbf{x}^* + \mathbf{\eta}) > f(\mathbf{x}^*) (\#eq:40)
266 | \end{equation}
267 | that is, $\mathbf{x}^*$ is a strict local minimizer of the objective function $f(\mathbf{x})$.
268 | ```
269 |
270 | ```{example}
271 | Let us now work with a new test function called Himmelblau's function, given by,
272 | \begin{equation}
273 | f(\mathbf{x}) = (x_1^2+x_2-11)^2+(x_1+x_2^2-7)^2 (\#eq:41)
274 | \end{equation}
275 |
276 | where, $\mathbf{x} \in \mathbb{R}^2$. We will check whether $\mathbf{x}^*=\begin{bmatrix} 3 \\ 2 \end{bmatrix}$ satisfies the second-order sufficient conditions satisfying the fact that it is a strong local minimizer. We will again use the `autograd` package to do the analyses for this objective function. Let us first define the function and the local minimizer as `x_star` in Python:
277 | ```
278 |
279 | ```{python}
280 | def Himm(x):
281 | return (x[0]**2 + x[1] - 11)**2 + (x[0] + x[1]**2 - 7)**2
282 |
283 | x_star = np.array([3, 2], dtype='float') #local minimizer
284 | ```
285 |
286 | We then check whether `x_star` is a minimizer.
287 |
288 | ```{python}
289 | print("function at x_star:", Himm(x_star))
290 | ```
291 |
292 | Now, we calculate the gradient vector and the Hessian matrix of the function at `x_star` and look at the results,
293 |
294 | ```{python}
295 | # import the necessary packages
296 | import autograd.numpy as au
297 | from autograd import grad, jacobian
298 |
299 | # gradient vector of the Himmelblau's function
300 | Himm_grad=grad(Himm)
301 | print("gradient vector at x_star:", Himm_grad(x_star))
302 |
303 | # Hessian matrix of the Himmelblau's function
304 | Himm_hess = jacobian(Himm_grad)
305 | M = Himm_hess(x_star)
306 | eigs = np.linalg.eigvals(M)
307 | print("The eigenvalues of M:", eigs)
308 |
309 | if (np.all(eigs>0)):
310 | print("M is positive definite")
311 | elif (np.all(eigs>=0)):
312 | print("M is positive semi-definite")
313 | else:
314 | print("M is negative definite")
315 | ```
316 |
317 | We see that `x1` satisfies the second order sufficient conditions and is a strong local minimizer. We wanted to perform the analyses using `autograd` package instead of `scipy.optimize`, because there might be cases when we need to use test functions that are not predefined in `scipy.optimize` package, unlike the `Rosenbrock function`.
318 |
319 | ## Algorithms for Solving Unconstrained Minimization Tasks
320 |
321 | An optimization algorithm for solving an unconstrained minimization problem requires an initial point $\mathbf{x}_0$ to start with. The choice of $\mathbf{x}_0$ depends either on the applicant who has some idea about the data and the task at hand or it can be set by the algorithm in charge. Starting with $\mathbf{x}_0$, the optimization algorithm iterates through a sequence of successive points $\{\mathbf{x}_0, \mathbf{x}_1, \ldots, \mathbf{x}_{\infty}\}$, which stops at the point where all the termination conditions are met for approximating the minimizer $\mathbf{x}^*$. The algorithm generates this sequence taking into consideration the objective function $f(\mathbf{x})$ at a particular point $f(\mathbf{x}_n)$. A new iterate $\mathbf{x}_{n+1}$ is added in the sequence if the condition $f(\mathbf{x}_{n+1}) < f(\mathbf{x}_n)$. Although in many special cases, the algorithm might fail to find a new point in each and every step following the above condition, it must satisfy that after some stipulated number $k$ of steps, the following condition is met: $$f(\mathbf{x}_{n+k}) < f(\mathbf{x}_n)$$. One of the important terminating conditions, for example, is to check whether the first order necessary condition is sufficiently accurate, for a smooth objective function, i.e, $\|\nabla f(\mathbf{x}_{\infty})\| < \epsilon$, where $\epsilon$ is the infinitesimal tolerance value. We will discuss these conditions further in the subsequent chapters.
322 |
323 | Fundamentally, there are two approaches available to generate $f(\mathbf{x}_{n+1})$ from $f(\mathbf{x}_n)$:
324 |
325 | * **Line Search Descent Method**: Using this method, the optimization algorithm first picks a direction $\mathbf{\delta}_n$ for the $n^{th}$ step and performs a search along this direction from the previous generated iterate $\mathbf{x}_{n-1}$ to find a new iterate $\mathbf{x}_n$ such that the condition $f(\mathbf{x}_n) < f(\mathbf{x}_{n-1})$ is satisfied. A direction $\mathbf{\delta}_n$ is selected for the next iterate if the following condition is satisfied:
326 |
327 | \begin{equation}
328 | \nabla^T f(\mathbf{x}_{n-1})\mathbf{\delta}_n < 0 (\#eq:42)
329 | \end{equation}
330 |
331 | i.e, if the directional derivative in the direction $\mathbf{\delta}_n$ is negative. Here $f$ is the objective function. In view of that, the algorithm then needs to ascertain a distance by which it has to move along the direction $\mathbf{\delta}_n$ to figure out $\mathbf{x}_n$. The distance $\beta >0$, which is called the *step length*, can be figured out by solving the one-dimensional minimization problem formulated as:
332 |
333 | \begin{equation}
334 | \underset{\beta > 0}{\min} \tilde{f}(\beta) = \underset{\beta > 0}{\min} f(\mathbf{x}_{n-1} + \beta \mathbf{\delta}_n) (\#eq:43)
335 |
336 | \end{equation}
337 |
338 | 
339 |
340 | * **Trust Region Method**: Using this method, the optimization algorithm develops a model function [refer to Nocedal & Wright], $M_n$, such that its behavior inside a boundary set around the current iterate $\mathbf{x}_n$ matches that of the objective function $f(\mathbf{x}_n)$ at that point. The model function is not expected to give a reasonable approximation to the behavior of the objective function at a point $\mathbf{x}_t$ which is far away from $\mathbf{x}_n$, i.e, not lying inside the boundary defined around $\mathbf{x}_n$. As a result, the algorithm obstructs the search for the minimizer of $M_n$ inside the boundary region, which is actually called the *trust region*, denoted by $\mathcal{T}$, before finding the step $\mathbf{\zeta}$, by solving the minimization problem formulated by:
341 |
342 | \begin{equation}
343 | \underset{\mathbf{\zeta}}{\min\ } M_n(\mathbf{x}_n+\mathbf{\zeta}),\text{ where }\mathbf{x}_n+\mathbf{\zeta}\in \mathcal{T} (\#eq:44)
344 | \end{equation}
345 | Using this $\mathbf{\zeta}$, if the decrease in the value of $f(\mathbf{x}_{n+1})$ from $f(\mathbf{x}_n)$ is not sufficient, it can be inferred that the selected trust region is unnecessarily large. The algorithm then reduces the size of $\mathcal{T}$ accordingly and re-solves the problem given by equation Eq.\@ref(eq:44). Most often, the trust region $\mathcal{T}$ is defined by a circle in case of a two dimensional problem or a sphere in case of a three dimensional problem of radius $\mathcal{T_r}>0$, which follows the condition $\|\mathbf{\zeta}\| \leq \mathcal{T_r}$. In special cases, the shape of the trust region might vary. The form of the model function is given by a quadratic function, given by,
346 | \begin{equation}
347 | M_n(\mathbf{x}_n+\mathbf{\zeta}) = f(\mathbf{x}_n) + \mathbf{\zeta}^T\nabla f(\mathbf{x}_n)+\frac{1}{2}\mathbf{\zeta}^T\mathbf{B} f(\mathbf{x}_n)\mathbf{\zeta} (\#eq:45)
348 | \end{equation}
349 | Where, $\mathbf{B}f(\mathbf{x}_n)$ is either the Hessian matrix $\mathbf{H}f(\mathbf{x}_n)$ or an approximation to it.
350 |
351 | Before moving into detailed discussions on line search descent methods and trust region methods in the later chapters, we will first deal with solving equation Eq.\@ref(eq:43) in the immediate next chapter, which is itself an unconstrained one dimensional minimization problem, where we have to solve for $$\underset{\beta>0}{\min\ }\tilde{f}(\beta)$$ and deduce the value of $\beta^*$, which is the minimizer for $\tilde{f}(\beta)$.
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Contributor Covenant Code of Conduct
2 |
3 | ## Our Pledge
4 |
5 | We as members, contributors, and leaders pledge to make participation in our
6 | community a harassment-free experience for everyone, regardless of age, body
7 | size, visible or invisible disability, ethnicity, sex characteristics, gender
8 | identity and expression, level of experience, education, socio-economic status,
9 | nationality, personal appearance, race, religion, or sexual identity
10 | and orientation.
11 |
12 | We pledge to act and interact in ways that contribute to an open, welcoming,
13 | diverse, inclusive, and healthy community.
14 |
15 | ## Our Standards
16 |
17 | Examples of behavior that contributes to a positive environment for our
18 | community include:
19 |
20 | * Demonstrating empathy and kindness toward other people
21 | * Being respectful of differing opinions, viewpoints, and experiences
22 | * Giving and gracefully accepting constructive feedback
23 | * Accepting responsibility and apologizing to those affected by our mistakes,
24 | and learning from the experience
25 | * Focusing on what is best not just for us as individuals, but for the
26 | overall community
27 |
28 | Examples of unacceptable behavior include:
29 |
30 | * The use of sexualized language or imagery, and sexual attention or
31 | advances of any kind
32 | * Trolling, insulting or derogatory comments, and personal or political attacks
33 | * Public or private harassment
34 | * Publishing others' private information, such as a physical or email
35 | address, without their explicit permission
36 | * Other conduct which could reasonably be considered inappropriate in a
37 | professional setting
38 |
39 | ## Enforcement Responsibilities
40 |
41 | Community leaders are responsible for clarifying and enforcing our standards of
42 | acceptable behavior and will take appropriate and fair corrective action in
43 | response to any behavior that they deem inappropriate, threatening, offensive,
44 | or harmful.
45 |
46 | Community leaders have the right and responsibility to remove, edit, or reject
47 | comments, commits, code, wiki edits, issues, and other contributions that are
48 | not aligned to this Code of Conduct, and will communicate reasons for moderation
49 | decisions when appropriate.
50 |
51 | ## Scope
52 |
53 | This Code of Conduct applies within all community spaces, and also applies when
54 | an individual is officially representing the community in public spaces.
55 | Examples of representing our community include using an official e-mail address,
56 | posting via an official social media account, or acting as an appointed
57 | representative at an online or offline event.
58 |
59 | ## Enforcement
60 |
61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be
62 | reported to the community leaders responsible for enforcement at
63 | Email.
64 | All complaints will be reviewed and investigated promptly and fairly.
65 |
66 | All community leaders are obligated to respect the privacy and security of the
67 | reporter of any incident.
68 |
69 | ## Enforcement Guidelines
70 |
71 | Community leaders will follow these Community Impact Guidelines in determining
72 | the consequences for any action they deem in violation of this Code of Conduct:
73 |
74 | ### 1. Correction
75 |
76 | **Community Impact**: Use of inappropriate language or other behavior deemed
77 | unprofessional or unwelcome in the community.
78 |
79 | **Consequence**: A private, written warning from community leaders, providing
80 | clarity around the nature of the violation and an explanation of why the
81 | behavior was inappropriate. A public apology may be requested.
82 |
83 | ### 2. Warning
84 |
85 | **Community Impact**: A violation through a single incident or series
86 | of actions.
87 |
88 | **Consequence**: A warning with consequences for continued behavior. No
89 | interaction with the people involved, including unsolicited interaction with
90 | those enforcing the Code of Conduct, for a specified period of time. This
91 | includes avoiding interactions in community spaces as well as external channels
92 | like social media. Violating these terms may lead to a temporary or
93 | permanent ban.
94 |
95 | ### 3. Temporary Ban
96 |
97 | **Community Impact**: A serious violation of community standards, including
98 | sustained inappropriate behavior.
99 |
100 | **Consequence**: A temporary ban from any sort of interaction or public
101 | communication with the community for a specified period of time. No public or
102 | private interaction with the people involved, including unsolicited interaction
103 | with those enforcing the Code of Conduct, is allowed during this period.
104 | Violating these terms may lead to a permanent ban.
105 |
106 | ### 4. Permanent Ban
107 |
108 | **Community Impact**: Demonstrating a pattern of violation of community
109 | standards, including sustained inappropriate behavior, harassment of an
110 | individual, or aggression toward or disparagement of classes of individuals.
111 |
112 | **Consequence**: A permanent ban from any sort of public interaction within
113 | the community.
114 |
115 | ## Attribution
116 |
117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage],
118 | version 2.0, available at
119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
120 |
121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct
122 | enforcement ladder](https://github.com/mozilla/diversity).
123 |
124 | [homepage]: https://www.contributor-covenant.org
125 |
126 | For answers to common questions about this code of conduct, see the FAQ at
127 | https://www.contributor-covenant.org/faq. Translations are available at
128 | https://www.contributor-covenant.org/translations.
129 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | This is the github repository for the online open source book **Introduction to Mathematical Optimization: with Python**. The link to the book: https://indrag49.github.io/Numerical-Optimization/
2 |
3 | *This repo was initially generated from a bookdown template available here: https://github.com/jtr13/bookdown-template.*
4 |
5 | # Book Description
6 |
7 | This is an open-source introductory book on numerical/mathematical optimization aimed at audience who want to start with the subject matter with hands on implementation of the methods with *Python*. I plan to make this an open source collaborative project which will grow with time with legitimate contributions from the comnmunity.
8 |
9 |
10 | This book aims to cover the following:
11 |
12 | - Introduction to Numerical Optimization
13 | - Introduction to Unconstrained Optimization
14 | - Solving one dimensional optimization problems
15 | - Line Search Descent Method
16 | - Conjugate Gradient Methods
17 | - Quasi-Newton Methods
18 |
19 | # Licence
20 |
21 | This work is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License.
22 |
23 |
24 |
25 |
26 |
27 |
--------------------------------------------------------------------------------
/_bookdown.yml:
--------------------------------------------------------------------------------
1 | book_filename: "bookdownproj"
2 | output_dir: docs
3 | delete_merged_file: true
4 | edit: https://github.com/indrag49/Numerical-Optimization/edit/master/%s
5 | view: https://github.com/indrag49/Numerical-Optimization/blob/master/%s
6 | language:
7 | ui:
8 | chapter_name: "Chapter "
9 |
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-1-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-1-1.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-1-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-1-3.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-10-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-10-1.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-10-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-10-2.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-10-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-10-3.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-10-5.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-10-5.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-1.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-11.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-11.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-13.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-13.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-15.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-15.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-17.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-17.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-19.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-19.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-21.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-21.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-23.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-23.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-25.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-25.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-27.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-27.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-29.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-29.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-3.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-31.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-31.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-33.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-33.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-35.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-35.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-5.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-5.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-7.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-7.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-9.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-2-9.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-1.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-11.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-11.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-13.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-13.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-15.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-15.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-17.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-17.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-19.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-19.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-21.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-21.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-23.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-23.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-25.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-25.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-27.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-27.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-29.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-29.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-3.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-31.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-31.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-33.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-33.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-35.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-35.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-37.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-37.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-39.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-39.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-41.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-41.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-43.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-43.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-45.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-45.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-47.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-47.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-48.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-48.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-49.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-49.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-5.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-5.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-51.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-51.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-53.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-53.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-55.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-55.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-57.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-57.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-59.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-59.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-61.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-61.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-63.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-63.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-65.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-65.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-7.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-7.png
--------------------------------------------------------------------------------
/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-9.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/indrag49/Numerical-Optimization/b051f28ee339382201a42f289c5dcc295510747f/_bookdown_files/bookdownproj_files/figure-html/unnamed-chunk-6-9.png
--------------------------------------------------------------------------------
/_output.yml:
--------------------------------------------------------------------------------
1 | bookdown::gitbook:
2 | css: style.css
3 | config:
4 | toc:
5 | before: |
6 |
This is an open-source introductory book on numerical/mathematical optimization aimed at audience who want to start with the subject matter with hands on implementation of the methods with Python. I plan to make this an open source collaborative project which will grow with time with legitimate contributions from the comnmunity.