from the training dataset
10 |
11 |
doesn’t have zero mean use the Gram matrix 
to substitute the kernel matrix .
14 |
15 | 
to solve for the vector 
.
18 |
19 | 4. Compute the kernel principal components 
20 |
21 | 
[1] Kernel Principal Component Analysis and its Applications in Face Recognition and Active Shape Models
24 | 25 | ## Resources 26 | 27 | - [Kernel Principal Component Analysis and its Applications in Face Recognition and Active Shape Models](https://arxiv.org/pdf/1207.3538.pdf) 28 | - [Kernel tricks and nonlinear dimensionality reduction via RBF kernel PCA](https://sebastianraschka.com/Articles/2014_kernel_pca.html) 29 | - [PCA and kernel PCA explained](https://nirpyresearch.com/pca-kernel-pca-explained/) 30 | - [What are the advantages of kernel PCA over standard PCA?](https://stats.stackexchange.com/questions/94463/what-are-the-advantages-of-kernel-pca-over-standard-pca) -------------------------------------------------------------------------------- /Algorithms/kernel_pca/README.tex.md: -------------------------------------------------------------------------------- 1 | # Kernel PCA 2 | 3 |  4 | 5 | Kernel PCA is an extension of [PCA](https://ml-explained.com/blog/principal-component-analysis-explained) that allows for the separability of nonlinear data by making use of kernels. The basic idea behind it is to project the linearly inseparable data onto a higher dimensional space where it becomes linearly separable. 6 | 7 | Kernel PCA can be summarized as a 4 step process [1]: 8 | 9 | 1. Construct the kernel matrix $K$ from the training dataset 10 | 11 | $$K_{i,j} = \kappa(\mathbf{x_i, x_j})$$ 12 | 13 | 2. If the projected dataset $\left\{\phi (\mathbf{x}_i) \right\}$ doesn’t have zero mean use the Gram matrix $\stackrel{\sim}{K}$ to substitute the kernel matrix $K$. 14 | 15 | $$\stackrel{\sim}{K} = K - \mathbf{1_N} K - K \mathbf{1_N} + \mathbf{1_N} K \mathbf{1_N}$$ 16 | 17 | 3. Use $K_{a_k} = \lambda_k N_{a_{k}}$ to solve for the vector $a_i$. 18 | 19 | 4. Compute the kernel principal components $y_k\left(x\right)$ 20 | 21 | $$y_k(\mathbf{x})= \phi \left(\mathbf{x}\right)^T \mathbf{v}_k = \sum_{i=1}^N a_{ki} \kappa(\mathbf{x_i, x_j})$$ 22 | 23 |[1] Kernel Principal Component Analysis and its Applications in Face Recognition and Active Shape Models
24 | 25 | ## Resources 26 | 27 | - [Kernel Principal Component Analysis and its Applications in Face Recognition and Active Shape Models](https://arxiv.org/pdf/1207.3538.pdf) 28 | - [Kernel tricks and nonlinear dimensionality reduction via RBF kernel PCA](https://sebastianraschka.com/Articles/2014_kernel_pca.html) 29 | - [PCA and kernel PCA explained](https://nirpyresearch.com/pca-kernel-pca-explained/) 30 | - [What are the advantages of kernel PCA over standard PCA?](https://stats.stackexchange.com/questions/94463/what-are-the-advantages-of-kernel-pca-over-standard-pca) -------------------------------------------------------------------------------- /Algorithms/kernel_pca/code/kernel_pca.py: -------------------------------------------------------------------------------- 1 | # based on https://sebastianraschka.com/Articles/2014_kernel_pca.html 2 | 3 | from __future__ import annotations 4 | from typing import Union 5 | import numpy as np 6 | from scipy.spatial.distance import pdist, squareform 7 | from scipy.linalg import eigh 8 | 9 | 10 | class KernelPCA: 11 | """KernelPCA 12 | Parameters: 13 | ----------- 14 | n_components: int = 2 15 | Number of components to keep. 16 | gamma: float = None 17 | Kernel coefficient 18 | """ 19 | def __init__(self, n_components: int = 2, gamma: float = None): 20 | self.n_components = n_components 21 | self.gamma = gamma 22 | self.alphas = None 23 | self.lambdas = None 24 | self.X = None 25 | 26 | def fit(self, X: Union[list, np.ndarray]) -> KernelPCA: 27 | if self.gamma == None: 28 | self.gamma = 1 / X.shape[1] 29 | 30 | sq_dists = pdist(X, 'sqeuclidean') 31 | 32 | mat_sq_dists = squareform(sq_dists) 33 | 34 | K = np.exp(-self.gamma * mat_sq_dists) 35 | 36 | N = K.shape[0] 37 | one_n = np.ones((N,N)) / N 38 | K_norm = K - one_n.dot(K) - K.dot(one_n) + one_n.dot(K).dot(one_n) 39 | 40 | eigenvalues, eigenvectors = eigh(K_norm) 41 | 42 | alphas = np.column_stack((eigenvectors[:,-i] for i in range(1, self.n_components+1))) 43 | lambdas = [eigenvalues[-i] for i in range(1, self.n_components+1)] 44 | 45 | self.alphas = alphas 46 | self.lambdas = lambdas 47 | self.X = X 48 | 49 | return self 50 | 51 | def fit_transform(self, X: Union[list, np.ndarray]) -> np.ndarray: 52 | self.fit(X) 53 | return self.alphas * np.sqrt(self.lambdas) 54 | 55 | def transform(self, X: Union[list, np.ndarray]) -> np.ndarray: 56 | # TODO: Rewrite as this is very inefficient 57 | def transform_row(X_r): 58 | pair_dist = np.array([np.sum((X_r-row)**2) for row in self.X]) 59 | k = np.exp(-self.gamma * pair_dist) 60 | return k.dot(self.alphas / self.lambdas) 61 | 62 | return np.array(list(map(transform_row, X))) 63 | 64 | 65 | 66 | 67 | if __name__ == '__main__': 68 | from sklearn.datasets import make_circles 69 | import matplotlib.pyplot as plt 70 | 71 | X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2) 72 | 73 | plt.figure(figsize=(8,6)) 74 | 75 | pca = KernelPCA(n_components=3) 76 | pca.fit(X) 77 | X = pca.transform(X) 78 | 79 | print(X) 80 | plt.plot(X[0], X[1]) 81 | plt.show() -------------------------------------------------------------------------------- /Algorithms/kernel_pca/doc/kernel_pca.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kernel_pca/doc/kernel_pca.png -------------------------------------------------------------------------------- /Algorithms/kmeans/doc/choose_k_value.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kmeans/doc/choose_k_value.jpeg -------------------------------------------------------------------------------- /Algorithms/kmeans/doc/elbow_method_using_yellowbrick.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kmeans/doc/elbow_method_using_yellowbrick.png -------------------------------------------------------------------------------- /Algorithms/kmeans/doc/k_means.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kmeans/doc/k_means.gif -------------------------------------------------------------------------------- /Algorithms/kmeans/doc/noisy_circles_with_true_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kmeans/doc/noisy_circles_with_true_output.png -------------------------------------------------------------------------------- /Algorithms/kmeans/doc/noisy_moons_with_true_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kmeans/doc/noisy_moons_with_true_output.png -------------------------------------------------------------------------------- /Algorithms/kmeans/doc/silhouette_analysis_3_clusters.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kmeans/doc/silhouette_analysis_3_clusters.jpeg -------------------------------------------------------------------------------- /Algorithms/kmeans/doc/silhouette_analysis_4_clusters.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kmeans/doc/silhouette_analysis_4_clusters.jpeg -------------------------------------------------------------------------------- /Algorithms/kmeans/doc/silhouette_analysis_5_clusters.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kmeans/doc/silhouette_analysis_5_clusters.jpeg -------------------------------------------------------------------------------- /Algorithms/kmeans/doc/two_lines.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Algorithms/kmeans/doc/two_lines.png -------------------------------------------------------------------------------- /Algorithms/kmeans/tex/44bc9d542a92714cac84e01cbbb7fd61.svg: -------------------------------------------------------------------------------- 1 |[1] Sebastian Ruder (2016). An overview of gradient descent optimization algorithms. arXiv preprint arXiv:1609.04747.
39 | 40 |[2] https://paperswithcode.com/method/adadelta
41 | 42 | ## Code 43 | 44 | * [Adadelta Numpy Implementation](code/adadelta.py) -------------------------------------------------------------------------------- /Optimizers/adadelta/code/adadelta.py: -------------------------------------------------------------------------------- 1 | # based on https://ruder.io/optimizing-gradient-descent/#adadelta 2 | # and https://github.com/eriklindernoren/ML-From-Scratch/blob/master/mlfromscratch/deep_learning/optimizers.py#L56 3 | 4 | import numpy as np 5 | 6 | 7 | class Adadelta: 8 | """Adadelta 9 | Parameters: 10 | ----------- 11 | rho: float = 0.95 12 | The decay rate. 13 | epsilon: float = 1e-07 14 | A small floating point value to avoid zero denominator. 15 | """ 16 | def __init__(self, rho: float = 0.95, epsilon: float = 1e-7) -> None: 17 | self.E_w_update = None # Running average of squared parameter updates 18 | self.E_grad = None # Running average of the squared gradient of w 19 | self.w_update = None # Parameter update 20 | self.epsilon = epsilon 21 | self.rho = rho 22 | 23 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray: 24 | if self.w_update is None: 25 | self.w_update = np.zeros(np.shape(w)) 26 | self.E_w_update = np.zeros(np.shape(w)) 27 | self.E_grad = np.zeros(np.shape(grad_wrt_w)) 28 | 29 | # Update average of gradients at w 30 | self.E_grad = self.rho * self.E_grad + (1 - self.rho) * np.power(grad_wrt_w, 2) 31 | 32 | # Calculate root mean squared error of the weight update and gradients 33 | RMS_delta_w = np.sqrt(self.E_w_update + self.epsilon) 34 | RMS_grad = np.sqrt(self.E_grad + self.epsilon) 35 | 36 | # Calculate adaptive learning rate 37 | adaptive_lr = RMS_delta_w / RMS_grad 38 | 39 | # Calculate the update 40 | self.w_update = adaptive_lr * grad_wrt_w 41 | 42 | # Update the running average of w updates 43 | self.E_w_update = self.rho * self.E_w_update + (1 - self.rho) * np.power(self.w_update, 2) 44 | 45 | return w - self.w_update 46 | -------------------------------------------------------------------------------- /Optimizers/adadelta/doc/adadelta_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/adadelta/doc/adadelta_example.png -------------------------------------------------------------------------------- /Optimizers/adadelta/tex/11c596de17c342edeed29f489aa4b274.svg: -------------------------------------------------------------------------------- 1 |[1] Duchi, J., Hazan, E., & Singer, Y. (2011). Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. Journal of Machine Learning Research, 12, 2121–2159. Retrieved from [http://jmlr.org/papers/v12/duchi11a.html](http://jml 24 |
[2] Dean, J., Corrado, G. S., Monga, R., Chen, K., Devin, M., Le, Q. V, … Ng, A. Y. (2012). Large Scale Distributed Deep Networks. NIPS 2012: Neural Information Processing Systems, 1–11. [http://papers.nips.cc/paper/4687-large-scale-distributed-deep-networks.pdf](http://papers.nips.cc/paper/4687-large-scale-distributed-deep-networks.pdf)
25 |[3] Sebastian Ruder (2016). An overview of gradient descent optimization algorithms. arXiv preprint arXiv:1609.04747.
26 | 27 | ## Code 28 | 29 | - [Adagrad Numpy Implementation](code/adagrad.py) -------------------------------------------------------------------------------- /Optimizers/adagrad/code/adagrad.py: -------------------------------------------------------------------------------- 1 | # based on https://ruder.io/optimizing-gradient-descent/#adagrad 2 | # and https://github.com/eriklindernoren/ML-From-Scratch/blob/master/mlfromscratch/deep_learning/optimizers.py#L41 3 | 4 | import numpy as np 5 | 6 | 7 | class Adagrad: 8 | """Adagrad 9 | Parameters: 10 | ----------- 11 | learning_rate: float = 0.001 12 | The step length used when following the negative gradient. 13 | initial_accumulator_value: float = 0.1 14 | Starting value for the accumulators, must be non-negative. 15 | epsilon: float = 1e-07 16 | A small floating point value to avoid zero denominator. 17 | """ 18 | def __init__(self, learning_rate: float = 0.001, initial_accumulator_value: float = 0.1, epsilon: float = 1e-07) -> None: 19 | self.learning_rate = learning_rate 20 | self.initial_accumulator_value = initial_accumulator_value 21 | self.epsilon = epsilon 22 | self.G = np.array([]) # Sum of squares of the gradients 23 | 24 | assert self.initial_accumulator_value > 0, "initial_accumulator_value must be non-negative" 25 | 26 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray: 27 | # Initialize w_update if not initialized yet 28 | if not self.G.any(): 29 | self.G = np.full(np.shape(w), self.initial_accumulator_value) 30 | # Add the square of the gradient of the loss function at w 31 | self.G += np.power(grad_wrt_w, 2) 32 | return w - self.learning_rate * grad_wrt_w / np.sqrt(self.G + self.epsilon) 33 | -------------------------------------------------------------------------------- /Optimizers/adagrad/doc/adagrad_example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/adagrad/doc/adagrad_example.gif -------------------------------------------------------------------------------- /Optimizers/adagrad/tex/4f4f4e395762a3af4575de74c019ebb5.svg: -------------------------------------------------------------------------------- 1 |[1] Diederik P. Kingma and Jimmy Ba (2014). Adam: A Method for Stochastic Optimization.
26 | 27 |[2] Sebastian Ruder (2016). An overview of gradient descent optimization algorithms. arXiv preprint arXiv:1609.04747.
28 | 29 | ## Code 30 | 31 | - [Adam Numpy Implementation](code/adam.py) 32 | 33 | ## Resources 34 | 35 | - [https://arxiv.org/abs/1412.6980](https://arxiv.org/abs/1412.6980) 36 | - [https://ruder.io/optimizing-gradient-descent/index.html#adam](https://ruder.io/optimizing-gradient-descent/index.html#adam) 37 | - [https://towardsdatascience.com/adam-latest-trends-in-deep-learning-optimization-6be9a291375c](https://towardsdatascience.com/adam-latest-trends-in-deep-learning-optimization-6be9a291375c) -------------------------------------------------------------------------------- /Optimizers/adam/code/adam.py: -------------------------------------------------------------------------------- 1 | # based on https://ruder.io/optimizing-gradient-descent/#adam 2 | # and https://github.com/eriklindernoren/ML-From-Scratch/blob/master/mlfromscratch/deep_learning/optimizers.py#L106 3 | 4 | import numpy as np 5 | 6 | 7 | class Adam: 8 | """Adam - Adaptive Moment Estimation 9 | Parameters: 10 | ----------- 11 | learning_rate: float = 0.001 12 | The step length used when following the negative gradient. 13 | beta_1: float = 0.9 14 | The exponential decay rate for the 1st moment estimates. 15 | beta_2: float = 0.999 16 | The exponential decay rate for the 2nd moment estimates. 17 | epsilon: float = 1e-07 18 | A small floating point value to avoid zero denominator. 19 | """ 20 | def __init__(self, learning_rate: float = 0.001, beta_1: float = 0.9, beta_2: float = 0.999, epsilon: float = 1e-7) -> None: 21 | self.learning_rate = learning_rate 22 | self.epsilon = epsilon 23 | self.beta_1 = beta_1 24 | self.beta_2 = beta_2 25 | 26 | self.t = 0 27 | self.m = None # Decaying averages of past gradients 28 | self.v = None # Decaying averages of past squared gradients 29 | 30 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray: 31 | self.t += 1 32 | if self.m is None: 33 | self.m = np.zeros(np.shape(grad_wrt_w)) 34 | self.v = np.zeros(np.shape(grad_wrt_w)) 35 | 36 | self.m = self.beta_1 * self.m + (1 - self.beta_1) * grad_wrt_w 37 | self.v = self.beta_2 * self.v + (1 - self.beta_2) * np.power(grad_wrt_w, 2) 38 | 39 | m_hat = self.m / (1 - self.beta_1**self.t) 40 | v_hat = self.v / (1 - self.beta_2**self.t) 41 | 42 | w_update = self.learning_rate * m_hat / (np.sqrt(v_hat) + self.epsilon) 43 | 44 | return w - w_update 45 | -------------------------------------------------------------------------------- /Optimizers/adam/doc/adam_example.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/adam/doc/adam_example.PNG -------------------------------------------------------------------------------- /Optimizers/adamax/README.tex.md: -------------------------------------------------------------------------------- 1 | # AdaMax 2 | 3 |  4 | 5 | In [Adam](https://ml-explained.com/blog/adam-explained), the update rule for individual weights is scaling their gradients inversely proportional to the $\ell_2$ norm of the past and current gradients. 6 | 7 | $$v_t = \beta_2 v_{t-1} + (1 - \beta_2) |g_t|^2$$ 8 | 9 | The L2 norm can be generalized to the $\ell_p$ norm. 10 | 11 | $$v_t = \beta_2^p v_{t-1} + (1 - \beta_2^p) |g_t|^p$$ 12 | 13 | Such variants generally become numerically unstable for large $p$, which is why $\ell_1$ and $\ell_2$ norms are most common in practice. However, in the special case where we let $p \rightarrow \infty$, a surprisingly simple and stable algorithm emerges. 14 | 15 | To avoid confusion with Adam, we use $u_t$ to denote the infinity norm-constrained $v_t$: 16 | 17 | $$ 18 | u_t = \beta_2^\infty v_{t-1} + (1 - \beta_2^\infty) |g_t|^\infty 19 | = \max(\beta_2 \cdot v_{t-1}, |g_t|) 20 | $$ 21 | 22 | We can now plug $u_t$ into the Adam update equation replacing $\sqrt{\hat{v}_t} + \epsilon$ to obtain the AdaMax update rule: 23 | 24 | $$\theta_{t+1} = \theta_{t} - \dfrac{\eta}{u_t} \hat{m}_t$$ 25 | 26 | ## Code 27 | 28 | - [AdaMax Numpy Implementation](code/adamax.py) 29 | 30 | ## Resources 31 | 32 | - [https://arxiv.org/abs/1412.6980](https://arxiv.org/abs/1412.6980) 33 | - [https://ruder.io/optimizing-gradient-descent/index.html#adamax](https://ruder.io/optimizing-gradient-descent/index.html#adamax) 34 | - [https://keras.io/api/optimizers/adamax/](https://keras.io/api/optimizers/adamax/) -------------------------------------------------------------------------------- /Optimizers/adamax/code/adamax.py: -------------------------------------------------------------------------------- 1 | # based on https://ruder.io/optimizing-gradient-descent/#adamax 2 | 3 | import numpy as np 4 | 5 | 6 | class AdaMax: 7 | """AdaMax 8 | Parameters: 9 | ----------- 10 | learning_rate: float = 0.001 11 | The step length used when following the negative gradient. 12 | beta_1: float = 0.9 13 | The exponential decay rate for the 1st moment estimates. 14 | beta_2: float = 0.999 15 | The exponential decay rate for the 2nd moment estimates. 16 | epsilon: float = 1e-07 17 | A small floating point value to avoid zero denominator. 18 | """ 19 | def __init__(self, learning_rate: float = 0.001, beta_1: float = 0.9, beta_2: float = 0.999, epsilon: float = 1e-7) -> None: 20 | self.learning_rate = learning_rate 21 | self.epsilon = epsilon 22 | self.beta_1 = beta_1 23 | self.beta_2 = beta_2 24 | 25 | self.t = 0 26 | self.m = None # Decaying averages of past gradients 27 | self.v = None # Decaying averages of past squared gradients 28 | 29 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray: 30 | self.t += 1 31 | if self.m is None: 32 | self.m = np.zeros(np.shape(grad_wrt_w)) 33 | self.v = np.zeros(np.shape(grad_wrt_w)) 34 | 35 | self.m = self.beta_1 * self.m + (1 - self.beta_1) * grad_wrt_w 36 | self.v = np.maximum(self.beta_2 * self.v, np.abs(grad_wrt_w)) 37 | 38 | m_hat = self.m / (1 - self.beta_1**self.t) 39 | 40 | w_update = self.learning_rate * m_hat / (self.v + self.epsilon) 41 | 42 | return w - w_update 43 | -------------------------------------------------------------------------------- /Optimizers/adamax/doc/adamax_example.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/adamax/doc/adamax_example.PNG -------------------------------------------------------------------------------- /Optimizers/adamw/README.md: -------------------------------------------------------------------------------- 1 | # AdamW 2 | 3 | AdamW is a stochastic optimization method that modifies the typical implementation of weight decay in Adam to combat Adam's known convergence problems by decoupling the weight decay from the gradient updates. 4 | 5 |  6 | 7 | ## Code 8 | 9 | - [AdamW Numpy Implementation](code/adamw.py) 10 | 11 | ## Resources 12 | 13 | - [https://arxiv.org/abs/1711.05101](https://arxiv.org/abs/1711.05101) 14 | - [https://paperswithcode.com/method/adamw](https://paperswithcode.com/method/adamw) 15 | - [https://www.fast.ai/2018/07/02/adam-weight-decay/](https://www.fast.ai/2018/07/02/adam-weight-decay/) 16 | - [https://towardsdatascience.com/why-adamw-matters-736223f31b5d](https://towardsdatascience.com/why-adamw-matters-736223f31b5d) -------------------------------------------------------------------------------- /Optimizers/adamw/README.tex.md: -------------------------------------------------------------------------------- 1 | # AdamW 2 | 3 | AdamW is a stochastic optimization method that modifies the typical implementation of weight decay in Adam to combat Adam's known convergence problems by decoupling the weight decay from the gradient updates. 4 | 5 |  6 | 7 | ## Code 8 | 9 | - [AdamW Numpy Implementation](code/adamw.py) 10 | 11 | ## Resources 12 | 13 | - [https://arxiv.org/abs/1711.05101](https://arxiv.org/abs/1711.05101) 14 | - [https://paperswithcode.com/method/adamw](https://paperswithcode.com/method/adamw) 15 | - [https://www.fast.ai/2018/07/02/adam-weight-decay/](https://www.fast.ai/2018/07/02/adam-weight-decay/) 16 | - [https://towardsdatascience.com/why-adamw-matters-736223f31b5d](https://towardsdatascience.com/why-adamw-matters-736223f31b5d) -------------------------------------------------------------------------------- /Optimizers/adamw/code/adamw.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class AdamW: 5 | """AdamW 6 | Parameters: 7 | ----------- 8 | learning_rate: float = 0.001 9 | The step length used when following the negative gradient. 10 | beta_1: float = 0.9 11 | The exponential decay rate for the 1st moment estimates. 12 | beta_2: float = 0.999 13 | The exponential decay rate for the 2nd moment estimates. 14 | epsilon: float = 1e-07 15 | A small floating point value to avoid zero denominator. 16 | weight_decay: float = 0.01 17 | Amount of weight decay to be applied. 18 | """ 19 | def __init__(self, learning_rate: float = 0.001, beta_1: float = 0.9, beta_2: float = 0.999, epsilon: float = 1e-7, weight_decay: float = 0.01) -> None: 20 | self.learning_rate = learning_rate 21 | self.epsilon = epsilon 22 | self.beta_1 = beta_1 23 | self.beta_2 = beta_2 24 | self.weight_decay = weight_decay 25 | 26 | self.t = 0 27 | self.m = None # Decaying averages of past gradients 28 | self.v = None # Decaying averages of past squared gradients 29 | 30 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray: 31 | self.t += 1 32 | if self.m is None: 33 | self.m = np.zeros(np.shape(grad_wrt_w)) 34 | self.v = np.zeros(np.shape(grad_wrt_w)) 35 | 36 | self.m = self.beta_1 * self.m + (1 - self.beta_1) * grad_wrt_w 37 | self.v = self.beta_2 * self.v + (1 - self.beta_2) * np.power(grad_wrt_w, 2) 38 | 39 | m_hat = self.m / (1 - self.beta_1**self.t) 40 | v_hat = self.v / (1 - self.beta_2**self.t) 41 | 42 | w_update = self.learning_rate * m_hat / (np.sqrt(v_hat) + self.epsilon) + self.weight_decay * grad_wrt_w 43 | 44 | return w - w_update 45 | -------------------------------------------------------------------------------- /Optimizers/adamw/doc/adamw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/adamw/doc/adamw.png -------------------------------------------------------------------------------- /Optimizers/amsgrad/README.md: -------------------------------------------------------------------------------- 1 | # AMSGrad 2 | 3 |  4 | 5 | The motivation for AMSGrad lies with the observation that [Adam](https://ml-explained.com/blog/adam-explained) fails to converge to an optimal solution for some data-sets and is outperformed by SDG with momentum. 6 | 7 | Reddi et al. (2018) [1] show that one cause of the issue described above is the use of the exponential moving average of the past squared gradients. 8 | 9 | To fix the above-described behavior, the authors propose a new algorithm called AMSGrad that keeps a running maximum of the squared gradients instead of an exponential moving average. 10 | 11 |
[1] Reddi, Sashank J., Kale, Satyen, & Kumar, Sanjiv. [On the Convergence of Adam and Beyond](https://arxiv.org/abs/1904.09237v1).
22 | 23 | ## Code 24 | 25 | - [AMSGrad Numpy Implementation](code/amsgrad.py) 26 | 27 | ## Resources 28 | 29 | - [https://arxiv.org/abs/1904.09237v1](https://arxiv.org/abs/1904.09237v1) 30 | - [https://paperswithcode.com/method/amsgrad](https://paperswithcode.com/method/amsgrad) 31 | - [https://ruder.io/optimizing-gradient-descent/index.html#amsgrad](https://ruder.io/optimizing-gradient-descent/index.html#amsgrad) -------------------------------------------------------------------------------- /Optimizers/amsgrad/README.tex.md: -------------------------------------------------------------------------------- 1 | # AMSGrad 2 | 3 |  4 | 5 | The motivation for AMSGrad lies with the observation that [Adam](https://ml-explained.com/blog/adam-explained) fails to converge to an optimal solution for some data-sets and is outperformed by SDG with momentum. 6 | 7 | Reddi et al. (2018) [1] show that one cause of the issue described above is the use of the exponential moving average of the past squared gradients. 8 | 9 | To fix the above-described behavior, the authors propose a new algorithm called AMSGrad that keeps a running maximum of the squared gradients instead of an exponential moving average. 10 | 11 | $$v_t = \beta_2 v_{t-1} + (1 - \beta_2) g_t^2$$ 12 | 13 | $$\hat{v}_t = \text{max}(\hat{v}_{t-1}, v_t)$$ 14 | 15 | For simplicity, the authors also removed the debiasing step, which leads to the following update rule: 16 | 17 | $$\begin{align} \begin{split} m_t &= \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ v_t &= \beta_2 v_{t-1} + (1 - \beta_2) g_t^2\\ \hat{v}_t &= \text{max}(\hat{v}_{t-1}, v_t) \\ \theta_{t+1} &= \theta_{t} - \dfrac{\eta}{\sqrt{\hat{v}_t} + \epsilon} m_t \end{split} \end{align}$$ 18 | 19 | For more information, check out the paper '[On the Convergence of Adam and Beyond](https://arxiv.org/abs/1904.09237v1)' and the [AMSGrad section](https://ruder.io/optimizing-gradient-descent/index.html#amsgrad) of the '[An overview of gradient descent optimization algorithms](https://ruder.io/optimizing-gradient-descent/index.html)' article. 20 | 21 |[1] Reddi, Sashank J., Kale, Satyen, & Kumar, Sanjiv. [On the Convergence of Adam and Beyond](https://arxiv.org/abs/1904.09237v1).
22 | 23 | ## Code 24 | 25 | - [AMSGrad Numpy Implementation](code/amsgrad.py) 26 | 27 | ## Resources 28 | 29 | - [https://arxiv.org/abs/1904.09237v1](https://arxiv.org/abs/1904.09237v1) 30 | - [https://paperswithcode.com/method/amsgrad](https://paperswithcode.com/method/amsgrad) 31 | - [https://ruder.io/optimizing-gradient-descent/index.html#amsgrad](https://ruder.io/optimizing-gradient-descent/index.html#amsgrad) -------------------------------------------------------------------------------- /Optimizers/amsgrad/code/amsgrad.py: -------------------------------------------------------------------------------- 1 | # based on https://ruder.io/optimizing-gradient-descent/#amsgrad 2 | 3 | import numpy as np 4 | 5 | 6 | class AMSGrad: 7 | """AMSGrad 8 | Parameters: 9 | ----------- 10 | learning_rate: float = 0.001 11 | The step length used when following the negative gradient. 12 | beta_1: float = 0.9 13 | The exponential decay rate for the 1st moment estimates. 14 | beta_2: float = 0.999 15 | The exponential decay rate for the 2nd moment estimates. 16 | epsilon: float = 1e-07 17 | A small floating point value to avoid zero denominator. 18 | """ 19 | def __init__(self, learning_rate: float = 0.001, beta_1: float = 0.9, beta_2: float = 0.999, epsilon: float = 1e-7) -> None: 20 | self.learning_rate = learning_rate 21 | self.epsilon = epsilon 22 | self.beta_1 = beta_1 23 | self.beta_2 = beta_2 24 | 25 | self.m = None # Decaying averages of past gradients 26 | self.v = None # Decaying averages of past squared gradients 27 | 28 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray: 29 | if self.m is None: 30 | self.m = np.zeros(np.shape(grad_wrt_w)) 31 | self.v = np.zeros(np.shape(grad_wrt_w)) 32 | 33 | self.m = self.beta_1 * self.m + (1 - self.beta_1) * grad_wrt_w 34 | v_1 = self.v 35 | self.v = self.beta_2 * self.v + (1 - self.beta_2) * np.power(grad_wrt_w, 2) 36 | 37 | v_hat = np.maximum(v_1, self.v) 38 | 39 | w_update = self.learning_rate * self.m / (np.sqrt(v_hat) + self.epsilon) 40 | 41 | return w - w_update 42 | -------------------------------------------------------------------------------- /Optimizers/amsgrad/doc/amsgrad_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/amsgrad/doc/amsgrad_example.png -------------------------------------------------------------------------------- /Optimizers/gradient_descent/code/gradient_descent_with_momentum.py: -------------------------------------------------------------------------------- 1 | # based on https://ruder.io/optimizing-gradient-descent/ 2 | # and https://github.com/eriklindernoren/ML-From-Scratch/blob/master/mlfromscratch/deep_learning/optimizers.py#L9 3 | 4 | import numpy as np 5 | 6 | 7 | class GradientDescent: 8 | """Gradient Descent with Momentum 9 | Parameters: 10 | ----------- 11 | learning_rate: float = 0.01 12 | The step length used when following the negative gradient. 13 | momentum: float = 0.0 14 | Amount of momentum to use. 15 | Momentum accelerates gradient descent in the relevant direction and dampens oscillations. 16 | """ 17 | def __init__(self, learning_rate: float = 0.01, momentum: float = 0.0) -> None: 18 | self.learning_rate = learning_rate 19 | self.momentum = momentum 20 | self.w_update = np.array([]) 21 | 22 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray: 23 | # Initialize w_update if not initialized yet 24 | if not self.w_update.any(): 25 | self.w_update = np.zeros(np.shape(w)) 26 | # Use momentum if set 27 | self.w_update = self.momentum * self.w_update + (1 - self.momentum) * grad_wrt_w 28 | # Move against the gradient to minimize loss 29 | return w - self.learning_rate * self.w_update 30 | -------------------------------------------------------------------------------- /Optimizers/gradient_descent/code/gradient_descent_with_nesterov_momentum.py: -------------------------------------------------------------------------------- 1 | # based on https://ruder.io/optimizing-gradient-descent/#nesterovacceleratedgradient 2 | # and https://github.com/eriklindernoren/ML-From-Scratch/blob/master/mlfromscratch/deep_learning/optimizers.py#L24 3 | 4 | from typing import Callable 5 | import numpy as np 6 | 7 | 8 | class NesterovAcceleratedGradientDescent: 9 | """Gradient Descent with Nesterov Momentum 10 | Parameters: 11 | ----------- 12 | learning_rate: float = 0.01 13 | The step length used when following the negative gradient. 14 | momentum: float = 0.0 15 | Amount of momentum to use. 16 | Momentum accelerates gradient descent in the relevant direction and dampens oscillations. 17 | """ 18 | def __init__(self, learning_rate: float = 0.01, momentum: float = 0.0) -> None: 19 | self.learning_rate = learning_rate 20 | self.momentum = momentum 21 | self.w_update = np.array([]) 22 | 23 | def update(self, w: np.ndarray, grad_func: Callable) -> np.ndarray: 24 | # Calculate the gradient of the loss a bit further down the slope from w 25 | approx_future_grad = np.clip(grad_func(w - self.momentum * self.w_update), -1, 1) 26 | # Initialize w_update if not initialized yet 27 | if not self.w_update.any(): 28 | self.w_update = np.zeros(np.shape(w)) 29 | 30 | self.w_update = self.momentum * self.w_update + self.learning_rate * approx_future_grad 31 | # Move against the gradient to minimize loss 32 | return w - self.w_update 33 | -------------------------------------------------------------------------------- /Optimizers/gradient_descent/doc/gradient_descent.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/gradient_descent/doc/gradient_descent.gif -------------------------------------------------------------------------------- /Optimizers/gradient_descent/doc/momentum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/gradient_descent/doc/momentum.png -------------------------------------------------------------------------------- /Optimizers/gradient_descent/doc/nesterov_accelerated_gradient.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/gradient_descent/doc/nesterov_accelerated_gradient.png -------------------------------------------------------------------------------- /Optimizers/gradient_descent/doc/pick_learning_rate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/gradient_descent/doc/pick_learning_rate.png -------------------------------------------------------------------------------- /Optimizers/gradient_descent/doc/variations_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/gradient_descent/doc/variations_comparison.png -------------------------------------------------------------------------------- /Optimizers/gradient_descent/tex/11c596de17c342edeed29f489aa4b274.svg: -------------------------------------------------------------------------------- 1 |
needs to be updated. For more information, check out [the paper](http://cs229.stanford.edu/proj2015/054_report.pdf) or the [Nadam section](https://ruder.io/optimizing-gradient-descent/index.html#nadam) of ['An overview of gradient descent optimization algorithms'](https://ruder.io/optimizing-gradient-descent/index.html).
6 |
7 | The final update rule looks as follows:
8 |
9 | 
divided by a weighted average of the mean squared gradients and the current squared gradient, weighting the current squared gradient with an immediate discount factor 
. [2]
8 |
9 | 
[1] Ma, J. and Yarats, D. Quasi-hyperbolic momentum and Adam for deep learning. arXiv preprint arXiv:1810.06801, 2018
12 | 13 | 14 | 15 |[3] John Chen. An updated overview of recent gradient descent algorithms
16 | 17 | ## Code 18 | 19 | - [QHAdam Numpy Implementation](code/qhadam.py) -------------------------------------------------------------------------------- /Optimizers/qhadam/README.tex.md: -------------------------------------------------------------------------------- 1 | # QHAdam (Quasi-Hyperbolic Adam) 2 | 3 |  4 | 5 | **Quasi-Hyperbolic Momentum Algorithm (QHM)** is a simple alteration of [SGD with momentum](https://paperswithcode.com/method/sgd-with-momentum), averaging a plain SGD step with a momentum step. **QHAdam (Quasi-Hyperbolic Adam)** is a QH augmented version of [Adam](https://ml-explained.com/blog/adam-explained) that replaces both of Adam's moment estimators with quasi-hyperbolic terms. Namely, QHAdam decouples the momentum term from the current gradient when updating the weights, and decouples the mean squared gradients term from the current squared gradient when updating the weights. [1, 2, 3] 6 | 7 | Essentially, it's a weighted average of the momentum and plain SGD, weighting the current gradient with an immediate discount factor $v_1$ divided by a weighted average of the mean squared gradients and the current squared gradient, weighting the current squared gradient with an immediate discount factor $v_2$. [2] 8 | 9 | $$ \theta_{t+1, i} = \theta_{t, i} - \eta\left[\frac{\left(1-v_{1}\right)\cdot{g_{t}} + v_{1}\cdot\hat{m}_{t}}{\sqrt{\left(1-v_{2}\right)g^{2}_{t} + v_{2}\cdot{\hat{v}_{t}}} + \epsilon}\right], \forall{t} $$ 10 | 11 |[1] Ma, J. and Yarats, D. Quasi-hyperbolic momentum and Adam for deep learning. arXiv preprint arXiv:1810.06801, 2018
12 | 13 | 14 | 15 |[3] John Chen. An updated overview of recent gradient descent algorithms
16 | 17 | ## Code 18 | 19 | - [QHAdam Numpy Implementation](code/qhadam.py) -------------------------------------------------------------------------------- /Optimizers/qhadam/code/qhadam.py: -------------------------------------------------------------------------------- 1 | # based on https://arxiv.org/pdf/1810.06801.pdf 2 | 3 | import numpy as np 4 | 5 | 6 | class QHAdam: 7 | """QHAdam - Quasi-Hyperbolic Adam 8 | Parameters: 9 | ----------- 10 | learning_rate: float = 0.001 11 | The step length used when following the negative gradient. 12 | beta_1: float = 0.9 13 | The exponential decay rate for the 1st moment estimates. 14 | beta_2: float = 0.999 15 | The exponential decay rate for the 2nd moment estimates. 16 | epsilon: float = 1e-07 17 | A small floating point value to avoid zero denominator. 18 | v_1: float = 0.7 19 | Immediate discount factor 20 | v_2: float = 1.0 21 | Immediate discount factor 22 | """ 23 | def __init__(self, learning_rate: float = 0.001, beta_1: float = 0.9, beta_2: float = 0.999, epsilon: float = 1e-7, v_1: float = 0.7, v_2: float = 1.0) -> None: 24 | self.learning_rate = learning_rate 25 | self.epsilon = epsilon 26 | self.beta_1 = beta_1 27 | self.beta_2 = beta_2 28 | self.v_1 = v_1 29 | self.v_2 = v_2 30 | 31 | self.t = 0 32 | self.m = None # Decaying averages of past gradients 33 | self.v = None # Decaying averages of past squared gradients 34 | 35 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray: 36 | self.t += 1 37 | if self.m is None: 38 | self.m = np.zeros(np.shape(grad_wrt_w)) 39 | self.v = np.zeros(np.shape(grad_wrt_w)) 40 | 41 | self.m = self.beta_1 * self.m + (1 - self.beta_1) * grad_wrt_w 42 | self.v = self.beta_2 * self.v + (1 - self.beta_2) * np.power(grad_wrt_w, 2) 43 | 44 | m_hat = self.m / (1 - self.beta_1**self.t) 45 | v_hat = self.v / (1 - self.beta_2**self.t) 46 | 47 | w_update = self.learning_rate * ((1 - self.v_1) * grad_wrt_w + self.v_1 * m_hat) / (np.sqrt((1 - self.v_2) * np.power(grad_wrt_w, 2) + self.v_2 * v_hat) + self.epsilon) 48 | 49 | return w - w_update 50 | -------------------------------------------------------------------------------- /Optimizers/qhadam/doc/qhadam_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/qhadam/doc/qhadam_example.png -------------------------------------------------------------------------------- /Optimizers/qhadam/tex/41922e474070adc90e7c1379c28d22fe.svg: -------------------------------------------------------------------------------- 1 |
from the current gradient 
when updating the weights.
6 |
7 | 
and 
as a good starting point. For more information about QHM, check out the resources below.
12 |
13 | ## Code
14 |
15 | - [QHM Numpy Implementation](code/qhm.py)
16 |
17 | ## Resources
18 |
19 | - [https://arxiv.org/pdf/1810.06801.pdf](https://arxiv.org/pdf/1810.06801.pdf)
20 | - [https://paperswithcode.com/method/qhadam](https://paperswithcode.com/method/qhadam)
21 | - [https://johnchenresearch.github.io/demon/](https://johnchenresearch.github.io/demon/)
22 | - [https://facebookresearch.github.io/qhoptim/](https://facebookresearch.github.io/qhoptim/)
--------------------------------------------------------------------------------
/Optimizers/qhm/README.tex.md:
--------------------------------------------------------------------------------
1 | # QHM (Quasi-Hyperbolic Momentum)
2 |
3 | 
4 |
5 | Quasi-Hyperbolic Momentum Algorithm (QHM) is a simple alteration of SGD with momentum, averaging a plain SGD step with a momentum step, thereby decoupling the momentum term $\beta$ from the current gradient $\nabla_t$ when updating the weights.
6 |
7 | $$g_{t + 1} \leftarrow \beta \cdot g_t + (1 - \beta) \cdot \nabla_t$$
8 |
9 | $$\theta_{t + 1} \leftarrow \theta_t + \alpha \left[ (1 - \nu) \cdot \nabla_t + \nu \cdot g_{t + 1} \right]$$
10 |
11 | The authors recommend $\nu=0.7$ and $\beta=0.999$ as a good starting point. For more information about QHM, check out the resources below.
12 |
13 | ## Code
14 |
15 | - [QHM Numpy Implementation](code/qhm.py)
16 |
17 | ## Resources
18 |
19 | - [https://arxiv.org/pdf/1810.06801.pdf](https://arxiv.org/pdf/1810.06801.pdf)
20 | - [https://paperswithcode.com/method/qhadam](https://paperswithcode.com/method/qhadam)
21 | - [https://johnchenresearch.github.io/demon/](https://johnchenresearch.github.io/demon/)
22 | - [https://facebookresearch.github.io/qhoptim/](https://facebookresearch.github.io/qhoptim/)
--------------------------------------------------------------------------------
/Optimizers/qhm/code/qhm.py:
--------------------------------------------------------------------------------
1 | # based on https://arxiv.org/pdf/1810.06801.pdf
2 |
3 | import numpy as np
4 |
5 |
6 | class QHM:
7 | """QHM -Quasi-Hyperbolic Momentum
8 | Parameters:
9 | -----------
10 | learning_rate: float = 0.001
11 | The step length used when following the negative gradient.
12 | beta: float = 0.999
13 | Momentum factor.
14 | v: float = 0.7
15 | Immediate discount factor.
16 | """
17 | def __init__(self, learning_rate: float = 0.001, beta: float = 0.999, v: float = 0.7) -> None:
18 | self.learning_rate = learning_rate
19 | self.beta = beta
20 | self.v = v
21 |
22 | self.g_t = np.array([])
23 |
24 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray:
25 | if not self.g_t.any():
26 | self.g_t = np.zeros(np.shape(w))
27 |
28 | self.g_t = self.beta * self.g_t + (1 - self.beta) * grad_wrt_w
29 |
30 | return w - self.learning_rate * ((1 - self.v) * grad_wrt_w + self.v * self.g_t)
31 |
--------------------------------------------------------------------------------
/Optimizers/qhm/doc/qhm_update_rule.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/qhm/doc/qhm_update_rule.PNG
--------------------------------------------------------------------------------
/Optimizers/qhm/tex/8217ed3c32a785f0b5aad4055f432ad8.svg:
--------------------------------------------------------------------------------
1 | 
entirely and replaces it by the root mean squared error of parameter updates.
11 |
12 | [1] Sebastian Ruder (2016). An overview of gradient descent optimization algorithms. arXiv preprint arXiv:1609.04747.
13 | 14 | ## Code 15 | 16 | * [RMSprop Numpy Implementation](code/rmsprop.py) -------------------------------------------------------------------------------- /Optimizers/rmsprop/README.tex.md: -------------------------------------------------------------------------------- 1 | # RMSprop 2 | 3 |  4 | 5 | RMSprop is an unpublished, adaptive learning rate optimization algorithm first proposed by [Geoff Hinton](https://en.wikipedia.org/wiki/Geoffrey_Hinton) in lecture 6 of his online class "[Neural Networks for Machine Learning](http://www.cs.toronto.edu/~hinton/coursera/lecture6/lec6.pdf)". RMSprop and Adadelta have been developed independently around the same time, and both try to resolve Adagrad's diminishing learning rate problem. [1] 6 | 7 | $$E[g^2]_t = 0.9 E[g^2]_{t-1} + 0.1 g^2_t$$ 8 | 9 | $$\theta_{t+1} = \theta_{t} - \dfrac{\eta}{\sqrt{E[g^2]_t + \epsilon}} g_{t}$$ 10 | 11 | The difference between Adadelta and RMSprop is that Adadelta removes the learning rate $\eta$ entirely and replaces it by the root mean squared error of parameter updates. 12 | 13 |[1] Sebastian Ruder (2016). An overview of gradient descent optimization algorithms. arXiv preprint arXiv:1609.04747.
14 | 15 | ## Code 16 | 17 | * [RMSprop Numpy Implementation](code/rmsprop.py) -------------------------------------------------------------------------------- /Optimizers/rmsprop/code/rmsprop.py: -------------------------------------------------------------------------------- 1 | # based on https://ruder.io/optimizing-gradient-descent/#rmsprop 2 | # and https://github.com/eriklindernoren/ML-From-Scratch/blob/master/mlfromscratch/deep_learning/optimizers.py#L88 3 | 4 | import numpy as np 5 | 6 | 7 | class RMSprop: 8 | """RMSprop 9 | Parameters: 10 | ----------- 11 | learning_rate: float = 0.001 12 | The step length used when following the negative gradient. 13 | rho: float = 0.9 14 | Discounting factor for the history/coming gradient. 15 | epsilon: float = 1e-07 16 | A small floating point value to avoid zero denominator. 17 | """ 18 | def __init__(self, learning_rate: float = 0.001, rho: float = 0.9, epsilon: float = 1e-7) -> None: 19 | self.learning_rate = learning_rate 20 | self.rho = rho 21 | self.epsilon = epsilon 22 | self.E_grad = None # Running average of the square gradients at w 23 | 24 | def update(self, w: np.ndarray, grad_wrt_w: np.ndarray) -> np.ndarray: 25 | if self.E_grad is None: 26 | self.E_grad = np.zeros(np.shape(grad_wrt_w)) 27 | 28 | # Update average of gradients at w 29 | self.E_grad = self.rho * self.E_grad + (1 - self.rho) * np.power(grad_wrt_w, 2) 30 | 31 | return w - self.learning_rate * grad_wrt_w / np.sqrt(self.E_grad + self.epsilon) 32 | -------------------------------------------------------------------------------- /Optimizers/rmsprop/doc/rmsprop_example.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TannerGilbert/Machine-Learning-Explained/c671f4cdb68eb316f39a8f6142c8d3cffb838c11/Optimizers/rmsprop/doc/rmsprop_example.PNG -------------------------------------------------------------------------------- /Optimizers/rmsprop/tex/1d0496971a2775f4887d1df25cea4f7e.svg: -------------------------------------------------------------------------------- 1 |