├── .gitignore ├── LICENSE ├── README.md ├── notebooks ├── 1. 1st-order methods.ipynb ├── 2. 2nd-order and quasi-Newton.ipynb ├── 3. Log-sum-exp.ipynb ├── 5. Accelerated methods.ipynb ├── jupyter_utils.py └── numpy_and_scipy.ipynb ├── optmethods ├── __init__.py ├── datasets │ ├── 14_Tumors.mat │ ├── __init__.py │ ├── a1a │ ├── a5a │ ├── a9a │ ├── covtype.bz2 │ ├── movielens_100k.data │ ├── mushrooms │ ├── news20.bz2 │ ├── utils.py │ └── w8a ├── first_order │ ├── __init__.py │ ├── adagrad.py │ ├── adgd.py │ ├── adgd_accel.py │ ├── gd.py │ ├── heavy_ball.py │ ├── ig.py │ ├── nest_line.py │ ├── nesterov.py │ ├── ogm.py │ ├── polyak.py │ └── rest_nest.py ├── line_search │ ├── __init__.py │ ├── armijo.py │ ├── best_grid.py │ ├── goldstein.py │ ├── line_search.py │ ├── nest_armijo.py │ ├── reg_newton_ls.py │ └── wolfe.py ├── loss │ ├── __init__.py │ ├── bounded_l2.py │ ├── linear_regression.py │ ├── log_sum_exp.py │ ├── logistic_regression.py │ ├── loss_oracle.py │ ├── regularizer.py │ └── utils.py ├── opt_trace.py ├── optimizer.py ├── quasi_newton │ ├── __init__.py │ ├── bfgs.py │ ├── dfp.py │ ├── lbfgs.py │ ├── shorr.py │ └── sr1.py ├── second_order │ ├── __init__.py │ ├── arc.py │ ├── cubic.py │ ├── newton.py │ └── reg_newton.py ├── stochastic_first_order │ ├── __init__.py │ ├── root_sgd.py │ ├── sgd.py │ ├── shuffling.py │ └── svrg.py └── utils.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # MacOS 132 | .DS_Store 133 | 134 | # Methods that have not been tested 135 | experimental/* 136 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Konstantin Mishchenko 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Optimization methods 2 | This is a package containing implementations of different loss functions and optimization algorithms. The main goal of this package is to have a unified and easy-to-use comparison of iteration complexities of the algorithms, so the time comparison of methods is approximate. If you are interested in finding the best implementation of a solver for your problem, you may find the [BenchOpt package](https://benchopt.github.io/index.html) more useful. 3 | ## Structure 4 | Currently, the methods are structured as follows: first-order, quasi-Newton, second-order, and stochastic first-order methods. A number of universal line search procedures is implemented. 5 | ### First-order 6 | Gradient-based algorithms: 7 | [Gradient Descent (GD)](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/gd.py), [Polyak's Heavy-ball](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/heavy_ball.py), [Incremental Gradient (IG)](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/ig.py), [Nesterov's Acceleration](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/nesterov.py), [Nesterov's Acceleration with a special line search](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/nest_line.py), [Nesterov's Acceleration with restarts (RestNest)](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/rest_nest.py), [Optimized Gradient Method (OGM)](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/ogm.py). 8 | Adaptive: [AdaGrad](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/adagrad.py), [Adaptive GD (AdGD)](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/adgd.py), [Accelerated AdGD (AdgdAccel)](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/adgd_accel.py), [Polyak](https://github.com/konstmish/opt_methods/blob/master/optmethods/first_order/polyak.py). 9 | ### Quasi-Newton 10 | [BFGS](https://github.com/konstmish/opt_methods/blob/master/optmethods/quasi_newton/bfgs.py), [DFP](https://github.com/konstmish/opt_methods/blob/master/optmethods/quasi_newton/dfp.py), [L-BFGS](https://github.com/konstmish/opt_methods/blob/master/optmethods/quasi_newton/lbfgs.py), [Shor's R](https://github.com/konstmish/opt_methods/blob/master/optmethods/quasi_newton/shorr.py), [SR1](https://github.com/konstmish/opt_methods/blob/master/optmethods/quasi_newton/sr1.py). 11 | ### Second-order 12 | Algorithms that use second-order information (second derivatives) or their approximations. 13 | [Newton](https://github.com/konstmish/opt_methods/blob/master/optmethods/second_order/newton.py), [Cubic Newton](https://github.com/konstmish/opt_methods/blob/master/optmethods/second_order/cubic.py), and [Regularized (Global) Newton](https://github.com/konstmish/opt_methods/blob/master/optmethods/second_order/reg_newton.py). 14 | ### Stochastic first-order 15 | [SGD](https://github.com/konstmish/opt_methods/blob/master/optmethods/stochastic_first_order/sgd.py), [Root-SGD](https://github.com/konstmish/opt_methods/blob/master/optmethods/stochastic_first_order/root_sgd.py), [Stochastic Variance Reduced Gradient (SVRG)](https://github.com/konstmish/opt_methods/blob/master/optmethods/stochastic_first_order/svrg.py), [Random Reshuffling (RR)](https://github.com/konstmish/opt_methods/blob/master/optmethods/stochastic_first_order/shuffling.py). 16 | ### Notebooks 17 | 1. [Deterministic first-order methods](https://github.com/konstmish/opt_methods/blob/master/notebooks/1.%201st-order%20methods.ipynb): GD, acceleration, adaptive algorithms. 18 | 2. [Second-order methods and quasi-Newton algorithms](https://github.com/konstmish/opt_methods/blob/master/notebooks/2.%202nd-order%20and%20quasi-Newton.ipynb): Newton, Levenberg-Marquardt, BFGS, SR1, DFP. 19 | 3. [Example of running the methods on log-sum-exp problem](https://github.com/konstmish/opt_methods/blob/master/notebooks/3.%20Log-sum-exp.ipynb): a hard problem where quasi-Newton methods may either outperform all first-order method or fail due to high nonsmoothness of the problem. One can change the problem difficulty by adjusting the smoothing parameters of the objective. 20 | To be added: benchmarking wall-clock time of some numpy and scipy operations to show how losses should be implemented. 21 | -------------------------------------------------------------------------------- /notebooks/5. Accelerated methods.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "13a733fd-8d67-46c8-adf4-f3b6da6d6bda", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "# Jupyter notebooks have problems with imports from parent folder, so let's change the path\n", 11 | "from jupyter_utils import change_path_to_parent\n", 12 | "change_path_to_parent()\n", 13 | "\n", 14 | "import matplotlib\n", 15 | "import numpy as np\n", 16 | "import seaborn as sns\n", 17 | "\n", 18 | "import matplotlib.pyplot as plt\n", 19 | "from scipy.sparse import csc_matrix\n", 20 | "\n", 21 | "from optmethods.datasets import get_dataset\n", 22 | "from optmethods.first_order import Adgd, AdgdAccel, Nesterov, Ogm, Polyak, RestNest\n", 23 | "from optmethods.loss import LogisticRegression\n", 24 | "\n", 25 | "sns.set(style=\"whitegrid\", context=\"talk\", palette=sns.color_palette(\"bright\"), color_codes=False)\n", 26 | "matplotlib.rcParams['font.family'] = 'sans-serif'\n", 27 | "matplotlib.rcParams['font.sans-serif'] = 'DejaVu Sans'\n", 28 | "matplotlib.rcParams['mathtext.fontset'] = 'cm'\n", 29 | "matplotlib.rcParams['figure.figsize'] = (9, 6)" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "id": "58c64ae4-62c2-40e7-8baa-e0c6b45e3fef", 35 | "metadata": { 36 | "tags": [] 37 | }, 38 | "source": [ 39 | "## Importance of controlling momentum" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": null, 45 | "id": "a07e7e94-7e4e-47eb-8680-07c46415e680", 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "dataset = 'w8a'\n", 50 | "A, b = get_dataset(dataset)\n", 51 | "\n", 52 | "loss = LogisticRegression(A, b, l1=0, l2=0)\n", 53 | "n, dim = A.shape\n", 54 | "L = loss.smoothness\n", 55 | "l2 = L * 1e-6\n", 56 | "loss.l2 = l2\n", 57 | "x0 = csc_matrix((dim, 1))\n", 58 | "n_epoch = 150\n", 59 | "trace_len = 300\n", 60 | "trace_path = f'../results/log_reg_{dataset}_l2_{relative_round(l2)}/'" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 10, 66 | "id": "ebe6ea60-f84e-4903-ad54-0b6b09ffe055", 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "data": { 71 | "text/plain": [ 72 | "" 73 | ] 74 | }, 75 | "execution_count": 10, 76 | "metadata": {}, 77 | "output_type": "execute_result" 78 | } 79 | ], 80 | "source": [ 81 | "from first_order import NestLine\n", 82 | "n1 = NestLine(loss=loss, start_with_small_momentum=False, mu=l2)\n", 83 | "n1.run(x0=x0_np.copy(), it_max=2100)" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 11, 89 | "id": "cbc1e60c-9c1a-4659-8e3a-533a2f4114f2", 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "data": { 94 | "text/plain": [ 95 | "" 96 | ] 97 | }, 98 | "execution_count": 11, 99 | "metadata": {}, 100 | "output_type": "execute_result" 101 | } 102 | ], 103 | "source": [ 104 | "n2 = NestLine(loss=loss, start_with_small_momentum=True, mu=l2)\n", 105 | "n2.run(x0=x0_np.copy(), it_max=2100)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 19, 111 | "id": "b47dabe1-39ea-4aac-82cb-41a82a3786cd", 112 | "metadata": {}, 113 | "outputs": [ 114 | { 115 | "data": { 116 | "text/plain": [ 117 | "" 118 | ] 119 | }, 120 | "execution_count": 19, 121 | "metadata": {}, 122 | "output_type": "execute_result" 123 | }, 124 | { 125 | "data": { 126 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiAAAAF0CAYAAAAafoJgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAB3lUlEQVR4nO3dd1gUVxfA4d9SFqlW1IC9gBUL2Ii9xd5LrFHzGY0txmhimolJ7FETNWqMGrtGYzfGbtREjaCosWEXELuoIH2Z749xF5CusMvCeZ9nnh1mhp2zd1j27J1bNIqiKAghhBBCGJGFqQMQQgghRO4jCYgQQgghjE4SECGEEEIYnSQgQgghhDA6SUCEEEIIYXRWpg4gtzh58iQAlpaWJo5ECCGEMA6dTgeAp6dnkn1SA2LGdDqd4eIK45PyNz25BqYl5W9a5l7+UgNiJPqaj+rVq2fac/r7+wPg7u6eac8p0k/K3/TkGpiWlL9pmUP5nz59OsV9UgMihBBCCKOTBEQIIYQQRicJiBBCCCGMThIQIYQQQhidJCBCCCGEMDpJQIQQQghhdJKACCGEEMLoJAERQgghhNFJAiKEEEIIo5MERAghhBBGJwmIEEIIIYxOEhAhhBBCoCjwJNR455MERAghhBDMXAcF28HK3cY5nyQgQgghhGD9QfXxerBxzicJiBBCCJHLRUXD2Wvqek0345xTEhAhhBAilzt7DWJi1XWvCsY5pyQgQgghRC7n668+ujrDGwWNc05JQIQQQohczveS+ujlbrxzSgIihBBC5HInX9SAGOv2C0gCIoQQQuRq4ZFw/qa6LgmIEEIIIYzC7wrExanrnkbqAQOSgAghhBC5mv72SxkXKJjXeOeVBEQIIYTIxXz0DVCNePsFJAFJ1aNHjxg2bBheXl7UrVuXOXPmEKevpxJCCCFyAEMDVCP2gAGwMu7pzMvo0aMpUqQIR44c4eHDh7z//vvY29vz7rvvmjo0IYQQ4rU9DQP/AHVdakCyiVu3bnHixAnGjRuHra0txYsXZ9iwYaxatcrUoQkhhBCZ4rcD6qNGY7wh2PWkBiQFV65cIV++fBQpUsSwrWzZsgQHB/Ps2TOcnJxMGJ0QQgjxahQF/rsOi7bBgi3qtobVwNHOuHFIApKC58+fY2trm2ib/ufw8HBJQIQQQpidv89C/0lw6278tqY1YdWXxo9FEpAU2NnZERERkWib/md7e3tThCSEEEK8lgVb4pOPkkVhcDv4uDdYWho/FklAUlC+fHmePHnCw4cPKVSoEADXrl2jaNGiODo6mjg6IYQQIuOuBKmP4/vAd4PVth+mIo1QU1CqVCk8PT2ZPHkyYWFhBAYGMn/+fLp162bq0IQQQogMUxS4HKiuVy9v2uQDcmgCcv/+fTw9PVPssRITE8Mvv/xCq1at8PDwoEWLFixatAidTpfouDlz5hAbG0uzZs3o0aMHDRo0YNiwYcZ4CUIIIUSmuvcYQsPVdbfipo0FcuAtmIiICEaNGkVYWFiKx0yYMIFNmzZRu3Ztmjdvjq+vLzNnzuTatWtMmzbNcFyhQoWYM2eOMcIWQgghstTloPj1cq6mi0MvRyUgwcHBjBw5knPnzqV4jI+PD5s2baJDhw7MmDEDAEVRGDNmDFu2bKFbt27UqlXLWCELIYQQRqG//VLMGextUz/WGHJMArJs2TLmzJlDZGQkdevW5fjx48ket379eoBEt1I0Gg1jxoxh586dbNy4McsSEJ1Oh7+/f6Y9X3i4WpeWmc8p0k/K3/TkGpiWlL9pZbT8/z3jDBTAteBz/P2D0jw+M+h0OixT6GKTY9qArFixAldXV1atWkXHjh1TPO706dM4OztTunTpRNuLFy+Oq6srJ06cyOpQhRBCCKO7ec8agFJFYkwciSrH1IBMnDgRb29vLC0tuXnzZrLHxMbGEhAQQM2aNZPd7+rqio+PD9HR0Wi12kyP0dLSEnf3zJvtR5/1ZuZzivST8jc9uQamJeVvWhkt/ztP1MfaVfLh7p4va4J6yenTp1Pcl2NqQBo0aJBiNY+evmFqSqOYOjg4oChKqg1YhRBCCHOj08HV2+p6+WzQAwZyUAKSHvqRTFOq3dBvj46ONlpMQgghRFa7dQ9iYtX17NAFF3LQLZj0sLGxAdRxQJKjTzxengNGCCGEMEe7T8DOY1CvivqzlSWUKmramPRyVQLi4OCAhYUFoaGhye7X33pxcHAwZlhCCCFElhg+C27cgdV71Z/LuIB1Nvnkz1W3YLRaLS4uLgQFJd/9KCgoiJIlS6bZliQ7ePIghGvrFxN88ZqpQxFCCJENPQ1Tkw+AkBffu8sXM108L8tVCQiAp6cnd+/eJTAwMNH2wMBAgoODqV69umkCy6Czm9fQJvp77I/NNHUoQgghsqHzN5Nuyy7tPyAXJiCdOnUCYNasWSiKAqgjoc6aNQuAHj16mCq0DFFiIgFwUB6bOBIhhBDZ0X/X1UerBJX62aUHDOSyNiAA3t7etGnThp07d3Lnzh28vLzw9fXFz8+PLl264OXlZeoQ00VjnQcALZEmjkQIIUR2dP5FAtLcCwrnh8OnoV09k4aUSK5LQACmTZtG2bJl2bx5M8uXL8fFxYWPPvqIgQMHmjq0dNNYqz11tBpJQIQQQiR17ob6WLUMTB1q2liSkyMTkC5dutClS5cU92u1WkaMGMGIESOMGFXmstC+qAHRRJk4EiGEENmNosTfgqlSxrSxpCTXtQHJKSxfJCA2UgMihBDiJXcfw+Nn6npVSUBEZrK0UROQPBYRJo5ECCFEdqOv/bC0hAolTBtLSiQBMVNWLxIQW4tIlDjFxNEIIYTITs69SEDcioFN5s+tmikkATFT+gQEIDpS5q4RQggRT5+AVC5t2jhSIwmImbLOEz9fTWSEtAMRQggRL2EPmOxKEhAzZZ0nQQ1IuCQgQgghVDGxcOGmup5de8CAJCBmS2sbn4BESQ2IEEIIYM1ecO8DES9GaKiSjW/B5MhxQHIDG9sEt2CePzdhJEIIIbKD01eg33fqukYDA9tAWVfTxpQaSUDMVP4iBQ3rYQ/uA5VMF4wQQgiTm7JKfXQvAVunZK+Zb5Mjt2DMlLWNNQ9jCgEQ/vCOiaMRQghhSpduwcZD6vrn/bN/8gGSgJi1R3FFAYh5ctfEkQghhDCV68HQ8yt1+PUyLtCziakjSh9JQMzYU00RAJRnUgMihBC50bGLdtQZona7tbKEmSPAykwaV5hJmCI5z62cAbCOkARECCFymyvBWt77sRi6OHDOB+u/gYbVTB1V+kkCYsaitIUhFmxj5BaMEELkNvv9HNDFaSjmDIfnQcmipo4oY+QWjBnT2aqNUJ3ipAZECCFym1NX7QBoW8/8kg+QBMSsaRzUBKSARhIQIYTITXQ6OH1NHZDSu6qJg3lFkoCYMeu8LxIQq8dEhUeZOBohhBDGcu4GhEVaAvCmJCDC2PIUKGBYfxh8z4SRCCGEMKaj/6mPhfPFUMoMb7+AJCBmzdE5v2H9yR1piCqEELnF3y8SkBplI9BoTBvLq5IExIzZOtoTprMHIOx+sImjEUIIYSxHz6mPNctFmDaQ1yAJiJm7F6vONBT1INDEkQghhDCGwPsQ8OKuuyQgwmQeadQB/+OeSAIihBC5wfHz6qOtNg73YubbAUESEDMXaq3WgGjDA0wciRBCCGM46a8+ViwRiZWlaWN5HZKAmLkoWxcAHKMlARFCiNzA95L6WKVkpGkDeU2SgJg5xVHtf1UISUCEECKni4uDk5fV9cqlJAERJmSVX01AilgFExMVY+JohBBCZKUrQfDsubpeVRIQYUp2hdQZcS00Cvdu3jZxNEIIITLLkTPw8QJ48CR+m++L9h9O9lDC2by/dEoCYubyFnU2rD8KlJ4wQghhDnQ6mL4G9vqkfMywWTBzHbT9GELD1W36Bqhe7mBh5p/gZh6+sLGz4UFMYQDC7ko7ECGEMAf7TsKnP0Pvb0BR1G1TVsKHc9Wfw8Lh4i11+0l/6D4BomPiG6B6upsm7sxkZeoAxOu7RxmcuU+eC8vRxbyNpbUZ98sSQohc4NqLO+aPn8HtB6C1hi8Wq9u6N1GTEH1iAmpNSbtP4NSLBqg5IQGRGpAc4Fn1DwHwtNzLkQUzTByNEEKItAQ9iF+/FKA2LtXzu6wuAKWKwsRB6vr+kxDxYtwxrwrGiTMrSQ1IDuD9dg8Of/cXDaMWUPvuNwRf641L2RKmDksIIUQKgu7Hr1+6BY728T+fvqp2twWoXh4+7w9FCqi1IEEPoIEHlH4D/J8ZN+bMJglIDlF92FTuTd9MEeu7nFk+Fpdv1ps6JCGEECm4/VINSAGn+J9PX4lPQGq6gUYDg9urS04it2ByCKcCTlypNB2AesoGfLb9aeKIhBBCpOT2w/j1S7fi24QAnLsB52+q69XLGzUso5IEJAfx7tOX07GNACh8dCRKnJLGbwghhDA2RUnaBiRhAhIdAzGx6noNSUCEObCw1GDd9nsASlpfI/DyDRNHJIQQ4mUhofGNSQHuPIqv8UiocH54o6DRwjI6SUBymAp1avAkNh8AD1YMJSrcfKdqFkKInChh7Yde+ItR1Qvljd9Wvbza/iOnkgQkh7G0tuSs20xA7ZbrN6U3sdGxJo5KCCGEnj4B0VqDc77E+zo1iF+vmYNvv4AkIDlSw4GD+Kuw2iC1Lps4NvV9E0ckhBBCT98DxrUQVCoVv72AEzSuEf9zTm6ACpKA5FiNR47jL8dPAWgQs5iTf+wycURCCCEgPgEp5gzuCYZsKuuSeICxnDDYWGokAUmFv78/gwYNonbt2rz55pt8/PHHPH782NRhpVujsZPwi20CQP4jo4mJijZxREIIIfS3YFydoULCBMQVyheDeaNh/kfqYGM5mSQgKYiMjOR///sf1atX5++//2bHjh2EhITw2WefmTq0dNNYaLDv+iM6xYIy1v4cWzzX1CEJIUSuF5SgBqRCyfjt5VzVx/c7w5AOxo/L2CQBSUFwcDDu7u4MHz4crVZL/vz5efvtt/HxSWXu5GzIzasqf+dR24B43/mEw/NmyPggQghhQglvwSSsASnjapp4TCXXJiCRkZHcunUr2SUiIoIyZcqwePFiLC3jZ5bdvXs3lStXNmHUr8bjvW/xj66GlUZHw3sfc+ybdyQJEUIIE9HXgLg4Q/HC6gJQK4e3+XhZrp0L5syZM/Tv3z/ZfQsXLqRJkyaGnxVF4YcffuDgwYOsWrXKWCFmmvyF82PzxTEOzxxOQ92veOtW8vey+tQf9J6pQxNCiFwlNByePVfXizmDhQUc+QnuhyTuEZMb5NoEpE6dOvj7+6d5XFhYGJ9++innz59n1apVuLu7GyG6zGfnaEuDCUs4+tVzvFmP57UPuH76TcpUN78aHSGEMFcJJ6Er5qw+JqwFyU1y7S2Y9AgICKBr166EhYXx+++/m23yoaex0FBp1M8ExZTE1iISu3UtuXDkuKnDEkKIXEN/+8XSEooWMG0spiYJSAqePn3KO++8Q82aNVmyZAkFCuSMv5R8zvl43Po3QnUOFLUOptyuhvyzZL46O5IQQogspU9AihZQk5DczCwTkPv37+Pp6Zlie4yYmBh++eUXWrVqhYeHBy1atGDRokXodLp0n2PTpk0EBwfz559/4unpSY0aNQyLufNoVIe73X24Gl0RrUUMb94czpEfvjZ1WEIIkeMl7AGT25ldG5CIiAhGjRpFWFhYisdMmDCBTZs2Ubt2bZo3b46vry8zZ87k2rVrTJs2LV3nGThwIAMHDsyssLOd8jUq8LTkCY7OfAdvi000ePwNx1aWpF6/QaYOTQghcixDD5hCpo0jOzCrGpDg4GD69u2Ln59fisf4+PiwadMmOnTowMqVKxk7dixr166lTZs2bNmyxezG8chKeQs44PXlOnxi3wKglv97+G3fbuKohBAi57rzSH10lQTEfGpAli1bxpw5c4iMjKRu3bocP55848n169cDMGzYMMM2jUbDmDFj2LlzJxs3bqRWrVpGifllOp0uXT1v0is8PBzgtZ/TovMULv5+h4o2Z6l8vCt7HyygxJv1MyPEHC2zyl+8OrkGpiXln3E3b5cAbLFSHuDv/3pTe5hD+et0ukTjaSVkNjUgK1aswNXVlVWrVtGxY8cUjzt9+jTOzs6ULl060fbixYvj6urKiRMnsjpUs+OQ146n7RbhH1UZrUUMja8N5da+ndIwVQghMtmDp+r3/kJOsSaOxPTMpgZk4sSJeHt7Y2lpyc2bN5M9JjY2loCAAGrWrJnsfldXV3x8fIiOjkar1WZhtMmztLTM1K68+qw3U57THe6WOsiln5pTQXuWlnfHcHr1LlwGLaJw6ZJp/34ulKnlL16JXAPTkvLPmLg4ePhMXa9R+Q3c3V9vtjlzKP/Tp0+nuM9sakAaNGiQYjWOnr5hqpOTU7L7HRwcUBQl1QasuVnREs7kH3mAv2O7AFCdPVgsqsWVo/+YODIhhDB/j59B7IvOmG8UNG0s2YHZJCDpERERAZBi7YZ+e3S0TEufkiLFCvLm5I38VWkLj2MLUMjqASX+aIrfBvMbgl4IIbITfQNUkAQEclgCYmNjA6jjgCRHn3jY2toaLSZzpNFA414dedDjX65FuWNjEU2Ns/3wmfs5ii7O1OEJIYRZuvOizamlJRTKa9pYsoMclYA4ODhgYWFBaGhosvv1t14cHByMGZbZcq9RDocPjnEiujkAte5P5vSkHsSGPzdxZEIIYX70NSBF8quT0OV2OaoItFotLi4uBAUFJbs/KCiIkiVLptmWRMQr4pqfql/tZDdqt+YaMRu5Makhz+7cNnFkQghhXu69qAGR2y+qHJWAAHh6enL37l0CAwMTbQ8MDCQ4OJjq1aubJjAzZmtnTYuJP7E9/1x0igXlLU4RMac2t8+cNHVoQghhNvQ1ILl9Ejq9HJeAdOrUCYBZs2ahvBjHQlEUZs2aBUCPHj1MFZpZs7CA9mNGcKDSHzyNdaKIVTD51zfgyvblMl6IEEKkgz4BkRoQldmMA5Je3t7etGnThp07d3Lnzh28vLzw9fXFz8+PLl264OXlZeoQzVqL3q3wOXwM5x3tKGVzg/InBnDn0nLeGDAfnCuYOjwhhMi27koNSCI5rgYEYNq0aYwcOZIHDx6wfPlyQkJC+Oijj/jmm29MHVqOUKthJeIG/8ue510BeOPZQWLnePB4/88mjkwIIbIvwy0YqQEBzLQGpEuXLnTp0iXF/VqtlhEjRjBixAgjRpW7lCnvTMFvfmfipJ30ixlBGdsbFPhrKHuPXqb+2OnY2kpDXyGESOiuNEJNJEfWgAjjyOsAX05qwx81z7LnWXsAWkTP4uzXrbkb+NDE0QkhRPYRFg5h6liZkoC8IAmIeC0WFjCytwONpm3Gp8BHANTR7iV2nicX/vE1cXRCCJE93E0w8a20AVFJAiIyhU0eS2p9+D2na6zluc6OYtoAyu58E9+VC9UZmIQQIhcLDY9fzydjYQKSgIhMVr3L29ztfoIb0W7YWETjdfl9HnxbkbgTv0BMpKnDE0IIkwiPil+3y2O6OLITSUBEpitbozJ5x/pwMPZtAJxjL2Ox/T10P9WEp4Fp/LYQQuQ84S++f1lagrVZdv/IfJKAiCxRwNmJBt+u5fuC/7HiXn9i4qywfHSR6AVvwgN/U4cnhBBGpa8BsbMxbRzZiSQgIstYWcHY0VWw6r6cTv67CY11QPs8kKj59VD8/zR1eEIIYTT6GhC5/RJPEhCR5Xq3gO8mN6Xf7QPcj3bGJjYEZWVbHm/9RhqoCiFyBakBSUoSEGEUNdxg8ZxafGZ1iuPP6mChUSjg+xUBsztCxBNThyeEEFkqUp+ASA2IgSQgwmgK5YPFk4rxoOshVoUMBaDEkx08nOZFXPBZ0wYnhBBZyHALRmpADNJMQOJeqiJ/+WchMqp9Qxs6Tl/AD7G/EqHLQyHdNWIW1CXaZ4XMrCuEyJH0t2BsJQExSDMBWbx4MRs2bABgw4YNLF68OMuDEjmfox2M+HYAcwsc5UZkKWyIQLvtHcIWtoQHl0wdnhBCZCp9DYgkIPHSTED69OnD+vXrefToEb/99ht9+/Y1RlwiF7Cygo/H1mCv50k2P+wMgEPwPnRzPNDt+hSin5s4QiGEyBzh0gYkiVQTEB8fHy5cuICXlxdvv/02tWrV4vz58/j4+BgrPpELvPd2AfK/t4nBwTu4HlEaS2Kw/GcqcT9WgjtnTB2eEEK8NmkDklSq47Ft2rQJgHv37hEUFMTly5d58uQJALVq1cry4ETu0bgG1JrXlm8WN8XOZxqfFJ9KnmcBxC1phEW/P6Dkm6YOUQghXlmE1IAkkWoNyJQpU5gyZQparZbvv/8erVZr2CZEZrO3hWkjbSnT52u8z/gQHPUGFlFPiVvWAi7LwGVCCPMlt2CSSrMNyJ49eyhUqBBt27bF2dmZPXv2GCMukYv1ewsmflqVZhf+4WpEWSxiI1BWdYCza00dmhBCvBJDI1StaePITtJMQIoWLcqoUaMAGDVqFEWLFs3yoIRo/yYs/K40bS8f4WxYVTRKLMqGPvDvT6YOTQghMkxuwSSVZgLi4eFB4cKFAShUqBAeHh5ZHpQQAI2qw9rv36D7rUP889QbDQrsGAH7J8h4IUIIsyKNUJPK0Eiop0+fJjIyMqtiESKJmm6w7Yf8/O/uHnY+aqNu/Otb4rYOhTidaYMTQoh0MgxEJjUgBhlKQN5++22++eabrIpFiGS5l4B9P9kzjS2suNcfAIuTi9Ct6Q4xkhALIbK/CJmMLokMJSAajQYlQdV3586dWbduXaYHJcTLXJ1h74/W+FVZxozAcQBY+m8mbmlzuCvzyAghsjfpBZNUhhIQBwcHnj17Zvj54sWLnDkjA0UJ49Baw+xRGixbT2fste8BsAj6B+Wn6rD5XYiNMm2AQgiRAhmKPakMJSDlypXj2LFjnD0r3ziF6YzpCa4dP6L9uR1ceF5RbZx6aims6yZJiBAi24mLk1swyclQAtKnTx/Cw8Pp2bMn7dq1AyAgIABfX1/CwsKyJEAhkvNhT+j6Xlu8L/zHlze/VTf674C1XSUJEUJkK5HR8etyCyZeqkOxv6xdu3bodDrmz5/P1atX0Wg0nDp1in79+gFQvHhxKlasSKVKlQxLwYIFsyRwIQa0hgYelvSY8AXR17VMK/MJXP5DTUJ6bQQr+aohhDC98ARt5aUGJF6GEhCAjh070rFjRwICAmjZsiXly5enRIkSXLhwgYCAAAICAti9ezcajQZQxw6pXLkylSpVMgxoJkRmKesKu2ZCwxEfw3UkCRFCZDsRUgOSrAwnIHolSpQAoEqVKoa5YZ48ecKFCxc4f/48Fy5cMCQlf/31F4cOHZIERGQJ53yw63to8HISsqYT9NoE1ramDlEIkYslqgGRBMTglRMQgC1btqDTxQ8GlS9fPry9vfH29jZse/78ORcvXuTChQuvcyohUlWyqJqENBr5MXHXLZhRZhxc2QWrO0DvraC1M3WIQohcKmECInPBxHutBKRChQppHmNvb4+XlxdeXl6vcyoh0lSpFOyYBi3GjCUmzpofyo2Ga/tQVrZF03c72DiYOkQhRC4UnqBdvNSAxMtQLxghsrs6lWDPTNiq+4DhV9SJ6zQ3/4KVrSEq1LTBCSFyJX0NiKUlWL/W1/6cRRIQkePUrQx+SyCw1DCGXF5EnKKBW3/DkkZw77ypwxNC5DLhCcYAedE/QyAJiMih8jnChm/gustg/nd5qZqE3PGDBTXh6GyZTVcIYTQRMgx7siQBETmWjRY2fgf/5R1As7MHuR5ZBnTR8OcY2DVWHZ5QCCGymP4WjIwBkpgkICJHc7JXG6YGOzSimu9ZNof0UHccnQVbBoEu1rQBCiFyPH0NiMwDk5gkICLHK1JAHazMKZ89Pf5bwy/3hqo7/Jar88fERKb+BEII8RqkBiR5mZqAjB07lr59+2bmUwqRKUq/AftmQxlXS4b6z+e7W1+oOy5thRWtIPKpaQMUQuRY4dIGJFmZmoCcP3+ekydPZuZTCpFpKpaCf3+G1nU1fHXrWz68NlvdcfMQLG0CYfdNGp8QImfS14DILZjE5BaMyFXyOcK2KTCqG8y5PZoBl5ajw1LtIbO4PoTcMHWIQogcRnrBJE8SEJHrWFjArBEwsA2svN+fLuc2E0MeeHQFFtZWxwwRQohMknAcEBFPEpB00Ol09OvXj/Hjx5s6FJFJNBpY+BH0bAo7Hren2em9hFIQwh/Cr83g9EpThyiEyCHkFkzyJAFJh3nz5uHr62vqMEQms7KClV/AsM7wz7P61DzxL4FKBXWskI39Ye9n0k1XCPHapBFq8iQBScOxY8fYs2cPLVu2NHUoIgtYWsKcD+CrgXA9sizVjh7jREwLdefhKerIqTcPmzZIIYRZk264ycu1CUhkZCS3bt1KdomIiADg0aNHfP7558ycORNbW1sTRyyyikYDEwbAvNHwLC4f9Y/tZMG90cShgXv/qXPIbBoAUWEmjlQIYY6kEWrycu28fGfOnKF///7J7lu4cCGNGjVi3LhxDBw4kAoVKhg5OmEK73eGgnnh3WlWjPCfza+3+7C48nA8bE6og5YF+UCvjeAsfw9CiPSLkEaoycq1CUidOnXw9/dPcf+CBQvQarX069fPiFEJU+vRFJrUhJ82wfS1XtT89xhfVpjHhCIfoXlwAebXgGbfQr3Rpg5VCGEm9G1AbKUGJJFcm4CkZevWrdy/fx8vLy9AvWUDsG/fPmmQmsM554OvB0HD6tDpMwu+uTSKv+7XYmuNt3GKCYDd4+C/ddjU+JKo/FIbIoRInfSCSV6ubQOSll27dnHq1Cl8fX3x9fWlXbt2tGvXTpKPXKRpTdg/G0oWhcOP61H8r3NsjRmu7gw+Scmd3Sh06nuICjVtoEKIbE3GAUmeWSYg9+/fx9PTk1WrViW7PyYmhl9++YVWrVrh4eFBixYtWLRoETqdzsiRCnNXqyKcXgoDWkOYzpEux+bR2v8IT23d0SixFLywGH5wgxML4fkDU4crhMiGDL1g5BZMIpmagFStWtVwyyKrREREMGrUKMLCUu6RMGHCBL7//nucnZ3p378/BQsWZObMmXz22WevfN6pU6cyderUV/59Yb6c7GHJeNg0CQrnhz336vPGntN8f38C0dhC2F3Y/j5MKwor28KDlNsWCSFyF0WRRqgpydQEZPr06axcmXUjSAYHB9O3b1/8/PxSPMbHx4dNmzbRoUMHVq5cydixY1m7di1t2rRhy5Yt+Pj4ZFl8ImfrWB/OLoPuTSBKycMnlyZS/l9/tj1/B52VPShxcHkn/FRVHcQs+rmpQxZCmFhkdPy61IAkZjaNUJctW8acOXOIjIykbt26HD9+PNnj1q9fD8CwYcMM2zQaDWPGjGHnzp1s3LiRWrVqGSXml+l0ulR73mRUeHg4QKY+p0jbxF4wqJk1S3c5svGfYnQ+uYw8Fgv41vs3hjl8SZ7wIDg8hdh/f+Zx5Xd54tYLxcrO1GHnSPIeMC0p/7Q9CbMAygNw/84N/C2jU/+FDDCH8tfpdFhaWia7z2zagKxYsQJXV1dWrVpFx44dUzzu9OnTODs7U7p06UTbixcvjqurKydOnMjqUEUuULJwDJ90C2Lp6Mu4uUYRGWfLuL8HUPyvS+yxHUuchRarqMcUPjWDMpubk//8EizD75s6bCGEkUVEx3/M2mgVE0aS/ZhNDcjEiRPx9vbG0tKSmzdvJntMbGwsAQEB1KxZM9n9rq6u+Pj4EB0djVarzcJok2dpaYm7u3umPZ8+683M5xTp5+/vT+2K8N8KG37ZAV8thUdP7Wm9ewa1XT9kRePplLuzUE1E/GZQ2G8GFPGAuiOhxjtgaW3ql2D25D1gWlL+6RAQv1qlYhlcCmXeU5tD+Z8+fTrFfWZTA9KgQYMUq3H09A1TnZyckt3v4OCAoiipNmAVIqOsrOD9TnB5NXzYA6ws4cRtFyqs/oGuD69zo9QHKFoH9eB7Z2HrYJhTCc6shjjpmSVETqbvAQPSCPVlZpOApId+DpeUajf026OjM+8enBB6+Rzh++Hw33Jo/6a6betZF8qt+IHqlx+xq8ph4qr1B40FPL4Kv/eFeR7guxgiQkwbvBAiS+jHAAFphPqyHJWA2Nio6WVMTEyy+/WJh0wsJ7KSW3HYMhn2zIJmnuq2cwFa2s5vQLXty9lb9xxKlR7qjgcX1BqRaUVgdSe4ulfttyeEyBH0XXAtLcHabBo9GEeOSkAcHBywsLAgNDT5kSn1t14cHByMGZbIpZp5qknI6aXQs6m67cJNaDWtInX2/8bRhqdRPHqDtR3oYuDSVljeEn6qrk5+Fys1dUKYu4SjoGo0po0lu8mUfMzf35+LFy/y6NEjnj17Rt68eSlQoACVKlXCzc0tM06RLlqtFhcXF4KCgpLdHxQURMmSJdNsSyJEZqpaFtZ8BZ/0gQlLYMdROOkPDb6rRsNqq/m0x3Oa22/F4tTPcPOw2k5k0wDY+ynUGgqe74KTq6lfhhDiFcg8MCl75QTk7NmzrF27loMHD/L06VPDdkVR0CRI8/LmzUuTJk3o1asXHh4erxdtOnh6erJ161YCAwMpXry4YXtgYCDBwcGpduEVIitVKwdbp8Dx8/DFL3DQDw6fgcNn7HF17s3A1r0Z8bYvzudnwvkNEHoHDnylLkU8oFwLqNgZSnjLVykhzIRhGHZJQJLI8C0YHx8funfvTs+ePdm8eTNhYWFUrFiRtm3b0qdPH4YOHUqfPn1o27YtFSpUICwsjM2bN9OzZ0+6d++e5eNwdOrUCYBZs2ahvLiXrigKs2bNAqBHjx5Zen4h0lK3Muz7Qb0907i6uu32A/huBRQb6UW/y2s50/461P8Y7F702bt3Fv6ZCYvrw8914ORSCH9sqpcghEgnwzDs0gA1iQzVgIwcOZJ9+/ZhY2NDmzZt6NSpE7Vq1SJPnpRLNjIykhMnTrB161YOHDjAO++8Q/PmzZk7d+5rB58cb29v2rRpw86dO7lz5w5eXl74+vri5+dHly5dsnyuGiHSq5mnuly7Dcv+hF+2w4MnsGYvrNlbgnqVpzGsw7d0LX0Um8B94P8H3D0Nt33UZdsQKNMMqnSHip3ArqCJX5EQ4mVSA5KyDCUgR48eZejQoQwaNAhHR8d0/U6ePHlo2LAhDRs2JDQ0lCVLlmTpfDEA06ZNo2zZsmzevJnly5fj4uLCRx99xMCBA7P0vEK8irKu8O3/4PN+sHY/zPkdzl6DY+fh2HktIxwa83bTxgxs/S1edkfRnJgHl7ZBTDhc3a0u24ZCmaZqIlKmGRQsL7dphMgGIl60JZcakKQ0ipL+Pn9PnjwhX758r33SzHoec6IfDa569eqZ9pzmMApeTpZV5a8o8JcfzN8C2/+BmNj4faWKQof60KlOOPXz/InlxQ1weUfSie+cikHZZlC6qfqYQxuxynvAtKT80zZ+IcxYC2/Vhp0zMve5zaH8U/vsy1ANyMtJw4IFC3j//fczHFBuSz6EyAiNBprUVJcHT2D1Hli6E87fgJt31RqSOb/bkd+xK63rdqVj3XBaF9iF/bXf4dpeCH8Iz4LUrrx+y9UnLeSu1pCUbgrFakHeElJDIoQRSC+YlL1WN9wff/yRkJAQPvvss8yKRwiRgHM+GN0DPugO/12HbX/Dtn/Ubrwhofr2InZYWXahYbUutKkTR5sy/1E++gAWN/bDzUMQHQYP/dXlxAL1iW3zQ9Hq8Eb1F481wLmCzE8jRCYLl0aoKXqtBKRy5cqsXLmSJ0+eMGXKlBTH13j8+DHz5s1jwoQJr3M6IXItjQY8yqrLF+9A0H3YflS9RXPQD6Jj4MApOHDKgrFUI79jNcoX+xB31xhal/Wlru1+ioXux/L2MYiNUod+v3FQXfQstVCkSnxiUsRDbUvi+IbUlgjxiqQRaspeKwFZuXIlw4cPZ9u2bTx79owff/zRMBw6QHh4OEuXLmXp0qVERERIAiJEJilWWJ0A7/1OEBYOe33hj2OwzxcC76u1IycuwomL1qykHlAPS8sv8CgZQ4Ni/tTOe5pylqdxiT1NoXA/bGIfgy4agk+pS0LWtpC/DBQoC44ukCefWoNimz9+PU9+sM2nPubJCxYy2J8QIDUgqXmtBMTOzo5Fixbx8ccf8+effzJo0CAWLlyInZ0d69atY/78+Tx+/BgLCwvD+BxCiMzlYAedG6qLosDV2+BzEW7cgXM3wPcSXA8GnQ78rlvjd70KUAXo++IZFIrZBFHd/jTVHE4bHsvaXld3x0TA/fPqkl42TqB1UBebF49ax5d+TmaxcQCbvODkAg5vgFXyE0sKYS4iEgzFLhJ77aHYra2tmT17NgUKFGD16tX06tWLmJgYAgICUBSFli1b8sEHH1C2bNnMiFcIkQqNBsoXU5eEHj0FX384c1VNRm7ehcfP4EkYPAnTcCesOEGPi7PjcXvD79hahFM6zw3K2V6lTJ5rlLW9RmHr++SzCiGf1RPyv3jMZ/UES01c4hNGPVOX12VfWO3B41QM8haDAuXUBrXOFSBfqdd/fiGymAxElrJMm5uvY8eO/Pnnn1y7dg0ALy8vPvnkE6pUqZJZpxBCvKKCedVugG/VTn6/osDzCDUhCQmFZ+Hw9LkdT8Mq8/R5ZZ49h+DncPG5uv9GMFy8Ak/DQEMcjpahhqQkv1UIRe2f4lk6jEouYXiVDqOQbZjaGDY6DKJC49ejwyAqLPHPugST8D2/ry53/JIGbamllEMJop1Kw21vdfyTYnWk1kRkK9ILJmWvnYBcuXKFH374gQMHDqAoCoUKFeLhw4eEhITg7OycGTEKIbKYRqPeynGwU9uXpIeiwN3HcOmWBZdu5eXirbycvVaSw+cg7imsC45/7o71oW9LaN4QHO3SeOLo5xAaDM9uJ16eBsDDy/D4ijp7sC4am6dXsXl6FQL3wsGJoLWHkg2hbHM1ISlSFSxy1KTfwszo24BIApLUayUgY8eOZefOncTFxVG4cGFGjhxJ165dmTNnDgsXLqRXr14sXbqUUqVKZVK4QojsQqOBNwqqS5Oa8dvvh6gz/h47D3t91EaxW46oi9YahnSACQOggFMKT6y1V3vfFCyf/H5dLDy5CQ/9uX/hCNpn18n37CLcP6cmL1f+VBcAe2d4o6bau8etLZRqJAmJMKqQUPUxr71p48iOXisB2bFjBw4ODvzvf/9jwIABhjlhRo8eTcGCBZk8eTK9e/fm559/pmrVqpkSsBAieyucHwa1VZfYWNh4GBZvV2f+jY6BuRth1R74agAM7QTWGf0vZGkFBctBwXKEUA6AfO7uEHoXrh+A6/vg2j54GgjPH8QPV//PTHUAtmp9oXp/cM6+o0eKnCE2Vh1MEKCoTNWUxGt9Fejbty979uxh6NChSSak69evH9OnT+fZs2cMGDCAo0ePvlagQgjzY2UFPZvC3tnwYDtMGwpO9uq3wtFzodpAtftw+ieESIVjUajWGzovhY9uwejL0PEXqPcBFHvR+OVpAByeDHMqwM914d/5EP4oE04uRFIPn8b/bRctYNpYsqPXSkC++OILChRIuVTbt2/PTz/9hE6nY+jQoa9zKiGEmXOyh7G9wH81vNdBvRPiHwAdxkPrcepQ85lGo1Fv4Xj9D9r8AEP+hQ/8odEXai0IQNC/sGM4TC0Mi+rBoSmSjIhMdfdx/HqR/KaLI7vK8puhjRo14tdff8XW1jarTyWEMAOF88OCj+DUYmjmqW7b6wM13oVPFqi9cbJEITdo/i2MuQGDDkLNgerYI0ocBB6HfZ/B9yVg52h4ciuLghC5iT4BsbaC/OmbQD5XMUprrBo1arB69WpjnEoIYSaqloXdM2HLZHXcEp0Ovl8HVd5R57vJMhYWULqxeqvmk3vQ/0+oO0odwTUmHI79CLPLwoa+cPdsFgYicjp9AlIkv7R9To7RiqRcuXLGOpUQwkxoNND+TTjzK0wcBDZaCLgHnT+Dzp+r61lKawflW0HbH+GjAGj1vTrwWZwOzq6Gn6rBitZwYQvERGZxMCKnuR+iPhaR9h/JylACsnz5cqKjo9M+MBXR0dEsW7bstZ5DCJGz2GjVSfbO/grNvdRt2/6G6oPg9BUjBZHHCd78CD68rtaOOFdUt1/ZBWs7w7QisPdziMyEEV5FrmCoAZEEJFkZSkCmTJlCy5YtWbFiBY8fP077FxIICQlh+fLltGjRgmnTpmXod4UQuUO5YrDre1jzldpW5GkYtP0Yjp4zYhBWWrV9yIhz0GcblG0BGgt1aPnDk+GHcmrvGV2MEYMS5ujuizbN0gMmeRlKQJYuXYqTkxOTJ0+mYcOGvP/++yxfvpwzZ84QFhaW6NiwsDBOnz7NsmXLGDp0KA0aNGDKlCk4OTmxZMmSTH0RQoicQ6NRu+7umw35HNRvkY1Gwqc/Q9TrVcBmjIUFVGgPA/bAx3eg8QSwtlPHFtkxHOZWgYtbM6kPsciJ9LdgJAFJXoaGALKzs+O3335j165drFy5koMHD/LXX38Z9ltaWmJnZ0d4eDg6nc6wXVEUKlWqRN++fenUqRMW0hpHCJGGyqXh0FzoP0mdRG/6GjhwEvbMgrwORg7GoTA0mwi1hsCBr+DUUnh0GdZ0gpINoMUUKOGtZk9CvJCwEapIKkMJSK9evejcuTOTJ0+mc+fO/Pfff+zfv58TJ05w8eJFIiIiePZMvT9qa2tLpUqVqFWrFk2bNsXDwyNLXoAQIueqUgaOL4Rvl8PU1eqMvt2+hB3T1HYjRufkAp1eDG62+2N1yPdbR2BxfchXEqr1g4afqo1bRa4nbUBSl+Gh2JUE1Y0TJkygR48erFmzBoCIiAhCQ0NxcnJKMjKqEEK8Cq01fPs/tavuwClw4BR0/RKWfGLCoIpUgf474dp+2PMxBJ9Sxw459B2c3wBdlkPxOiYMUJhaVHT8PDByCyZ5GboX4uDgYKjhALh48SJnz8b3k7e1taVw4cKSfAghMl3/VjBliLr+53Go3B92nzT2vZiXlG0GQ31h5Hlo8AlYWMFDf/ilHmwepM5PI3Kl+0/i16UGJHkZSkDKlSvHsWPHEiUdQghhLON6wdJP1VElQ0Lho0UuHDxj4mlGNRooXAlaToUhJ6CIh9ow9dSvao+ZP0bBo6umjVEY3d0Eo/pLDUjyMpSA9OnTh/DwcHr27Em7du0ACAgIwNfXN0kvGCGEyGwaDbzTCs6vAE93iFM0jFnkwpEzpo7sBZca8P5JaL8A7ApC9HM4Phd+dIM1XeCBv6kjFEaib/9hawOO0iQoWRlqA9KuXTt0Oh3z58/n6tWraDQaTp06Rb9+/QAoXrw4FStWpFKlSoalYEGZg1gIkbmKFFAbotYeHE3gAy0tP4LZI2FIh2zQEcXSCmoPBY9ecHKxmoA8uQUXN4P/dqg9DJpMUBMUkWPdS9AF1+R/k9lUhhuhduzYkY4dOxIQEEDLli0pX748JUqU4MKFCwQEBBAQEMDu3bvRvCjxQoUKUblyZSpVqsSoUaMy/QUIIXKnwvlh8egg3vuxGLfuaxk+C/75DxaMAYfs8I0zT151ZNW6H8CFjbD3Uwi5AcfnwOkVL/aNUkdgFTnOPemCm6ZXHpCjRAl1SusqVarw008/cfDgQY4fP87SpUv56KOPaNWqFSVKlODhw4f89ddfLFiwINOCFkIIgOLOMWz4/BbdGqs/r9kLdYfCxZumjOolllZQtSeMughvzQAbJ4h8Avu/hJkl4c8xcPOIOv+MyDGkC27aMlwDktCWLVsSDTiWL18+vL298fb2Nmx7/vw5Fy9e5MKFC69zKiGESJaDbRzrvoZ5G2HcArh4C+oMhfF94INuYG9r6ghfsLKB+mOhxjvw9wz49yc1ETk6W13yl4G6I6DmILX2RJg1fQIiDVBT9lpDklaoUIHKlSuneoy9vT1eXl7079//dU4lhBAp0mhgZDf4aw4ULwzPI+DLxVC+N8z6TZ1TJtuwd4a3psNHN6HpN1C0uro95LpaGzLDFdb3gnO/q41YhVm6LzUgaXqtGhAhhMhO6lYGv6UwbTXM3ajehx83X01GGlSD5p5QvTxULAlvFFSnezEZe2do8qW6PLikNlb1W6YmHf+tUxdrWyjTTE1SilSBN2pAwfLSqtEMyC2YtEkCIoTIUfI7wtShMKKLOn/Msl1qjcheH3XRs7CAvPbq8fkd1Ynv9OsJt71REKqVU2tWsuxz37kCtP8Jmn8H53+H8xvh+n6IiQD/HeqiZ+8MRatB/tKQr7T6aF9IbVuidVQbtWodQetg4gwrd5NbMGmTBEQIkSMVKwxzRsM378IeH9jrC0fOwLVgiItTl5DQ+OGy01LACWqUV2tQqpeDmm7q8PCWlhmLS1HUcz58Ck/CIFanJjquhSCvQ37wGqwuESFwaTvcOgz3z6tLVKg6G++1fek7mdZBTUxsHEFrD9b26jat/YvFIX6bbX41ubErpC72zmpXYSubjL1AwfMICItQ16UXTMokARFC5Gj5HKFHU3UBiIyCK0HqUNn6BORJKDwOjf85JFRNDkJC4fZDdV6Px89g/0l10bPLA9XKgnsJKJQXCuZVExWtFUTFqL935xEEPYDbD9THoAcQEZV8rO4loE4lGNoR6lTKDzX6qwuoGdPDSxBwVB3uPeQGPLkBITchMkTNbF4WHaYu6UyykmXjCHbOai2LPjGxLwyOb+AYqhBr6wwFNeD4hnpsVtLFqLeoosPiH2OeQ1ys+vqVOOBFOVjliV9sHMGhqHpLywj0Y4AAFJXhXlIkCYgQIlfJYwNVy6b/+JhYuHQL/K7A6avgd1l9fPYcwiPh2Hl1yQz+AeqyYhc094LP+0PDai92WlioQ74XrpT0FxUFYsIh6plaS/LyY3Toiw/sBB/aL3+QRzxWa1ciHiVOZqJC1SXkepLTuuhX9r54tLZTP+yt7V7UrNir64ke7dVEQIlTEwf9EhulxvJybAl/1kW/XgHbOKmJkuMb6uzFBcqrbWoKuYFzxUyr7XkcP2UaBWWYlxRJAiKEEKmwtlITlqplQd+XLy4Obtx5kZRcUdcfPVM/eB49VW+r2GjBxlqtgnd1hmLO6m2hYs7qz4Xzqe1MLC3U2pbLQeBzEdbuU593n6+6NK4Ov3wCZVxSCVKjib+t4vjG673gOJ3aPfj5Qwh/8OLxxfL8wYvlHoTeIfZJIFZRT+J/NyZcXUxBY6GWg8ZCTaDiYpMeE/VMXR4mMyS+VR4oVgdKNoBSDaB4vVeu0dHffgGwl7lZUyQJiBBCZJCFBZR1VRf9IGivo6iNWlXfsBqM6Qk7j8OkFfDvBfjrNNR+D9Z+BS1qvf650mRhqbb9sCsIuKd66DV/f9BF4+6SF0LvQNg9tcYiJjy+xiUmPOm22AjQWKqzB1u8eLTUxrdPsbYHG4f4WhRDuxWHl/Y5qDUrlsl8lMXpIDZSXSKfQthdNcbQOxAarN7CenRFXaKeqcfdPKQuh16UQ9HqakJStjmUeyv58yQj9EUOZpcn422EchNJQIQQIhvRaKBtPWhTF/78FwZNgQdPoM3HsGgcDGxj6ghfYqmFfCXUJTuxsIyvFbIrCAXKJH+coqjJScBRuHUEbh6Gu2fUBCb4pLoc+wEcXaDmQPB8V+15lAp9DYhMQpc6SUCEECIb0mjUJMTnF+j2Bfj6w/DZ6izAHhlowyLSoHnRgLZyV3UBtcYk4Cjc+lvthRTwj1prcmgSHJ4MZZqrPZUqdAQrbZKn1CcgDtllFN5sShIQIYTIxooXhr2zwfN/cD0Y+nwD//6sVu+LLJInL7i1VhdQexqdXAKnlqqJyLW96mLvDNXfUS+Oc/ztqrAXt2AcJQFJlYxSI4QQ2ZyTPayeAFaWcOEmfPSTqSPKZfKXgubfwke3oM82qNBBvcXz/AH88z3MqQDrusOTAEBqQNJLEhAhhDADtSvCd/9T1xdtg42HTBtPrmRpBRXaQ5+tajLS7DvIV0rdd/53NRE5+C2R4ZFANpoIMZuSBCQVjx8/ZsyYMdSuXZs6derw8ccf8/y5TA4lhDCNj96GZp7q+nvTIeCeaePJ1ZxcofHn8OE16LpSbUcSEwEHJvDhvUp0KLgVR9tkBocTBpKApGLYsGHodDoOHDjArl27CAoKYsaMGaYOSwiRS1lYwPLPwTmfOnZIv29BpzN1VLmchQVU7wsf+EP9cWBpTaG4G2yu3IkvLFpDwDFTR5htSQKSgrNnz3LhwgUmTZqEg4MD+fPn58cff2TAgAGmDk0IkYu9URB+/VRd//s/mLbGtPGIF2wc4a3pMPw//JS3AKiq2w2/eMMib7jta+IAs59cm4BERkZy69atZJeIiAj+++8/ypUrx9q1a2nWrBkNGjTg559/pkiRIqYOXQiRy7Wuq872CzDxV3UEVZFNOLvzSfifdDq3ldvWNdVtgcdgUV3Y+xnEvuZw8jlIrk1Azpw5Q8uWLZNdjh8/ztOnT/H39yc4OJjt27ezbt06Tp48ybRp00wduhBCMHUoVC6tDvve7zt1XhqRPYRFaNj+uAMrS/vCwAPgXEkd2OzwFFhcXx2FVeTecUDq1KmDv38y8wG8cP26OvHSp59+ilarxc7OjiFDhvD111/z9ddfGylKIYRInq0NrPwC6gxRZ/ddvANGdcuac8XGqnPVXA+Gm3fihxrXReWlhHM0jgXBpVDWnNscGbrh2mmgTBMYdgoOToQjU+G2D8yvAR0XQ5UsumBmItcmIGkpV64csbGxhIWFUaBAAQDi4uJMHJUQQsSrVg4GtYGft8HMdTC0I2itM+/5I6JgyQ6YsRaCHiR3RFH1YRZUKgV9WqhJUG4fJE2foDnoh2K3soEWk6FMM/i9jzpnzm/d4cYwaDUTrHNngeXaWzBpqVevHsWLF+e7774jIiKCu3fv8vPPP9OhQwdThyaEEAZje6kTngU9gNV7M+c5o6Lh+7VQpid8MCc++bDRgnsJqFsJ6lSCovljDL9z4SZ8/gtUeQe2/q1OsZJbpTgQWdlmMPyMOrkdwIn5aiPVkJvGDC/bMMsE5P79+3h6erJq1apk98fExPDLL7/QqlUrPDw8aNGiBYsWLUKXgf5qWq2WVatWERsbS7NmzejYsSOenp589NFHmfUyhBDitZVxgZ5N1fXpayAmmVnoM+LERXWak08Wwv0QtUZlSAe4sBLCdquP/yyAowvgwLTrnJp3mX/mqzUfWmu4dRe6fA4dPlVv2eRGqY6E6lAE+u9SBzHTWMAdP1jgCVczKXs0I2aXgERERDBq1CjCwsJSPGbChAl8//33ODs7079/fwoWLMjMmTP57LPPMnSuokWLMmfOHI4ePcq///7LhAkTsLGxed2XIIQQmeqT3urj5UD4cO6rPYeiwHfL4c1ham2GpSWM7ApX18L8j9SaD4tkPjHyaBXqVobZI+HMr9Cilrp95zG1NuTnba8Wj7mKjlEXSGU2XAtLdRCzAXvBrhBEPIYVreDw1FxVdWRWbUCCg4MZOXIk586dS/EYHx8fNm3aRIcOHQyDhimKwpgxY9iyZQvdunWjVq1axgo5EZ1Ol2rD14wKD1dvNGbmc4r0k/I3PbkGKmtgaNuCLPyjEAu2QCG7e/Rq/CTdvx8RpeHz5UXZ5esEQHmXKCYPvEPlklGEPQb/x8n/XnLl/8P/YG9NB6auL8zdEGuGz1KIi7hN0+q5YxTpJ88tgPIAPLx3E39tVCpHu2L11npcDn+A7aP/YO+nhF46wN16U4jTOqR5LnP4+9fpdFhaWia7z2xqQJYtW0a7du24ePEidevWTfG49evXA+oopnoajYYxY8YAsHHjxqwNVAghTGBE+0c0rxEKwJR1hbkclHSa+ORcDdbSZ3oJQ/LRrf4TNnx+i8olU/vgTJlGAy09w9g+8QbVykSgKBrGLXHhUmDuqD0Oj4z/WLXPk3bHhVh7FwJbruJJWbVHjGPgXkrs6oH26fUsizG7MJsakBUrVuDq6srEiRO5efMmx48fT/a406dP4+zsTOnSpRNtL168OK6urpw4ccIY4SbL0tISd3f3tA9MJ33Wm5nPKdJPyt/05BoktmmKOjP8lSAN0zeV5tDc5G+b6M3fDGPnq41OLSxg1nAY0TUfGk2+dJ0vrfL/cybUHQoB9yz4eGkpfBenclsih4hNMMSHR+UyFM6fzl+stAF8f4EdI7B5dp3Sf3aGOiOgwXiwT76Pszn8/Z8+fTrFfWZTAzJx4kS2bNlCzZo1UzwmNjaWgIAAihcvnux+V1dXgoODiY6WkeiEEDmPva3aXgPg6DlY8kfyx8XFwUfzYOQPavJR+g048AOM7KbWYGSWIgVg8yS198zV2zBiduY9d3alb4AKKTRCTY3XYHj3MOQtDrFR8M9MmFUajs/NkW1DzCYBadCgQYr3kfT0DVOdnJyS3e/g4ICiKKk2YBVCCHPWtKY6HgfApz+rPVkSioyCPt/CDxvUn7s0Ar+l0KBa1sRTvTzMeF9dX7UHVu7OmvNkF/oExMJCHSwuw4rXgVEXocUUyJMPosPgj1GwfRjoXrOLU2ri4uBpUNae4yVmk4CkR0SEeuW12uTvfeq3Sw2IECIn+3445HeEkFDo+gWcuQpPQtVRTBuPgvUH1ONGdIF1X2X9bZFhnaH9m+r60O9h2z9Zez5TMgxCZvsatUlae2g4HsbcgCo91G0+C9Vh3O+cyZQ4AbVW5epeWNcDphWG74vDtvcy7/nTkKMSEH0X2ZiYmGT36xMPW9uM1osJIYT5KJwfZo5Q14+eg5rvQsF2UPZt8LmkfjBOfx9+GKV2t81qGg0s+QTKuUJktJoULfsz689rCqmOAZJRtvmg+1po9Ln6c9C/sNATdn4Izx+++vOG3oG/v4c5FWF5Szi/AcIfqfscir522OllNo1Q08PBwQELCwtCQ0OT3a+/9eLgkHb3JiGEMGfvtALnfDByNty8G789rwOsmQCt6hg3noJ54chP0P4T8PVXa0Ka1oQSOWyCcX0Ckmm1ShYW0Pw7dfTUbUPhoT8c+wFOLuYNl8aEuTaG4s5gVyD154l8Bpf/gNMr4OoeUBL00CnVCDx6Q+nGUMgtkwJPW45KQLRaLS4uLgQFBSW7PygoiJIlS6bZlkQIIXKCNnWh5Wq1AWhktLpUKgVO9qaJp3B+2PcDVOoHwQ9h2mr4aYxpYskqYQluwWSq0o3VYdyPzoYj0yDyCU43d+B0cwcc/RhKeEOpxuD4Btjmh9hItabjyU14cAkCj0FcgvYdtgWg6ttQawgU9cjkYNMnRyUgAJ6enmzdupXAwMBEvWECAwMJDg6mY8eOJoxOCCGMy8oKKpQ0dRTxHO3g414weq7aS2d8Xyhe2NRRZZ5MvQXzMisbtW1IraFwdg1hp9Zjd+84FroouPW3uqTG0hrKt4Ea74BbG/X5TCjHJSCdOnVi69atzJo1i1mzZqHRaFAUhVmzZgHQo0cPE0cohBC52//aw9TVcPcxTFkZ33U4J8jSBETPNh/UGcbtfM3QxEbgZnVbvb1y75w6027kE7C2U8cPyVca8peC4t7qZHg2jlkYWMbkuATE29ubNm3asHPnTu7cuYOXlxe+vr74+fnRpUsXvLy8TB2iEELkarY28Ekfdd6aRduhWxO1PUhOEJpVt2BSoFjZgntbdTEzOaoXjN60adMYOXIkDx48YPny5YSEhPDRRx/xzTffmDo0IYQQwNCO4OWu9gTt9y3cS2G+GXNjlBqQHMIsa0C6dOlCly5dUtyv1WoZMWIEI0aMMGJUQggh0ktrDWu/VoeOv/sY+k+CP2ekPnS8Ocj0XjA5mJlfaiGEEOaqjAssGqeu7/OFHzeYNp7MEB6pPtrlMW0c5kASECGEECbTvQkMbq+uf7cCHj8zbTyvK/LFQNuvNAx7LiMJiBBCCJP67n/q2CRPwtTeMeYsIkp9tE1+RhCRgCQgQgghTKpQPnVsEIB5myDgnnHP/+AJ/LQJGo+EPM2gUDtoOQb+Ppvx59InIHkkAUmTJCBCCCFMblQ3eKMgREXDiNnGm33+j2NQsS+M+hGOnIWYWHUSv/0nodFI6DBeTVDSS38LJo/cgkmTJCBCCCFMzt4WZgxT1/84po4PkpXi4mDCEjXBCAlVbwG90wrWfwMrvoDaFeNj6fRpfM1GWiL0CYjUgKRJEhAhhBDZQq/m8HYzdf2jeXDuetacR1FgzDyYtEL92bsKnFsOSz+Fro2gTws4ugCWjFdn8j1+Qe0mHBeX+vOCNELNCElAhBBCZBs/fQgli6o1Dj2+ih9ZNDNNWw1zN6rr77aFAz+Cq3PiYzQaGNAaZg5Xf950SO2lkxZpA5J+koAIIYTINvI5wrqvwdoK/ANgyIzMbQ+y5A/4/Bd1vUdTWDhWPVdKRnWD9zup69PXpD5ia1wcRMeo61IDkjZJQIQQQmQrtSvG1zz8dgDGL8ycJGTr3zD0e3W9uRcs/yztkVc1GpgyBArmVWs3pq9J+Vj97ReQGpD0MMuh2HODiIgInj17RlRUFEoK77zISHXIvVu3bhkzNPGClL/pyTUwPo1Gg42NDU5OTll6nmGd4cRFWLUHvl+njhEyfwxYWr7a8/17AXpPVGspvNzh92/V4eDTw9EOxr0N43+GhVvho7fBpVDS4xImIFIDkjapAcmGHj9+zM2bNwkJCUGn06V4nKOjI46O2Wdq5dxGyt/05BoYn06nIyQkhJs3bxITE5Nl59FoYOl4tY0GwOId8NXSV3suRYHRc9QEwa047Jie8blahnUG53zqc0xLYbC0hD1lpAYkbVIDks1ERERw7949HB0deeONN7BMJd3Xf/vLk0cmHTAFKX/Tk2tgGjqdjjt37vDo0aNU/0e9LktL+Hmc2kZj4VaYsgpquKk9VTJi53G1NgVg8SdqIpFR9rbwcW8YNx9+2QETB6ntVRKSGpCMkRqQbObZs2doNJo0kw8hhDAVS0tLw/+o2NjYLD2XRgM/jIJG1dWfB07JWPdcRYGJv6rrLWrBm1VfPZb32quJRVQ0bPsn6X6pAckYSUCymaioKGxsbCT5EEJka5aWluTJkyfFNmqZydpK7RlTzBmeR0CHT+Huo/T97u9/wUl/df3rga8Xh4MdtKqjrm88lHR/wgRE5oJJmyQg2YyiKFik1SxbCCGyAQsLC6MkIACF88Pmyeo097fuqklIWBpjhFwPhiEver20rgt1K79+HN0aq497fOBpWOJ90gsmY+STTgghhFmo6QZrv1K7zp70h97fQEp3gB48gS6fq0lCkQKwaFzmxNC2Htho1fE+th9NvE8/DLuVJVhJC8s0SQIihBDCbLTzhrmj1fU/jsHouUnHCAm8r85s+991tSHr2q+S7zb7KhztoGUtdX2fb+J9Mgx7xkgCIoQQwqwM7QjjeqnrC7aoXWwD7oFOpzZQbTgCLgWo43xs+Ca+AWtmqfNiorqz1xJvl2HYM0YSEJEtzZ07F3d3dwYOTLnVWFBQEO7u7owaNcqIkeVM9+7dY9OmTUY737Zt27h9+7bRzpcTSJklNvk9dSh1gHmboHQP0DaFagPVZMTeFv6YDh3rZ/65Pcqpjxduxg+9DlIDklGSgIhs7ejRo2zZssXUYeRojx49onXr1vz1119GOd+MGTMYN24cz58/N8r5cgIps6QsLGDZp/DlO2obj4Sc88G+2dC0Ztacu1pZ9TEmFi4mGIRXakAyRprJiGxv6tSpNGzYkAIFCqR9sMiwiIgIo36wPXqUzv6TwkDKLHk2Wvh6EHzeH/75T20LUsAJyhdTe8tkFVdn9TyPn8GZq1DtRY1IlNSAZIjUgIhsrVKlSoSEhDB16lRThyKEyKasraBxDWhSU00GsjL5AHVwNH0tSMJ2IFIDkjGSgIhsbciQIZQsWZKtW7dy9OjRtH8BiIuLY+XKlbRv3x4PDw/q1KnDBx98wPXrSYdPfPToEVOmTKFVq1Z4eHhQvXp1OnTowK+//ppofAN9m5Rjx47RpUsXqlSpQrdu3dI1F8bevXvp06cPNWvWpF69egwePJizZ88miXnFihW0b9+eqlWrUqtWLYYMGcKZM2cSHbdp0ybc3d05ceIEP//8My1atKBKlSq89dZbLF68OMmYDIcPH6Z///7UrVuXatWq0bFjR5YsWWKYY2jTpk00a9YMgN27d+Pu7p6oLci+ffsYNGgQderUoXLlynh7ezNq1CiuXUvc+s7d3Z3PP/8cHx8f+vTpQ/Xq1alTpw5jx47l3r17huOaNm3K5s2bAWjfvj1NmzZNtezc3d2ZMGECx44d4+2336ZatWo0aNCAefPmoSgKFy5c4N1336V69eo0bdqUOXPmJBmZMyoqinnz5vHWW29RpUoV6tWrx4cffpjkNeivcUBAAJMnT6Z+/fpUr16dvn374u/vT0xMDHPmzKFRo0bUqFGDfv36cenSpSQxX79+nQ8//JC6detStWpV2rdvz4oVK4iLi8vUMtPHe/DgwSQxvPnmm4nKNrNem4inbweSMAExtAGRBCRdJAER2ZqNjQ0TJ04E4KuvvjLM/ZGacePG8d1332FhYUGvXr1o2rQphw4dokePHly4cMFw3LNnz+jevTurV6/G3d2dd955h9atWxMUFMTUqVP5+eefkzz32LFjcXBwoG/fvtSqVQtr69Sn01y4cCEjRozg1q1btG3blpYtW3Ly5En69OljSC7i4uIYPXo0kyZNIjo6mp49e9KwYUP+/fdfevfuzZ49e5I879SpU1m4cCG1atWid+/ehIaGMmPGDFasWGE45sSJEwwbNoy7d+/Svn17evfuTUxMDNOnTzfUKFWsWJH+/fsDULZsWUaMGEHFimoT/2XLljF8+HBu375Nhw4d6N+/PyVKlGD37t306dOH0NDQRDGdPXuWgQMHotVq6dOnD2XKlGH79u0MGTLEcEz//v2pUKECAL169TKcOzV+fn4MHjwYZ2dn3n77bSwsLJg7dy5ff/01gwYNwtbWll691C4RP/30E2vXrjX8bmRkJO+88w5z587F1taW3r174+Xlxe7du+nWrRt+fn5Jzjdq1Cj27dtHu3btqF+/Pj4+Prz33nuMGzeO33//nRYtWtC4cWNOnDjBkCFDEv1N/vfff3Tv3p39+/dTv359+vfvj7W1NZMmTeLTTz9Ncq6sKrOUvM5rE4l5vKgBOXM1vhuwvgbERhKQ9FGEUfj5+Sl+fn5pHnfz5k3l5s2bKe6PjlGUG8HqculGpHLpRqTh5+yyRMe8fnnNmTNHcXNzUw4cOKAoiqJ8/PHHipubmzJjxgzDMYGBgYqbm5sycuRIw7Y//vhDcXNzU8aPH6/ExsYatvv7+ytVq1ZVOnXqZNi2YMECxc3NTdm8eXOic1+7dk1xd3dX2rVrlySeHj16KDqdTlEURYmIiFAiIiJSfA3Xr19XKlWqpLRv31559OiRYfv58+eVSpUqKf3791cURVE2bdqkuLm5KUOGDEn0fJcuXVJq1KiheHp6Ks+ePVMURVE2btyouLm5KbVq1VICAgIMx964cUOpWLGi0rp1a8O2ESNGKNWrV1dCQ0MN2yIjI5WWLVsqHh4eSlRUVIrlGBUVpdSoUUNp06aNEhkZmeh1jRkzRnFzc1N27dpl2Obm5qa4ubkpv/76q2GbTqdT+vbtq7i5uSlnzpwxbP/kk08UNzc3xd/fP8Wye/l5V65cadh28eJFw/Y5c+YYyiwoKEhxc3NTevbsaTh27ty5ipubm/Lll18m+ns4evSoUqFCBaV58+aG7fpr3Lx5c0N5K4qiDB8+XHFzc1O8vb2Vx48fG7aPHz9ecXNzU44cOaIoiqLExcUpbdq0UapVq6ZcunTJcFxcXJwybtw4xc3NTdm3b1+mldnL75GEvL29lSZNmiQ59lVfW0quXbumnD59OtVjciq/y4pi0VBdbj9Qt42crf7c5XPjxHDp0qVEf2vZUWqffdII1YzExELl/nDN0BMve7Z0KusK51eo92Uzy/jx4zl06BC//vor7dq1M3wjfNnGjRvRaDSMHz8+0Xw6bm5utG/fnt9//x1/f3/c3d1p2LAh+fPnp127domeo0yZMhQqVIjHjx8nef4WLVqke6j8Xbt2ERsby/DhwxM1oK1UqRIff/wxVi+GSty8eTMajYavvvoq0ayu7u7u9O/fnwULFrBnzx66du1q2NeqVSuKFy9u+LlUqVKUKFGCgIAAwzZFUYiMjOT8+fPUqaNOYGFjY8OKFSuwt7dHq035a1psbCzfffcdRYoUwcYm8d9Z7dq12bFjR5LysbOzo0+fPoafLSwsaNCgASdOnCAgIAAPD490ldvL8uTJw9tvv234uUKFCtjZ2REREUHfvn0N211dXSlUqFCirqqbN2/G3t6eTz/9NNHfQ7169WjXrh3btm3Dx8eHunXrGvZ16dIFR8f4aU5r1qzJ3r176dy5M/nz5zds9/DwYNOmTQQHBwNw+vRprl69yjvvvIO7u7vhOI1Gw5gxY9i6dStbtmwx3PLKyjJLyau+NpFUxZLqiKexOjh9VR3oTH8LRtqApI8kIMIs5M+fn/Hjx/PJJ5/w5Zdf8ttvvyV73Pnz58mTJ0+iWxF6d+/eBeDixYu4u7tTqVIlKlWqRFhYGBcvXuTWrVvcuHGDs2fP8ujRI/LmzZvkOVxdXdMds/4eenIfIu+8845h3d/fHxcXF954440kx3l6ehqOSahUqVJJjnV0dEzUJqV79+7s27eP/v374+bmRoMGDWjYsCG1atVKc7JDOzs72rRpA6htGq5du0ZAQAD+/v6Gtjgvt2koVqxYkltSDg4OAOlqK5MSFxcXQ7KWMD57e3tsbW0TbbexsTH06AkLCyMoKIjatWsnOQ7Ust22bRv+/v6JEpCSJUsmOk7/u8WKFUtyLoDoaPVTR3977+bNm8ydOzfJ+bRaLRcvXky0LavKLCWv+tpEUjZaNQn57zqcvQpt6sYPxS69YNJHEhAzYm2l1izcfqD+HBWl3nB8+Ruqqbk6Z27th16nTp3Ytm0b//zzD6tWrUq2AWNoaCixsbHMmzcvxefRt12IiopixowZrF+/3lCWrq6u1K5dG39//2Qn2UpYQwHg4+PD6dOnE21zdHRkwIABPHv2DIj/QElJWFgYRYoUSXZf4cKFAbWrbELJ1V5oNJpEPzdq1Ihly5axdOlSjh49yuXLl1myZAmFChXiww8/pFu3bqnG9e+//zJ58mRDImVra2tI2g4dOpSkfJJrD6OPKbmyTK/kkgdIvgwS0iciKZV/SmX7qufTX+9Dhw5x6FAyU6VCknYzWVVmKXnV1yaS51FWTUDOvGiIKjUgGSMJiJmxtoJSL74oR0aq/6DyZHGXs+xk4sSJtGvXjtmzZ1OpUqUk++3s7MibNy/79u1L87mmTp3KmjVraNu2Lb169cLNzc1Q61G/fv10fQP19fVl4cKFiba5uroyYMAA7OzsAPWDMGG1N6iNI21sbNBoNNjb23P//v1kn1//oZYvX740Y0lO3bp1qVu3LuHh4fj4+HDw4EG2bNnC559/TpkyZahZM/mRmm7fvs17771Hnjx5mDx5MjVq1KBUqVJYWFiwbt26FD9gsxN7e3uALCvbl+mv97Rp0+jUqVOmPGdqUktUwsPDs90Xk5yoWjlYvVetAYH4RqhSA5I+0gtGmJXixYszfPhwwsPDmTRpUpL97u7uBAcHJ9t+Y9u2bcydO5fAwEAAduzYgbOzM7NmzaJWrVqG5CMkJCTdAz+9//77+Pv7J1oOHDgAqO1OQO0Z8bKxY8dSs2ZNQkNDqVChAiEhIcl2E/bx8QGgXLly6YpHT6fT8fPPP7N48WJA/XBs1KgRX3/9NZ999hkAJ0+eBJLWnIDa/TYyMpIPP/yQrl27UqZMGUPbF3331Vf9hp7c+bKCg4MDxYoV4/r16zx58iTJ/lct25To232cO3cuyb7Q0FAmTZr0yqP6Jldm+tqT8PDEc9I/ePAgyTaRNfRfBu+++HcTJTUgGSIJiDA7gwYNokKFCom61Op17NgRnU7HpEmTEtVg3Lp1i2+//ZalS5cavvHa2NgQFRWVqFo8Ojqab775hri4uNe+B9+2bVs0Gg0LFiwwfNsGtW3IoUOHqFq1Ko6OjoZvy5MnTzbcCgK13ceyZctwdHSkSZMmGTq3paUlu3btYu7cuUkSm6CgICC+PYu+fUXC16v/9vzw4cNEv3v69Gk2bNgAkGS8jYzE9jq/nxGdOnUiPDyc6dOnG8Y+ATh27Bhbt26lWLFiKdYCZVStWrVwdXVl/fr1SZLO2bNns2LFCm7evPlKz51cmZUpUwYgyRD6P/300yudQ2RcXrWSjWfhaldcqQHJGLkFI8yOlZUV3377LT179kzSELJr167s37+fHTt2cOnSJerVq0dkZCR//vknz58/Z8qUKYbbIe3bt2fp0qV07dqVZs2aERMTw19//cXt27fJly8fz549Izo6+pXvj5crV47hw4czb948OnbsSOPGjYmJieGPP/7AysqKr776CoDOnTuzb98+9u/fT4cOHWjQoAFPnjxh3759xMbGMnPmTJycnDJ8/jFjxvDee+/RrVs3WrVqRYECBbh8+TKHDx+mSpUqtGjRAlAb+Gq1Wo4dO8a0adNo3rw5TZo0YebMmcyfP5+rV69SvHhxrl27xqFDh8ibNy8RERHJ1iqkR9GiRQGYMmUKdevWZfjw4a/0POnx3nvvcfjwYTZu3MjFixepVasWd+/eZd++feTJk4fp06enu1dTWiwtLZk6dSqDBw+mV69eNG/eHBcXF06dOoWfnx8VK1bkf//73ys9d3Jl1qhRIwoXLsz27dt5+vQpbm5u+Pr6cvPmTcqVK5ekbYvIfPoEJC4OwiJkILKMkhoQYZY8PDwSdV/Us7CwYN68eYwfPx4LCwvWr1/P/v378fDw4Ndff6Vz586GYz/88ENGjhwJwOrVq9m/fz9ly5ZlxYoV9OnTh7i4OP7555/XinPkyJHMmDGDggULsmnTJv744w9q167N2rVrKVtWHclIo9EwZ84cxo8fj1ar5bfffuPvv/+mfv36rFmzhrfeeuuVzt2gQQN+/fVXatSoweHDh1m+fDk3b95k8ODBLF++3FCFr9VqmTBhAo6OjqxatYpjx45RpEgRli5dSp06dTh69Chr164lKCiIwYMHs2PHDrRaLUeOHHmluHr37s2bb77JmTNnWLFiRZbeLtB3Ox42bBjPnz9nzZo1nDp1ivbt27Nx40ZDL6PMUrt2bdavX0+zZs04fvw4K1euJCQkhCFDhrBixYo0GySnJLky02q1rFixgmbNmnHy5EnWrl1L/vz5WbdunSFhEVkrb4LL+TRMhmLPKI2SFU2tRRL6nhLVq1dP9bhbt9SpFV/uLpcc/SiFL/fMEMYh5W96cg1M6/r164SGhlKtWjVTh2IS9x6Dy4vvNGeXQYfxcPMuLP4EBrbJ+vPru+cnHHcmu0nts09qQIQQQohXoL8FA/D0efw4IFIDkj6SgAghhBCvII8NaF8M5ZLwFow0Qk0fSUCEEEKIV6SvBXn6XAYiyyhJQIQQQohXpE9AQkIh+kVPdqkBSR9JQIQQQohXpO8Jcz8kfpvUgKSPJCCpOH/+PH369MHLy4v69eszadIkmZxJCCGEgb4G5F6CwZelBiR9JAFJQVxcHEOGDOGtt97ixIkT/P777xw5coRffvnF1KEJIYTIJpxeJCB3EyQgUgOSPpKApODp06c8ePCAuLg4w5wXFhYWKc4mKYQQIvfR14AkvAUjNSDpk2uHYo+MjOTevXvJ7itcuDD58+dnwIABTJs2zTCPRLNmzRgwYIBxAxVCCJFt6duASA1IxuXaBOTMmTP0798/2X0LFy6kUaNG2NjY8MUXX9C9e3du3brFiBEjmDNnDqNHjzZusEIIIbIlQxuQhDUgkoCkS65NQOrUqWMYxjY5u3fvZs+ePezatQuA8uXLM3z4cCZNmiQJiBBCCCA+AQmPjN8mNSDpI21AUnDnzp0k04VbWVkZJvASQgghnF6aX9DKEqxy7Vf7jJEEJAX169fn3r17LFq0CJ1OR2BgIAsWLKB9+/amDk0IIUQ2kXA+GJDaj4wwywTk/v37eHp6smrVqmT3x8TE8Msvv9CqVSs8PDxo0aKFIZFIr3LlyvHzzz+zb98+6tSpQ//+/WnatCkffvhhZr0MkYqgoCDc3d3p16+fqUMxmX79+uHu7s7z589NHUqud+TIEc6dO2fqMEQ29HICIj1g0s/sKooiIiIYNWoUYWFhKR4zYcIENm3aRO3atWnevDm+vr7MnDmTa9euMW3atHSfy9vbG29v78wIW4gM69y5M7Vr15bbfia2Zs0aJk6cyMKFC00disiG8r50C0YSkPQzqwQkODiYkSNHpvpNxMfHh02bNtGhQwdmzJgBgKIojBkzhi1bttCtWzdq1aplrJCFeGVdunQxdQgCePTokalDENmY3IJ5dWaTgCxbtow5c+YQGRlJ3bp1OX78eLLHrV+/HoBhw4YZtmk0GsaMGcPOnTvZuHGjyRIQnU6Xas8bUMcncXR0JDIyMtXjQB2tVf87OU1UlDqvdVxcXLZ9fTm5/M2FMa6BvjF6dHS0XOuXKIpCbGxsmv/XcrIHTy2BcoafNUok/v63jHLu8PBwgGxd/jqdDktLy2T3mU0bkBUrVuDq6sqqVavo2LFjisedPn0aZ2dnSpcunWh78eLFcXV15cSJE1kdqsgiPj4+VKtWjR07dvD777/TuXNnatWqxVtvvcWcOXOSnadn//79DBw4EG9vbxo3bszw4cP577//DPu3bt1KtWrV2LdvH4MHD6ZWrVq0adPG8K332bNnfP/997Ru3RovLy9atWrFjBkzCA0NTXIuf39/xo8fT4sWLfD09OTNN9/k3Xff5ciRI4mOi4mJ4aeffqJr167UqVOHhg0bMmLECE6dOpXouHfffZdq1aoZ/snoY/X19WXJkiW0a9cOLy8v2rdvz7Jlywwj9iY8z8KFC2nbti21a9emR48eHDhwgK+//ppq1aqlWd6tW7fm/fff59KlSwwZMoS6devSuHFjpkyZQnR0NIGBgYwaNYp69erRvHlzJk+eTERERKLniIuLY/Xq1Yaax/r16zNixAjOnj2b6Dj9azt16hQ///wzrVq1onbt2vTu3RsfHx8URWHNmjW0a9eOOnXq0Lt372Tfy/fu3WPixIk0b97cUDYLFy5M8rfRunVr3nvvPa5cucKwYcMMt1tHjRrF9evXE10D/a2XUaNGGcpNH+/atWuTxNC3b99E5ZtZr01kT462cYl+trFWUjhSvMxsakAmTpyIt7c3lpaW3Lx5M9ljYmNjCQgIoGbNmsnud3V1xcfHh+joaLRa49eTWVpa4u7unuoxt26pmXOePHnSfD79t7H0HGtubGzUG6kWFhaG16e/ZmvWrOHy5cu89dZbNG7cmN27d7NkyRJiY2P57LPPDM+xcOFCZs+ejbOzM23btsXCwoLt27czaNAgVq1aRbVq1QztK6ZMmUKRIkXo27cv9+/fx9XVlWfPnjFgwACuXbtG/fr1adOmDdeuXWP16tWcOHGCZcuWYW9vT548efDz8+Odd97B1taWFi1akC9fPm7cuMGBAwc4efIkq1atwsvLC4BJkyaxfv16GjVqRJMmTXj8+DF//vknx48fZ82aNXh4eBheu74s8uTJY4h11qxZ3Lhxg9atW+Pg4MCOHTuYPXs2tra2vPPOO4bXP3r0aA4ePEjlypVp2bIl/v7+jBkzBhcXFyDtvxuNRsPt27cZOHAgNWvWpFevXvz111+sW7eO58+fc+zYMcqUKUOvXr04cuQIv/32G3Z2dowfPx5Qk4/Ro0eze/duSpUqRc+ePQkJCWH//v0cO3aM2bNn07JlSwDDa5s+fTr37t2jbdu2hIaGsm3bNkaNGkX79u3ZtWsXrVu3JiYmhq1bt/LBBx+wbds2nJ2dyZMnD0FBQfTp04dHjx7RvHlzSpYsyZkzZ1iwYAF+fn4sWbIEqxf9IzUaDXfv3mXAgAGUK1eOnj17cvXqVQ4dOsSFCxc4cOAAWq2Wrl27YmFhwYkTJ2jXrh2lSpVKdC2sra2TlKP+uum3v+pr2717N4ULF071GpmaRqPBysoqzf9rOZmigLUVxLwYtaFAXlujlYe+5iM7l//p06dT3Gc2CUiDBg3SPEbfMNXJySnZ/Q4ODiiKQlhYGAUKFMjU+IxGFwPPbgOgeXGbApts1urJyRUss67hpL+/P2vXrjV8UL/33nu0bNmSjRs3Mm7cOKytrblx4wZz587F3d2dZcuWGa53z5496d69O7NmzWL58uWG57SxsWHNmjWJPkz0DZcnTZpEt27dDNs3bNjAF198wfz58xk3bhwAc+fOJS4ujt9++41SpUoZjv3tt9+YMGECf/75J15eXoSGhvL777/TsWNHpk+fbjiuffv2DBgwgHXr1hleV0qCgoLYtm0bxYsXB6B37960adOG3377zZCA7Ny5k4MHD9KmTRu+//57QxXozJkzWbRoUbrL+tatWwwaNIhPPvkEgMGDB9OoUSO2b99O9+7d+e677wD1lmejRo3YsWOHIQHZunUru3fvpkmTJvzwww+GsvX396dXr1589tln1KtXD0dHR8P5bt++zfbt2ylatCgA+fLlY9myZWzZsoXt27dTsmRJAFxcXJg7dy6HDh0yXJuvv/6aR48esXjxYt58803Dc/7www8sWLCANWvWJBr9+NatW/Tv35/PP//csO3jjz9m69at7Nu3jzZt2tClSxdu375tSECaNGmS7rJ7WUZf28GDB+nZs+crn08Yh0ajtgN5+FT92UbagKSb2SQg6aGv/k2pdkO/PbmqerOgi4E5FeHxNQCyWdoRr0BZGHUxy5KQunXrJvqQzp8/Px4eHvz999+EhIRQuHBhdu3aRWxsLMOHD0+UbFaqVImPP/7Y8E1Yr1GjRomSj9jYWLZt20bFihUTJR8A3bt355dffmHHjh2MHTsWgIEDB9KjR49EyQdA7dq1AXj8OH6iCEVRuHbtGk+ePCFfvnwA1KtXj7179xpqJ1LTqlUrQ/IBUKpUKUqUKEFAQIBh27Zt29BoNIwbNy7R/df333+f9evX8+TJkzTPo5dw/qMCBQpQqlQpLl++zMCBAw3bHRwcKFu2LGfPnjXUMG7evBmNRsNXX32VqGzd3d3p378/CxYsYM+ePXTt2tWw76233jJ8QAPUrFmTZcuW0axZM8MHNGC4/nfu3AHUWy9HjhyhefPmiZIP/Wtevnw5W7ZsSTL9wuDBgxP93KhRI7Zu3UpgYGC6yye9MvragoODMz0GkTXyOsQnINILJv1yVAKir7aPiYlJdr8+8ZAZbc3byx/ygOFbtP7aX7p0CSDZ2oSEtyn0XF1dE/1848YNwsPDiYmJYe7cuUmOt7Cw4MmTJ9y7d49SpUoZauju3buHv78/gYGBXL16FR8fHwDDGDSOjo60adOGP/74g4YNG1K7dm0aNGhAkyZNKFGixGu9/oR/9+fOnaNQoUJJEho7Ozvc3d35999/03UuGxsbihQpkmib/v2TMAnSH6soCjExMWi1Wvz9/XFxceGNN95I8ryenp5A0sZzL5eB/lzFihVLci6If09fvHgRUHusJHe97Ozs8Pf3R1EUNBqNYdvLtzgcHNQ+lSn9D3kdr/raRPaXsCeM9IJJvxyVgDg4OGBhYZFsA0GIv0Wj/ydjdiyt1ZqFF7dg9D1FbHLZLZjkarj0Hyr6hpjPnj0D0n+tX76Pr//9q1evMm/evBR/T/+3dvv2bb799lv++usvFEXB0tKSsmXLUrVqVa5cuZLod6ZOnUrlypXZtGkTR44c4ciRI0yePBlPT0++++47ypQpk2qsqb1+vZCQEMqVK5fkOABnZ+dUnz+h1JL1tNpRhYWFJUle9PQf/C83WrWzs3ulc+mvl5+fH35+fike9/z5c8PfRHr+jjLTq742kf0lTECkBiT9clQCotVqcXFxISgoKNn9QUFBlCxZMsUuQWbB0hrylwJA0XcJzIGNUF+X/p/98+fPE7UxALXxro2NTZIP7YTs7dX/KF27dmXy5MnJHqNvBKwoCkOHDuXatWsMHz6cpk2bUq5cOWxsbLh+/TqbNm1K9HtarZZ3332Xd999l9u3b3P06FH++OMPjh07xrBhw/jzzz9TjS097O3tUxxB1Vgjq9rb23P//v1k9+kTBv0tqNelv96jRo1i+PDhmfKcqUktUXk5qRI5n5PUgLwSs+mGm16enp7cvXs3yT3cwMBAgoODqV69umkCE0bl5uYGkKjLrd7YsWOpWbNmijVlAKVLl8ba2prz588nu3/evHksWbKEmJgY/P39uXz5Mq1atWLkyJFUrlzZUCul79Kp/6C6cuUK06dPx9fXF1Bv/XTv3p1ly5ZRo0YNbty4kai9yKuqVKkSt2/fTvJccXFxRhtSvEKFCoSEhCTq1qqnvzWVUi1NRul7AST32nQ6HdOmTWPlypWv9NzJJYP6ni36LtJ6sbGxKX4BEjlXwtFQpQYk/XJcAtKpUydA7aqo/6evKAqzZs0CoEePHqYKTRhR27Zt0Wg0LFiwwPBtG9S2IYcOHaJq1apJakYSsrGxoXXr1ly6dCnJnEO7du1i7ty5HDlyBGtra0MV+sOHDxMd9+DBA8Pfnb5NgU6nY8mSJfz000+GQbRArU15+PAh9vb25M2b9/VePOr7IC4uju+//z7ReRYvXsyDBw9e+/nTGwPA5MmTDbcLQW33sWzZMhwdHV+rV0lCxYsXx9PTkwMHDnDgwIFE+5YtW8bSpUuTjD2SXvoa04SzY+tvkx0+fDhR+f76669JkhKR80kbkFeTo27BgDp/S5s2bdi5cyd37tzBy8sLX19f/Pz86NKli2EsBpGzlStXjuHDhzNv3jw6duxI48aNiYmJ4Y8//sDKyoqvvvoqzef45JNPOHXqFN9++y179+6lUqVKBAUFsX//fpycnAzdN0uVKoWHhwf//vsvffv2pXr16jx69Ih9+/YB6rdlfa+TChUq0K5dO3bs2EHHjh3x9vYmLi6OQ4cOERgYyKeffpqkh86r6NChA5s3b2bjxo34+/vj5eXF5cuX+ffff3Fyckp1LqXM0rlzZ/bt28f+/fvp0KEDDRo04MmTJ+zbt4/Y2FhmzpyZYpf5V/Htt9/Su3dvhg0bRuPGjSlTpgz+/v78/fffuLi4GHosZZS+58r8+fM5e/YsI0eOpFKlSlSuXBlfX1/69OmDp6cnFy9exMfHh6pVqyZb8yZyLmkD8mpyXA0IwLRp0xg5ciQPHjxg+fLlhISE8NFHH/HNN9+YOjRhRCNHjmTGjBkULFiQTZs28ccff1C7dm3Wrl1L2bJl0/z9QoUKsWHDBvr3709gYCArV67kv//+o02bNmzYsIHy5csDao+Y+fPn06lTJ27dusXKlSs5deoUzZs3Z/PmzVSvXp1z584ZkpApU6bwySefoNFo2LBhA7///juFChXixx9/TNTl9XVYWFiwYMECBg0axIMHD1i9ejVPnjxh4cKFhsG0sppGo2HOnDmMHz8erVbLb7/9xt9//039+vVZs2YNb731Vqaer2zZsmzcuJHOnTtz7tw5VqxYwa1bt+jVqxfr1q1LsUFsWtq0aUPr1q25ceMGa9asMdxiWbhwIR07duT69eusWrWKmJgYVq1aRaVKlTLzZQkzkPAWjNSApJ9GyYrm3iIJ/WhwabVB0Y+EmnBcgJTk5JFQzUF2Lv87d+7g6OiYbC+gJk2aYGNjw65du0wQWebKztcgN7h+/TqhoaHpGto/J1u8A4aoc58ybzS839k45zWnkVCT++zLkTUgQuR2ixYtwtPTM0m7hz///JPg4GDDAGlCiNeX8BaMjISafjmuDYgQArp06cKGDRsYOHAgb731FgULFuTGjRscPHgQZ2dnRo4caeoQhcgxpBfMq5EERIgcqGrVqqxdu5ZFixYZhqgvVKgQ3bp1Y/jw4RkajEwIkTpphPpqJAERIoeqWrVqssOSCyEyl1OCQW6lEWr6SRsQIYQQ4jXILZhXIwmIEEII8RoKOMY3Pi30+uMI5hpyCyab0Wg0hplThRAiO4uLi3vteYtygjw2sGYCBD+EyqVNHY35kAQkm7GxsSEkJASdTmfek+YJIXI0nU5HZGSkJCAvdGpg6gjMj9yCyWacnJxQFIU7d+5ITYgQIlvS6XSG/1GZMXWAyJ3kLyebsbW1pUiRIty7d4+wsDBsbGywsEg+T9QnKFJTYhpS/qYn18D44uLiiIqKQlEULC0tU/z/JERaJAHJhgoUKICtrS2hoaFERkaS0mj5+unk8+XLZ8TohJ6Uv+nJNTA+S0tLChQogKOjIwEBAaYOR5gxSUCyKVtbW2xtbVM9Rj8PRnrmjRGZT8rf9OQaCGG+pO5MCCGEEEYnCYgQQgghjE4SECGEEEIYnSQgQgghhDA6SUCEEEIIYXSSgAghhBDC6CQBEUIIIYTRaZSURrkSmerkyZNA5o7YKKNAmpaUv+nJNTAtKX/TMofy18fo6emZZJ8MRGbGsvMfXW4g5W96cg1MS8rftMy9/KUGRAghhBBGJ21AhBBCCGF0koAIIYQQwugkARFCCCGE0UkCIoQQQgijkwRECCGEEEYnCYgQQgghjE4SECGEEEIYnSQgQgghhDA6SUCEEEIIYXSSgAghhBDC6CQBEUIIIYTRSQJihmJiYvjll19o1aoVHh4etGjRgkWLFhlmHRQZ9+233+Lu7p7sMnv2bMNxGSl7uU5pu3//Pp6enqxatSrJvqwqa7ku8VIr//S+J0DKP6Pu3LnDp59+Sv369alSpQoNGzbkq6++4vHjx4mOy+nvAZkN1wxNmDCBTZs2Ubt2bZo3b46vry8zZ87k2rVrTJs2zdThmSV/f3/y5s1Lv379kuzz8vIyrGek7OU6pS4iIoJRo0YRFhaW7P6sKmu5Lqq0yj+97wmQ8s+I4OBgunfvzsOHD2nSpAmlS5fm4sWLrFu3jqNHj7Jhwwby5csH5IL3gCLMyokTJxQ3Nzdl7Nixhm1xcXHK6NGjFTc3N+XEiRMmjM581apVSxk4cGCqx2Sk7OU6pe727dtKly5dFDc3N8XNzU1ZuXJlov1ZVdZyXVRplb+ipO89oShS/hk1duxYxc3NTdm8eXOi7fPnz1fc3NyUyZMnK4qSO94DcgvGzKxfvx6AYcOGGbZpNBrGjBkDwMaNG00Slzm7e/cuT58+xc3NLdXjMlL2cp1StmzZMtq1a8fFixepW7dussdkVVnLdUlf+af3PQFS/hmhKAr79++nZMmSdOrUKdG+wYMHY2Njw+HDh4Hc8R6QWzBm5vTp0zg7O1O6dOlE24sXL46rqysnTpwwUWTmy9/fH4Dy5cunelxGyl6uU8pWrFiBq6srEydO5ObNmxw/fjzJMVlV1nJd0lf+6X1PgJR/RsTExDBq1CgcHByS7LO0tMTS0pLw8HAgd7wHpAbEjMTGxhIQEEDx4sWT3e/q6kpwcDDR0dFGjsy86f/ZBgcH06dPH2rWrEndunX55JNPuHv3LpCxspfrlLqJEyeyZcsWatasmez+rCpruS6qtMof0veeAHlfZJRWq2XAgAF069Ytyb5jx44RHh5OuXLlcs17QBIQM6JvLObk5JTsfgcHBxRFSbFRmUie/p/tokWLKFKkCD179qRMmTJs2bKFbt26cefOnQyVvVyn1DVo0ABLS8sU92dVWct1UaVV/pC+9wRk3bXKbSIjI5kyZQoAPXv2zDXvAbkFY0YiIiIANYtOjn57Tv4GkRW0Wi3Fixfnhx9+oEqVKobtCxYs4IcffmDy5Ml89tlnhmNTeg5Qy15RlHQfK5LKyN95Rsparkv6pec9MXfu3Cy7VrlJTEwMH374IZcvX6Z58+a0bNnSkODl9PeAJCBmxMbGBlD/YJOj/6OxtbU1Wkw5gf6bx8uGDBnC77//zsGDBw0JSHrKXt+XXq7Tq8nI33lGylquS/ql5z0RGRmZZdcqt4iMjGT06NEcPHiQypUrG7rA5pb3gCQgZsTBwQELCwtCQ0OT3a+vNkuugZPIOAsLCypUqEBQUBCRkZHpLnudTifX6TVk5O88I2Ut1+X1JXxP3Lt3jzfeeEPK/xU9ffqUIUOG4OfnR5UqVViyZInhteeW94AkIGZEq9Xi4uJCUFBQsvuDgoIoWbJkmvd3Rbzo6GguXbqEpaUllStXTrI/MjISUL8RpLfsLS0t5Tq9hoz8nWekrOW6pE963xM2NjZZdq1yugcPHjBw4ECuXLlCnTp1mD9/fqIP/dzyHpBGqGbG09OTu3fvEhgYmGh7YGAgwcHBVK9e3TSBmanIyEh69OjByJEjk+yLiIjgwoULODs7U7Ro0QyVvVyn15NVZS3XJW0ZeU+AlH9GhYWF8e6773LlyhWaNWvG4sWLk61xyA3vAUlAzIx+8JpZs2YZGhQpisKsWbMA6NGjh6lCM0tOTk54e3tz+/Zt1q5da9iuKAozZ87k8ePH9OrVC8hY2ct1ej1ZVdZyXdKWkfcESPln1NSpU/H396dBgwbMmTMnxQahueE9ILdgzIy3tzdt2rRh586d3LlzBy8vL3x9ffHz86NLly5J5mgQafv888/p1asXX3/9NYcOHaJUqVL4+vry33//Ubt2bQYPHgxkrOzlOr2erCpruS7pk973BEj5Z0RQUBCbNm0C1IG/FixYkOSYPHnyMHjw4FzxHtAo+hRImI3o6GgWLVrE5s2buX//Pi4uLnTt2pWBAwdibW1t6vDMUmBgID/88ANHjx4lNDQUV1dX2rdvbxgeWS8jZS/XKW2bNm3i008/5csvv6Rv376J9mVVWct1iZda+af3PQFS/um1ceNGQ4+6lOTLl49///0XyPnvAUlAhBBCCGF00gZECCGEEEYnCYgQQgghjE4SECGEEEIYnSQgQgghhDA6SUCEEEIIYXSSgAghhBDC6CQBEUIIIYTRSQIihBBCCKOTBEQIIYQQRicJiBBCCCGM7v8Zwnzmd7+YXAAAAABJRU5ErkJggg==\n", 127 | "text/plain": [ 128 | "
" 129 | ] 130 | }, 131 | "metadata": {}, 132 | "output_type": "display_data" 133 | } 134 | ], 135 | "source": [ 136 | "n1.trace.plot_losses(label='Near-constant momentum')\n", 137 | "n2.trace.plot_losses(label='Increasing momentum')\n", 138 | "plt.yscale('log')\n", 139 | "plt.legend()" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "id": "fad0ed6c-bc35-4c72-8629-819bc0799e10", 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "id": "85c5bf80-970e-401a-8851-9d86c5d877ca", 154 | "metadata": {}, 155 | "outputs": [], 156 | "source": [] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "id": "448ed22c-9904-4cc3-bee2-89adb3f449b9", 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [] 165 | } 166 | ], 167 | "metadata": { 168 | "kernelspec": { 169 | "display_name": "Python 3 (ipykernel)", 170 | "language": "python", 171 | "name": "python3" 172 | }, 173 | "language_info": { 174 | "codemirror_mode": { 175 | "name": "ipython", 176 | "version": 3 177 | }, 178 | "file_extension": ".py", 179 | "mimetype": "text/x-python", 180 | "name": "python", 181 | "nbconvert_exporter": "python", 182 | "pygments_lexer": "ipython3", 183 | "version": "3.9.7" 184 | } 185 | }, 186 | "nbformat": 4, 187 | "nbformat_minor": 5 188 | } 189 | -------------------------------------------------------------------------------- /notebooks/jupyter_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | def change_path_to_parent(): 5 | module_path = os.path.abspath(os.path.join('..')) 6 | if module_path not in sys.path: 7 | sys.path.append(module_path) 8 | -------------------------------------------------------------------------------- /notebooks/numpy_and_scipy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Let us test the efficiency of numpy and scipy functions" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np\n", 17 | "\n", 18 | "import scipy.sparse\n", 19 | "\n", 20 | "from scipy.sparse import csc_matrix, csr_matrix\n", 21 | "\n", 22 | "from jupyter_utils import change_path_to_parent\n", 23 | "change_path_to_parent()\n", 24 | "\n", 25 | "from datasets import get_dataset" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "metadata": {}, 31 | "source": [ 32 | "### Generate sparse random data" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 2, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "n = 5000\n", 42 | "dim = 2000\n", 43 | "A = csr_matrix(scipy.sparse.random(n, dim))\n", 44 | "x = csr_matrix(scipy.sparse.random(dim, 1))\n", 45 | "A_ = csc_matrix(A)\n", 46 | "x_ = csc_matrix(x)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "### Compare csc_matrix and csr_matrix" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 3, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "name": "stdout", 63 | "output_type": "stream", 64 | "text": [ 65 | "318 µs ± 13.8 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n", 66 | "369 µs ± 6.28 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n", 67 | "151 µs ± 2.42 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n", 68 | "99 µs ± 2.24 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" 69 | ] 70 | } 71 | ], 72 | "source": [ 73 | "%timeit A.dot(x)\n", 74 | "%timeit A.dot(x_)\n", 75 | "%timeit A_.dot(x)\n", 76 | "%timeit A_.dot(x_)" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "#### Let's check that the results match." 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 4, 89 | "metadata": {}, 90 | "outputs": [ 91 | { 92 | "data": { 93 | "text/plain": [ 94 | "0.0" 95 | ] 96 | }, 97 | "execution_count": 4, 98 | "metadata": {}, 99 | "output_type": "execute_result" 100 | } 101 | ], 102 | "source": [ 103 | "abs(A.dot(x) - A.dot(x_)).sum()" 104 | ] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "metadata": {}, 109 | "source": [ 110 | "#### Is Numpy faster?" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 5, 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "x = x.toarray().squeeze()\n", 120 | "A = A.toarray()" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 6, 126 | "metadata": {}, 127 | "outputs": [ 128 | { 129 | "name": "stdout", 130 | "output_type": "stream", 131 | "text": [ 132 | "4.23 ms ± 60.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" 133 | ] 134 | } 135 | ], 136 | "source": [ 137 | "%timeit A @ x" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": {}, 143 | "source": [ 144 | "#### Now let us sample rows (used in SGD)" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": 7, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "def fake_stochastic_gradient(A, x):\n", 154 | " n = A.shape[0]\n", 155 | " i = np.random.choice(n)\n", 156 | " return A[i].dot(x)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 8, 162 | "metadata": {}, 163 | "outputs": [ 164 | { 165 | "name": "stdout", 166 | "output_type": "stream", 167 | "text": [ 168 | "5.74 µs ± 56.7 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n", 169 | "72.2 ms ± 3.32 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n", 170 | "229 µs ± 8.1 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n", 171 | "322 µs ± 9.77 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n" 172 | ] 173 | } 174 | ], 175 | "source": [ 176 | "%timeit fake_stochastic_gradient(A, x)\n", 177 | "%timeit fake_stochastic_gradient(A, x_)\n", 178 | "%timeit fake_stochastic_gradient(A_, x)\n", 179 | "%timeit fake_stochastic_gradient(A_, x_)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "metadata": {}, 185 | "source": [ 186 | "## Conclusions:\n", 187 | "### 1. Use csc for deterministic (full batch) gradient computation\n", 188 | "### 2. Use csr if stochastic gradients are required" 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": {}, 194 | "source": [ 195 | "## Efficiencty of other sparse-vector operations" 196 | ] 197 | }, 198 | { 199 | "cell_type": "markdown", 200 | "metadata": {}, 201 | "source": [ 202 | "#### Sparse vector" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 9, 208 | "metadata": {}, 209 | "outputs": [ 210 | { 211 | "data": { 212 | "text/plain": [ 213 | "<1x1000001 sparse matrix of type ''\n", 214 | "\twith 1 stored elements in Compressed Sparse Row format>" 215 | ] 216 | }, 217 | "execution_count": 9, 218 | "metadata": {}, 219 | "output_type": "execute_result" 220 | } 221 | ], 222 | "source": [ 223 | "x = csr_matrix([1] + list(np.zeros(1000000)))\n", 224 | "x" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 10, 230 | "metadata": {}, 231 | "outputs": [ 232 | { 233 | "name": "stdout", 234 | "output_type": "stream", 235 | "text": [ 236 | "77 µs ± 2.56 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n", 237 | "39.2 µs ± 2.46 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n", 238 | "35.9 µs ± 1.61 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n", 239 | "38.1 µs ± 1.24 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n", 240 | "4.41 µs ± 140 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n" 241 | ] 242 | } 243 | ], 244 | "source": [ 245 | "%timeit x + x\n", 246 | "%timeit x * 2\n", 247 | "%timeit abs(x)\n", 248 | "%timeit x.minimum(0.5)\n", 249 | "%timeit x.eliminate_zeros()" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "#### Dense vector" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 11, 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "data": { 266 | "text/plain": [ 267 | "<1x1000000 sparse matrix of type ''\n", 268 | "\twith 999999 stored elements in Compressed Sparse Row format>" 269 | ] 270 | }, 271 | "execution_count": 11, 272 | "metadata": {}, 273 | "output_type": "execute_result" 274 | } 275 | ], 276 | "source": [ 277 | "x = csr_matrix(np.arange(1000000))\n", 278 | "x" 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": 12, 284 | "metadata": {}, 285 | "outputs": [ 286 | { 287 | "name": "stdout", 288 | "output_type": "stream", 289 | "text": [ 290 | "4.27 ms ± 130 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", 291 | "1.9 ms ± 111 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n", 292 | "1.81 ms ± 36.3 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n", 293 | "2.52 ms ± 197 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", 294 | "680 µs ± 39.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n" 295 | ] 296 | } 297 | ], 298 | "source": [ 299 | "%timeit x + x\n", 300 | "%timeit x * 2\n", 301 | "%timeit abs(x)\n", 302 | "%timeit x.minimum(0.5)\n", 303 | "%timeit x.eliminate_zeros()" 304 | ] 305 | }, 306 | { 307 | "cell_type": "code", 308 | "execution_count": null, 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "execution_count": null, 316 | "metadata": {}, 317 | "outputs": [], 318 | "source": [ 319 | "A, b = get_dataset('news20_class1')\n", 320 | "# A = A.toarray()\n", 321 | "l1 = 1e-4\n", 322 | "loss = LogisticRegression(A, b, l1=l1, l2=0)" 323 | ] 324 | }, 325 | { 326 | "cell_type": "code", 327 | "execution_count": 269, 328 | "metadata": {}, 329 | "outputs": [], 330 | "source": [ 331 | "def stochastic_gradient1(loss, x, idx=None, batch_size=1, replace=False, normalization=None):\n", 332 | " if idx is None:\n", 333 | " idx = np.random.choice(loss.n, size=batch_size, replace=replace)\n", 334 | " else:\n", 335 | " batch_size = 1 if np.isscalar(idx) else len(idx)\n", 336 | " if normalization is None:\n", 337 | " normalization = batch_size\n", 338 | " z = loss.A[idx] @ x\n", 339 | " if scipy.sparse.issparse(z):\n", 340 | " z = z.toarray().ravel()\n", 341 | " activation = scipy.special.expit(z)\n", 342 | " error = (activation-loss.b[idx]) / normalization\n", 343 | " stoch_grad = safe_sparse_add(loss.A[idx].T@error, loss.l2*x)\n", 344 | " return scipy.sparse.csr_matrix(stoch_grad).T\n", 345 | "\n", 346 | "def stochastic_gradient2(loss, x, idx=None, batch_size=1, replace=False, normalization=None):\n", 347 | " if idx is None:\n", 348 | " idx = np.random.choice(loss.n, size=batch_size, replace=replace)\n", 349 | " else:\n", 350 | " batch_size = 1 if np.isscalar(idx) else len(idx)\n", 351 | " A_idx = loss.A[idx]\n", 352 | " if normalization is None:\n", 353 | " normalization = batch_size\n", 354 | " z = A_idx @ x\n", 355 | " if scipy.sparse.issparse(z):\n", 356 | " z = z.toarray().ravel()\n", 357 | " activation = scipy.special.expit(z)\n", 358 | " if scipy.sparse.issparse(x):\n", 359 | " error = csr_matrix(activation-loss.b[idx]) / normalization\n", 360 | " else:\n", 361 | " error = (activation-loss.b[idx]) / normalization\n", 362 | " return loss.l2*x + (error@A_idx).T\n", 363 | "\n", 364 | "def stochastic_gradient3(loss, x, idx=None, batch_size=1, replace=False, normalization=None):\n", 365 | " if idx is None:\n", 366 | " idx = np.random.choice(loss.n, size=batch_size, replace=replace)\n", 367 | " else:\n", 368 | " batch_size = 1 if np.isscalar(idx) else len(idx)\n", 369 | " if normalization is None:\n", 370 | " normalization = batch_size\n", 371 | " z = loss.A[idx] @ x\n", 372 | " if scipy.sparse.issparse(z):\n", 373 | " z = z.toarray().ravel()\n", 374 | " activation = scipy.special.expit(z)\n", 375 | " error = csc_matrix(activation-loss.b[idx]) / normalization\n", 376 | " stoch_grad = safe_sparse_add(loss.l2*x.T, error@loss.A[idx])\n", 377 | " return scipy.sparse.csr_matrix(stoch_grad).T\n", 378 | "\n", 379 | "def stochastic_gradient5(loss, x, idx=None, batch_size=1, replace=False, normalization=None):\n", 380 | " if idx is None:\n", 381 | " idx = np.random.choice(loss.n, size=batch_size, replace=replace)\n", 382 | " else:\n", 383 | " batch_size = 1 if np.isscalar(idx) else len(idx)\n", 384 | " if normalization is None:\n", 385 | " normalization = batch_size\n", 386 | " z = loss.A[idx] @ x\n", 387 | " if scipy.sparse.issparse(z):\n", 388 | " z = z.toarray().ravel()\n", 389 | " activation = scipy.special.expit(z)\n", 390 | " error = csc_matrix(activation-loss.b[idx]) / normalization\n", 391 | " stoch_grad = safe_sparse_add(loss.l2*x, (error@loss.A[idx]).T)\n", 392 | " return scipy.sparse.csr_matrix(stoch_grad)\n", 393 | "\n", 394 | "def grad_step(loss, x, lr, batch_size=32, option=1):\n", 395 | " if option == 1:\n", 396 | " grad = stochastic_gradient1(loss, x, batch_size=batch_size)\n", 397 | " elif option == 2:\n", 398 | " grad = stochastic_gradient2(loss, x, batch_size=batch_size)\n", 399 | " elif option == 3:\n", 400 | " grad = stochastic_gradient3(loss, x, batch_size=batch_size)\n", 401 | " elif option == 4:\n", 402 | " grad = stochastic_gradient4(loss, x, batch_size=batch_size)\n", 403 | " elif option == 5:\n", 404 | " grad = stochastic_gradient5(loss, x, batch_size=batch_size)\n", 405 | " elif option == 6:\n", 406 | " grad = stochastic_gradient6(loss, x, batch_size=batch_size)\n", 407 | " elif option == 7:\n", 408 | " grad = stochastic_gradient7(loss, x, batch_size=batch_size)\n", 409 | " elif option == 8:\n", 410 | " grad = stochastic_gradient8(loss, x, batch_size=batch_size)\n", 411 | " elif option == 9:\n", 412 | " grad = stochastic_gradient9(loss, x, batch_size=batch_size)\n", 413 | " return x - lr * grad\n", 414 | "\n", 415 | "def prox_grad_step(loss, x, lr, batch_size=32, option=1):\n", 416 | " return loss.regularizer.prox(grad_step(loss, x, lr, batch_size, option), self.lr)" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": 154, 422 | "metadata": {}, 423 | "outputs": [], 424 | "source": [ 425 | "lr = 1 / loss.smoothness()" 426 | ] 427 | }, 428 | { 429 | "cell_type": "code", 430 | "execution_count": 270, 431 | "metadata": {}, 432 | "outputs": [ 433 | { 434 | "name": "stdout", 435 | "output_type": "stream", 436 | "text": [ 437 | "2.03 ms ± 35.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", 438 | "1.59 ms ± 29.5 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n", 439 | "1.95 ms ± 92.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n", 440 | "1.56 ms ± 29.1 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n", 441 | "2.17 ms ± 47.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" 442 | ] 443 | } 444 | ], 445 | "source": [ 446 | "%timeit grad_step(loss, gd.x, lr_, option=1)\n", 447 | "%timeit grad_step(loss, gd.x, lr_, option=2)\n", 448 | "%timeit grad_step(loss, gd.x, lr_, option=3)\n", 449 | "%timeit grad_step(loss, gd.x, lr_, option=5)" 450 | ] 451 | }, 452 | { 453 | "cell_type": "code", 454 | "execution_count": null, 455 | "metadata": {}, 456 | "outputs": [], 457 | "source": [] 458 | }, 459 | { 460 | "cell_type": "markdown", 461 | "metadata": {}, 462 | "source": [ 463 | "## Random number generation" 464 | ] 465 | }, 466 | { 467 | "cell_type": "markdown", 468 | "metadata": {}, 469 | "source": [ 470 | "### Todo" 471 | ] 472 | }, 473 | { 474 | "cell_type": "code", 475 | "execution_count": null, 476 | "metadata": {}, 477 | "outputs": [], 478 | "source": [] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": null, 483 | "metadata": {}, 484 | "outputs": [], 485 | "source": [] 486 | } 487 | ], 488 | "metadata": { 489 | "kernelspec": { 490 | "display_name": "Python 3", 491 | "language": "python", 492 | "name": "python3" 493 | }, 494 | "language_info": { 495 | "codemirror_mode": { 496 | "name": "ipython", 497 | "version": 3 498 | }, 499 | "file_extension": ".py", 500 | "mimetype": "text/x-python", 501 | "name": "python", 502 | "nbconvert_exporter": "python", 503 | "pygments_lexer": "ipython3", 504 | "version": "3.8.5" 505 | } 506 | }, 507 | "nbformat": 4, 508 | "nbformat_minor": 4 509 | } 510 | -------------------------------------------------------------------------------- /optmethods/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/konstmish/opt_methods/3f3cf99a64cafb0d3d033d5819889c0451b4100b/optmethods/__init__.py -------------------------------------------------------------------------------- /optmethods/datasets/14_Tumors.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/konstmish/opt_methods/3f3cf99a64cafb0d3d033d5819889c0451b4100b/optmethods/datasets/14_Tumors.mat -------------------------------------------------------------------------------- /optmethods/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils import get_dataset 2 | -------------------------------------------------------------------------------- /optmethods/datasets/covtype.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/konstmish/opt_methods/3f3cf99a64cafb0d3d033d5819889c0451b4100b/optmethods/datasets/covtype.bz2 -------------------------------------------------------------------------------- /optmethods/datasets/news20.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/konstmish/opt_methods/3f3cf99a64cafb0d3d033d5819889c0451b4100b/optmethods/datasets/news20.bz2 -------------------------------------------------------------------------------- /optmethods/datasets/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sklearn 3 | 4 | from sklearn.datasets import load_svmlight_file 5 | 6 | 7 | def get_dataset(dataset, data_path='../optmethods/datasets/'): 8 | if len(data_path) > 0 and data_path[-1] != '/': 9 | data_path += '/' 10 | if dataset in ['news20', 'real-sim', 'webspam', 'YearPredictionMSD']: 11 | return load_svmlight_file(data_path + dataset + '.bz2') 12 | elif dataset in ['a1a', 'a5a', 'a9a', 'mushrooms', 'gisette', 'w8a']: 13 | return load_svmlight_file(data_path + dataset) 14 | elif dataset == 'covtype': 15 | return sklearn.datasets.fetch_covtype(return_X_y=True) 16 | elif dataset == 'covtype_binary': 17 | A, b = sklearn.datasets.fetch_covtype(return_X_y=True) 18 | # Following paper "A Parallel Mixture of SVMs for Very Large Scale Problems" 19 | # we make the problem binary by splittong the data into class 2 and the rest. 20 | b = (b == 2) 21 | elif dataset == 'YearPredictionMSD_binary': 22 | A, b = load_svmlight_file(data_path + dataset[:-7] + '.bz2') 23 | b = b > 2000 24 | elif dataset == 'news20_more_features': 25 | A, b = sklearn.datasets.fetch_20newsgroups_vectorized(return_X_y=True) 26 | elif dataset == 'news20_binary': 27 | A, b = load_svmlight_file(data_path + 'news20' + '.bz2') 28 | b = (b == 1) 29 | elif dataset == 'rcv1': 30 | return sklearn.datasets.fetch_rcv1(return_X_y=True) 31 | elif dataset == 'rcv1_binary': 32 | A, b = sklearn.datasets.fetch_rcv1(return_X_y=True) 33 | freq = np.asarray(b.sum(axis=0)).squeeze() 34 | main_class = np.argmax(freq) 35 | b = (b[:, main_class] == 1) * 1. 36 | b = b.toarray().squeeze() 37 | else: 38 | raise ValueError(f'The dataset {dataset} is not supported. Consider loading it yourself.') 39 | return A, b 40 | -------------------------------------------------------------------------------- /optmethods/first_order/__init__.py: -------------------------------------------------------------------------------- 1 | from .adagrad import Adagrad 2 | from .adgd import Adgd 3 | from .adgd_accel import AdgdAccel 4 | from .gd import Gd 5 | from .ig import Ig 6 | from .nesterov import Nesterov 7 | from .nest_line import NestLine 8 | from .ogm import Ogm 9 | from .polyak import Polyak 10 | from .rest_nest import RestNest 11 | -------------------------------------------------------------------------------- /optmethods/first_order/adagrad.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | class Adagrad(Optimizer): 8 | """ 9 | Implement Adagrad from Duchi et. al, 2011 10 | "Adaptive Subgradient Methods for Online Learning and Stochastic Optimization" 11 | http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf 12 | This implementation only supports deterministic gradients and use dense vectors. 13 | 14 | Arguments: 15 | primal_dual (boolean, optional): if true, uses the dual averaging method of Nesterov, 16 | otherwise uses gradient descent update (default: False) 17 | lr (float, optional): learning rate coefficient, which needs to be tuned to 18 | get better performance (default: 1.) 19 | delta (float, optional): another learning rate parameter, slows down performance if 20 | chosen too large, so a small value is recommended, otherwise requires tuning (default: 0.) 21 | """ 22 | def __init__(self, primal_dual=False, lr=1., delta=0., *args, **kwargs): 23 | super(Adagrad, self).__init__(*args, **kwargs) 24 | self.primal_dual = primal_dual 25 | self.lr = lr 26 | self.delta = delta 27 | 28 | def estimate_stepsize(self): 29 | self.s = np.sqrt(self.s**2 + self.grad**2) 30 | self.inv_lr = self.delta + self.s 31 | 32 | def step(self): 33 | self.grad = self.loss.gradient(self.x) 34 | self.estimate_stepsize() 35 | if self.primal_dual: 36 | self.sum_grad += self.grad 37 | self.x = self.x0 - self.lr * np.divide(self.sum_grad, self.inv_lr, out=0. * self.x, where=self.inv_lr != 0) 38 | else: 39 | self.x -= self.lr * np.divide(self.grad, self.inv_lr, out=0. * self.x, where=self.inv_lr != 0) 40 | 41 | def init_run(self, *args, **kwargs): 42 | super(Adagrad, self).init_run(*args, **kwargs) 43 | if type(self.x) is not np.ndarray: 44 | self.x = self.x.toarray().ravel() 45 | self.x0 = copy.deepcopy(self.x) 46 | self.s = 0. * self.x 47 | self.sum_grad = 0. * self.x 48 | -------------------------------------------------------------------------------- /optmethods/first_order/adgd.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | class Adgd(Optimizer): 8 | """ 9 | Gradient descent with adaptive stepsize estimation 10 | using local values of smoothness (gradient Lipschitzness). 11 | 12 | Arguments: 13 | lr0 (float, optional): a small value that idealy should be smaller than the 14 | inverse (local) smoothness constant. Does not affect performance too much. 15 | """ 16 | def __init__(self, lr0=1e-6, *args, **kwargs): 17 | super(Adgd, self).__init__(*args, **kwargs) 18 | self.lr0 = lr0 19 | 20 | def step(self): 21 | self.grad = self.loss.gradient(self.x) 22 | self.estimate_new_stepsize() 23 | self.x_old = copy.deepcopy(self.x) 24 | self.grad_old = copy.deepcopy(self.grad) 25 | self.x -= self.lr * self.grad 26 | if self.use_prox: 27 | self.x = self.loss.regularizer.prox(self.x, self.lr) 28 | 29 | def estimate_new_stepsize(self): 30 | if self.grad_old is not None: 31 | L = self.loss.norm(self.grad-self.grad_old) / self.loss.norm(self.x-self.x_old) 32 | if L == 0: 33 | lr_new = np.sqrt(1+self.theta) * self.lr 34 | else: 35 | lr_new = min(np.sqrt(1+self.theta) * self.lr, 0.5/L) 36 | self.theta = lr_new / self.lr 37 | self.lr = lr_new 38 | 39 | def init_run(self, *args, **kwargs): 40 | super(Adgd, self).init_run(*args, **kwargs) 41 | self.lr = self.lr0 42 | self.trace.lrs = [self.lr] 43 | self.theta = 1e12 44 | self.grad_old = None 45 | 46 | def update_trace(self): 47 | super(Adgd, self).update_trace() 48 | self.trace.lrs.append(self.lr) 49 | -------------------------------------------------------------------------------- /optmethods/first_order/adgd_accel.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | class AdgdAccel(Optimizer): 8 | """ 9 | Accelerated gradient descent with adaptive stepsize and momentum estimation 10 | using local values of smoothness (gradient Lipschitzness) and strong convexity. 11 | Momentum is used as given by Nesterov's acceleration. 12 | 13 | Arguments: 14 | lr0 (float, optional): a small value that idealy should be smaller than the 15 | inverse (local) smoothness constant. Does not affect performance too much. 16 | start_with_small_momentum (bool, optional): momentum gradually increases. Helps with 17 | ill-conditioned problems when the estimated momentum gets large too quickly. 18 | """ 19 | def __init__(self, lr0=1e-6, max_momentum=1-1e-6, *args, **kwargs): 20 | super(AdgdAccel, self).__init__(*args, **kwargs) 21 | self.lr0 = lr0 22 | 23 | def step(self): 24 | self.grad = self.loss.gradient(self.x_nest) 25 | self.estimate_new_stepsize() 26 | self.estimate_new_momentum() 27 | self.x_nest_old = copy.deepcopy(self.x_nest) 28 | self.x_old = copy.deepcopy(self.x) 29 | self.grad_old = copy.deepcopy(self.grad) 30 | self.x = self.x_nest - self.lr*self.grad 31 | if self.use_prox: 32 | self.x = self.loss.regularizer.prox(self.x, self.lr) 33 | self.x_nest = self.x + self.momentum*(self.x - self.x_old) 34 | 35 | def estimate_new_stepsize(self): 36 | if self.grad_old is not None: 37 | self.L = self.loss.norm(self.grad-self.grad_old) / self.loss.norm(self.x_nest-self.x_nest_old) 38 | if self.L == 0: 39 | lr_new = np.sqrt(1+0.5*self.theta) * self.lr 40 | else: 41 | lr_new = min(np.sqrt(1+0.5*self.theta) * self.lr, 0.5/self.L) 42 | self.theta = lr_new / self.lr 43 | self.lr = lr_new 44 | 45 | def estimate_new_momentum(self): 46 | alpha_new = 0.5 * (1 + np.sqrt(1 + 4*self.alpha**2)) 47 | self.momentum = (self.alpha - 1) / alpha_new 48 | self.alpha = alpha_new 49 | if self.grad_old is not None: 50 | if self.L == 0: 51 | mu_new = self.mu / 10 52 | else: 53 | mu_new = min(np.sqrt(1+0.5*self.theta_mu) * self.mu, 0.5*self.L) 54 | self.theta_mu = mu_new / self.mu 55 | self.mu = mu_new 56 | kappa = 1 / (self.lr*self.mu) 57 | self.momentum = min(self.momentum, 1 - 2 / (1+np.sqrt(kappa))) 58 | 59 | def init_run(self, *args, **kwargs): 60 | super(AdgdAccel, self).init_run(*args, **kwargs) 61 | self.x_nest = copy.deepcopy(self.x) 62 | self.momentum = 0 63 | self.lr = self.lr0 64 | self.mu = 1 / self.lr 65 | self.trace.lrs = [self.lr] 66 | self.trace.momentums = [0] 67 | self.theta = 1e12 68 | self.theta_mu = 1 69 | self.grad_old = None 70 | self.alpha = 1. 71 | 72 | def update_trace(self): 73 | super(AdgdAccel, self).update_trace() 74 | self.trace.lrs.append(self.lr) 75 | self.trace.momentums.append(self.momentum) 76 | -------------------------------------------------------------------------------- /optmethods/first_order/gd.py: -------------------------------------------------------------------------------- 1 | from optmethods.optimizer import Optimizer 2 | 3 | 4 | class Gd(Optimizer): 5 | """ 6 | Gradient descent with constant learning rate or a line search procedure. 7 | 8 | Arguments: 9 | lr (float, optional): an estimate of the inverse smoothness constant 10 | """ 11 | def __init__(self, lr=None, *args, **kwargs): 12 | super(Gd, self).__init__(*args, **kwargs) 13 | self.lr = lr 14 | 15 | def step(self): 16 | self.grad = self.loss.gradient(self.x) 17 | if self.line_search is None: 18 | self.x -= self.lr * self.grad 19 | if self.use_prox: 20 | self.x = self.loss.regularizer.prox(self.x, self.lr) 21 | else: 22 | self.x = self.line_search(x=self.x, direction=-self.grad) 23 | 24 | def init_run(self, *args, **kwargs): 25 | super(Gd, self).init_run(*args, **kwargs) 26 | if self.lr is None: 27 | self.lr = 1 / self.loss.smoothness 28 | -------------------------------------------------------------------------------- /optmethods/first_order/heavy_ball.py: -------------------------------------------------------------------------------- 1 | from optmethods.optimizer import Optimizer 2 | 3 | 4 | class HeavyBall(Optimizer): 5 | """ 6 | Gradient descent with Polyak's heavy-ball momentum 7 | For details, see, e.g., https://vsokolov.org/courses/750/files/polyak64.pdf 8 | 9 | Arguments: 10 | lr (float, optional): an estimate of the inverse smoothness constant 11 | momentum (float, optional): momentum value. For quadratics, 12 | it should be close to 1-sqrt(λ_min/λ_max), where λ_min and 13 | λ_max are the smallest/largest eigenvalues of the quadratic matrix 14 | """ 15 | def __init__(self, lr=None, strongly_convex=False, momentum=None, *args, **kwargs): 16 | super(HeavyBall, self).__init__(*args, **kwargs) 17 | self.lr = lr 18 | if momentum < 0: 19 | raise ValueError("Invalid momentum: {}".format(mu)) 20 | self.strongly_convex = strongly_convex 21 | if self.strongly_convex: 22 | self.momentum = momentum 23 | 24 | def step(self): 25 | if not self.strongly_convex: 26 | self.momentum = self.it / (self.it+1) 27 | x_copy = self.x.copy() 28 | self.grad = self.loss.gradient(self.x) 29 | if self.use_prox: 30 | self.x = self.loss.regularizer.prox(self.x - self.lr*self.grad, self.lr) + self.momentum*(self.x-self.x_old) 31 | else: 32 | self.x = self.x - self.lr * self.grad + self.momentum*(self.x-self.x_old) 33 | self.x_old = x_copy 34 | 35 | def init_run(self, *args, **kwargs): 36 | super(HeavyBall, self).init_run(*args, **kwargs) 37 | if self.lr is None: 38 | self.lr = 1 / self.loss.smoothness 39 | self.x_old = self.x.copy() 40 | -------------------------------------------------------------------------------- /optmethods/first_order/ig.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from optmethods.optimizer import Optimizer 4 | 5 | 6 | class Ig(Optimizer): 7 | """ 8 | Incremental gradient descent (IG) with decreasing or constant learning rate. 9 | 10 | For a formal description and convergence guarantees, see Section 10 in 11 | https://arxiv.org/abs/2006.05988 12 | 13 | The method is sensitive to finishing the final epoch, so it will terminate earlier 14 | than it_max if it_max is not divisible by the number of steps per epoch. 15 | 16 | Arguments: 17 | prox_every_it (bool, optional): whether to use proximal operation every iteration 18 | or only at the end of an epoch. Theory supports the latter. Only used if the loss includes 19 | a proximal regularizer (default: False) 20 | lr0 (float, optional): an estimate of the inverse smoothness constant, this step-size 21 | is used for the first epoch_start_decay epochs. If not given, it will be set 22 | with the value in the loss. 23 | lr_max (float, optional): a maximal step-size never to be exceeded (default: np.inf) 24 | lr_decay_coef (float, optional): the coefficient in front of the number of finished epochs 25 | in the denominator of step-size. For strongly convex problems, a good value 26 | is mu/3, where mu is the strong convexity constant 27 | lr_decay_power (float, optional): the power to exponentiate the number of finished epochs 28 | in the denominator of step-size. For strongly convex problems, a good value is 1 (default: 1) 29 | epoch_start_decay (int, optional): how many epochs the step-size is kept constant 30 | By default, will be set to have about 2.5% of iterations with the step-size equal to lr0 31 | batch_size (int, optional): the number of samples from the function to be used at each iteration 32 | update_trace_at_epoch_end (bool, optional): save progress only at the end of an epoch, which 33 | avoids bad iterates 34 | """ 35 | def __init__(self, prox_every_it=False, lr0=None, lr_max=np.inf, lr_decay_coef=0, lr_decay_power=1, 36 | epoch_start_decay=None, batch_size=1, update_trace_at_epoch_end=True, *args, **kwargs): 37 | super(Ig, self).__init__(*args, **kwargs) 38 | self.prox_every_it = prox_every_it 39 | self.lr0 = lr0 40 | self.lr_max = lr_max 41 | self.lr_decay_coef = lr_decay_coef 42 | self.lr_decay_power = lr_decay_power 43 | self.epoch_start_decay = epoch_start_decay 44 | self.batch_size = batch_size 45 | self.update_trace_at_epoch_end = update_trace_at_epoch_end 46 | 47 | if epoch_start_decay is None and np.isfinite(self.epoch_max): 48 | self.epoch_start_decay = 1 + self.epoch_max // 40 49 | elif epoch_start_decay is None: 50 | self.epoch_start_decay = 1 51 | self.steps_per_epoch = math.ceil(self.loss.n/batch_size) 52 | 53 | def step(self): 54 | i_max = min(self.loss.n, self.i+self.batch_size) 55 | idx = np.arange(self.i, i_max) 56 | self.i += self.batch_size 57 | if self.i >= self.loss.n: 58 | self.i = 0 59 | normalization = self.loss.n / self.steps_per_epoch 60 | self.grad = self.loss.stochastic_gradient(self.x, idx=idx, normalization=normalization) 61 | 62 | denom_const = 1 / self.lr0 63 | it_decrease = self.steps_per_epoch * max(0, self.finished_epochs-self.epoch_start_decay) 64 | lr_decayed = 1 / (denom_const + self.lr_decay_coef*it_decrease**self.lr_decay_power) 65 | self.lr = min(lr_decayed, self.lr_max) 66 | 67 | self.x -= self.lr * self.grad 68 | end_of_epoch = self.i == 0 69 | self.finished_epochs += end_of_epoch 70 | if self.prox_every_it and self.use_prox: 71 | self.x = self.loss.regularizer.prox(self.x, self.lr) 72 | elif end_of_epoch and self.use_prox: 73 | self.x = self.loss.regularizer.prox(self.x, self.lr * self.steps_per_epoch) 74 | 75 | def init_run(self, *args, **kwargs): 76 | super(Ig, self).init_run(*args, **kwargs) 77 | self.finished_epochs = 0 78 | if self.lr0 is None: 79 | self.lr0 = 1 / self.loss.batch_smoothness(batch_size) 80 | self.i = 0 81 | -------------------------------------------------------------------------------- /optmethods/first_order/nest_line.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from optmethods.line_search import NestArmijo 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | class NestLine(Optimizer): 8 | """ 9 | Accelerated gradient descent with line search proposed by Nesterov. 10 | For details, see, equation (4.9) in 11 | http://www.optimization-online.org/DB_FILE/2007/09/1784.pdf 12 | The method does not support increasing momentum, which may limit 13 | its efficiency on ill-conditioned problems. 14 | 15 | Arguments: 16 | line_search (optmethods.LineSearch, optional): a callable line search, here it should be None or 17 | an instance of NestArmijo class. If None, line search is intialized automatically (default: None) 18 | lr (float, optional): an estimate of the inverse smoothness constant 19 | strongly_convex (bool, optional): use the variant for strongly convex functions, 20 | which requires mu to be provided (default: False) 21 | mu (float, optional): strong-convexity constant or a lower bound on it (default: 0) 22 | start_with_small_momentum (bool, optional): momentum gradually increases. 23 | Only used if mu>0 (default: True) 24 | """ 25 | def __init__(self, line_search=None, lr=None, mu=0, start_with_small_momentum=True, *args, **kwargs): 26 | if mu < 0: 27 | raise ValueError("Invalid mu: {}".format(mu)) 28 | if line_search is None: 29 | line_search = NestArmijo(mu=mu, start_with_small_momentum=start_with_small_momentum) 30 | super(NestLine, self).__init__(line_search=line_search, *args, **kwargs) 31 | self.lr = lr 32 | self.mu = mu 33 | self.start_with_small_momentum = start_with_small_momentum 34 | 35 | def step(self): 36 | self.x, a = self.line_search(self.x, self.v, self.A) 37 | self.A += a 38 | self.grad = self.loss.gradient(self.x) 39 | self.v -= a * self.grad 40 | 41 | def init_run(self, *args, **kwargs): 42 | super(NestLine, self).init_run(*args, **kwargs) 43 | self.v = copy.deepcopy(self.x) 44 | self.A = 0 45 | -------------------------------------------------------------------------------- /optmethods/first_order/nesterov.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | class Nesterov(Optimizer): 8 | """ 9 | Accelerated gradient descent with constant learning rate. 10 | For details, see, e.g., 11 | http://mpawankumar.info/teaching/cdt-big-data/nesterov83.pdf 12 | 13 | Arguments: 14 | lr (float, optional): an estimate of the inverse smoothness constant 15 | strongly_convex (bool, optional): use the variant for strongly convex functions, 16 | which requires mu to be provided (default: False) 17 | max_momentum (float, optional): the target value of momentum. If start_with_small_momentum 18 | is True, the value of momentum in the method will be increased from 0 to the value 19 | provided in this parameter. Otherwise, it is used in all iterations 20 | mu (float, optional): strong-convexity constant or a lower bound on it. Ignored if 21 | momentum is provided (default: 0) 22 | start_with_small_momentum (bool, optional): momentum gradually increases. Only used if 23 | strongly_convex is set to True (default: True) 24 | """ 25 | def __init__(self, lr=None, strongly_convex=False, max_momentum=None, mu=0, 26 | start_with_small_momentum=True, *args, **kwargs): 27 | super(Nesterov, self).__init__(*args, **kwargs) 28 | self.lr = lr 29 | self.max_momentum = max_momentum 30 | if strongly_convex: 31 | self.mu = mu 32 | if mu <= 0 and max_momentum is None: 33 | raise ValueError("""Mu must be larger than 0 for strongly_convex=True, 34 | invalid value: {}""".format(mu)) 35 | self.strongly_convex = strongly_convex 36 | self.start_with_small_momentum = start_with_small_momentum 37 | 38 | def step(self): 39 | if not self.strongly_convex or self.start_with_small_momentum: 40 | alpha_new = 0.5 * (1 + np.sqrt(1 + 4*self.alpha**2)) 41 | self.momentum = (self.alpha - 1) / alpha_new 42 | self.alpha = alpha_new 43 | self.momentum = min(self.momentum, self.max_momentum) 44 | else: 45 | self.momentum = self.max_momentum 46 | self.x_old = copy.deepcopy(self.x) 47 | self.grad = self.loss.gradient(self.x_nest) 48 | self.x = self.x_nest - self.lr*self.grad 49 | if self.use_prox: 50 | self.x = self.loss.regularizer.prox(self.x, self.lr) 51 | self.x_nest = self.x + self.momentum*(self.x-self.x_old) 52 | 53 | def init_run(self, *args, **kwargs): 54 | super(Nesterov, self).init_run(*args, **kwargs) 55 | if self.lr is None: 56 | self.lr = 1 / self.loss.smoothness 57 | self.x_nest = copy.deepcopy(self.x) 58 | self.alpha = 1. 59 | if self.strongly_convex and self.max_momentum is None: 60 | kappa = (1/self.lr)/self.mu 61 | self.max_momentum = (np.sqrt(kappa)-1) / (np.sqrt(kappa)+1) 62 | elif self.max_momentum is None: 63 | self.max_momentum = 1. - 1e-8 64 | -------------------------------------------------------------------------------- /optmethods/first_order/ogm.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | class Ogm(Optimizer): 8 | """ 9 | Optimized (accelerated) gradient method with constant learning rate. 10 | For a simple convergence proof, see, e.g., 11 | https://arxiv.org/abs/2102.07366 12 | 13 | Arguments: 14 | lr (float, optional): an estimate of the inverse smoothness constant 15 | strongly_convex (bool, optional): use the variant for strongly convex functions, 16 | which requires mu to be provided (default: False) 17 | mu (float, optional): strong-convexity constant or a lower bound on it (default: 0) 18 | start_with_small_momentum (bool, optional): momentum gradually increases. Only used if 19 | strongly_convex is set to True (default: True) 20 | """ 21 | def __init__(self, lr=None, strongly_convex=False, mu=0, start_with_small_momentum=True, *args, **kwargs): 22 | super(Ogm, self).__init__(*args, **kwargs) 23 | self.lr = lr 24 | if strongly_convex: 25 | self.mu = mu 26 | if mu <= 0: 27 | raise ValueError("""Mu must be larger than 0 for strongly_convex=True, 28 | invalid value: {}""".format(mu)) 29 | self.strongly_convex = strongly_convex 30 | self.start_with_small_momentum = start_with_small_momentum 31 | 32 | 33 | def step(self): 34 | if not self.strongly_convex or self.start_with_small_momentum: 35 | alpha_new = 0.5 * (1 + np.sqrt(1 + 4*self.alpha**2)) 36 | self.momentum1 = (self.alpha - 1) / alpha_new 37 | self.momentum2 = self.alpha / alpha_new 38 | self.alpha = alpha_new 39 | self.momentum1 = min(self.momentum1, self.max_momentum) 40 | self.momentum2 = min(self.momentum2, self.max_momentum) 41 | else: 42 | self.momentum1 = self.momentum2 = self.max_momentum 43 | self.x_old = copy.deepcopy(self.x) 44 | self.grad = self.loss.gradient(self.x_nest) 45 | self.x = self.x_nest - self.lr*self.grad 46 | if self.use_prox: 47 | self.x = self.loss.regularizer.prox(self.x, self.lr) 48 | self.x_nest = self.x + self.momentum1*(self.x-self.x_old) + self.momentum2*(self.x-self.x_nest) 49 | 50 | def init_run(self, *args, **kwargs): 51 | super(Ogm, self).init_run(*args, **kwargs) 52 | if self.lr is None: 53 | self.lr = 1 / self.loss.smoothness 54 | self.x_nest = copy.deepcopy(self.x) 55 | self.alpha = 1. 56 | if self.strongly_convex: 57 | kappa = (1/self.lr)/self.mu 58 | self.gamma = (np.sqrt(8*kappa+1) + 3) / (2*kappa-2) 59 | self.max_momentum = 1 / (2*self.gamma + 1) 60 | else: 61 | self.max_momentum = 1. 62 | -------------------------------------------------------------------------------- /optmethods/first_order/polyak.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from optmethods.optimizer import Optimizer 4 | 5 | 6 | class Polyak(Optimizer): 7 | """ 8 | Polyak adaptive gradient descent, proposed in 9 | (B. T. Poyal, "Introduction to Optimization") 10 | which can be accessed, e.g., here: 11 | https://www.researchgate.net/publication/342978480_Introduction_to_Optimization 12 | 13 | Arguments: 14 | f_opt (float): precise value of the objective's minimum. If an underestimate is given, 15 | the algorirthm can be unstable; if an overestimate is given, will not converge below 16 | the overestimate. 17 | lr_min (float, optional): the smallest step-size, useful when 18 | an overestimate of the optimal value is given (default: 0) 19 | lr_max (float, optional): the laregest allowed step-size, useful when 20 | an underestimate of the optimal value is given (defaul: np.inf) 21 | """ 22 | def __init__(self, f_opt, lr_min=0, lr_max=np.inf, *args, **kwargs): 23 | super(Polyak, self).__init__(*args, **kwargs) 24 | self.f_opt = f_opt 25 | self.lr_min = lr_min 26 | self.lr_max = lr_max 27 | 28 | def step(self): 29 | self.grad = self.loss.gradient(self.x) 30 | self.estimate_new_stepsize() 31 | self.x -= self.lr * self.grad 32 | 33 | def estimate_new_stepsize(self): 34 | loss_gap = self.loss.value(self.x) - self.f_opt 35 | self.lr = loss_gap / self.loss.norm(self.grad)**2 36 | self.lr = min(self.lr, self.lr_max) 37 | self.lr = max(self.lr, self.lr_min) 38 | 39 | def init_run(self, *args, **kwargs): 40 | super(Polyak, self).init_run(*args, **kwargs) 41 | self.trace.lrs = [] 42 | 43 | def update_trace(self): 44 | super(Polyak, self).update_trace() 45 | self.trace.lrs.append(self.lr) 46 | -------------------------------------------------------------------------------- /optmethods/first_order/rest_nest.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | class RestNest(Optimizer): 8 | """ 9 | Accelerated gradient descent with constant learning rate. 10 | For details, see, e.g., https://arxiv.org/abs/1204.3982 11 | 12 | Arguments: 13 | lr (float, optional): an estimate of the inverse smoothness constant 14 | it_before_first_rest (int, optional): number of iterations with increasing momentum when 15 | restart is not allowed to happen (default: 10) 16 | func_condition (bool, optional): whether objective (function) decrease should 17 | be checked as restart condition (default: False) 18 | doubling (bool, optional): instead of checking conditions, just double 19 | the number of iterations until next restart 20 | after every new restart (default: False) 21 | """ 22 | def __init__(self, lr=None, it_before_first_rest=10, func_condition=False, doubling=False, *args, **kwargs): 23 | super(RestNest, self).__init__(*args, **kwargs) 24 | self.lr = lr 25 | self.it_before_first_rest = it_before_first_rest 26 | self.func_condition = func_condition 27 | self.doubling = doubling 28 | 29 | def step(self): 30 | self.x_old = copy.deepcopy(self.x) 31 | self.grad = self.loss.gradient(self.x_nest) 32 | self.x = self.x_nest - self.lr*self.grad 33 | if self.use_prox: 34 | self.x = self.loss.regularizer.prox(self.x, self.lr) 35 | if self.restart_condition(): 36 | self.n_restarts += 1 37 | self.alpha = 1. 38 | self.it_without_rest = 0 39 | self.potential_old = np.inf 40 | else: 41 | self.it_without_rest += 1 42 | alpha_new = 0.5 * (1 + np.sqrt(1 + 4*self.alpha**2)) 43 | self.momentum = (self.alpha - 1) / alpha_new 44 | self.alpha = alpha_new 45 | self.x_nest = self.x + self.momentum*(self.x-self.x_old) 46 | 47 | def restart_condition(self): 48 | if self.it_without_rest < self.it_before_first_rest: 49 | return False 50 | if self.doubling: 51 | if self.it_without_rest >= self.it_until_rest: 52 | self.it_until_rest *= 2 53 | return True 54 | return False 55 | if self.func_condition: 56 | potential = self.loss.value(self.x) 57 | restart = potential > self.potential_old 58 | self.potential_old = potential 59 | return restart 60 | direction_is_bad = self.loss.inner_prod(self.x - self.x_old, self.grad) > 0 61 | return direction_is_bad 62 | 63 | def init_run(self, *args, **kwargs): 64 | super(RestNest, self).init_run(*args, **kwargs) 65 | if self.lr is None: 66 | self.lr = 1 / self.loss.smoothness 67 | self.x_nest = self.x 68 | self.alpha = 1. 69 | self.it_without_rest = 0 70 | self.potential_old = np.inf 71 | self.n_restarts = 0 72 | if self.doubling: 73 | self.it_until_rest = self.it_before_first_rest 74 | -------------------------------------------------------------------------------- /optmethods/line_search/__init__.py: -------------------------------------------------------------------------------- 1 | from .armijo import Armijo 2 | from .best_grid import BestGrid 3 | from .goldstein import Goldstein 4 | from .nest_armijo import NestArmijo 5 | from .reg_newton_ls import RegNewtonLS 6 | from .wolfe import Wolfe 7 | -------------------------------------------------------------------------------- /optmethods/line_search/armijo.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from .line_search import LineSearch 4 | 5 | class Armijo(LineSearch): 6 | """ 7 | Armijo line search with optional resetting of the initial stepsize 8 | at each iteration. If resetting is used, the previous value is optionally 9 | multiplied by 1/backtracking and used as the first stepsize to try at the 10 | new iteration. Otherwise, it starts with the maximal stepsize. 11 | Arguments: 12 | armijo_const (float, optional): proportionality constant for the armijo condition (default: 0.5) 13 | start_with_prev_lr (boolean, optional): initialize lr with the previous value (default: True) 14 | increase_lr (boolean, optional): multiply the previous lr by 1/backtracking (default: True) 15 | backtracking (float, optional): constant by which the current stepsize is multiplied (default: 0.5) 16 | """ 17 | 18 | def __init__(self, armijo_const=0.5, start_with_prev_lr=True, increase_lr=True, backtracking=0.5, *args, **kwargs): 19 | super(Armijo, self).__init__(*args, **kwargs) 20 | self.armijo_const = armijo_const 21 | self.start_with_prev_lr = start_with_prev_lr 22 | self.increase_lr = increase_lr 23 | self.backtracking = backtracking 24 | self.x_prev = None 25 | self.val_prev = None 26 | 27 | def condition(self, gradient, x, x_new): 28 | value_new = self.loss.value(x_new) 29 | self.val_prev = value_new 30 | descent = self.armijo_const * self.loss.inner_prod(gradient, x - x_new) 31 | return value_new <= self.current_value - descent + self.tolerance 32 | 33 | def __call__(self, x=None, x_new=None, gradient=None, direction=None): 34 | if gradient is None: 35 | gradient = self.optimizer.grad 36 | if x is None: 37 | x = self.optimizer.x 38 | if direction is None: 39 | direction = (x_new - x) / self.lr 40 | if self.start_with_prev_lr: 41 | self.lr = self.lr / self.backtracking if self.increase_lr else self.lr 42 | else: 43 | self.lr = self.lr0 44 | if x_new is None: 45 | x_new = x + self.lr * direction 46 | if self.loss.is_equal(x, self.x_prev): 47 | self.current_value = self.val_prev 48 | else: 49 | self.current_value = self.loss.value(x) 50 | 51 | armijo_condition_met = self.condition(gradient, x, x_new) 52 | it_extra = 0 53 | it_max = min(self.it_max, self.optimizer.ls_it_max - self.it) 54 | while not armijo_condition_met and it_extra < it_max: 55 | self.lr *= self.backtracking 56 | x_new = x + self.lr * direction 57 | armijo_condition_met = self.condition(gradient, x, x_new) 58 | it_extra += 1 59 | 60 | self.x_prev = copy.deepcopy(x_new) 61 | self.it += self.it_per_call + it_extra 62 | return x_new 63 | -------------------------------------------------------------------------------- /optmethods/line_search/best_grid.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from .line_search import LineSearch 5 | 6 | class BestGrid(LineSearch): 7 | """ 8 | Find the best stepsize either over values {lr_max * backtracking ** pow: pow=0, 1, ...} or 9 | over {lr * backtracking ** (1 - pow): pow=0, 1, ...} where lr is the previous value 10 | Arguments: 11 | lr_max (float, optional): the maximal stepsize, useful for second-order 12 | and quasi-Newton methods (default: np.inf) 13 | functional (boolean, optional): use functional values to check optimality. 14 | Otherwise, gradient norm is used (default: True) 15 | start_with_prev_lr (boolean, optional): initialize lr with the previous value (default: True) 16 | increase_lr (boolean, optional): multiply the previous lr by 1/backtracking (default: True) 17 | increase_many_times (boolean, optional): multiply the lr by 1/backtracking until it's the best (default: False) 18 | backtracking (float, optional): constant to multiply the estimated stepsize by (default: 0.5) 19 | """ 20 | 21 | def __init__(self, lr_max=np.inf, functional=True, start_with_prev_lr=False, increase_lr=True, 22 | increase_many_times=True, backtracking=0.5, *args, **kwargs): 23 | super(BestGrid, self).__init__(*args, **kwargs) 24 | self.lr_max = lr_max 25 | self.functional = functional 26 | self.start_with_prev_lr = start_with_prev_lr 27 | self.increase_lr = increase_lr 28 | self.increase_many_times = increase_many_times 29 | self.backtracking = backtracking 30 | self.x_prev = None 31 | self.val_prev = None 32 | 33 | def condition(self, proposed_value, proposed_next): 34 | return proposed_value <= proposed_next + self.tolerance 35 | 36 | def metric_value(self, x): 37 | if self.functional: 38 | return self.loss.value(x) 39 | return self.loss.norm(self.loss.gradient(x)) 40 | 41 | def __call__(self, x, x_new=None, direction=None): 42 | if x is None: 43 | x = self.optimizer.x 44 | if direction is None: 45 | direction = x_new - x 46 | self.lr = 1 47 | elif self.start_with_prev_lr: 48 | self.lr = self.lr / self.backtracking if self.increase_lr else self.lr 49 | self.lr = min(self.lr, self.lr_max) 50 | else: 51 | self.lr = self.lr0 52 | if x_new is None: 53 | x_new = x + self.lr * direction 54 | if self.loss.is_equal(x, self.x_prev): 55 | self.current_value = self.val_prev 56 | else: 57 | self.current_value = self.metric_value(x) 58 | 59 | it_extra = 0 60 | proposed_value = self.metric_value(x_new) 61 | need_to_decrease_lr = proposed_value > self.current_value 62 | if not need_to_decrease_lr: 63 | x_next = x + self.lr * self.backtracking * direction 64 | proposed_next = self.metric_value(x_next) 65 | if not self.condition(proposed_value, proposed_next): 66 | need_to_decrease_lr = True 67 | self.lr *= self.backtracking 68 | proposed_value = proposed_next 69 | it_extra += 1 70 | found_best = not need_to_decrease_lr and not self.increase_many_times 71 | it_max = min(self.it_max, self.optimizer.ls_it_max - self.it) 72 | while not found_best and it_extra < it_max: 73 | if need_to_decrease_lr: 74 | lr_next = self.lr * self.backtracking 75 | else: 76 | lr_next = min(self.lr / self.backtracking, self.lr_max) 77 | x_next = x + lr_next * direction 78 | proposed_next = self.metric_value(x_next) 79 | found_best = self.condition(proposed_value, proposed_next) 80 | it_extra += 1 81 | if not found_best or it_extra == self.it_max: 82 | self.lr = lr_next 83 | proposed_value = proposed_next 84 | 85 | x_new = x + self.lr * direction 86 | self.val_prev = proposed_value 87 | self.x_prev = copy.deepcopy(x_new) 88 | self.it += self.it_per_call + it_extra 89 | return x_new 90 | -------------------------------------------------------------------------------- /optmethods/line_search/goldstein.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from .line_search import LineSearch 4 | 5 | class Goldstein(LineSearch): 6 | """ 7 | Goldstein line search with optional resetting of the initial stepsize 8 | at each iteration. Combines Armijo condition with an extra condition to make sure 9 | that the stepsize is not too small. If resetting is used, the previous value 10 | is used as the first stepsize to try at this iteration. Otherwise, 11 | it starts with the maximal stepsize. 12 | Arguments: 13 | goldstein_const (float, optional): proportionality constant for both conditions (default: 0.05) 14 | start_with_prev_lr (boolean, optional): sets the reset option from (default: True) 15 | backtracking (float, optional): constant by which the current stepsize is multiplied (default: 0.5) 16 | """ 17 | 18 | def __init__(self, goldstein_const=0.05, start_with_prev_lr=True, backtracking=0.5, *args, **kwargs): 19 | super(Goldstein, self).__init__(*args, **kwargs) 20 | self.goldstein_const = goldstein_const 21 | self.start_with_prev_lr = start_with_prev_lr 22 | self.backtracking = backtracking 23 | self.x_prev = None 24 | self.val_prev = None 25 | 26 | def armijo_condition(self, gradient, x, x_new): 27 | value_new = self.loss.value(x_new) 28 | self.x_prev = copy.deepcopy(x_new) 29 | self.val_prev = value_new 30 | descent = self.goldstein_const * self.loss.inner_prod(gradient, x - x_new) 31 | return value_new <= self.current_value - descent + self.tolerance 32 | 33 | def goldstein_condition(self, gradient, x, x_new): 34 | value_new = self.loss.value(x_new) 35 | self.x_prev = copy.deepcopy(x_new) 36 | self.val_prev = value_new 37 | descent = (1-self.goldstein_const) * self.loss.inner_prod(gradient, x - x_new) 38 | return value_new >= self.current_value - descent 39 | 40 | def __call__(self, x=None, x_new=None, gradient=None, direction=None): 41 | if gradient is None: 42 | gradient = self.optimizer.grad 43 | if x is None: 44 | x = self.optimizer.x 45 | if direction is None: 46 | direction = (x_new - x) / self.lr 47 | self.lr = self.lr if self.start_with_prev_lr else self.lr0 48 | if x_new is None: 49 | x_new = x + self.lr * direction 50 | if self.loss.is_equal(x, self.x_prev): 51 | self.current_value = self.val_prev 52 | else: 53 | self.current_value = self.loss.value(x) 54 | 55 | armijo_condition = self.armijo_condition(gradient, x, x_new) 56 | goldstein_condition = self.goldstein_condition(gradient, x, x_new) 57 | it_extra = 0 58 | it_max = min(self.it_max, self.optimizer.ls_it_max - self.it) 59 | while not armijo_condition and it_extra < it_max: 60 | self.lr *= self.backtracking 61 | x_new = x + self.lr * direction 62 | armijo_condition = self.armijo_condition(gradient, x, x_new) 63 | it_extra += 1 64 | if it_extra == 0: 65 | while not goldstein_condition and it_extra < self.it_max: 66 | self.lr /= self.backtracking 67 | x_new = x + self.lr * direction 68 | goldstein_condition = self.goldstein_condition(gradient, x, x_new) 69 | it_extra += 1 70 | 71 | self.it += self.it_per_call + it_extra 72 | return x_new 73 | -------------------------------------------------------------------------------- /optmethods/line_search/line_search.py: -------------------------------------------------------------------------------- 1 | class LineSearch(): 2 | """ 3 | A universal Line Search class that allows for finding the best 4 | scalar alpha such that x + alpha * delta is a good 5 | direction for optimization. The goodness of the new point can 6 | be measured in many ways: decrease of functional values, 7 | smaller gradient norm, Lipschitzness of an operator, etc. 8 | Arguments: 9 | lr0 (float, optional): the initial estimate (default: 1.0) 10 | count_first_it (bool, optional): to count the first iteration as requiring effort. 11 | This should be False for methods that reuse information, such as objective value, from the previous 12 | line search iteration. In contrast, most stochastic line searches 13 | should count the initial iteration too as information can't be reused (default: False) 14 | count_last_it (bool, optional): to count the last iteration as requiring effort. 15 | Not true for line searches that can use the produced information, such as matrix-vector 16 | product, to compute the next gradient or other important quantities. However, even then, 17 | it is convenient to set to False to account for gradient computation (default: True) 18 | it_max (int, optional): maximal number of innert iterations per one call. 19 | Prevents the line search from running for too long and from 20 | running into machine precision issues (default: 50) 21 | tolerance (float, optional): the allowed amount of condition violation (default: 0) 22 | """ 23 | 24 | def __init__(self, lr0=1.0, count_first_it=False, count_last_it=True, it_max=50, tolerance=0): 25 | self.lr0 = lr0 26 | self.lr = lr0 27 | self.count_first_it = count_first_it 28 | self.count_last_it = count_last_it 29 | self.it = 0 30 | self.it_max = it_max 31 | self.tolerance = tolerance 32 | 33 | @property 34 | def it_per_call(self): 35 | return self.count_first_it + self.count_last_it 36 | 37 | def reset(self, optimizer): 38 | self.lr = self.lr0 39 | self.it = 0 40 | self.optimizer = optimizer 41 | self.loss = optimizer.loss 42 | self.use_prox = optimizer.use_prox 43 | 44 | def __call__(self, x=None, direction=None, x_new=None): 45 | pass 46 | -------------------------------------------------------------------------------- /optmethods/line_search/nest_armijo.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from .line_search import LineSearch 4 | 5 | class NestArmijo(LineSearch): 6 | """ 7 | This line search implements procedure (4.9) from the following paper by Nesterov: 8 | http://www.optimization-online.org/DB_FILE/2007/09/1784.pdf 9 | Arguments: 10 | mu (float, optional): strong convexity constant (default: 0.0) 11 | start_with_prev_lr (boolean, optional): initialize lr with the previous value (default: True) 12 | backtracking (float, optional): constant by which the current stepsize is multiplied (default: 0.5) 13 | start_with_small_momentum (bool, optional): momentum gradually increases. 14 | Only used if mu>0 (default: True) 15 | """ 16 | 17 | def __init__(self, mu=0, start_with_prev_lr=True, backtracking=0.5, start_with_small_momentum=True, *args, **kwargs): 18 | super(NestArmijo, self).__init__(count_first_it=True, *args, **kwargs) 19 | self.mu = mu 20 | self.start_with_prev_lr = start_with_prev_lr 21 | self.backtracking = backtracking 22 | self.start_with_small_momentum = start_with_small_momentum 23 | self.global_calls = 0 24 | 25 | def condition(self, y, x_new): 26 | grad_new = self.loss.gradient(x_new) 27 | return self.loss.inner_prod(grad_new, y - x_new) >= self.lr * self.loss.norm(grad_new)**2 - self.tolerance 28 | 29 | def __call__(self, x, v, A): 30 | self.global_calls += 1 31 | self.lr = self.lr / self.backtracking if self.start_with_prev_lr else self.lr0 32 | # Find $a$ from quadratic equation a^2/(A+a) = 2*lr*(1 + mu*A) 33 | discriminant = (self.lr * (1+self.mu*A)) ** 2 + A * self.lr * (1 + self.mu*A) 34 | a = self.lr * (1+self.mu*A) + np.sqrt(discriminant) 35 | if self.start_with_small_momentum: 36 | a_small = self.lr + np.sqrt(self.lr**2 + A * self.lr) 37 | a = min(a, a_small) 38 | y = (A*x + a*v) / (A+a) 39 | gradient = self.loss.gradient(y) 40 | x_new = y - self.lr * gradient 41 | nest_condition_met = self.condition(y, x_new) 42 | 43 | it_extra = 0 44 | it_max = min(2 * self.it_max, self.optimizer.ls_it_max - self.it) 45 | while not nest_condition_met and it_extra < it_max: 46 | self.lr *= self.backtracking 47 | discriminant = (self.lr * (1+self.mu*A)) ** 2 + A * self.lr * (1+self.mu*A) 48 | a = self.lr * (1+self.mu*A) + np.sqrt(discriminant) 49 | if self.start_with_small_momentum: 50 | a_small = self.lr + np.sqrt(self.lr**2 + A * self.lr) 51 | a = min(a, a_small) 52 | y = A / (A+a) * x + a / (A+a) *v 53 | gradient = self.loss.gradient(y) 54 | x_new = y - self.lr * gradient 55 | nest_condition_met = self.condition(y, x_new) 56 | it_extra += 2 57 | if self.lr * self.backtracking == 0: 58 | break 59 | 60 | self.it += self.it_per_call + it_extra 61 | return x_new, a 62 | 63 | def reset(self, *args, **kwargs): 64 | super(NestArmijo, self).reset(*args, **kwargs) 65 | self.global_calls = 0 66 | -------------------------------------------------------------------------------- /optmethods/line_search/reg_newton_ls.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import numpy.linalg as la 4 | 5 | from .line_search import LineSearch 6 | 7 | class RegNewtonLS(LineSearch): 8 | """ 9 | This line search estimates the Hessian Lipschitz constant for the Global Regularized Newton. 10 | See the following paper for the details and convergence proof: 11 | "Regularized Newton Method with Global O(1/k^2) Convergence" 12 | https://arxiv.org/abs/2112.02089 13 | For consistency with other line searches, 'lr' parameter is used to denote the inverse of regularization. 14 | Arguments: 15 | decrease_reg (boolean, optional): multiply the previous regularization parameter by 1/backtracking (default: True) 16 | backtracking (float, optional): constant by which the current regularization is divided (default: 0.5) 17 | """ 18 | 19 | def __init__(self, decrease_reg=True, backtracking=0.5, H0=None, *args, **kwargs): 20 | super(RegNewtonLS, self).__init__(*args, **kwargs) 21 | self.decrease_reg = decrease_reg 22 | self.backtracking = backtracking 23 | self.H0 = H0 24 | self.H = self.H0 25 | self.attempts = 0 26 | 27 | def condition(self, x_new, x, grad, identity_coef): 28 | if self.f_prev is None: 29 | self.f_prev = self.loss.value(x) 30 | self.f_new = self.loss.value(x_new) 31 | r = self.loss.norm(x_new - x) 32 | condition_f = self.f_new <= self.f_prev - 2/3 * identity_coef * r**2 33 | grad_new = self.loss.gradient(x_new) 34 | condition_grad = self.loss.norm(grad_new) <= 2 * identity_coef * r 35 | self.attempts = self.attempts + 1 if not condition_f or not condition_grad else 0 36 | return condition_f and condition_grad 37 | 38 | def __call__(self, x, grad, hess): 39 | if self.decrease_reg: 40 | self.H *= self.backtracking 41 | grad_norm = self.loss.norm(grad) 42 | identity_coef = np.sqrt(self.H * grad_norm) 43 | 44 | x_new = x - np.linalg.solve(hess + identity_coef*np.eye(self.loss.dim), grad) 45 | condition_met = self.condition(x_new, x, grad, identity_coef) 46 | self.it += self.it_per_call 47 | it_extra = 0 48 | it_max = min(self.it_max, self.optimizer.ls_it_max - self.it) 49 | while not condition_met and it_extra < it_max: 50 | self.H /= self.backtracking 51 | identity_coef = np.sqrt(self.H * grad_norm) 52 | x_new = x - np.linalg.solve(hess + identity_coef*np.eye(self.loss.dim), grad) 53 | condition_met = self.condition(x_new, x, grad, identity_coef) 54 | it_extra += 1 55 | if self.backtracking / self.H == 0: 56 | break 57 | self.f_prev = self.f_new 58 | self.it += it_extra 59 | self.lr = 1 / identity_coef 60 | return x_new 61 | 62 | def reset(self, *args, **kwargs): 63 | super(RegNewtonLS, self).reset(*args, **kwargs) 64 | self.f_prev = None 65 | -------------------------------------------------------------------------------- /optmethods/line_search/wolfe.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | from .line_search import LineSearch 5 | 6 | class Wolfe(LineSearch): 7 | """ 8 | Wolfe line search with optional resetting of the initial stepsize 9 | at each iteration. If resetting is used, the previous value is used 10 | as the first stepsize to try at this iteration. Otherwise, it starts 11 | with the maximal stepsize. 12 | Arguments: 13 | armijo_const (float, optional): proportionality constant for the armijo condition (default: 0.5) 14 | wolfe_const (float, optional): second proportionality constant for the wolfe condition (default: 0.5) 15 | start_with_prev_lr (boolean, optional): sets the reset option from (default: True) 16 | backtracking (float, optional): constant by which the current stepsize is multiplied (default: 0.5) 17 | """ 18 | 19 | def __init__(self, armijo_const=0.1, wolfe_const=0.9, strong=False, 20 | start_with_prev_lr=True, backtracking=0.5, *args, **kwargs): 21 | super(Wolfe, self).__init__(*args, **kwargs) 22 | self.armijo_const = armijo_const 23 | self.wolfe_const = wolfe_const 24 | self.strong = strong 25 | self.start_with_prev_lr = start_with_prev_lr 26 | self.backtracking = backtracking 27 | self.x_prev = None 28 | self.val_prev = None 29 | 30 | def armijo_condition(self, gradient, x, x_new): 31 | value_new = self.loss.value(x_new) 32 | self.x_prev = copy.deepcopy(x_new) 33 | self.val_prev = value_new 34 | descent = self.armijo_const * self.loss.inner_prod(gradient, x - x_new) 35 | return value_new <= self.current_value - descent + self.tolerance 36 | 37 | def curvature_condition(self, gradient, x, x_new): 38 | grad_new = self.loss.gradient(x_new) 39 | curv_x = self.loss.inner_prod(gradient, x - x_new) 40 | curv_x_new = self.loss.inner_prod(grad_new, x - x_new) 41 | if self.strong: 42 | curv_x, curv_x_new = np.abs(curv_x), np.abs(curv_x_new) 43 | return curv_x_new <= self.wolfe_const * curv_x + self.tolerance 44 | 45 | def __call__(self, x=None, x_new=None, gradient=None, direction=None): 46 | if gradient is None: 47 | gradient = self.optimizer.grad 48 | if x is None: 49 | x = self.optimizer.x 50 | if direction is None: 51 | direction = (x_new - x) / self.lr 52 | self.lr = self.lr if self.start_with_prev_lr else self.lr0 53 | if x_new is None: 54 | x_new = x + self.lr * direction 55 | if self.loss.is_equal(x, self.x_prev): 56 | self.current_value = self.val_prev 57 | else: 58 | self.current_value = self.loss.value(x) 59 | 60 | armijo_condition = self.armijo_condition(gradient, x, x_new) 61 | curvature_condition = self.curvature_condition(gradient, x, x_new) 62 | it_extra = 0 63 | it_max = min(self.it_max, self.optimizer.ls_it_max - self.it) 64 | while not armijo_condition and it_extra < it_max: 65 | self.lr *= self.backtracking 66 | x_new = x + self.lr * direction 67 | armijo_condition = self.armijo_condition(gradient, x, x_new) 68 | it_extra += 1 69 | if it_extra == 0: 70 | while not curvature_condition and it_extra < it_max: 71 | self.lr /= self.backtracking 72 | x_new = x + self.lr * direction 73 | curvature_condition = self.curvature_condition(gradient, x, x_new) 74 | it_extra += 1 75 | 76 | self.it += self.it_per_call + it_extra 77 | return x_new 78 | -------------------------------------------------------------------------------- /optmethods/loss/__init__.py: -------------------------------------------------------------------------------- 1 | from .bounded_l2 import Boundedl2 2 | from .logistic_regression import LogisticRegression 3 | from .log_sum_exp import LogSumExp 4 | -------------------------------------------------------------------------------- /optmethods/loss/bounded_l2.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | 3 | from .regularizer import Regularizer 4 | 5 | 6 | class Boundedl2(Regularizer): 7 | """ 8 | The bounded l2 regularization is equal to 9 | R(x) = sum_{i=1}^d x_i^2 / (x_i^2 + 1) 10 | where x=(x_1, ..., x_d) is from R^d. This penalty is attractive for benchmarking 11 | purposes since it is smooth (has Lipschitz gradient) and nonconvex. 12 | 13 | See 14 | https://arxiv.org/pdf/1905.05920.pdf 15 | https://arxiv.org/pdf/1810.10690.pdf 16 | for examples of using this penalty for benchmarking. 17 | """ 18 | def __init__(self, coef): 19 | self.coef = coef 20 | 21 | def value(x, x2=None): 22 | if not scipy.sparse.issparse(x): 23 | if x2 is None: 24 | x2 = x * x 25 | return self.coef * 0.5 * np.sum(x2 / (x2 + 1)) 26 | if x2 is None: 27 | x2 = x.multiply(x) 28 | ones_where_nonzero = x2.sign() 29 | return self.coef * 0.5 * (x2 / (x2 + ones_where_nonzero)).sum() 30 | 31 | def prox(x, lr=None): 32 | raise NotImplementedError('Exact proximal operator for bounded l2 does not exist. Consider using gradients.') 33 | 34 | def grad(self, x): 35 | if not scipy.sparse.issparse(x): 36 | return self.coef * x / (x**2+1)**2 37 | ones_where_nonzero = abs(x.sign()) 38 | x2_plus_one = x.multiply(x) + ones_where_nonzero 39 | denominator = x2_plus_one.multiply(x2_plus_one) 40 | return self.coef * x.multiply(ones_where_nonzero / denominator) 41 | 42 | @property 43 | def smoothness(self): 44 | return self.coef 45 | -------------------------------------------------------------------------------- /optmethods/loss/linear_regression.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import numpy as np 3 | 4 | import numpy.linalg as la 5 | from sklearn.utils.extmath import row_norms, safe_sparse_dot 6 | 7 | from .loss_oracle import Oracle 8 | from .utils import safe_sparse_add, safe_sparse_multiply, safe_sparse_norm 9 | 10 | 11 | class LinearRegression(Oracle): 12 | """ 13 | Linear regression oracle that returns loss values, gradients and Hessians. 14 | """ 15 | def __init__(self, A, b, store_mat_vec_prod=True, *args, **kwargs): 16 | super(LinearRegression, self).__init__(*args, **kwargs) 17 | self.A = A 18 | b = np.asarray(b) 19 | if (np.unique(b) == [1, 2]).all(): 20 | # Transform labels {1, 2} to {0, 1} 21 | self.b = b - 1 22 | elif (np.unique(b) == [-1, 1]).all(): 23 | # Transform labels {-1, 1} to {0, 1} 24 | self.b = (b+1) / 2 25 | else: 26 | assert (np.unique(b) == [0, 1]).all() 27 | self.b = b 28 | self.n, self.dim = A.shape 29 | self.store_mat_vec_prod = store_mat_vec_prod 30 | self.x_last = 0 #np.zeros(self.dim) 31 | self.mat_vec_prod = 0 #np.zeros(self.n) 32 | 33 | def value(self, x): 34 | z = self.mat_vec_product(x) 35 | regularization = self.l1*safe_sparse_norm(x, ord=1) + self.l2/2*safe_sparse_norm(x)**2 36 | return np.mean((1-self.b)*z-logsig(z)) + regularization 37 | 38 | def gradient(self, x): 39 | z = self.mat_vec_product(x) 40 | if scipy.sparse.issparse(z): 41 | z = z.toarray() 42 | activation = scipy.special.expit(z) 43 | grad = safe_sparse_add(self.A.T@(activation-self.b)/self.n, self.l2*x) 44 | if scipy.sparse.issparse(x): 45 | grad = scipy.sparse.csr_matrix(grad).T 46 | return grad 47 | 48 | def hessian(self, x): 49 | z = self.mat_vec_product(x) 50 | if scipy.sparse.issparse(z): 51 | z = z.toarray() 52 | activation = scipy.special.expit(z) 53 | weights = activation * (1-activation) 54 | A_weighted = safe_sparse_multiply(self.A.T, weights) 55 | return A_weighted@self.A/self.n + self.l2*np.eye(self.dim) 56 | 57 | def stochastic_gradient(self, x, idx=None, batch_size=1, replace=False): 58 | if idx is None: 59 | idx = np.random.choice(self.n, size=batch_size, replace=replace) 60 | z = self.A[idx] @ x 61 | if scipy.sparse.issparse(z): 62 | z = z.toarray() 63 | activation = scipy.special.expit(z) 64 | stoch_grad = safe_sparse_add(self.A[idx].T@(activation-self.b[idx])/len(idx), self.l2*x) 65 | return stoch_grad 66 | 67 | def mat_vec_product(self, x): 68 | if not self.store_mat_vec_prod or safe_sparse_norm(x-self.x_last) != 0: 69 | z = self.A @ x 70 | if self.store_mat_vec_prod: 71 | self.mat_vec_prod = z 72 | self.x_last = x.copy() 73 | 74 | return self.mat_vec_prod 75 | 76 | def smoothness(self): 77 | covariance = self.A.T@self.A/self.n 78 | if scipy.sparse.issparse(covariance): 79 | covariance = covariance.toarray() 80 | return 0.25*np.max(la.eigvalsh(covariance)) + self.l2 81 | 82 | def max_smoothness(self): 83 | max_squared_sum = row_norms(self.A, squared=True).max() 84 | return 0.25*max_squared_sum + self.l2 85 | 86 | def average_smoothness(self): 87 | ave_squared_sum = row_norms(self.A, squared=True).mean() 88 | return 0.25*ave_squared_sum + self.l2 89 | 90 | @staticmethod 91 | def norm(x, ord=None): 92 | return safe_sparse_norm(x, ord=ord) 93 | 94 | @staticmethod 95 | def inner_prod(x, y): 96 | return safe_sparse_inner_prod(x, y) 97 | -------------------------------------------------------------------------------- /optmethods/loss/log_sum_exp.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import numpy as np 3 | import warnings 4 | 5 | from sklearn.utils.extmath import row_norms 6 | 7 | from .loss_oracle import Oracle 8 | 9 | 10 | class LogSumExp(Oracle): 11 | """ 12 | Logarithm of the sum of exponentials plus two optional quadratic terms: 13 | log(sum_{i=1}^n exp(-b_i)) + 1/2*||Ax - b||^2 + l2/2*||x||^2 14 | See, for instance, 15 | https://arxiv.org/pdf/2002.00657.pdf 16 | https://arxiv.org/pdf/2002.09403.pdf 17 | for examples of using the objective to benchmark second-order methods. 18 | 19 | Due to the potential under- and overflow, log-sum-exp and softmax 20 | functions might be unstable. This implementation has not been tested 21 | for stability and an alternative choice of functions may lead to 22 | more precise results. See 23 | https://academic.oup.com/imajna/advance-article/doi/10.1093/imanum/draa038/5893596 24 | for a discussion of possible ways to increase stability. 25 | 26 | Sparse matrices are currently not supported as it is not clear if it is relevant to practice. 27 | 28 | Arguments: 29 | max_smoothing (float, optional): the smoothing constant of the log-sum-exp term 30 | least_squares_term (bool, optional): add term 0.5*||Ax-b||^2 to the objective (default: False) 31 | """ 32 | 33 | def __init__(self, max_smoothing=1, least_squares_term=False, A=None, b=None, n=None, dim=None, store_mat_vec_prod=True, store_softmax=True, *args, **kwargs): 34 | super(LogSumExp, self).__init__(*args, **kwargs) 35 | self.max_smoothing = max_smoothing 36 | self.least_squares_term = least_squares_term 37 | self.A = A 38 | self.b = np.asarray(b) 39 | if b is None: 40 | self.b = self.rng.normal(-1, 1, size=n) 41 | if A is None: 42 | self.A = self.rng.uniform(-1, 1, size=(n, dim)) 43 | # for the first gradient computation, we have to set 44 | # the values of self.store_mat_vec_prod and self.store_softmax 45 | self.store_mat_vec_prod = False 46 | self.store_softmax = False 47 | self.A -= self.gradient(np.zeros(dim)) 48 | self.value(np.zeros(dim)) 49 | self.store_mat_vec_prod = store_mat_vec_prod 50 | self.store_softmax = store_softmax 51 | 52 | self.n, self.dim = self.A.shape 53 | self.x_last_mv = 0. 54 | self.x_last_soft = 0. 55 | self._mat_vec_prod = np.zeros(self.n) 56 | 57 | def _value(self, x): 58 | Ax = self.mat_vec_product(x) 59 | regularization = 0 60 | if self.l2 != 0: 61 | regularization = self.l2/2 * self.norm(x)**2 62 | if self.least_squares_term: 63 | regularization += 1/2 * np.linalg.norm(Ax)**2 64 | return self.max_smoothing*scipy.special.logsumexp(((Ax-self.b)/self.max_smoothing)) + regularization 65 | 66 | def gradient(self, x): 67 | Ax = self.mat_vec_product(x) 68 | softmax = self.softmax(Ax=Ax) 69 | if self.least_squares_term: 70 | grad = (softmax + Ax) @ self.A 71 | else: 72 | grad = softmax @ self.A 73 | 74 | if self.l2 == 0: 75 | return grad 76 | return grad + self.l2 * x 77 | 78 | def hessian(self, x): 79 | Ax = self.mat_vec_product(x) 80 | softmax = self.softmax(x=x, Ax=Ax) 81 | hess1 = self.A.T * (softmax/self.max_smoothing) @ self.A 82 | grad = softmax @ self.A 83 | hess2 = -np.outer(grad, grad) / self.max_smoothing 84 | return hess1 + hess2 + self.l2 * np.eye(self.dim) 85 | 86 | def stochastic_hessian(self, x, idx=None, batch_size=1, replace=False, normalization=None): 87 | pass 88 | 89 | def mat_vec_product(self, x): 90 | if self.store_mat_vec_prod and self.is_equal(x, self.x_last_mv): 91 | return self._mat_vec_prod 92 | 93 | Ax = self.A @ x 94 | if self.store_mat_vec_prod: 95 | self._mat_vec_prod = Ax 96 | self.x_last_mv = x.copy() 97 | return Ax 98 | 99 | def softmax(self, x=None, Ax=None): 100 | if x is None and Ax is None: 101 | raise ValueError("Either x or Ax must be provided to compute softmax.") 102 | if self.store_softmax and self.is_equal(x, self.x_last_soft): 103 | return self._softmax 104 | if Ax is None: 105 | Ax = self.mat_vec_product(x) 106 | 107 | softmax = scipy.special.softmax((Ax-self.b) / self.max_smoothing) 108 | if self.store_softmax and x is not None: 109 | self._softmax = softmax 110 | self.x_last_soft = x.copy() 111 | return softmax 112 | 113 | def hess_vec_prod(self, x, v, grad_dif=False, eps=None): 114 | pass 115 | 116 | @property 117 | def smoothness(self): 118 | if self._smoothness is not None: 119 | return self._smoothness 120 | matrix_coef = 1 + self.least_squares_term 121 | if self.dim > 20000 and self.n > 20000: 122 | warnings.warn("The matrix is too large to estimate the smoothness constant, so Frobeniius estimate is used instead.") 123 | self._smoothness = matrix_coef*np.linalg.norm(self.A, ord='fro')**2 + self.l2 124 | else: 125 | sing_val_max = scipy.sparse.linalg.svds(self.A, k=1, return_singular_vectors=False)[0] 126 | self._smoothness = matrix_coef*sing_val_max**2 + self.l2 127 | return self._smoothness 128 | 129 | @property 130 | def hessian_lipschitz(self): 131 | if self._hessian_lipschitz is None: 132 | row_norms = np.linalg.norm(self.A, axis=1) 133 | max_row_norm = np.max(row_norms) 134 | self._hessian_lipschitz = 2 * max_row_norm / self.max_smoothing * self.smoothness 135 | return self._hessian_lipschitz 136 | 137 | @staticmethod 138 | def norm(x): 139 | return np.linalg.norm(x) 140 | 141 | @staticmethod 142 | def inner_prod(x, y): 143 | return x @ y 144 | 145 | @staticmethod 146 | def outer_prod(x, y): 147 | return np.outer(x, y) 148 | 149 | @staticmethod 150 | def is_equal(x, y): 151 | return np.array_equal(x, y) 152 | -------------------------------------------------------------------------------- /optmethods/loss/logistic_regression.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import numpy as np 3 | import warnings 4 | 5 | from numba import njit 6 | from sklearn.utils.extmath import row_norms, safe_sparse_dot 7 | 8 | from .loss_oracle import Oracle 9 | from .utils import safe_sparse_add, safe_sparse_multiply, safe_sparse_norm, safe_sparse_inner_prod 10 | 11 | 12 | @njit 13 | def logsig(x): 14 | """ 15 | Compute the log-sigmoid function component-wise. 16 | See http://fa.bianp.net/blog/2019/evaluate_logistic/ for more details. 17 | """ 18 | out = np.zeros_like(x) 19 | idx0 = x < -33 20 | out[idx0] = x[idx0] 21 | idx1 = (x >= -33) & (x < -18) 22 | out[idx1] = x[idx1] - np.exp(x[idx1]) 23 | idx2 = (x >= -18) & (x < 37) 24 | out[idx2] = -np.log1p(np.exp(-x[idx2])) 25 | idx3 = x >= 37 26 | out[idx3] = -np.exp(-x[idx3]) 27 | return out 28 | 29 | 30 | class LogisticRegression(Oracle): 31 | """ 32 | Logistic regression oracle that returns loss values, gradients, Hessians, 33 | their stochastic analogues as well as smoothness constants. Supports both 34 | sparse and dense iterates but is far from optimal for dense vectors. 35 | """ 36 | 37 | def __init__(self, A, b, store_mat_vec_prod=True, *args, **kwargs): 38 | super(LogisticRegression, self).__init__(*args, **kwargs) 39 | self.A = A 40 | b = np.asarray(b) 41 | b_unique = np.unique(b) 42 | # check that only two unique values exist in b 43 | if len(b_unique) == 1: 44 | warnings.warn('The labels have only one unique value.') 45 | self.b = b 46 | if len(b_unique) > 2: 47 | raise ValueError('The number of classes must be no more than 2 for binary classification.') 48 | self.b = b 49 | if len(b_unique) == 2 and (b_unique != [0, 1]).any(): 50 | if (b_unique == [1, 2]).all(): 51 | print('The passed labels have values in the set {1, 2}. Changing them to {0, 1}') 52 | self.b = b - 1 53 | elif (b_unique == [-1, 1]).all(): 54 | print('The passed labels have values in the set {-1, 1}. Changing them to {0, 1}') 55 | self.b = (b+1) / 2 56 | else: 57 | print(f'Changing the labels from {b[0]} to 1s and the rest to 0s') 58 | self.b = 1. * (b == b[0]) 59 | self.store_mat_vec_prod = store_mat_vec_prod 60 | 61 | self.n, self.dim = A.shape 62 | self.x_last = 0. 63 | self._mat_vec_prod = np.zeros(self.n) 64 | 65 | def _value(self, x): 66 | Ax = self.mat_vec_product(x) 67 | regularization = 0 68 | if self.l2 != 0: 69 | regularization = self.l2 / 2 * safe_sparse_norm(x)**2 70 | return np.mean(safe_sparse_multiply(1-self.b, Ax)-logsig(Ax)) + regularization 71 | 72 | def partial_value(self, x, idx, include_reg=True, normalization=None, return_idx=False): 73 | batch_size = 1 if np.isscalar(idx) else len(idx) 74 | if normalization is None: 75 | normalization = batch_size 76 | Ax = self.A[idx] @ x 77 | if scipy.sparse.issparse(Ax): 78 | Ax = Ax.toarray().ravel() 79 | regularization = 0 80 | if include_reg: 81 | regularization = self.l2 / 2 * safe_sparse_norm(x)**2 82 | value = np.sum(safe_sparse_multiply(1-self.b[idx], Ax)-logsig(Ax))/normalization + regularization 83 | if return_idx: 84 | return (value, idx) 85 | return value 86 | 87 | def gradient(self, x): 88 | Ax = self.mat_vec_product(x) 89 | activation = scipy.special.expit(Ax) 90 | if self.l2 == 0: 91 | grad = self.A.T@(activation-self.b)/self.n 92 | else: 93 | grad = safe_sparse_add(self.A.T@(activation-self.b)/self.n, self.l2*x) 94 | if scipy.sparse.issparse(x): 95 | grad = scipy.sparse.csr_matrix(grad).T 96 | return grad 97 | 98 | def stochastic_gradient(self, x, idx=None, batch_size=1, replace=False, normalization=None, 99 | importance_sampling=False, p=None, rng=None, return_idx=False): 100 | """ 101 | normalization (int, optional): this parameter is needed for Shuffling optimizer 102 | to remove the bias of the last (incomplete) minibatch 103 | """ 104 | if batch_size is None or batch_size == self.n: 105 | return (self.gradient(x), np.arange(self.n)) if return_idx else self.gradient(x) 106 | if idx is None: 107 | if rng is None: 108 | rng = self.rng 109 | if p is None and importance_sampling: 110 | if self._importance_probs is None: 111 | self._importance_probs = self.individ_smoothness 112 | self._importance_probs /= sum(self._importance_probs) 113 | p = self._importance_probs 114 | idx = np.random.choice(self.n, size=batch_size, replace=replace, p=p) 115 | else: 116 | batch_size = 1 if np.isscalar(idx) else len(idx) 117 | if normalization is None: 118 | if p is None: 119 | normalization = batch_size 120 | else: 121 | normalization = batch_size * p[idx] * self.n 122 | A_idx = self.A[idx] 123 | Ax = A_idx @ x 124 | if scipy.sparse.issparse(Ax): 125 | Ax = Ax.toarray().ravel() 126 | activation = scipy.special.expit(Ax) 127 | if scipy.sparse.issparse(x): 128 | error = scipy.sparse.csr_matrix((activation-self.b[idx]) / normalization) 129 | else: 130 | error = (activation-self.b[idx]) / normalization 131 | if not np.isscalar(error): 132 | grad = self.l2*x + (error@A_idx).T 133 | else: 134 | grad = self.l2*x + error*A_idx.T 135 | if return_idx: 136 | return (grad, idx) 137 | return grad 138 | 139 | def hessian(self, x): 140 | Ax = self.mat_vec_product(x) 141 | activation = scipy.special.expit(Ax) 142 | weights = activation * (1-activation) 143 | A_weighted = safe_sparse_multiply(self.A.T, weights) 144 | return A_weighted@self.A/self.n + self.l2*np.eye(self.dim) 145 | 146 | def stochastic_hessian(self, x, idx=None, batch_size=1, replace=False, normalization=None, 147 | rng=None, return_idx=False): 148 | if batch_size == self.n: 149 | return (self.hessian(x), np.arange(self.n)) if return_idx else self.hessian(x) 150 | if idx is None: 151 | if rng is None: 152 | rng = self.rng 153 | idx = rng.choice(self.n, size=batch_size, replace=replace) 154 | else: 155 | batch_size = 1 if np.isscalar(idx) else len(idx) 156 | if normalization is None: 157 | normalization = batch_size 158 | A_idx = self.A[idx] 159 | Ax = A_idx @ x 160 | if scipy.sparse.issparse(Ax): 161 | Ax = Ax.toarray().ravel() 162 | activation = scipy.special.expit(Ax) 163 | weights = activation * (1-activation) 164 | A_weighted = safe_sparse_multiply(A_idx.T, weights) 165 | hess = A_weighted@A_idx/normalization + self.l2*np.eye(self.dim) 166 | if return_idx: 167 | return (hess, idx) 168 | return hess 169 | 170 | def mat_vec_product(self, x): 171 | if self.store_mat_vec_prod and self.is_equal(x, self.x_last): 172 | return self._mat_vec_prod 173 | Ax = self.A @ x 174 | if scipy.sparse.issparse(Ax): 175 | Ax = Ax.toarray() 176 | Ax = Ax.ravel() 177 | if self.store_mat_vec_prod: 178 | self._mat_vec_prod = Ax 179 | self.x_last = x.copy() 180 | return Ax 181 | 182 | def hess_vec_prod(self, x, v, grad_dif=False, eps=None): 183 | if grad_dif: 184 | grad_x = self.gradient(x) 185 | grad_x_v = self.gradient(x + eps * v) 186 | return (grad_x_v - grad_x) / eps 187 | return safe_sparse_dot(self.hessian(x), v) 188 | 189 | @property 190 | def smoothness(self): 191 | if self._smoothness is not None: 192 | return self._smoothness 193 | if self.dim > 20000 and self.n > 20000: 194 | warnings.warn("The matrix is too large to estimate the smoothness constant, so Frobenius estimate is used instead.") 195 | if scipy.sparse.issparse(self.A): 196 | self._smoothness = 0.25*scipy.sparse.linalg.norm(self.A, ord='fro')**2/self.n + self.l2 197 | else: 198 | self._smoothness = 0.25*np.linalg.norm(self.A, ord='fro')**2/self.n + self.l2 199 | else: 200 | sing_val_max = scipy.sparse.linalg.svds(self.A, k=1, return_singular_vectors=False)[0] 201 | self._smoothness = 0.25*sing_val_max**2/self.n + self.l2 202 | return self._smoothness 203 | 204 | @property 205 | def max_smoothness(self): 206 | if self._max_smoothness is not None: 207 | return self._max_smoothness 208 | max_squared_sum = row_norms(self.A, squared=True).max() 209 | self._max_smoothness = 0.25*max_squared_sum + self.l2 210 | return self._max_smoothness 211 | 212 | @property 213 | def average_smoothness(self): 214 | if self._ave_smoothness is not None: 215 | return self._ave_smoothness 216 | ave_squared_sum = row_norms(self.A, squared=True).mean() 217 | self._ave_smoothness = 0.25*ave_squared_sum + self.l2 218 | return self._ave_smoothness 219 | 220 | def batch_smoothness(self, batch_size): 221 | "Smoothness constant of stochastic gradients sampled in minibatches" 222 | L = self.smoothness 223 | L_max = self.max_smoothness 224 | L_batch = self.n / (self.n-1) * (1-1/batch_size) * L + (self.n/batch_size-1) / (self.n-1) * L_max 225 | return L_batch 226 | 227 | @property 228 | def individ_smoothness(self): 229 | if self._individ_smoothness is not None: 230 | return self._individ_smoothness 231 | self._individ_smoothness = row_norms(self.A) 232 | return self._individ_smoothness 233 | 234 | @property 235 | def hessian_lipschitz(self): 236 | if self._hessian_lipschitz is not None: 237 | return self._hessian_lipschitz 238 | # Estimate the norm of tensor T = sum_i f_i(x)''' * [a_i, a_i, a_i] as ||T|| <= max||a_i|| * max|f_i'''| * ||A||^2 239 | a_max = row_norms(self.A, squared=False).max() 240 | A_norm = (self.smoothness - self.l2) * 4 241 | self._hessian_lipschitz = A_norm * a_max / (6*np.sqrt(3)) 242 | return self._hessian_lipschitz 243 | 244 | @staticmethod 245 | def norm(x, ord=None): 246 | return safe_sparse_norm(x, ord=ord) 247 | 248 | @staticmethod 249 | def inner_prod(x, y): 250 | return safe_sparse_inner_prod(x, y) 251 | 252 | @staticmethod 253 | def outer_prod(x, y): 254 | return np.outer(x, y) 255 | 256 | @staticmethod 257 | def is_equal(x, y): 258 | if x is None: 259 | return y is None 260 | if y is None: 261 | return False 262 | x_sparse = scipy.sparse.issparse(x) 263 | y_sparse = scipy.sparse.issparse(y) 264 | if (x_sparse and not y_sparse) or (y_sparse and not x_sparse): 265 | return False 266 | if not x_sparse and not y_sparse: 267 | return np.array_equal(x, y) 268 | if x.nnz != y.nnz: 269 | return False 270 | return (x!=y).nnz == 0 271 | 272 | @staticmethod 273 | def density(x): 274 | if hasattr(x, "toarray"): 275 | dty = float(x.nnz) / (x.shape[0]*x.shape[1]) 276 | else: 277 | dty = 0 if x is None else float((x!=0).sum()) / x.size 278 | return dty 279 | -------------------------------------------------------------------------------- /optmethods/loss/loss_oracle.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | import warnings 4 | 5 | from .regularizer import Regularizer 6 | 7 | 8 | class Oracle(): 9 | """ 10 | Base class for all objectives. Can provide objective values, 11 | gradients and its Hessians as functions that take parameters as input. 12 | Takes as input the values of l1 and l2 regularization. 13 | """ 14 | def __init__(self, l1=0, l2=0, l2_in_prox=False, regularizer=None, seed=42): 15 | if l1 < 0.0: 16 | raise ValueError("Invalid value for l1 regularization: {}".format(l1)) 17 | if l2 < 0.0: 18 | raise ValueError("Invalid value for l2 regularization: {}".format(l2)) 19 | if l2 == 0. and l2_in_prox: 20 | warnings.warn("The value of l2 is set to 0, so l2_in_prox is changed to False.") 21 | l2_in_prox = False 22 | self.l1 = l1 23 | self.l2 = 0 if l2_in_prox else l2 24 | self.l2_in_prox = l2_in_prox 25 | self.x_opt = None 26 | self.f_opt = np.inf 27 | self.regularizer = regularizer 28 | self.seed = seed 29 | 30 | if (l1 > 0 or l2_in_prox) and regularizer is None: 31 | l2_prox = l2 if l2_in_prox else 0 32 | self.regularizer = Regularizer(l1=l1, l2=l2_prox) 33 | self.rng = np.random.default_rng(seed) 34 | self._smoothness = None 35 | self._max_smoothness = None 36 | self._ave_smoothness = None 37 | self._importance_probs = None 38 | self._individ_smoothness = None 39 | self._hessian_lipschitz = None 40 | 41 | def set_seed(self, seed): 42 | self.seed = seed 43 | self.rng = np.random.default_rng(seed) 44 | 45 | def value(self, x): 46 | value = self._value(x) 47 | if self.regularizer is not None: 48 | value += self.regularizer(x) 49 | if value < self.f_opt: 50 | self.x_opt = copy.deepcopy(x) 51 | self.f_opt = value 52 | return value 53 | 54 | def gradient(self, x): 55 | pass 56 | 57 | def hessian(self, x): 58 | pass 59 | 60 | def hess_vec_prod(self, x, v, grad_dif=False, eps=None): 61 | pass 62 | 63 | @property 64 | def smoothness(self): 65 | pass 66 | 67 | @property 68 | def max_smoothness(self): 69 | pass 70 | 71 | @property 72 | def average_smoothness(self): 73 | pass 74 | 75 | def batch_smoothness(self, batch_size): 76 | pass 77 | 78 | @staticmethod 79 | def norm(x): 80 | pass 81 | 82 | @staticmethod 83 | def inner_prod(x, y): 84 | pass 85 | 86 | @staticmethod 87 | def outer_prod(x, y): 88 | pass 89 | 90 | @staticmethod 91 | def is_equal(x, y): 92 | pass 93 | -------------------------------------------------------------------------------- /optmethods/loss/regularizer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy 3 | 4 | from .utils import safe_sparse_norm 5 | 6 | 7 | class Regularizer(): 8 | """ 9 | A simple oracle class for regularizers that have 10 | proximal operator and can be evaluated during the training. 11 | By default, l1+l2 regularization is implemented. 12 | """ 13 | def __init__(self, l1=0, l2=0, coef=None): 14 | self.l1 = l1 15 | self.l2 = l2 16 | self.coef = coef 17 | 18 | def __call__(self, x): 19 | return self.value(x) 20 | 21 | def value(self, x): 22 | return self.l1*safe_sparse_norm(x, ord=1) + self.l2/2*safe_sparse_norm(x)**2 23 | 24 | def prox_l1(self, x, lr=None): 25 | abs_x = abs(x) 26 | if scipy.sparse.issparse(x): 27 | prox_res = abs_x - abs_x.minimum(self.l1 * lr) 28 | prox_res.eliminate_zeros() 29 | prox_res = prox_res.multiply(x.sign()) 30 | else: 31 | prox_res = abs_x - np.minimum(abs_x, self.l1 * lr) 32 | prox_res *= np.sign(x) 33 | return prox_res 34 | 35 | def prox_l2(self, x, lr=None): 36 | return x / (1 + lr * self.l2) 37 | 38 | def prox(self, x, lr): 39 | """ 40 | The proximal operator of l1||x||_1 + l2/2 ||x||^2 is equal 41 | to the combination of the proximal operator of l1||x||_1 and then 42 | the proximal operator of l2/2 ||x||^2 43 | """ 44 | prox_l1 = self.prox_l1(x, lr) 45 | return self.prox_l2(prox_l1, lr) 46 | -------------------------------------------------------------------------------- /optmethods/loss/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy 3 | 4 | 5 | def safe_sparse_add(a, b): 6 | """ 7 | Implemenet a+b compatible with different types of input. 8 | Supports scalars, numpy arrays and scipy.sparse objects. 9 | """ 10 | both_sparse = scipy.sparse.issparse(a) and scipy.sparse.issparse(b) 11 | one_is_scalar = isinstance(a, (int, float)) or isinstance(b, (int, float)) 12 | if both_sparse or one_is_scalar: 13 | # both are sparse, keep the result sparse 14 | return a + b 15 | else: 16 | # one of them is non-sparse, convert 17 | # everything to dense. 18 | if scipy.sparse.issparse(a): 19 | a = a.toarray() 20 | if a.ndim == 2 and b.ndim == 1: 21 | b.ravel() 22 | elif scipy.sparse.issparse(b): 23 | b = b.toarray() 24 | if b.ndim == 2 and a.ndim == 1: 25 | b = b.ravel() 26 | return a + b 27 | 28 | 29 | def safe_sparse_inner_prod(a, b): 30 | if scipy.sparse.issparse(a) and scipy.sparse.issparse(b): 31 | if a.ndim == 2 and a.shape[1] == b.shape[0]: 32 | return (a @ b)[0, 0] 33 | if a.shape[0] == b.shape[0]: 34 | return (a.T @ b)[0, 0] 35 | return (a @ b.T)[0, 0] 36 | if scipy.sparse.issparse(a): 37 | a = a.toarray() 38 | elif scipy.sparse.issparse(b): 39 | b = b.toarray() 40 | return a @ b 41 | 42 | 43 | def safe_sparse_multiply(a, b): 44 | if scipy.sparse.issparse(a) and scipy.sparse.issparse(b): 45 | return a.multiply(b) 46 | if scipy.sparse.issparse(a): 47 | a = a.toarray() 48 | elif scipy.sparse.issparse(b): 49 | b = b.toarray() 50 | return np.multiply(a, b) 51 | 52 | 53 | def safe_sparse_norm(a, ord=None): 54 | if scipy.sparse.issparse(a): 55 | return scipy.sparse.linalg.norm(a, ord=ord) 56 | return np.linalg.norm(a, ord=ord) 57 | -------------------------------------------------------------------------------- /optmethods/opt_trace.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import numpy.linalg as la 3 | import matplotlib.pyplot as plt 4 | import os 5 | from pathlib import Path 6 | import pickle 7 | import warnings 8 | 9 | 10 | class Trace: 11 | """ 12 | Stores the logs of running an optimization method 13 | and plots the trajectory. 14 | 15 | Arguments: 16 | loss (Oracle): the optimized loss class 17 | label (string, optional): label for convergence plots (default: None) 18 | """ 19 | def __init__(self, loss, label=None): 20 | self.loss = loss 21 | self.label = label 22 | 23 | self.xs = [] 24 | self.ts = [] 25 | self.its = [] 26 | self.loss_vals = [] 27 | self.its_converted_to_epochs = False 28 | self.ls_its = None 29 | 30 | def compute_loss_of_iterates(self): 31 | if len(self.loss_vals) == 0: 32 | self.loss_vals = np.asarray([self.loss.value(x) for x in self.xs]) 33 | else: 34 | warnings.warn('Loss values have already been computed. Set .loss_vals = [] to recompute.') 35 | 36 | def convert_its_to_epochs(self, batch_size=1): 37 | if self.its_converted_to_epochs: 38 | warnings.warn('The iteration count has already been converted to epochs.') 39 | return 40 | its_per_epoch = self.loss.n / batch_size 41 | self.its = np.asarray(self.its) / its_per_epoch 42 | self.its_converted_to_epochs = True 43 | 44 | def plot_losses(self, its=None, f_opt=None, label=None, markevery=None, use_ls_its=True, time=False, *args, **kwargs): 45 | if label is None: 46 | label = self.label 47 | if its is None: 48 | if use_ls_its and self.ls_its is not None: 49 | print(f'Line search iteration counter is used for plotting {label}') 50 | its = self.ls_its 51 | elif time: 52 | its = self.ts 53 | else: 54 | its = self.its 55 | if len(self.loss_vals) == 0: 56 | self.compute_loss_of_iterates() 57 | if f_opt is None: 58 | f_opt = self.loss.f_opt 59 | if markevery is None: 60 | markevery = max(1, len(self.loss_vals)//20) 61 | 62 | plt.plot(its, self.loss_vals - f_opt, label=label, markevery=markevery, *args, **kwargs) 63 | plt.ylabel(r'$f(x)-f^*$') 64 | 65 | def plot_distances(self, its=None, x_opt=None, label=None, markevery=None, use_ls_its=True, time=False, *args, **kwargs): 66 | if its is None: 67 | if use_ls_its and self.ls_its is not None: 68 | its = self.ls_its 69 | elif time: 70 | its = self.ts 71 | else: 72 | its = self.its 73 | if x_opt is None: 74 | if self.loss.x_opt is None: 75 | x_opt = self.xs[-1] 76 | else: 77 | x_opt = self.loss.x_opt 78 | if label is None: 79 | label = self.label 80 | if markevery is None: 81 | markevery = max(1, len(self.xs)//20) 82 | 83 | dists = [self.loss.norm(x-x_opt)**2 for x in self.xs] 84 | plt.plot(its, dists, label=label, markevery=markevery, *args, **kwargs) 85 | plt.ylabel(r'$\Vert x-x^*\Vert^2$') 86 | 87 | @property 88 | def best_loss_value(self): 89 | if len(self.loss_vals) == 0: 90 | self.compute_loss_of_iterates() 91 | return np.min(self.loss_vals) 92 | 93 | def save(self, file_name, path='./results/'): 94 | # To make the dumped file smaller, remove the loss 95 | loss_ref_copy = self.loss 96 | self.loss = None 97 | Path(path).mkdir(parents=True, exist_ok=True) 98 | with open(path + file_name, 'wb') as f: 99 | pickle.dump(self, f) 100 | self.loss = loss_ref_copy 101 | 102 | @classmethod 103 | def from_pickle(cls, path, loss=None): 104 | if not os.path.isfile(path): 105 | return None 106 | with open(path, 'rb') as f: 107 | trace = pickle.load(f) 108 | trace.loss = loss 109 | if loss is not None: 110 | loss.f_opt = min(self.best_loss_value, loss.f_opt) 111 | return trace 112 | 113 | 114 | class StochasticTrace: 115 | """ 116 | Class that stores the logs of running a stochastic 117 | optimization method and plots the trajectory. 118 | """ 119 | def __init__(self, loss, label=None): 120 | self.loss = loss 121 | self.label = label 122 | 123 | self.xs_all = {} 124 | self.ts_all = {} 125 | self.its_all = {} 126 | self.loss_vals_all = {} 127 | self.its_converted_to_epochs = False 128 | self.loss_is_computed = False 129 | 130 | def init_seed(self): 131 | self.xs = [] 132 | self.ts = [] 133 | self.its = [] 134 | self.loss_vals = None 135 | 136 | def append_seed_results(self, seed): 137 | self.xs_all[seed] = self.xs.copy() 138 | self.ts_all[seed] = self.ts.copy() 139 | self.its_all[seed] = self.its.copy() 140 | self.loss_vals_all[seed] = self.loss_vals.copy() if self.loss_vals else None 141 | 142 | def compute_loss_of_iterates(self): 143 | for seed, loss_vals in self.loss_vals_all.items(): 144 | if loss_vals is None: 145 | self.loss_vals_all[seed] = np.asarray([self.loss.value(x) for x in self.xs_all[seed]]) 146 | else: 147 | warnings.warn("""Loss values for seed {} have already been computed. 148 | Set .loss_vals_all[{}] = [] to recompute.""".format(seed, seed)) 149 | self.loss_is_computed = True 150 | 151 | @property 152 | def best_loss_value(self): 153 | if not self.loss_is_computed: 154 | self.compute_loss_of_iterates() 155 | return np.min([np.min(loss_vals) for loss_vals in self.loss_vals_all.values()]) 156 | 157 | def convert_its_to_epochs(self, batch_size=1): 158 | if self.its_converted_to_epochs: 159 | return 160 | self.its_per_epoch = self.loss.n / batch_size 161 | for seed, its in self.its_all.items(): 162 | self.its_all[seed] = np.asarray(its) / self.its_per_epoch 163 | self.its = np.asarray(self.its) / self.its_per_epoch 164 | self.its_converted_to_epochs = True 165 | 166 | def plot_losses(self, its=None, f_opt=None, log_std=True, label=None, markevery=None, alpha=0.25, *args, **kwargs): 167 | if not self.loss_is_computed: 168 | self.compute_loss_of_iterates() 169 | if its is None: 170 | its = np.mean([np.asarray(its_) for its_ in self.its_all.values()], axis=0) 171 | if f_opt is None: 172 | f_opt = self.loss.f_opt 173 | if log_std: 174 | y_log = [np.log(loss_vals-f_opt) for loss_vals in self.loss_vals_all.values()] 175 | y_log_ave = np.mean(y_log, axis=0) 176 | y_log_std = np.std(y_log, axis=0, ddof=1) 177 | lower, upper = np.exp(y_log_ave - y_log_std), np.exp(y_log_ave + y_log_std) 178 | y_ave = np.exp(y_log_ave) 179 | else: 180 | y = [loss_vals-f_opt for loss_vals in self.loss_vals_all.values()] 181 | y_ave = np.mean(y, axis=0) 182 | y_std = np.std(y, axis=0, ddof=1) 183 | lower, upper = y_ave - y_std, y_ave + y_std 184 | if label is None: 185 | label = self.label 186 | if markevery is None: 187 | markevery = max(1, len(y_ave)//20) 188 | 189 | plot = plt.plot(its, y_ave, label=label, markevery=markevery, *args, **kwargs) 190 | if len(self.loss_vals_all.keys()) > 1: 191 | plt.fill_between(its, lower, upper, alpha=alpha, color=plot[0].get_color()) 192 | plt.ylabel(r'$f(x)-f^*$') 193 | 194 | def plot_distances(self, its=None, x_opt=None, log_std=True, label=None, markevery=None, alpha=0.25, *args, **kwargs): 195 | if its is None: 196 | its = np.mean([np.asarray(its_) for its_ in self.its_all.values()], axis=0) 197 | if x_opt is None: 198 | if self.loss.x_opt is None: 199 | x_opt = self.xs[-1] 200 | else: 201 | x_opt = self.loss.x_opt 202 | 203 | dists = [np.asarray([self.loss.norm(x-x_opt)**2 for x in xs]) for xs in self.xs_all.values()] 204 | if log_std: 205 | y_log = [np.log(dist) for dist in dists] 206 | y_log_ave = np.mean(y_log, axis=0) 207 | y_log_std = np.std(y_log, axis=0, ddof=1) 208 | lower, upper = np.exp(y_log_ave - y_log_std), np.exp(y_log_ave + y_log_std) 209 | y_ave = np.exp(y_log_ave) 210 | else: 211 | y = dists 212 | y_ave = np.mean(y, axis=0) 213 | y_std = np.std(y, axis=0, ddof=1) 214 | lower, upper = y_ave - y_std, y_ave + y_std 215 | if label is None: 216 | label = self.label 217 | if markevery is None: 218 | markevery = max(1, len(y_ave)//20) 219 | 220 | plot = plt.plot(its, y_ave, label=label, markevery=markevery, *args, **kwargs) 221 | if len(self.xs_all.keys()) > 1: 222 | plt.fill_between(its, lower, upper, alpha=alpha, color=plot[0].get_color()) 223 | plt.ylabel(r'$\Vert x-x^*\Vert^2$') 224 | 225 | def save(self, file_name=None, path='./results/'): 226 | if file_name is None: 227 | file_name = self.label 228 | if path[-1] != '/': 229 | path += '/' 230 | self.loss = None 231 | Path(path).mkdir(parents=True, exist_ok=True) 232 | f = open(path + file_name, 'wb') 233 | pickle.dump(self, f) 234 | f.close() 235 | 236 | @classmethod 237 | def from_pickle(cls, path, loss): 238 | if not os.path.isfile(path): 239 | return None 240 | with open(path, 'rb') as f: 241 | trace = pickle.load(f) 242 | trace.loss = loss 243 | return trace 244 | -------------------------------------------------------------------------------- /optmethods/optimizer.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | import numpy.linalg as la 4 | import scipy 5 | import time 6 | 7 | from tqdm.notebook import tqdm 8 | 9 | from optmethods.opt_trace import Trace, StochasticTrace 10 | from optmethods.utils import set_seed 11 | 12 | 13 | SEED = 42 14 | MAX_SEED = 10000000 15 | 16 | 17 | class Optimizer: 18 | """ 19 | Base class for optimization algorithms. Provides methods to run them, 20 | save the trace and plot the results. 21 | 22 | Arguments: 23 | loss (required): an instance of class Oracle, which will be used to produce gradients, 24 | loss values, or whatever else is required by the optimizer 25 | trace_len (int, optional): the number of checkpoints that will be stored in the 26 | trace. Larger value may slowdown the runtime (default: 200) 27 | use_prox (bool, optional): whether the optimizer should treat the regularizer 28 | using prox (default: True) 29 | tolerance (float, optional): stationarity level at which the method should interrupt. 30 | Stationarity is computed using the difference between 31 | two consecutive iterates(default: 0) 32 | line_search (optional): an instance of class LineSearch, which is used to tune stepsize, 33 | or other parameters of the optimizer (default: None) 34 | save_first_iterations (int, optional): how many of the very first iterations should be 35 | saved as checkpoints in the trace. Useful when 36 | optimizer converges fast at the beginning 37 | (default: 5) 38 | label (string, optional): label to be passed to the Trace attribute (default: None) 39 | seeds (list, optional): random seeds to be used to create random number generator (RNG). 40 | If None, a single random seed 42 will be used (default: None) 41 | tqdm (bool, optional): whether to use tqdm to report progress of the run (default: True) 42 | """ 43 | def __init__(self, loss, trace_len=200, use_prox=True, tolerance=0, line_search=None, 44 | save_first_iterations=5, label=None, seeds=None, tqdm=True): 45 | self.loss = loss 46 | self.trace_len = trace_len 47 | self.use_prox = use_prox and (self.loss.regularizer is not None) 48 | self.tolerance = tolerance 49 | self.line_search = line_search 50 | self.save_first_iterations = save_first_iterations 51 | self.label = label 52 | self.tqdm = tqdm 53 | 54 | self.initialized = False 55 | self.x_old_tol = None 56 | self.trace = Trace(loss=loss, label=label) 57 | if seeds is None: 58 | self.seeds = [42] 59 | else: 60 | self.seeds = seeds 61 | self.finished_seeds = [] 62 | 63 | def run(self, x0, t_max=np.inf, it_max=np.inf, ls_it_max=None): 64 | if t_max is np.inf and it_max is np.inf: 65 | it_max = 100 66 | print(f'{self.label}: The number of iterations is set to {it_max}.') 67 | self.t_max = t_max 68 | self.it_max = it_max 69 | 70 | for seed in self.seeds: 71 | if seed in self.finished_seeds: 72 | continue 73 | if len(self.seeds) > 1: 74 | print(f'{self.label}: Running seed {seed}') 75 | self.rng = np.random.default_rng(seed) 76 | if ls_it_max is None: 77 | self.ls_it_max = it_max 78 | if not self.initialized: 79 | self.init_run(x0) 80 | self.initialized = True 81 | 82 | it_criterion = self.ls_it_max is not np.inf 83 | tqdm_total = self.ls_it_max if it_criterion else self.t_max 84 | tqdm_val = 0 85 | with tqdm(total=tqdm_total) as pbar: 86 | while not self.check_convergence(): 87 | if self.tolerance > 0: 88 | self.x_old_tol = copy.deepcopy(self.x) 89 | self.step() 90 | self.save_checkpoint() 91 | if it_criterion and self.line_search is not None: 92 | tqdm_val_new = self.ls_it 93 | elif it_criterion: 94 | tqdm_val_new = self.it 95 | else: 96 | tqdm_val_new = self.t 97 | pbar.update(tqdm_val_new - tqdm_val) 98 | tqdm_val = tqdm_val_new 99 | self.finished_seeds.append(seed) 100 | self.initialized = False 101 | 102 | return self.trace 103 | 104 | def check_convergence(self): 105 | no_it_left = self.it >= self.it_max 106 | if self.line_search is not None: 107 | no_it_left = no_it_left or (self.line_search.it >= self.ls_it_max) 108 | no_time_left = time.perf_counter()-self.t_start >= self.t_max 109 | if self.tolerance > 0: 110 | tolerance_met = self.x_old_tol is not None and self.loss.norm(self.x-self.x_old_tol) < self.tolerance 111 | else: 112 | tolerance_met = False 113 | return no_it_left or no_time_left or tolerance_met 114 | 115 | def step(self): 116 | pass 117 | 118 | def init_run(self, x0): 119 | self.dim = x0.shape[0] 120 | self.x = copy.deepcopy(x0) 121 | self.trace.xs = [copy.deepcopy(x0)] 122 | self.trace.its = [0] 123 | self.trace.ts = [0] 124 | if self.line_search is not None: 125 | self.trace.ls_its = [0] 126 | self.trace.lrs = [self.line_search.lr] 127 | self.it = 0 128 | self.t = 0 129 | self.t_start = time.perf_counter() 130 | self.time_progress = 0 131 | self.iterations_progress = 0 132 | self.max_progress = 0 133 | if self.line_search is not None: 134 | self.line_search.reset(self) 135 | 136 | def should_update_trace(self): 137 | if self.it <= self.save_first_iterations: 138 | return True 139 | self.time_progress = int((self.trace_len-self.save_first_iterations) * self.t / self.t_max) 140 | self.iterations_progress = int((self.trace_len-self.save_first_iterations) * (self.it / self.it_max)) 141 | if self.line_search is not None: 142 | ls_it = self.line_search.it 143 | self.iterations_progress = max(self.iterations_progress, int((self.trace_len-self.save_first_iterations) * (ls_it / self.it_max))) 144 | enough_progress = max(self.time_progress, self.iterations_progress) > self.max_progress 145 | return enough_progress 146 | 147 | def save_checkpoint(self): 148 | self.it += 1 149 | if self.line_search is not None: 150 | self.ls_it = self.line_search.it 151 | self.t = time.perf_counter() - self.t_start 152 | if self.should_update_trace(): 153 | self.update_trace() 154 | self.max_progress = max(self.time_progress, self.iterations_progress) 155 | 156 | def update_trace(self): 157 | self.trace.xs.append(copy.deepcopy(self.x)) 158 | self.trace.ts.append(self.t) 159 | self.trace.its.append(self.it) 160 | if self.line_search is not None: 161 | self.trace.ls_its.append(self.line_search.it) 162 | self.trace.lrs.append(self.line_search.lr) 163 | 164 | def compute_loss_of_iterates(self): 165 | self.trace.compute_loss_of_iterates() 166 | 167 | def reset(self): 168 | self.initialized = False 169 | self.x_old_tol = None 170 | self.trace = Trace(loss=loss, label=self.label) 171 | 172 | 173 | class StochasticOptimizer(Optimizer): 174 | """ 175 | Base class for stochastic optimization algorithms. 176 | The class has the same methods as Optimizer and, in addition, uses 177 | multiple seeds to run the experiments. 178 | """ 179 | def __init__(self, loss, n_seeds=1, seeds=None, label=None, *args, **kwargs): 180 | super(StochasticOptimizer, self).__init__(loss=loss, *args, **kwargs) 181 | self.seeds = seeds 182 | if not seeds: 183 | np.random.seed(SEED) 184 | self.seeds = np.random.choice(MAX_SEED, size=n_seeds, replace=False) 185 | self.label = label 186 | self.finished_seeds = [] 187 | self.trace = StochasticTrace(loss=loss, label=label) 188 | self.seed = None 189 | 190 | def run(self, *args, **kwargs): 191 | for seed in self.seeds: 192 | if seed in self.finished_seeds: 193 | continue 194 | if self.line_search is not None: 195 | self.line_search.reset() 196 | set_seed(seed) 197 | self.seed = seed 198 | self.trace.init_seed() 199 | super(StochasticOptimizer, self).run(*args, **kwargs) 200 | self.trace.append_seed_results(seed) 201 | self.finished_seeds.append(seed) 202 | self.initialized = False 203 | self.seed = None 204 | return self.trace 205 | 206 | def add_seeds(self, n_extra_seeds=1): 207 | np.random.seed(SEED) 208 | n_seeds = len(self.seeds) + n_extra_seeds 209 | self.seeds = np.random.choice(MAX_SEED, size=n_seeds, replace=False) 210 | self.loss_is_computed = False 211 | if self.trace.its_converted_to_epochs: 212 | # TODO: create a bool variable for each seed 213 | pass 214 | -------------------------------------------------------------------------------- /optmethods/quasi_newton/__init__.py: -------------------------------------------------------------------------------- 1 | from .bfgs import Bfgs 2 | from .dfp import Dfp 3 | from .lbfgs import Lbfgs 4 | from .shorr import Shorr 5 | from .sr1 import Sr1 6 | -------------------------------------------------------------------------------- /optmethods/quasi_newton/bfgs.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from optmethods.optimizer import Optimizer 4 | 5 | 6 | class Bfgs(Optimizer): 7 | """ 8 | Broyden–Fletcher–Goldfarb–Shanno algorithm. See 9 | https://arxiv.org/pdf/2004.14866.pdf 10 | for a convergence proof and see 11 | https://en.wikipedia.org/wiki/BFGS 12 | for a general description. 13 | 14 | Arguments: 15 | L (float, optional): an upper bound on the smoothness constant 16 | to initialize the Hessian estimate 17 | hess_estim (float array of shape (dim, dim), optional): initial Hessian estimate 18 | lr (float, optional): stepsize (default: 1) 19 | """ 20 | 21 | def __init__(self, L=None, hess_estim=None, lr=1, store_hess_estimate=False, *args, **kwargs): 22 | super(Bfgs, self).__init__(*args, **kwargs) 23 | if L is None and hess_estim is None: 24 | L = self.loss.smoothness 25 | if L is None: 26 | raise ValueError("Either smoothness constant L or Hessian estimate must be provided") 27 | self.L = L 28 | self.B = hess_estim 29 | self.lr = lr 30 | self.store_hess_estimate = store_hess_estimate 31 | 32 | def step(self): 33 | self.grad = self.loss.gradient(self.x) 34 | x_new = self.x - self.lr * self.B_inv @ self.grad 35 | if self.line_search is not None: 36 | x_new = self.line_search(self.x, x_new) 37 | 38 | s = x_new - self.x 39 | grad_new = self.loss.gradient(x_new) 40 | y = grad_new - self.grad 41 | self.grad = grad_new 42 | 43 | y_s = y @ s 44 | if self.store_hess_estimate: 45 | Bs = self.B @ s 46 | sBs = s @ Bs 47 | self.B += np.outer(y, y)/y_s - np.outer(Bs, Bs)/sBs 48 | if y_s > 0: 49 | B_inv_y = self.B_inv @ y 50 | y_B_inv_y = y @ B_inv_y 51 | self.B_inv += (y_s + y_B_inv_y) * np.outer(s, s) / y_s**2 52 | self.B_inv -= (np.outer(B_inv_y, s) + np.outer(s, B_inv_y)) / y_s 53 | self.x = x_new 54 | 55 | def init_run(self, *args, **kwargs): 56 | super(Bfgs, self).init_run(*args, **kwargs) 57 | if self.B is None: 58 | if self.store_hess_estimate: 59 | self.B = self.L * np.eye(self.loss.dim) 60 | self.B_inv = 1 / self.L * np.eye(self.loss.dim) 61 | else: 62 | self.B_inv = np.linalg.pinv(self.B) 63 | self.grad = self.loss.gradient(self.x) 64 | -------------------------------------------------------------------------------- /optmethods/quasi_newton/dfp.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from optmethods.optimizer import Optimizer 4 | 5 | 6 | class Dfp(Optimizer): 7 | """ 8 | Davidon–Fletcher–Powell algorithm. See 9 | https://arxiv.org/pdf/2004.14866.pdf 10 | for a convergence proof and see 11 | https://en.wikipedia.org/wiki/Davidon-Fletcher-Powell_formula 12 | for a general description. 13 | 14 | Arguments: 15 | L (float, optional): an upper bound on the smoothness constant 16 | to initialize the Hessian estimate 17 | hess_estim (float array of shape (dim, dim)): initial Hessian estimate 18 | lr (float, optional): stepsize (default: 1) 19 | """ 20 | 21 | def __init__(self, L=None, hess_estim=None, lr=1, *args, **kwargs): 22 | super(Dfp, self).__init__(*args, **kwargs) 23 | if L is None and hess_estim is None: 24 | L = self.loss.smoothness 25 | if L is None: 26 | raise ValueError("Either smoothness constant L or Hessian estimate must be provided") 27 | self.lr = lr 28 | self.L = L 29 | self.B = hess_estim 30 | 31 | def step(self): 32 | self.grad = self.loss.gradient(self.x) 33 | x_new = self.x - self.lr * self.B_inv @ self.grad 34 | if self.line_search is not None: 35 | x_new = self.line_search(self.x, x_new) 36 | 37 | s = x_new - self.x 38 | grad_new = self.loss.gradient(x_new) 39 | y = grad_new - self.grad 40 | self.grad = grad_new 41 | Bs = self.B @ s 42 | sBs = s @ Bs 43 | y_s = y @ s 44 | self.B += (1 + sBs/y_s) / y_s * np.outer(y, y) - (np.outer(y, Bs) + np.outer(Bs, y)) / y_s 45 | B_inv_y = self.B_inv @ y 46 | y_B_inv_y = y @ B_inv_y 47 | self.B_inv += np.outer(s, s) / y_s 48 | self.B_inv -= np.outer(B_inv_y, B_inv_y) / y_B_inv_y 49 | self.x = x_new 50 | 51 | def init_run(self, *args, **kwargs): 52 | super(Dfp, self).init_run(*args, **kwargs) 53 | if self.B is None: 54 | self.B = self.L * np.eye(self.loss.dim) 55 | self.B_inv = 1 / self.L * np.eye(self.loss.dim) 56 | else: 57 | self.B_inv = np.linalg.pinv(self.B) 58 | self.grad = self.loss.gradient(self.x) 59 | -------------------------------------------------------------------------------- /optmethods/quasi_newton/lbfgs.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | class Lbfgs(Optimizer): 8 | """ 9 | Limited-memory Broyden–Fletcher–Goldfarb–Shanno algorithm. See 10 | p. 177 in (J. Nocedal and S. J. Wright, "Numerical Optimization", 2nd edtion) 11 | or 12 | https://en.wikipedia.org/wiki/Limited-memory_BFGS 13 | for a general description. 14 | 15 | Arguments: 16 | L (float, optional): an upper bound on the smoothness constant 17 | to initialize the Hessian estimate 18 | hess_estim (float array of shape (dim, dim), optional): initial Hessian estimate 19 | lr (float, optional): stepsize (default: 1) 20 | mem_size (int, optional): memory size (default: 1) 21 | adaptive_init (bool, optional): whether to use 22 | """ 23 | 24 | def __init__(self, L=None, hess_estim=None, inv_hess_estim=None, lr=1, mem_size=1, adaptive_init=False, *args, **kwargs): 25 | super(Lbfgs, self).__init__(*args, **kwargs) 26 | if L is None and hess_estim is None and inv_hess_estim is None: 27 | L = self.loss.smoothness 28 | if L is None: 29 | raise ValueError("Either smoothness constant L or Hessian/inverse-Hessian estimate must be provided") 30 | self.L = L 31 | self.lr = lr 32 | self.mem_size = mem_size 33 | self.adaptive_init = adaptive_init 34 | self.B = hess_estim 35 | self.B_inv = inv_hess_estim 36 | if inv_hess_estim is None and hess_estim is not None: 37 | self.B_inv = np.linalg.pinv(self.B) 38 | self.x_difs = [] 39 | self.grad_difs = [] 40 | self.rhos = [] 41 | 42 | def step(self): 43 | self.grad = self.loss.gradient(self.x) 44 | q = copy.deepcopy(self.grad) 45 | alphas = [] 46 | for s, y, rho in zip(reversed(self.x_difs), reversed(self.grad_difs), reversed(self.rhos)): 47 | alpha = rho * self.loss.inner_prod(s, q) 48 | alphas.append(alpha) 49 | q -= alpha * y 50 | if self.B_inv is not None: 51 | r = self.B_inv @ q 52 | else: 53 | if self.adaptive_init and len(self.x_difs) > 0: 54 | y = self.grad_difs[-1] 55 | y_norm = self.loss.norm(y) 56 | self.L_local = y_norm**2 * self.rhos[-1] 57 | r = q / self.L_local 58 | else: 59 | r = q / self.L 60 | for s, y, rho, alpha in zip(self.x_difs, self.grad_difs, self.rhos, reversed(alphas)): 61 | beta = rho * self.loss.inner_prod(y, r) 62 | r += s * (alpha - beta) 63 | 64 | x_new = self.x - self.lr * r 65 | if self.line_search is not None: 66 | x_new = self.line_search(self.x, x_new) 67 | grad_new = self.loss.gradient(x_new) 68 | 69 | s_new = x_new - self.x 70 | y_new = grad_new - self.grad 71 | rho_inv = self.loss.inner_prod(s_new, y_new) 72 | if rho_inv > 0: 73 | self.x_difs.append(s_new) 74 | self.grad_difs.append(y_new) 75 | self.rhos.append(1 / rho_inv) 76 | 77 | if len(self.x_difs) > self.mem_size: 78 | self.x_difs.pop(0) 79 | self.grad_difs.pop(0) 80 | self.rhos.pop(0) 81 | self.x = x_new 82 | self.grad = grad_new 83 | -------------------------------------------------------------------------------- /optmethods/quasi_newton/shorr.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from optmethods.line_search import BestGrid 5 | from optmethods.optimizer import Optimizer 6 | 7 | 8 | class Shorr(Optimizer): 9 | """ 10 | Shor's r-algorithm. For a convergence analysis, see 11 | https://www.researchgate.net/publication/243084304_The_Speed_of_Shor's_R-algorithm 12 | In general, the method won't work without a line search. The best line search is 13 | probably a grid search that starts at lr=1. 14 | 15 | Arguments: 16 | gamma (float, optional): the closer it is to 1, the faster the Hessian 17 | estimate changes. Must be from (0, 1) (default: 0.5) 18 | L (float, optional): an upper bound on the smoothness constant 19 | to initialize the Hessian estimate 20 | """ 21 | 22 | def __init__(self, gamma=0.5, L=None, *args, **kwargs): 23 | super(Shorr, self).__init__(*args, **kwargs) 24 | if not 0.0 < gamma < 1.0: 25 | raise ValueError("Invalid gamma: {}".format(gamma)) 26 | if L is None: 27 | L = self.loss.smoothness 28 | if L is None: 29 | L = 1 30 | self.gamma = gamma 31 | self.L = L 32 | self.B = 1/np.sqrt(self.L) * np.eye(self.loss.dim) 33 | if self.line_search is None: 34 | self.line_search = BestGrid(lr0=1.0, start_with_prev_lr=False, 35 | increase_many_times=True) 36 | self.line_search.loss = self.loss 37 | 38 | def step(self): 39 | self.grad = self.loss.gradient(self.x) 40 | if self.line_search.lr != 1.0: 41 | # avoid machine precision issues 42 | self.B *= np.sqrt(self.line_search.lr) 43 | self.line_search.lr = 1.0 44 | r = self.B.T @ (self.grad - self.grad_old) 45 | if self.loss.norm(r) > 0: 46 | r /= self.loss.norm(r) 47 | self.B -= self.gamma * self.B @ self.loss.outer_prod(r, r) 48 | x_new = self.x - self.B @ (self.B.T @ self.grad) 49 | self.x = self.line_search(self.x, x_new=x_new) 50 | self.grad_old = copy.deepcopy(self.grad) 51 | 52 | def init_run(self, *args, **kwargs): 53 | super(Shorr, self).init_run(*args, **kwargs) 54 | self.grad_old = self.loss.gradient(self.x) 55 | self.x -= 1 / self.L * self.grad_old 56 | self.save_checkpoint() 57 | -------------------------------------------------------------------------------- /optmethods/quasi_newton/sr1.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from optmethods.optimizer import Optimizer 4 | 5 | 6 | class Sr1(Optimizer): 7 | """ 8 | Quasi-Newton algorithm with Symmetric Rank 1 (SR1) update. See 9 | https://arxiv.org/pdf/2002.00657.pdf 10 | for a formal description and convergence proof of a similar method. 11 | 12 | The stability condition is from 13 | p. 145 in (J. Nocedal and S. J. Wright, "Numerical Optimization", 2nd edtion) 14 | 15 | Arguments: 16 | L (float, optional): an upper bound on the smoothness constant 17 | to initialize the Hessian estimate 18 | hess_estim (float array of shape (dim, dim)): initial Hessian estimate 19 | lr (float, optional): stepsize (default: 1) 20 | stability_const (float, optional): a constant from [0, 1) that ensures a curvature 21 | condition before updating the Hessian-inverse estimate (default: 1e-8) 22 | """ 23 | 24 | def __init__(self, L=None, hess_estim=None, lr=1, stability_const=1e-8, *args, **kwargs): 25 | super(Sr1, self).__init__(*args, **kwargs) 26 | if L is None and hess_estim is None: 27 | L = self.loss.smoothness 28 | if L is None: 29 | raise ValueError("Either smoothness constant L or Hessian estimate must be provided") 30 | if not 0 <= stability_const < 1: 31 | raise ValueError("Invalid stability parameter: {}".format(stability_const)) 32 | self.lr = lr 33 | self.L = L 34 | self.B = hess_estim 35 | self.stability_const = stability_const 36 | 37 | def step(self): 38 | self.grad = self.loss.gradient(self.x) 39 | x_new = self.x - self.lr * self.B_inv @ self.grad 40 | if self.line_search is not None: 41 | x_new = self.line_search(self.x, x_new) 42 | 43 | s = x_new - self.x 44 | grad_new = self.loss.gradient(x_new) 45 | y = grad_new - self.grad 46 | self.grad = grad_new 47 | Bs = self.B @ s 48 | sBs = s @ Bs 49 | B_inv_y = self.B_inv @ y 50 | y_B_inv_y = y @ B_inv_y 51 | y_s = y @ s 52 | if abs(y_s-sBs) > self.stability_const * self.loss.norm(s) * self.loss.norm(y-Bs) and y_s!=y_B_inv_y: 53 | self.B += np.outer(y-Bs, y-Bs) / (y_s-sBs) 54 | self.B_inv += np.outer(s-B_inv_y, s-B_inv_y) / (y_s-y_B_inv_y) 55 | self.x = x_new 56 | 57 | def init_run(self, *args, **kwargs): 58 | super(Sr1, self).init_run(*args, **kwargs) 59 | if self.B is None: 60 | self.B = self.L * np.eye(self.loss.dim) 61 | self.B_inv = 1 / self.L * np.eye(self.loss.dim) 62 | else: 63 | self.B_inv = np.linalg.pinv(self.B) 64 | self.grad = self.loss.gradient(self.x) 65 | -------------------------------------------------------------------------------- /optmethods/second_order/__init__.py: -------------------------------------------------------------------------------- 1 | from .arc import Arc 2 | from .cubic import Cubic 3 | from .newton import Newton 4 | from .reg_newton import RegNewton 5 | -------------------------------------------------------------------------------- /optmethods/second_order/arc.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import numpy.linalg as la 3 | 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | class MockLineSearch(): 8 | #TODO: replace with a proper line search 9 | def __init__(self): 10 | self.lr = None 11 | 12 | def reset(self, optimizer): 13 | self.it = 0 14 | 15 | 16 | def arc_cubic_solver(x, g, H, M, it_max=100, epsilon=1e-8, loss=None): 17 | """ 18 | Solve min_z + 1/2 + M/3 ||z-x||^3 19 | 20 | For explanation of Cauchy point, see "Gradient Descent 21 | Efficiently Finds the Cubic-Regularized Non-Convex Newton Step" 22 | https://arxiv.org/pdf/1612.00547.pdf 23 | Other potential implementations can be found in paper 24 | "Adaptive cubic regularisation methods" 25 | https://people.maths.ox.ac.uk/cartis/papers/ARCpI.pdf 26 | """ 27 | solver_it = 1 28 | newton_step = -np.linalg.solve(H, g) 29 | if M == 0: 30 | return x + newton_step, solver_it 31 | def cauchy_point(g, H, M): 32 | if la.norm(g) == 0 or M == 0: 33 | return 0 * g 34 | g_dir = g / la.norm(g) 35 | H_g_g = H @ g_dir @ g_dir 36 | R = -H_g_g / (2*M) + np.sqrt((H_g_g/M)**2/4 + la.norm(g)/M) 37 | return -R * g_dir 38 | 39 | def conv_criterion(s, r): 40 | """ 41 | The convergence criterion is an increasing and concave function in r 42 | and it is equal to 0 only if r is the solution to the cubic problem 43 | """ 44 | s_norm = la.norm(s) 45 | return 1/s_norm - 1/r 46 | 47 | # Solution s satisfies ||s|| >= Cauchy_radius 48 | r_min = la.norm(cauchy_point(g, H, M)) 49 | 50 | if loss is not None: 51 | x_new = x + newton_step 52 | if loss.value(x) > loss.value(x_new): 53 | return x_new, solver_it 54 | 55 | r_max = la.norm(newton_step) 56 | if r_max - r_min < epsilon: 57 | return x + newton_step, solver_it 58 | id_matrix = np.eye(len(g)) 59 | for _ in range(it_max): 60 | # run bisection on the regularization using conv_criterion 61 | r_try = (r_min + r_max) / 2 62 | lam = r_try * M 63 | s_lam = -np.linalg.solve(H + lam*id_matrix, g) 64 | solver_it += 1 65 | crit = conv_criterion(s_lam, r_try) 66 | if np.abs(crit) < epsilon: 67 | return x + s_lam, solver_it 68 | if crit < 0: 69 | r_min = r_try 70 | else: 71 | r_max = r_try 72 | if r_max - r_min < epsilon: 73 | break 74 | return x + s_lam, solver_it 75 | 76 | 77 | class Arc(Optimizer): 78 | """ 79 | Adaptive Regularisation algorithm using Cubics (ARC) is a second-order optimizer based on Cubic Newton. 80 | This implementation is based on the paper by Cartis et al., 81 | "Adaptive cubic regularisation methods for unconstrained optimization. 82 | Part I: motivation, convergence and numerical results" 83 | We use the same rules for initializing eta1, eta2, sigma and updating sigma as given in the paper. 84 | 85 | Arguments: 86 | eta1 (float, optional): parameter to identify very successful iterations (default: 0.1) 87 | eta2 (float, optional): parameter to identify unsuccessful iterations (default: 0.9) 88 | sigma_eps (float, optional): minimal value of the cubic-penalty coefficient (default: 1e-16) 89 | sigma (float, optional): an estimate of the Hessian's Lipschitz constant 90 | solver_it_max (int, optional): subsolver hard limit on iteration number (default: 100) 91 | solver_eps (float, optional): subsolver precision parameter (default: 1e-4) 92 | cubic_solver (callable, optional): subsolver (default: None) 93 | """ 94 | def __init__(self, eta1=0.1, eta2=0.9, sigma_eps=1e-16, sigma=None, solver_it_max=100, 95 | solver_eps=1e-4, cubic_solver=None, *args, **kwargs): 96 | super(Arc, self).__init__(*args, **kwargs) 97 | self.eta1 = eta1 98 | self.eta2 = eta2 99 | self.sigma_eps = sigma_eps 100 | self.sigma = sigma 101 | self.cubic_solver = cubic_solver 102 | self.solver_it = 0 103 | self.solver_it_max = solver_it_max 104 | self.solver_eps = solver_eps 105 | if sigma is None: 106 | self.sigma = self.loss.hessian_lipschitz / 2 107 | if self.sigma is None: 108 | self.sigma = 1. 109 | if cubic_solver is None: 110 | self.cubic_solver = arc_cubic_solver 111 | self.f_prev = None 112 | self.line_search = MockLineSearch() 113 | 114 | def step(self): 115 | if self.f_prev is None: 116 | self.f_prev = self.loss.value(self.x) 117 | self.grad = self.loss.gradient(self.x) 118 | grad_norm = self.loss.norm(self.grad) 119 | self.hess = self.loss.hessian(self.x) 120 | solver_eps = min(self.solver_eps, np.sqrt(grad_norm)) * grad_norm 121 | x_cubic, solver_it, lam = self.cubic_solver(self.x, self.grad, self.hess, self.sigma, self.solver_it_max, solver_eps) 122 | s = x_cubic - self.x 123 | model_value = self.f_prev + self.loss.inner_prod(s, self.grad) + 0.5 * self.hess @ s @ s + self.sigma/3 * self.loss.norm(s)**3 124 | f_new = self.loss.value(x_cubic) 125 | rho = (self.f_prev - f_new) / (self.f_prev - model_value) 126 | if rho > self.eta1: 127 | self.x = x_cubic 128 | self.f_prev = f_new 129 | else: 130 | self.sigma *= 2 131 | if rho > self.eta2: 132 | self.sigma = max(self.sigma_eps, min(self.sigma / 2, grad_norm)) 133 | self.line_search.it += solver_it 134 | self.line_search.lr = 1 / lam if lam > 0 else np.inf 135 | 136 | def init_run(self, *args, **kwargs): 137 | super(Arc, self).init_run(*args, **kwargs) 138 | self.trace.sigmas = [self.sigma] 139 | 140 | def update_trace(self): 141 | super(Arc, self).update_trace() 142 | self.trace.sigmas.append(self.sigma) 143 | -------------------------------------------------------------------------------- /optmethods/second_order/cubic.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import numpy.linalg as la 3 | 4 | from optmethods.optimizer import Optimizer 5 | 6 | 7 | def ls_cubic_solver(x, g, H, M, it_max=100, epsilon=1e-8, loss=None): 8 | """ 9 | Solve min_z + 1/2 + M/3 ||z-x||^3 10 | 11 | For explanation of Cauchy point, see "Gradient Descent 12 | Efficiently Finds the Cubic-Regularized Non-Convex Newton Step" 13 | https://arxiv.org/pdf/1612.00547.pdf 14 | Other potential implementations can be found in paper 15 | "Adaptive cubic regularisation methods" 16 | https://people.maths.ox.ac.uk/cartis/papers/ARCpI.pdf 17 | """ 18 | solver_it = 1 19 | newton_step = -np.linalg.solve(H, g) 20 | if M == 0: 21 | return x + newton_step, solver_it 22 | def cauchy_point(g, H, M): 23 | if la.norm(g) == 0 or M == 0: 24 | return 0 * g 25 | g_dir = g / la.norm(g) 26 | H_g_g = H @ g_dir @ g_dir 27 | R = -H_g_g / (2*M) + np.sqrt((H_g_g/M)**2/4 + la.norm(g)/M) 28 | return -R * g_dir 29 | 30 | def conv_criterion(s, r): 31 | """ 32 | The convergence criterion is an increasing and concave function in r 33 | and it is equal to 0 only if r is the solution to the cubic problem 34 | """ 35 | s_norm = la.norm(s) 36 | return 1/s_norm - 1/r 37 | 38 | # Solution s satisfies ||s|| >= Cauchy_radius 39 | r_min = la.norm(cauchy_point(g, H, M)) 40 | 41 | if loss is not None: 42 | x_new = x + newton_step 43 | if loss.value(x) > loss.value(x_new): 44 | return x_new, solver_it 45 | 46 | r_max = la.norm(newton_step) 47 | if r_max - r_min < epsilon: 48 | return x + newton_step, solver_it 49 | id_matrix = np.eye(len(g)) 50 | for _ in range(it_max): 51 | r_try = (r_min + r_max) / 2 52 | lam = r_try * M 53 | s_lam = -np.linalg.solve(H + lam*id_matrix, g) 54 | solver_it += 1 55 | crit = conv_criterion(s_lam, r_try) 56 | if np.abs(crit) < epsilon: 57 | return x + s_lam, solver_it 58 | if crit < 0: 59 | r_min = r_try 60 | else: 61 | r_max = r_try 62 | if r_max - r_min < epsilon: 63 | break 64 | return x + s_lam, solver_it 65 | 66 | 67 | class Cubic(Optimizer): 68 | """ 69 | Newton method with cubic regularization for global convergence. 70 | The method was studied by Nesterov and Polyak in the following paper: 71 | "Cubic regularization of Newton method and its global performance" 72 | https://link.springer.com/article/10.1007/s10107-006-0706-8 73 | 74 | Arguments: 75 | reg_coef (float, optional): an estimate of the Hessian's Lipschitz constant 76 | """ 77 | def __init__(self, reg_coef=None, solver_it_max=100, solver_eps=1e-8, cubic_solver=None, *args, **kwargs): 78 | super(Cubic, self).__init__(*args, **kwargs) 79 | self.reg_coef = reg_coef 80 | self.cubic_solver = cubic_solver 81 | self.solver_it = 0 82 | self.solver_it_max = solver_it_max 83 | self.solver_eps = solver_eps 84 | if reg_coef is None: 85 | self.reg_coef = self.loss.hessian_lipschitz 86 | if cubic_solver is None: 87 | self.cubic_solver = ls_cubic_solver 88 | 89 | def step(self): 90 | self.grad = self.loss.gradient(self.x) 91 | self.hess = self.loss.hessian(self.x) 92 | self.x, solver_it = self.cubic_solver(self.x, self.grad, self.hess, self.reg_coef/2, self.solver_it_max, self.solver_eps) 93 | self.solver_it += solver_it 94 | 95 | def init_run(self, *args, **kwargs): 96 | super(Cubic, self).init_run(*args, **kwargs) 97 | self.trace.solver_its = [0] 98 | 99 | def update_trace(self): 100 | super(Cubic, self).update_trace() 101 | self.trace.solver_its.append(self.solver_it) 102 | -------------------------------------------------------------------------------- /optmethods/second_order/newton.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from optmethods.optimizer import Optimizer 4 | 5 | 6 | class Newton(Optimizer): 7 | """ 8 | Newton algorithm for convex minimization. 9 | 10 | Arguments: 11 | lr (float, optional): dampening constant (default: 1) 12 | """ 13 | def __init__(self, lr=1, *args, **kwargs): 14 | super(Newton, self).__init__(*args, **kwargs) 15 | self.lr = lr 16 | 17 | def step(self): 18 | self.grad = self.loss.gradient(self.x) 19 | self.hess = self.loss.hessian(self.x) 20 | inv_hess_grad_prod = np.linalg.solve(self.hess, self.grad) 21 | if self.line_search is None: 22 | self.x -= self.lr * inv_hess_grad_prod 23 | else: 24 | self.x = self.line_search(x=self.x, direction=-inv_hess_grad_prod) 25 | -------------------------------------------------------------------------------- /optmethods/second_order/reg_newton.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | import warnings 4 | 5 | from optmethods.line_search import RegNewtonLS 6 | from optmethods.optimizer import Optimizer 7 | 8 | 9 | def empirical_hess_lip(grad, grad_old, hess, x, x_old, loss): 10 | grad_error = grad - grad_old - hess@(x - x_old) 11 | r2 = loss.norm(x - x_old)**2 12 | if r2 > 0: 13 | return 2 * loss.norm(grad_error) / r2 14 | return np.finfo(float).eps 15 | 16 | 17 | class RegNewton(Optimizer): 18 | """ 19 | Regularized Newton algorithm for second-order minimization. 20 | By default returns the Regularized Newton method from paper 21 | "Regularized Newton Method with Global O(1/k^2) Convergence" 22 | https://arxiv.org/abs/2112.02089 23 | 24 | Arguments: 25 | loss (optmethods.loss.Oracle): loss oracle 26 | identity_coef (float, optional): initial regularization coefficient (default: None) 27 | hess_lip (float, optional): estimate for the Hessian Lipschitz constant. 28 | If not provided, it is estimated or a small value is used (default: None) 29 | adaptive (bool, optional): use decreasing regularization based on either empirical Hessian-Lipschitz constant 30 | or a line-search procedure 31 | line_search (optmethods.LineSearch, optional): a callable line search. If None, line search is intialized 32 | automatically as an instance of RegNewtonLS (default: None) 33 | use_line_search (bool, optional): use line search to estimate the Lipschitz constan of the Hessian. 34 | If adaptive is True, line search will be non-monotonic and regularization may decrease (default: False) 35 | backtracking (float, optional): backtracking constant for the line search if line_search is None and 36 | use_line_search is True (default: 0.5) 37 | """ 38 | def __init__(self, loss, identity_coef=None, hess_lip=None, adaptive=False, line_search=None, 39 | use_line_search=False, backtracking=0.5, *args, **kwargs): 40 | if hess_lip is None: 41 | hess_lip = loss.hessian_lipschitz 42 | if loss.hessian_lipschitz is None: 43 | hess_lip = 1e-5 44 | warnings.warn(f"No estimate of Hessian-Lipschitzness is given, so a small value {hess_lip} is used as a heuristic.") 45 | self.hess_lip = hess_lip 46 | 47 | self.H = hess_lip / 2 48 | 49 | if use_line_search and line_search is None: 50 | if adaptive: 51 | line_search = RegNewtonLS(decrease_reg=adaptive, backtracking=backtracking, H0=self.H) 52 | else: 53 | # use a more optimistic initial estimate since hess_lip is often too optimistic 54 | line_search = RegNewtonLS(decrease_reg=adaptive, backtracking=backtracking, H0=self.H / 100) 55 | super(RegNewton, self).__init__(loss=loss, line_search=line_search, *args, **kwargs) 56 | 57 | self.identity_coef = identity_coef 58 | self.adaptive = adaptive 59 | self.use_line_search = use_line_search 60 | 61 | def step(self): 62 | self.grad = self.loss.gradient(self.x) 63 | if self.adaptive and self.hess is not None and not self.use_line_search: 64 | self.hess_lip /= 2 65 | empirical_lip = empirical_hess_lip(self.grad, self.grad_old, self.hess, self.x, self.x_old, self.loss) 66 | self.hess_lip = max(self.hess_lip, empirical_lip) 67 | self.hess = self.loss.hessian(self.x) 68 | 69 | if self.use_line_search: 70 | self.x = self.line_search(self.x, self.grad, self.hess) 71 | else: 72 | if self.adaptive: 73 | self.H = self.hess_lip / 2 74 | grad_norm = self.loss.norm(self.grad) 75 | self.identity_coef = (self.H * grad_norm)**0.5 76 | self.x_old = copy.deepcopy(self.x) 77 | self.grad_old = copy.deepcopy(self.grad) 78 | delta_x = -np.linalg.solve(self.hess + self.identity_coef*np.eye(self.loss.dim), self.grad) 79 | self.x += delta_x 80 | 81 | def init_run(self, *args, **kwargs): 82 | super(RegNewton, self).init_run(*args, **kwargs) 83 | self.x_old = None 84 | self.hess = None 85 | self.trace.lrs = [] 86 | 87 | def update_trace(self, *args, **kwargs): 88 | super(RegNewton, self).update_trace(*args, **kwargs) 89 | if not self.use_line_search: 90 | self.trace.lrs.append(1 / self.identity_coef) 91 | -------------------------------------------------------------------------------- /optmethods/stochastic_first_order/__init__.py: -------------------------------------------------------------------------------- 1 | from .root_sgd import Rootsgd 2 | from .sgd import Sgd 3 | from .shuffling import Shuffling 4 | from .svrg import Svrg 5 | -------------------------------------------------------------------------------- /optmethods/stochastic_first_order/root_sgd.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | 4 | from optmethods.optimizer import StochasticOptimizer 5 | 6 | 7 | class Rootsgd(StochasticOptimizer): 8 | """ 9 | Recursive One-Over-T SGD with decreasing or constant learning rate. 10 | Based on the paper 11 | https://arxiv.org/pdf/2008.12690.pdf 12 | 13 | Arguments: 14 | lr0 (float, optional): an estimate of the inverse smoothness constant 15 | """ 16 | def __init__(self, lr0=None, lr_max=np.inf, lr_decay_coef=0, lr_decay_power=1, it_start_decay=None, 17 | first_batch=None, batch_size=1, avoid_cache_miss=True, *args, **kwargs): 18 | super(Rootsgd, self).__init__(*args, **kwargs) 19 | self.lr0 = lr0 20 | self.lr_max = lr_max 21 | self.lr_decay_coef = lr_decay_coef 22 | self.lr_decay_power = lr_decay_power 23 | self.it_start_decay = it_start_decay 24 | if it_start_decay is None and np.isfinite(self.it_max): 25 | self.it_start_decay = self.it_max // 40 if np.isfinite(self.it_max) else 0 26 | self.first_batch = first_batch 27 | if first_batch is None: 28 | self.first_batch = 10 * batch_size 29 | self.batch_size = batch_size 30 | self.avoid_cache_miss = avoid_cache_miss 31 | 32 | def step(self): 33 | denom_const = 1 / self.lr0 34 | lr_decayed = 1 / (denom_const + self.lr_decay_coef*max(0, self.it-self.it_start_decay)**self.lr_decay_power) 35 | if lr_decayed < 0: 36 | lr_decayed = np.inf 37 | self.lr = min(lr_decayed, self.lr_max) 38 | if self.it > 0: 39 | if self.avoid_cache_miss: 40 | i = np.random.choice(self.loss.n) 41 | idx = np.arange(i, i + self.batch_size) 42 | idx %= self.loss.n 43 | else: 44 | idx = np.random.choice(self.loss.n, size=self.batch_size, replace=False) 45 | self.grad_old = self.loss.stochastic_gradient(self.x_old, idx=idx) 46 | self.grad_estim -= self.grad_old 47 | self.grad_estim *= 1 - 1 / (self.it+self.first_batch/self.batch_size) 48 | self.grad = self.loss.stochastic_gradient(self.x, idx=idx) 49 | self.grad_estim += self.grad 50 | else: 51 | self.grad_estim = self.loss.stochastic_gradient(self.x, batch_size=self.first_batch) 52 | self.x_old = copy.deepcopy(self.x) 53 | self.x -= self.lr * self.grad_estim 54 | 55 | def init_run(self, *args, **kwargs): 56 | super(Rootsgd, self).init_run(*args, **kwargs) 57 | if self.lr0 is None: 58 | self.lr0 = 1 / self.loss.batch_smoothness(batch_size) 59 | -------------------------------------------------------------------------------- /optmethods/stochastic_first_order/sgd.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from scipy.sparse import csr_matrix 4 | 5 | from optmethods.optimizer import StochasticOptimizer 6 | 7 | 8 | class Sgd(StochasticOptimizer): 9 | """ 10 | Stochastic gradient descent with decreasing or constant learning rate. 11 | 12 | Arguments: 13 | lr (float, optional): an estimate of the inverse smoothness constant 14 | lr_decay_coef (float, optional): the coefficient in front of the number of finished iterations 15 | in the denominator of step-size. For strongly convex problems, a good value 16 | is mu/2, where mu is the strong convexity constant 17 | lr_decay_power (float, optional): the power to exponentiate the number of finished iterations 18 | in the denominator of step-size. For strongly convex problems, a good value is 1 (default: 1) 19 | it_start_decay (int, optional): how many iterations the step-size is kept constant 20 | By default, will be set to have about 2.5% of iterations with the step-size equal to lr0 21 | batch_size (int, optional): the number of samples from the function to be used at each iteration 22 | avoid_cache_miss (bool, optional): whether to make iterations faster by using chunks of the data 23 | that are adjacent to each other. Implemented by sampling an index and then using the next 24 | batch_size samples to obtain the gradient. May lead to slower iteration convergence (default: True) 25 | """ 26 | def __init__(self, lr0=None, lr_max=np.inf, lr_decay_coef=0, lr_decay_power=1, it_start_decay=None, 27 | batch_size=1, avoid_cache_miss=False, importance_sampling=False, *args, **kwargs): 28 | super(Sgd, self).__init__(*args, **kwargs) 29 | self.lr0 = lr0 30 | self.lr_max = lr_max 31 | self.lr_decay_coef = lr_decay_coef 32 | self.lr_decay_power = lr_decay_power 33 | self.it_start_decay = it_start_decay 34 | self.batch_size = batch_size 35 | self.avoid_cache_miss = avoid_cache_miss 36 | self.importance_sampling = importance_sampling 37 | 38 | def step(self): 39 | if self.avoid_cache_miss: 40 | i = np.random.choice(self.loss.n) 41 | idx = np.arange(i, i + self.batch_size) 42 | idx %= self.loss.n 43 | self.grad = self.loss.stochastic_gradient(self.x, idx=idx) 44 | else: 45 | self.grad = self.loss.stochastic_gradient(self.x, batch_size=self.batch_size, importance_sampling=self.importance_sampling) 46 | denom_const = 1 / self.lr0 47 | it_decrease = max(0, self.it-self.it_start_decay) 48 | lr_decayed = 1 / (denom_const + self.lr_decay_coef*it_decrease**self.lr_decay_power) 49 | if lr_decayed < 0: 50 | lr_decayed = np.inf 51 | self.lr = min(lr_decayed, self.lr_max) 52 | self.x -= self.lr * self.grad 53 | if self.use_prox: 54 | self.x = self.loss.regularizer.prox(self.x, self.lr) 55 | 56 | def init_run(self, *args, **kwargs): 57 | super(Sgd, self).init_run(*args, **kwargs) 58 | if self.lr0 is None: 59 | self.lr0 = 1 / self.loss.batch_smoothness(batch_size) 60 | if self.it_start_decay is None and np.isfinite(self.it_max): 61 | self.it_start_decay = self.it_max // 40 if np.isfinite(self.it_max) else 0 62 | -------------------------------------------------------------------------------- /optmethods/stochastic_first_order/shuffling.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | 4 | from optmethods.optimizer import StochasticOptimizer 5 | 6 | 7 | class Shuffling(StochasticOptimizer): 8 | """ 9 | Shuffling-based stochastic gradient descent with decreasing or constant learning rate. 10 | For a formal description and convergence guarantees, see 11 | https://arxiv.org/abs/2006.05988 12 | 13 | The method is sensitive to finishing the final epoch, so it will terminate earlier 14 | than it_max if it_max is not divisible by the number of steps per epoch. 15 | 16 | Arguments: 17 | reshuffle (bool, optional): whether to get a new permuation for every new epoch. 18 | For convex problems, only a single permutation should suffice and it can run faster (default: False) 19 | prox_every_it (bool, optional): whether to use proximal operation every iteration 20 | or only at the end of an epoch. Theory supports the latter. Only used if the loss includes 21 | a proximal regularizer (default: False) 22 | lr0 (float, optional): an estimate of the inverse smoothness constant, this step-size 23 | is used for the first epoch_start_decay epochs. If not given, it will be set 24 | with the value in the loss. 25 | lr_max (float, optional): a maximal step-size never to be exceeded (default: np.inf) 26 | lr_decay_coef (float, optional): the coefficient in front of the number of finished epochs 27 | in the denominator of step-size. For strongly convex problems, a good value 28 | is mu/3, where mu is the strong convexity constant 29 | lr_decay_power (float, optional): the power to exponentiate the number of finished epochs 30 | in the denominator of step-size. For strongly convex problems, a good value is 1 (default: 1) 31 | epoch_start_decay (int, optional): how many epochs the step-size is kept constant 32 | By default, will be set to have about 2.5% of iterations with the step-size equal to lr0 33 | batch_size (int, optional): the number of samples from the function to be used at each iteration 34 | importance_sampling (bool, optional): use importance sampling to speed up convergence 35 | update_trace_at_epoch_end (bool, optional): save progress only at the end of an epoch, which 36 | avoids bad iterates 37 | """ 38 | def __init__(self, reshuffle=False, prox_every_it=False, lr0=None, lr_max=np.inf, lr_decay_coef=0, 39 | lr_decay_power=1, epoch_start_decay=1, batch_size=1, importance_sampling=False, 40 | update_trace_at_epoch_end=True, *args, **kwargs): 41 | super(Shuffling, self).__init__(*args, **kwargs) 42 | self.reshuffle = reshuffle 43 | self.prox_every_it = prox_every_it 44 | self.lr0 = lr0 45 | self.lr_max = lr_max 46 | self.lr_decay_coef = lr_decay_coef 47 | self.lr_decay_power = lr_decay_power 48 | self.epoch_start_decay = epoch_start_decay 49 | self.batch_size = batch_size 50 | self.importance_sampling = importance_sampling 51 | self.update_trace_at_epoch_end = update_trace_at_epoch_end 52 | 53 | self.steps_per_epoch = math.ceil(self.loss.n/batch_size) 54 | self.epoch_max = self.it_max // self.steps_per_epoch 55 | if epoch_start_decay is None and np.isfinite(self.epoch_max): 56 | self.epoch_start_decay = 1 + self.epoch_max // 40 57 | elif epoch_start_decay is None: 58 | self.epoch_start_decay = 1 59 | if importance_sampling: 60 | self.sample_counts = self.loss.individ_smoothness / np.mean(self.loss.individ_smoothness) 61 | self.sample_counts = np.int64(np.ceil(self.sample_counts)) 62 | self.idx_with_copies = np.repeat(np.arange(self.loss.n), self.sample_counts) 63 | self.n_copies = sum(self.sample_counts) 64 | self.steps_per_epoch = math.ceil(self.n_copies / batch_size) 65 | 66 | def step(self): 67 | if self.it%self.steps_per_epoch == 0: 68 | # Start new epoch 69 | if self.reshuffle: 70 | if not self.importance_sampling: 71 | self.permutation = np.random.permutation(self.loss.n) 72 | else: 73 | self.permutation = np.random.permutation(self.idx_with_copies) 74 | self.sampled_permutations += 1 75 | self.i = 0 76 | i_max = min(len(self.permutation), self.i+self.batch_size) 77 | idx = self.permutation[self.i:i_max] 78 | self.i += self.batch_size 79 | # since the objective is 1/n sum_{i=1}^n (f_i(x) + l2/2*||x||^2) 80 | # any incomplete minibatch should be normalized by batch_size 81 | if not self.importance_sampling: 82 | normalization = self.loss.n / self.steps_per_epoch 83 | else: 84 | normalization = self.sample_counts[idx] * self.n_copies / self.steps_per_epoch 85 | self.grad = self.loss.stochastic_gradient(self.x, idx=idx, normalization=normalization) 86 | denom_const = 1 / self.lr0 87 | it_decrease = self.steps_per_epoch * max(0, self.finished_epochs-self.epoch_start_decay) 88 | lr_decayed = 1 / (denom_const + self.lr_decay_coef*it_decrease**self.lr_decay_power) 89 | self.lr = min(lr_decayed, self.lr_max) 90 | self.x -= self.lr * self.grad 91 | end_of_epoch = self.it%self.steps_per_epoch == self.steps_per_epoch-1 92 | if end_of_epoch and self.use_prox: 93 | self.x = self.loss.regularizer.prox(self.x, self.lr * self.steps_per_epoch) 94 | self.finished_epochs += 1 95 | 96 | def should_update_trace(self): 97 | if not self.update_trace_at_epoch_end: 98 | super(Shuffling, self).should_update_trace() 99 | if self.it <= self.save_first_iterations: 100 | return True 101 | if self.it%self.steps_per_epoch != 0: 102 | return False 103 | self.time_progress = int((self.trace_len-self.save_first_iterations) * self.t / self.t_max) 104 | self.iterations_progress = int((self.trace_len-self.save_first_iterations) * (self.it / self.it_max)) 105 | enough_progress = max(self.time_progress, self.iterations_progress) > self.max_progress 106 | return enough_progress 107 | 108 | def init_run(self, *args, **kwargs): 109 | super(Shuffling, self).init_run(*args, **kwargs) 110 | if self.lr0 is None: 111 | self.lr0 = 1 / self.loss.batch_smoothness(batch_size) 112 | self.finished_epochs = 0 113 | self.permutation = np.random.permutation(self.loss.n) 114 | self.sampled_permutations = 1 115 | -------------------------------------------------------------------------------- /optmethods/stochastic_first_order/svrg.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from optmethods.optimizer import StochasticOptimizer 4 | 5 | 6 | class Svrg(StochasticOptimizer): 7 | """ 8 | Stochastic variance-reduced gradient descent with constant stepsize. 9 | Reference: 10 | https://papers.nips.cc/paper/4937-accelerating-stochastic-gradient-descent-using-predictive-variance-reduction.pdf 11 | 12 | Arguments: 13 | lr (float, optional): an estimate of the inverse smoothness constant 14 | """ 15 | def __init__(self, lr=None, batch_size=1, avoid_cache_miss=False, loopless=True, 16 | loop_len=None, restart_prob=None, *args, **kwargs): 17 | super(Svrg, self).__init__(*args, **kwargs) 18 | self.lr = lr 19 | self.batch_size = batch_size 20 | self.avoid_cache_miss = avoid_cache_miss 21 | self.loopless = loopless 22 | self.loop_len = loop_len 23 | self.restart_prob = restart_prob 24 | if loopless and restart_prob is None: 25 | self.restart_prob = batch_size / self.loss.n 26 | elif not loopless and loop_len is None: 27 | self.loop_len = self.loss.n // batch_size 28 | 29 | def step(self): 30 | new_loop = self.loopless and np.random.uniform() < self.restart_prob 31 | if not self.loopless and self.loop_it == self.loop_len: 32 | new_loop = True 33 | if new_loop or self.it == 0: 34 | self.x_old = self.x.copy() 35 | self.full_grad_old = self.loss.gradient(self.x_old) 36 | self.vr_grad = self.full_grad_old.copy() 37 | if not self.loopless: 38 | self.loop_it = 0 39 | self.loops += 1 40 | else: 41 | if self.avoid_cache_miss: 42 | i = np.random.choice(self.loss.n) 43 | idx = np.arange(i, i + self.batch_size) 44 | idx %= self.loss.n 45 | else: 46 | idx = np.random.choice(self.loss.n, size=self.batch_size) 47 | stoch_grad = self.loss.stochastic_gradient(self.x, idx=idx) 48 | stoch_grad_old = self.loss.stochastic_gradient(self.x_old, idx=idx) 49 | self.vr_grad = stoch_grad - stoch_grad_old + self.full_grad_old 50 | 51 | self.x -= self.lr * self.vr_grad 52 | if self.use_prox: 53 | self.x = self.loss.regularizer.prox(self.x, self.lr) 54 | self.loop_it += 1 55 | 56 | def init_run(self, *args, **kwargs): 57 | super(Svrg, self).init_run(*args, **kwargs) 58 | self.loop_it = 0 59 | self.loops = 0 60 | if self.lr is None: 61 | self.lr = 0.5 / self.loss.batch_smoothness(self.batch_size) 62 | -------------------------------------------------------------------------------- /optmethods/utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import os 4 | import pickle 5 | import random 6 | 7 | 8 | def relative_round(x): 9 | """ 10 | A util that rounds the input to the most significant digits. 11 | Useful for storing the results as rounding float 12 | numbers may cause file name ambiguity. 13 | """ 14 | mantissa, exponent = math.frexp(x) 15 | return round(mantissa, 3) * 2**exponent 16 | 17 | 18 | def get_trace(path, loss): 19 | if not os.path.isfile(path): 20 | return None 21 | f = open(path, 'rb') 22 | trace = pickle.load(f) 23 | trace.loss = loss 24 | f.close() 25 | return trace 26 | 27 | 28 | def set_seed(seed=42): 29 | """ 30 | :param seed: 31 | :return: 32 | """ 33 | random.seed(seed) 34 | os.environ['PYTHONHASHSEED'] = str(seed) 35 | np.random.seed(seed) 36 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | numpy 3 | scipy 4 | setuptools 5 | sklearn 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | 7 | setuptools.setup( 8 | name='opt_methods', 9 | version='0.1.1', 10 | author='Konstantin Mishchenko', 11 | author_email='konsta.mish@gmail.com', 12 | description='A collection of optimization methods and' 13 | 'loss functions for comparing their' 14 | 'iteration convergence and plotting the results', 15 | long_description=long_description, 16 | long_description_content_type="text/markdown", 17 | license='MIT', 18 | packages=setuptools.find_packages(), 19 | url='https://github.com/konstmish/opt_methods' 20 | ) 21 | --------------------------------------------------------------------------------