├── requirements.txt ├── docs └── sources │ └── examples │ ├── images │ └── cyclical-lr.png │ ├── continuous_jaccard.ipynb │ └── CyclicalLearningRate.ipynb ├── mytorch ├── __init__.py ├── losses.py └── learning_rate.py ├── README.md ├── LICENSE ├── setup.py └── .gitignore /requirements.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/sources/examples/images/cyclical-lr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/mytorch/HEAD/docs/sources/examples/images/cyclical-lr.png -------------------------------------------------------------------------------- /mytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Sebastian Raschka 2018 2 | # mytorch 3 | # Author: Sebastian Raschka 4 | # 5 | # License: MIT 6 | 7 | __version__ = '0.1.0dev' 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mytorch 2 | 3 | Collection of PyTorch-related utility functions. Currently, this package and its documentation is kept simple, since it's mainly intended for personal use. 4 | 5 | ## Installation 6 | 7 | pip install git+https://github.com/rasbt/mytorch.git 8 | 9 | ## Documentation 10 | 11 | - [Continuous Jaccard Distance](docs/sources/examples/continuous_jaccard.ipynb) 12 | - [Cyclical Learning Rate Scheduler](docs/sources/examples/CyclicalLearningRate.ipynb) 13 | -------------------------------------------------------------------------------- /mytorch/losses.py: -------------------------------------------------------------------------------- 1 | # Sebastian Raschka 2018 2 | # mytorch 3 | # Author: Sebastian Raschka 4 | # 5 | # License: MIT 6 | 7 | import torch 8 | 9 | 10 | def continuous_jaccard(x, y): 11 | """ 12 | Implementation of the continuous version of the 13 | Jaccard distance: 14 | 15 | 1 - [sum_i min(x_i, y_i)] / [sum_i max(x_i, y_i)] 16 | """ 17 | c = torch.cat((x.view(-1).unsqueeze(1), y.view(-1).unsqueeze(1)), dim=1) 18 | 19 | numerator = torch.sum(torch.min(c, dim=1)[0]) 20 | denominator = torch.sum(torch.max(c, dim=1)[0]) 21 | 22 | return 1. - numerator/denominator 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Sebastian Raschka 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Sebastian Raschka 2018 2 | # mytorch 3 | # Author: Sebastian Raschka 4 | # 5 | # License: MIT 6 | 7 | from os.path import realpath, dirname, join 8 | from setuptools import setup, find_packages 9 | import mytorch 10 | 11 | VERSION = mytorch.__version__ 12 | PROJECT_ROOT = dirname(realpath(__file__)) 13 | 14 | REQUIREMENTS_FILE = join(PROJECT_ROOT, 'requirements.txt') 15 | 16 | with open(REQUIREMENTS_FILE) as f: 17 | install_reqs = f.read().splitlines() 18 | 19 | install_reqs.append('setuptools') 20 | 21 | 22 | setup(name='mytorch', 23 | version=VERSION, 24 | description='PyTorch-related utility functions', 25 | author='Sebastian Raschka', 26 | author_email='mail@sebastianraschka.com', 27 | url='https://github.com/rasbt/mytorch', 28 | packages=find_packages(), 29 | package_data={'': ['LICENSE.txt', 30 | 'README.md', 31 | 'requirements.txt'] 32 | }, 33 | include_package_data=True, 34 | install_requires=install_reqs, 35 | extras_require={'testing': ['nose'], 36 | 'docs': ['mkdocs']}, 37 | license='MIT', 38 | platforms='any', 39 | long_description=""" 40 | 41 | A library of PyTorch-related utility tools. 42 | Currently only intended for personal use. 43 | """) 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | -------------------------------------------------------------------------------- /docs/sources/examples/continuous_jaccard.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Continuous Jaccard Distance for PyTorch" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 5, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "from mytorch.losses import continuous_jaccard" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Concept" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "Implementation of the continuous version of the Jaccard Index/Distance. I.e., given 2 vectors, $x$ and $y$:\n", 31 | "\n", 32 | "$$\n", 33 | "J(x, y) = 1 - \\frac{\\sum_i \\min(x_i, y_i)}{\\sum_i \\max(x_i, y_i)}.\n", 34 | "$$\n", 35 | "\n", 36 | "\n", 37 | "### References\n", 38 | "\n", 39 | "\n", 40 | "- [1] https://en.wikipedia.org/wiki/Jaccard_index" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "## Example 1" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 1, 53 | "metadata": {}, 54 | "outputs": [ 55 | { 56 | "data": { 57 | "text/plain": [ 58 | "tensor(0.6275)" 59 | ] 60 | }, 61 | "execution_count": 1, 62 | "metadata": {}, 63 | "output_type": "execute_result" 64 | } 65 | ], 66 | "source": [ 67 | "import torch\n", 68 | "from mytorch.losses import continuous_jaccard\n", 69 | "\n", 70 | "\n", 71 | "\n", 72 | "import torch\n", 73 | "\n", 74 | "x = torch.tensor([7, 2, 3, 4, 5, 6]).float()\n", 75 | "y = torch.tensor([1, 8, 9, 10, 11, 4]).float()\n", 76 | "\n", 77 | "continuous_jaccard(x, y)" 78 | ] 79 | } 80 | ], 81 | "metadata": { 82 | "kernelspec": { 83 | "display_name": "Python 3", 84 | "language": "python", 85 | "name": "python3" 86 | }, 87 | "language_info": { 88 | "codemirror_mode": { 89 | "name": "ipython", 90 | "version": 3 91 | }, 92 | "file_extension": ".py", 93 | "mimetype": "text/x-python", 94 | "name": "python", 95 | "nbconvert_exporter": "python", 96 | "pygments_lexer": "ipython3", 97 | "version": "3.6.5" 98 | } 99 | }, 100 | "nbformat": 4, 101 | "nbformat_minor": 2 102 | } 103 | -------------------------------------------------------------------------------- /mytorch/learning_rate.py: -------------------------------------------------------------------------------- 1 | # Sebastian Raschka 2018 2 | # mytorch 3 | # Author: Sebastian Raschka 4 | # 5 | # License: MIT 6 | 7 | import numpy as np 8 | from torch.optim.lr_scheduler import _LRScheduler 9 | 10 | 11 | class CyclicalLearningRate(_LRScheduler): 12 | 13 | """Learning rate scheduler based on the cyclical 14 | learning rate concept introduced in 15 | Leslie N. Smith 16 | "Cyclical learning rates for training neural networks." 17 | Applications of Computer Vision (WACV), 18 | 2017 IEEE Winter Conference on. IEEE, 2017. 19 | 20 | Args: 21 | optimizer (Optimizer): Wrapped optimizer. 22 | step_size (int): Period of learning rate decay. 23 | max_lr (float): Maximum learning rate. 24 | mode (str): `'triangular'`, `'triangular2'`, or 25 | `'exp_range'` mode. Default: `'triangular'` 26 | gamma (float): Multiplicative factor of learning rate decay 27 | if mode=`exp_range`. Default: 0.999995. 28 | batch_count (int): The index of the most recent batch. 29 | Default: -1. 30 | 31 | Example: 32 | >>> num_epochs = 50 33 | >>> train_size = 50000 34 | >>> batch_size = 100 35 | >>> iterations_per_epoch = train_size // batch_size 36 | >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.01) 37 | >>> scheduler = CyclicalLearningRate( 38 | ... optimizer, 39 | ... step_size=step_size, 40 | ... max_lr=0.06) 41 | >>> 42 | >>> for epoch in range(num_epochs): 43 | >>> for batch in range(iterations_per_epoch): 44 | >>> # train(...) 45 | >>> # validate(...) 46 | >>> # note that the scheduler should be called 47 | >>> # after each batch (not only after each epoch) 48 | >>> scheduler.step() 49 | """ 50 | 51 | def __init__(self, 52 | optimizer, 53 | step_size, 54 | max_lr, 55 | mode='triangular', 56 | gamma=0.999995, 57 | batch_count=-1): 58 | 59 | self.step_size = step_size 60 | self.max_lr = max_lr 61 | self.mode = mode 62 | self.gamma = gamma 63 | self.batch_count = batch_count 64 | 65 | if self.batch_count == -1: 66 | for group in optimizer.param_groups: 67 | group.setdefault('initial_lr', group['lr']) 68 | else: 69 | for i, group in enumerate(optimizer.param_groups): 70 | if 'initial_lr' not in group: 71 | raise KeyError("param 'initial_lr' is not specified " 72 | "in param_groups[{}] when " 73 | "resuming an optimizer".format(i)) 74 | 75 | self.base_lrs = list(map(lambda group: group['initial_lr'], 76 | optimizer.param_groups)) 77 | super(CyclicalLearningRate, self).__init__(optimizer) 78 | 79 | def _compute_lr(self, base_lr): 80 | cycle = np.floor(1 + self.batch_count / (2. * self.step_size)) 81 | x = np.abs(self.batch_count / float(self.step_size) - 2 * cycle + 1) 82 | 83 | lr_delta = (self.max_lr - base_lr) * np.maximum(0, (1 - x)) 84 | 85 | if self.mode == 'triangular': 86 | pass 87 | elif self.mode == 'triangular2': 88 | lr_delta = lr_delta * 1 / (2. ** (cycle - 1)) 89 | elif self.mode == 'exp_range': 90 | lr_delta = lr_delta * (self.gamma**(self.batch_count)) 91 | else: 92 | raise ValueError('mode must be "triangular", ' 93 | '"triangular2", or "exp_range"') 94 | 95 | return base_lr + lr_delta 96 | 97 | def step(self, epoch=None): 98 | self.batch_count += 1 99 | for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): 100 | param_group['lr'] = lr 101 | 102 | def get_lr(self): 103 | return [self._compute_lr(lr) for lr in self.base_lrs] 104 | -------------------------------------------------------------------------------- /docs/sources/examples/CyclicalLearningRate.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Cyclical Learning Rate Scheduler for PyTorch" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 5, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "from mytorch.learning_rate import CyclicalLearningRate" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "## Concept" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "In his paper [1], Leslie N. Smith introduced the concept of cyclical learning rates, that is, learning rates that periodically alternative between a user-specified minimum and maximum learning rate. \n", 31 | "\n", 32 | "Varying the learning rate between between specified bounds, as implemented by Smith, is cheaper to compute than the nowadays popular approach using adaptive learning rates. Note that adaptive learning rate can also be combined with the concept of cyclical learning rates.\n", 33 | "\n", 34 | "The idea behind cyclical learning rates is that while increasing the learning rate can be harmful short term it can be beneficial in the long run. Concretely, the three methods introduced by Smith (and implemented in this notebook) are\n", 35 | "\n", 36 | "- `triangular`: The base approach, varying between a lower and an upper bound, as illustrated in the figure below\n", 37 | "- `triangular2`: Same as triangular, but learning rate difference is cut in half at the end of each cycle. This means the learning rate difference drops after each cycle\n", 38 | "-- `exp_range`: The learning rate varies between the minimum and maximum boundaries and each boundary value declines by an exponential factor of $gamma^{iteration}$\n", 39 | "\n", 40 | "\n", 41 | "![](./images/cyclical-lr.png)\n", 42 | "\n", 43 | "\n", 44 | "### References\n", 45 | "\n", 46 | "\n", 47 | "- [1] Smith, Leslie N. “[Cyclical learning rates for training neural networks](https://ieeexplore.ieee.org/abstract/document/7926641/).” Applications of Computer Vision (WACV), 2017 IEEE Winter Conference on. IEEE, 2017." 48 | ] 49 | }, 50 | { 51 | "cell_type": "markdown", 52 | "metadata": {}, 53 | "source": [ 54 | "## Example: triangular learning rate" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "### Placeholder model" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 42, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "import torch\n", 71 | "import torch.nn as nn\n", 72 | "import torch.nn.functional as F\n", 73 | "\n", 74 | "\n", 75 | "class Net(nn.Module):\n", 76 | " def __init__(self):\n", 77 | " super(Net, self).__init__()\n", 78 | " self.fc1 = nn.Linear(1, 1)\n", 79 | " def forward(self, x):\n", 80 | " x = self.fc1(x)\n", 81 | " return x\n", 82 | "\n", 83 | "model = Net()" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 43, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "num_epochs = 50\n", 93 | "train_size = 50000\n", 94 | "batch_size = 100\n", 95 | "iterations_per_epoch = train_size // batch_size\n", 96 | "\n", 97 | "# Set step size to 4 epochs (cycle length = 8 epochs):\n", 98 | "step_size = 4 * iterations_per_epoch\n", 99 | "\n", 100 | "collected_lr = []\n", 101 | "\n", 102 | "optimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n", 103 | "scheduler = CyclicalLearningRate(optimizer, step_size=step_size, max_lr=0.06)\n", 104 | "for epoch in range(num_epochs):\n", 105 | " for batch in range(iterations_per_epoch):\n", 106 | " # train(...)\n", 107 | " # validate(...)\n", 108 | " # note that the scheduler should be called\n", 109 | " # after each batch (not only after each epoch)\n", 110 | " scheduler.step()\n", 111 | " \n", 112 | " collected_lr.append(optimizer.param_groups[0]['lr'])" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 44, 118 | "metadata": {}, 119 | "outputs": [ 120 | { 121 | "data": { 122 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEjCAYAAADe/dHWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+0XWV95/H3x3tNwAgoUViuBEwkcWyoGDCNWMWxUmxAx+AUFmFwzLSsMlpZI0NpJ5QWlCVV+gOYLum0KEwjWglDxcaKIsMPta2NCRJ+REQukSlpXGYpCBIkeMN3/tjPrScn5969773nnP3r81rrrrvP3s85+3nOfs53P/vZPx5FBGZm1g4vKDsDZmY2PA76ZmYt4qBvZtYiDvpmZi3ioG9m1iIO+mZmLeKgb7Ukaa+krR1/6/r42YskPdCvz5tFPpZLOqXsfFizjJadAbMZ+mlELC87EwO2HFgB3FJ2Rqw53NK3RpH0qKTLJX0z/S1J818p6XZJ96X/R6b5h0u6WdK96e+X00eNSPqEpG2SviLpwB7r6vleSedLeiD9nZfm7XP0IOkCSR9K03d15Pm7kk6QNAe4FDgjHcmcMcjvzdrDQd/q6sCu7p3OoPhURKwEPg5cleZ9HPhURBwDfAb48zT/z4GvRsTrgOOAbWn+UuDqiDga+DHw6z3ysN97Jb0e+A3gDcDxwG9JOrZAeUZTns8DLomI54CLgQ0RsTwiNhT4DLNc7t6xupqqe+ezHf+vTNNvBP5jmr4e+OM0/TbgvQARsRd4UtJLge9FxNaU5m5gUY/19Hrvm4GbI2I3gKTPAScAG3PK87mcdZn1hVv61kQxyfRkaXrZ0zG9l+INJE0yf5x9f28HTLK+6azLbNoc9K2Jzuj4/400/U/AmjR9FvAPafp24P0AkkYkHTyN9fR679eAUyW9SNI84N3A14EfAIdJmi9pLvDOAp//E+CgaeTHLJeDvtVVd5/+xzqWzZW0Cfgg8N/TvP8G/Iak+4D/nJaR/v+KpPvJulaOnkYe9ntvRHwL+Gvgm8Am4JMRcU9E/IzsxOwm4O+B7xT4/DuBZT6Ra/0kP1rZmkTSo8CKiPhh2XkxqyK39M3MWsQtfTOzFnFL38ysRRoT9CWtkvSQpLF+PoelSiRdJ2lX152dh0q6TdLD6f9Ly8xjv0k6QtKdkh5Md8d+MM1vbLklHZDuzr03lfnDaf5iSZtSmTeku3YbI10BdY+kv0+vm17eRyXdn07Ub0nzBl6vGxH0JY0AVwMnA8uAMyUtKzdXA/HXwKqueeuA2yNiKdklhE3b4Y0DvxMRv0B2h+sH0rZtcrn3AG9Ld/ouB1ZJOh64HLgylfkJ4OwS8zgIHwQe7Hjd9PIC/Eq643pFej3wet2IoA+sBMYiYnu6ff0GYHXJeeq7iPga8HjX7NXA+jS9Hjh1qJkasIj4froMkoj4CVlQWECDyx2Zp9PLF6a/ILsD+KY0v1FllrQQeAfwyfRaNLi8Uxh4vW5K0F8APNbxekea1waHR8T3IQuQwGEl52dgJC0CjiW71r3R5U5dHVuBXcBtwCPAjyNiPCVpWh2/Cvg94Pn0ej7NLi9kO/KvSLpb0jlp3sDrdVNu9+5167svS2oQSS8G/hY4LyKeyhqCzZWe5bNc0kuAm4Ff6JVsuLkaDEnvBHZFxN2S3joxu0fSRpS3w5siYqekw4DbJBW5YW/WmtLS3wEc0fF6IbCzpLwM2w8kvQIg/d9Vcn76TtILyQL+ZyJi4sFkjS83QET8GLiL7HzGSyRNNNSaVMffBLwr3Vh3A1m3zlU0t7wARMTO9H8X2Y59JUOo100J+puBpels/xyyZ6zkPdWwKTYCa9P0WuDvSsxL36W+3WuBByPiio5FjS23pJenFj7KnuP/q2TnMu4ETkvJGlPmiLgwIhZGxCKy3+4dEXEWDS0vgKR5kg6amAbeDjzAEOp1Y27OUjas3FXACHBdRFxWcpb6TtJngbcCLyN7gNclwOeBG4EjgX8BTo+I7pO9tZUeVfx14H5+3t/7+2T9+o0st6RjyE7ijZA1zG6MiEslvYqsJXwocA/wnojYM/kn1U/q3rkgIt7Z5PKmst2cXo4CfxMRl0maz4DrdWOCvpmZ5WtK946ZmRXgoG9m1iIO+mZmLeKgb2bWIo0L+h13trVC28oLLnNbuMyD0bigD7StorStvOAyt4XLPABNDPpmZjaJyl2n/4IXvCAOPPDAGb9/fHyc0dGmPFIoX9vKCy5zW7jM0/PMM89EROQ25Cv3jR544IHs3r277GyYmdWKpJ8WSefuHTOzFnHQNzNrEQd9M7MWcdA3M2sRB30zsxZx0DczaxEHfTOzFnHQNzNrEQd9M7MWcdA3M2sRB30zsxZx0DczaxEHfTOzFnHQNzNrEQd9M7MWKRT0Ja2S9JCkMUnreiyfK2lDWr5J0qI0/yxJWzv+npe0vL9FMDOzonJHzpI0AnwXOAnYAWwGzoyIb3ek+W3gmIh4n6Q1wLsj4oyuz3kt8HcR8aqp1jdv3rzwICpmZtMj6ZmImJeXrkhLfyUwFhHbI+I54AZgdVea1cD6NH0TcKIkdaU5E/hsgfWZmdmAFAn6C4DHOl7vSPN6pomIceBJYH5XmjOYJOhLOkfSFklbxsfHi+TbzMxmoEjQ726xA3T3CU2ZRtIbgGci4oFeK4iIayJiRUSsaNtAyGZmw1Qk6O8Ajuh4vRDYOVkaSaPAIcDjHcvX4K4dM7PSFQn6m4GlkhZLmkMWwDd2pdkIrE3TpwF3RDpDLOkFwOlk5wLMzKxEuX0pETEu6VzgVmAEuC4itkm6FNgSERuBa4HrJY2RtfDXdHzEW4AdEbG9/9k3M7PpyL1kc9h8yaaZ2fT185JNMzNrCAd9M7MWcdA3M2sRB30zsxZx0DczaxEHfTOzFnHQNzNrEQd9M7MWcdA3M2sRB30zsxZx0DczaxEHfTOzFnHQNzNrEQd9M7MWcdA3M2sRB30zsxYpNAq5pFXA/yQbOeuTEfGxruVzgU8Brwd+BJwREY+mZccAfwUcDDwP/FJEPNuvAgzLMZd8maf27P231wfPHeG+D68qMUfTV/cy1D3/4DJUQd3zP1u5LX1JI8DVwMnAMuBMScu6kp0NPBERS4ArgcvTe0eBTwPvi4ijgbcCP+tb7oeku5IAPLVnL8dc8uWScjR9i9Z9sdZlaMI2aEIZmlqPFq37Ykk5Gr4i3TsrgbGI2B4Rz5ENcL66K81qYH2avgk4UZKAtwP3RcS9ABHxo4jYS810V5K8+VUzVYWuSxnqvg2g/mVYcmFz6xFM/TtpkiJBfwHwWMfrHWlezzQRMQ48CcwHXg2EpFslfUvS7/VagaRzJG2RtGV8fHy6ZRioP/j8/WVnwXK05cdatvFqDadtM1Qk6KvHvO7NP1maUeDNwFnp/7slnbhfwohrImJFRKwYHS10mmFoPv3P/zLl8s/f869DyonVWd6OyTsuG5YiQX8HcETH64XAzsnSpH78Q4DH0/yvRsQPI+IZ4BbguNlmukrO27C17CyYmRVWJOhvBpZKWixpDrAG2NiVZiOwNk2fBtwREQHcChwj6UVpZ/DvgW/3J+vWL1VvZVY9f5ap+naqev6GJTfopz76c8kC+IPAjRGxTdKlkt6Vkl0LzJc0BpwPrEvvfQK4gmzHsRX4VkT4mx8iV3Sz4k664q6yszBwhTrQI+IWsq6ZznkXd0w/C5w+yXs/TXbZptnALFr3RR792DvKzkZjtaXx8PCu3WVnYeB8R+4U2lLRbbCK1qM3XHbbgHNi5qDfF945WD/84CfPlZ0FawEHfQOq25fpHWq9VHV7VTVfZXDQb7DpdBe0oS/TrIim33vjoN9gbesucGtuMNr2vTb93hsH/Um0raLbYFS128zay0G/T7yTsF6m223memSD5qBvleUAWE9V226+FHZfDvr2b6r2YzXrh7ad28rjoN9QbQ3gi1ta7kFpaz1qcrkd9Hto8gZvOj/y3WxqDvp95J2FdZppfXA9skFy0LdKcjdNvXnEuepy0Ld9VKWV6W6aessbcW5YqlKfq8RBv4HaXtHP+sQ3ys5CI7S9HjW1/A76XRww6u8fH3m87CyYVZaDfpe8gJE3UId3Ggb5rcS8etTUVqaVr1DQl7RK0kOSxiSt67F8rqQNafkmSYvS/EWSfippa/r7y/5mv3rcypy9vB2nR8iqhrpvhzcddWjZWShFbtCXNAJcDZwMLAPOlLSsK9nZwBMRsQS4Eri8Y9kjEbE8/b2vT/m2Wah6K9M7zmYoux7lrf8zv/XGIeWkWoq09FcCYxGxPSKeA24AVnelWQ2sT9M3ASdKUv+yaUXNtluhLppSjqrKu+SyKd9/Xjlec9EtUy6voyJBfwHwWMfrHWlezzQRMQ48CcxPyxZLukfSVyWd0GsFks6RtEXSlvHx8WkVYJiuOmN52VmwgspuZdZdVS65LNuze5t38XCRoN+rxd79TUyW5vvAkRFxLHA+8DeSDt4vYcQ1EbEiIlaMjo4WyNJg5AWKU4/N9nUO/jaVokdbea3MJRd6x2X9VyTo7wCO6Hi9ENg5WRpJo8AhwOMRsScifgQQEXcDjwCvnm2myzYR/CfjVubgNKVboYjxGjQylx42r+wszEib+56LBP3NwFJJiyXNAdYAG7vSbATWpunTgDsiIiS9PJ0IRtKrgKXA9v5k3Wajqn2Z3mHWy23nv3XK5WVtz7z1fq9FjYduuUE/9dGfC9wKPAjcGBHbJF0q6V0p2bXAfEljZN04E5d1vgW4T9K9ZCd43xcRvjRjQPK6A6bTSq5DX+bhB80pOwut1LSjrbzyNO05QoWu04+IWyLi1RFxVERcluZdHBEb0/SzEXF6RCyJiJURsT3N/9uIODoiXhcRx0XEFwZXlMGqw2FsHboD+mnTRSdNudxHDTPj721fTTup7Ttyk7yK3n0YW4edgA3fMZd8ecrl3a3KprWarfoc9Geoqn2ZTVaH7pyn9uzt6+e5Hlm/Oei3WNVamXkBLq87x8pRtTu88wZCr1q9HzYH/Yb4/D3/OuXymVR0tzKtWx26NWcyEPp7jj9yADmpJgf9hjhvw9ays1CKqrUy626657aa4iOnvnbK5U2qRw76tOd5NVauyeqR7/C2YXLQnwW3Mq3TTLe37/C2YXLQb7mq9GX6aKve8rbPSVfcNZR8DOLcVtM46LfAVIG9TX2ZVp6Hd+0eynpmc25rtCUP5HHQb4C8wJwX2OvOQ1j2R9uPtsY+2o7u2tYH/bzDzqZX9DaowkhcefXogJGWNDOtdK0P+rM97Mz7MefdKGLNMNtW4HcuO2Wgn282ofVBf9BmcqPIsJXdl+mB0Juh6tupKhctlM1Bv+GKdBuU3ZdZhe4XG7xB16O2n9sqykG/5vIqel63QVNUvZVZdT63lWnDvTetDvq+prc9yvyxlt19VsSwLqm08hUK+pJWSXpI0pikdT2Wz5W0IS3fJGlR1/IjJT0t6YL+ZLs/+vW8Gu8c2i1vh5LXfTYhrx4tbkAr08qXG/TTGLdXAycDy4AzJS3rSnY28ERELAGuBC7vWn4l8KXZZ7eemnBIWBbvUH+uDgOjVfUpnDU42BqaIi39lcBYRGyPiOeAG4DVXWlWA+vT9E3AiZIEIOlUssHQt/UnyzYIZfVleofYLGUNLuSB0IsrEvQXAI91vN6R5vVMkwZSf5JsoPR5wP8APjzVCiSdI2mLpC3j4+NF8956/RwIvQnedNShZWehkdpWj5o+UHqRoN/ryKj7SHOyNB8GroyIp6daQURcExErImLF6OhogSwNXh0CSNsGQs/zmd9645TLfVTRm7+X6an7QOlFgv4O4IiO1wuBnZOlkTQKHAI8DrwB+GNJjwLnAb8v6dxZ5rkv8ip6XgDpVtW+TBusfj+vxs8RskErEvQ3A0slLZY0B1gDbOxKsxFYm6ZPA+6IzAkRsSgiFgFXAX8UER/vU94rxQOl918djraGzTey2WzlBv3UR38ucCvwIHBjRGyTdKmkd6Vk15L14Y8B5wP7XdZp1Tfsvsx+H21ZNQz7ooC8S1nbdk4iT6Hr9CPiloh4dUQcFRGXpXkXR8TGNP1sRJweEUsiYmVEbO/xGR+KiD/tb/bbKy8AD6Ki170v06avDt2Wgzi11eQhLFt9R26dOQD31obb6PuprQOh52nyEJatDPptHyzC+mNQj/Fw/bNBamXQHxS3MtulX4/xmC7XI5sNB33bx7BamT7aara87feai27py3o8FsP0VeNOKOurQVb0Reu+6B+Szdqze/tz+nWQl7AeftCcoQ2C1KsRNKjfmVv6NeTD+6nl/ViOueTLQ8pJtfloa2qbLjppyuX9+h1O9jmD+p23LujnHVa2vaK3wVN79g58HbOtRwfPHelTTsz21bqg36/Dysl4oPR2GPTR1n0fXlXq+m04ytiOrQv6ZavDQOmDviEnr3vFR1vNkLcd8y55HfT628pBv2H60S0w6OcIDaN7xapvtpe8+mhnZhz0ayavoud1C7SFW3lT87mtYpp4702rgr4HQrcJg/yxHn7QnIF9dr8M+tyWVVergv6w7qAsuy/TBitvh5F3qV9RTWxl2s+VdW6rVUG/Ksq6fb8KfLTVLu85/shS1nvASPWHQi/r3JaDvvU0qFZmm3d4bfSRU1875fKZ1qO8933nslNm9Llt4KBfI76Dcnr8fcyMv7d95X0fJ11x13Ay0icO+omH5muftvaJt7Xcg/Lwrt1lZ2FaCgV9SaskPSRpTNJ+QyFKmitpQ1q+SdKiNH+lpK3p715J7+5v9osb9tB8ZfVl2mAN+2jLd3g3Uxkj303IDfqSRoCrgZOBZcCZkpZ1JTsbeCIilgBXApen+Q8AKyJiObAK+CtJrXiy56D6MpusyUPUDUod7vCuw0nVYStz5LsiLf2VwFhEbI+I54AbgNVdaVYD69P0TcCJkhQRz6SB1QEOYDDDWdqA9LsvM29HlzdEndVT3knV6TaAfG5rdooE/QXAYx2vd6R5PdOkIP8kMB9A0hskbQPuB97XsRP4N5LOkbRF0pbx8f0WG/mH8WVU9Dr0ZY66kTktPtrqrUn33hQJ+r1+Nt0t9knTRMSmiDga+CXgQkkH7Jcw4pqIWBERK0ZHh9/7U4fAUIfD+Coa+6hvcOrko63BqNOlyEWC/g7giI7XC4Gdk6VJffaHAPsMaRMRDwK7gV+caWZnKq+i5wWGmXJfZrPkdWcN6mjL3RXtMuiLQIoE/c3AUkmLJc0B1gAbu9JsBNam6dOAOyIi0ntGASS9Evh3wKN9yXkN9Lsvs8nqsHusaneW61G95G2vvItAZis36Kc++HOBW4EHgRsjYpukSyW9KyW7FpgvaQw4H5i4rPPNwL2StgI3A78dET/sdyFscPrVysyr6N9za7bR8urRkguL7bg8EPrsFepAj4hbgFu65l3cMf0scHqP910PXD/LPFqOMiu6B0pvjjK7I8cLXtc3yIHQ87zpqENLXX+/+I7cGvDh++zk7ZQWt+T79fNqZifvBs66/E4bH/R9Ta/l6cfNI4OuR35MSDsM40rCxgf9srmV2Qxlt+Ka0spsu7KuJOzkoF+yOtyiPNuB0vN2bD7aaoe87Zz3PJrZfr5lHPRrbhiH/bMdKL0OOzYrX97zaHw00x8O+hU37KeDNlWTbqOfibxLIt1KLqYJQ1g2Ouj7ml4raja30Q/rMdqzOcdX9JJIa75GB/2qXFObt3PJ2zlZucq+g3JC3g1sdWhltllVzm01OujXRVV2ToPggdCt06C2dx0uaa3KwZaDfo0N8/7JmfZl1unpg1a+yeqRz231j4N+hfl5Nf3V1qOKMofma6K87+uYS748pJzMTGuDvgeLsG5N7RMvc2i+Nnpqz96yszClxgb9qg0W4Z1MPVXtMR4eKL2eqnQlYWODftXk7WSa2sqcirsV+q8OI6y1cXChKl2s4aBvheUF6ddctM/Tt1u5I7N80x1cqGpHW3XnoF9RVRwIPc+ze6tyUdrkDp47UnYWKqWK9agO6nyHd6GgL2mVpIckjUla12P5XEkb0vJNkhal+SdJulvS/en/2/qb/Zk5/KA5ZWchVx0O0+vovg+vmnJ5045Omlaeuqjypcq5QV/SCHA1cDKwDDhT0rKuZGcDT0TEEuBK4PI0/4fAf4iI15KNoTuUUbTyKvqmi04aRjb208a+zDrLu/SurFayW+fNMqzHeEwo0tJfCYxFxPaIeA64AVjdlWY1sD5N3wScKEkRcU9E7EzztwEHSJrbj4zXkQdK/7k6dLNU/dK7ybSpHtVBVR7jMaFI0F8APNbxekea1zNNGkj9SWB+V5pfB+6JiD3dK5B0jqQtkraMj48XzbuVoGgrM6+i53WzWLMVvcO7qkdbdVYk6Pfqk+g+YzdlGklHk3X5/NdeK4iIayJiRUSsGB0tNFZ7o9X5eTVuZdZHHbobq3y0VYfn/fRSJOjvAI7oeL0Q2DlZGkmjwCHA4+n1QuBm4L0R8chsM9wGVT4J1ARNeCZ6ER4IfbDqOoRlkaC/GVgqabGkOcAaYGNXmo1kJ2oBTgPuiIiQ9BLgi8CFEfGP/cr0VHxNrw1D2fXId3g3QxnntnKDfuqjPxe4FXgQuDEitkm6VNK7UrJrgfmSxoDzgYnLOs8FlgB/KGlr+jus76Wokba0Muuu6tvBd3jXQxXPbRXqQI+IW4BbuuZd3DH9LHB6j/d9BPjILPNoFfOe44+c8iFePtqyIh792DumrCs+iTsYviO3ZoZ9TW8vw77EzNqpyidxJ4xW/1z4fhz0a8YBtz/a3kpse/n7Zeyj9fseGxX0T7ririmXu6JbP1ThaAuGO3KaNUejgv7Du3aXnYVCvPOpt6ocbXnktHorKw40Kujb8My0L9M7POs00/pQlaOtOnLQr5EqHc7XsS/TmqMqR1t15KBfIz6c76+2HnW0tdyDUrfvszVBv24bxqqpjpfomXVqTNCv2x2I3gnVU9W6tVyP6qnM7daYoG/V5wBlvSw9bN600vtga3Yc9G3GHMStH247/63TSu9zW7PjoF8TDrCD0bZL/1yPBqNO32srgv50Dx+tPXzpn7VNI4J+3knc6R4+DktdR95pq6q25qqaL+ut7O3ViKBfV3kj7zSJd3Bm1eCgb7NStNXSph2cTV/RelR2K7kJHPRrwBXd+sHntgarLkNYFgr6klZJekjSmKR1PZbPlbQhLd8kaVGaP1/SnZKelvTx/mbdrD/aslOt6rmtpsgbwrIqcoO+pBHgauBkYBlwpqRlXcnOBp6IiCXAlcDlaf6zwB8CF/Qtx9PUlh+0DVbV61HV82eZKhxtFWnprwTGImJ7RDwH3ACs7kqzGlifpm8CTpSkiNgdEf9AFvytB/9YzdqjCkdbRYL+AuCxjtc70ryeaSJiHHgSmF80E5LOkbRF0pbx8fGib7OKyNtxecdmRbgeDUeRoN/rURcxgzSTiohrImJFRKwYHR0t+rZWcEU3q4/DD5pTdhZyFQn6O4AjOl4vBHZOlkbSKHAI8Hg/MljEZI+79WNwraim71ybXr6q2HTRSWVnIVeRoL8ZWCppsaQ5wBpgY1eajcDaNH0acEdEFG7pz9bYR9+xX4AfVfUegzuZyX6Qdfqhugzlq3v+wWUYBhWJzZJOAa4CRoDrIuIySZcCWyJio6QDgOuBY8la+GsiYnt676PAwcAc4MfA2yPi25Ota968ebF7dz0GODczqwpJz0RE7uVBhYL+MDnom5lNX9Gg7ztyzcxaxEHfzKxFHPTNzFrEQd/MrEUc9M3MWsRB38ysRRz0zcxaxEHfzKxFHPTNzFrEQd/MrEUc9M3MWsRB38ysRRz0zcxaxEHfzKxFHPTNzFrEQd/MrEUKBX1JqyQ9JGlM0roey+dK2pCWb5K0qGPZhWn+Q5J+rX9ZNzOz6coN+pJGgKuBk4FlwJmSlnUlOxt4IiKWAFcCl6f3LiMbU/doYBXwF+nzzMysBEVa+iuBsYjYHhHPATcAq7vSrAbWp+mbgBMlKc2/ISL2RMT3gLH0eWZmVoIiQX8B8FjH6x1pXs80ETEOPAnML/heJJ0jaYukLePj48Vzb2Zm01Ik6KvHvO7R1CdLU+S9RMQ1EbEiIlaMjo4WyJKZmc1EkaC/Azii4/VCYOdkaSSNAocAjxd8r5mZDUmRoL8ZWCppsaQ5ZCdmN3al2QisTdOnAXdERKT5a9LVPYuBpcA3+5N1MzObrty+lIgYl3QucCswAlwXEdskXQpsiYiNwLXA9ZLGyFr4a9J7t0m6Efg2MA58ICL2DqgsZmaWQ1mDvDrmzZsXu3fvLjsbZma1IumZiJiXl8535JqZtYiDvplZizjom5m1iIO+mVmLOOibmbWIg76ZWYs46JuZtYiDvplZizjom5m1iIO+mVmLOOibmbWIg76ZWYs46JuZtYiDvplZi1Tu0cqSngd+OouPGCV7dn9btK284DK3hcs8PQdGRG5DvnJBf7YkbYmIFWXnY1jaVl5wmdvCZR4Md++YmbWIg76ZWYs0MehfU3YGhqxt5QWXuS1c5gFoXJ++mZlNroktfTMzm4SDvplZizQm6EtaJekhSWOS1pWdn9mS9Kik+yVtlbQlzTtU0m2SHk7/X5rmS9Kfp7LfJ+m4js9Zm9I/LGltWeXpRdJ1knZJeqBjXt/KKOn16TscS+/VcEu4v0nK/CFJ/5q29VZJp3QsuzDl/yFJv9Yxv2d9l7RY0qb0XWyQNGd4pdufpCMk3SnpQUnbJH0wzW/sdp6izNXYzhFR+z9gBHgEeBUwB7gXWFZ2vmZZpkeBl3XN+2NgXZpeB1yepk8BvgQIOB7YlOYfCmxP/1+apl9adtk6yvMW4DjggUGUEfgm8Mb0ni8BJ1e0zB8CLuiRdlmqy3OBxamOj0xV34EbgTVp+i+B95dc3lcAx6Xpg4DvpnI1djtPUeZKbOemtPRXAmMRsT0ingNuAFaXnKdBWA2sT9PrgVM75n8qMv8MvETSK4BfA26LiMcj4gngNmDVsDM9mYj4GvB41+y+lDEtOzgivhHZL+NTHZ9VmknKPJnVwA0RsScivgeMkdX1nvU9tXDfBtyU3t/5/ZUiIr4fEd9K0z/54VXuAAAFs0lEQVQBHgQW0ODtPEWZJzPU7dyUoL8AeKzj9Q6m/pLrIICvSLpb0jlp3uER8X3IKhZwWJo/Wfnr+L30q4wL0nT3/Ko6N3VnXDfR1cH0yzwf+HFEjHfNrwRJi4BjgU20ZDt3lRkqsJ2bEvR79eHV/VrUN0XEccDJwAckvWWKtJOVv0nfy3TLWKey/y/gKGA58H3gz9L8xpRZ0ouBvwXOi4inpkraY15TylyJ7dyUoL8DOKLj9UJgZ0l56YuI2Jn+7wJuJjvU+0E6nCX935WST1b+On4v/SrjjjTdPb9yIuIHEbE3Ip4HPkG2rWH6Zf4hWXfIaNf8Ukl6IVnw+0xEfC7NbvR27lXmqmznpgT9zcDSdEZ7DrAG2FhynmZM0jxJB01MA28HHiAr08RVC2uBv0vTG4H3pisfjgeeTIfMtwJvl/TSdCj59jSvyvpSxrTsJ5KOT32g7+34rEqZCH7Ju8m2NWRlXiNprqTFwFKyk5Y963vq074TOC29v/P7K0X67q8FHoyIKzoWNXY7T1bmymznMs9y9/OP7Kz/d8nOdl9Udn5mWZZXkZ2pvxfYNlEesr6824GH0/9D03wBV6ey3w+s6Pis3yQ7MTQG/EbZZesq52fJDnN/RtaqObufZQRWpB/WI8DHSXegV7DM16cy3ZcCwCs60l+U8v8QHVelTFbfU935Zvou/g8wt+Tyvpms6+E+YGv6O6XJ23mKMldiO/sxDGZmLdKU7h0zMyvAQd/MrEUc9M3MWsRB38ysRRz0zcxaxEHfKkHS3vTkwXslfUvSL+ekf4mk3y7wuXdJmtFA05J+fybvK4Ok5Z1PbTSbjIO+VcVPI2J5RLwOuBD4aE76lwC5QX+WahP0yW7td9C3XA76VkUHA09A9vwSSben1v/9kiaenvox4Kh0dPAnKe3vpTT3SvpYx+edLumbkr4r6YTulUl6haSvpc96QNIJ6f0HpnmfSenekz5nq6S/kjSS5j8t6c9SHm+X9PIe6zhc0s0pb/dOHMlIOj+t8wFJ56V5i7Tv8/YvkPShNH2XpMs7y5Pu1rwUOCPl7YzZff3WaGXfoeg//0UEwF6yOxe/AzwJvD7NHyV7dC7Ay8juQBSwiH2fSX8y8E/Ai9LriTs87wL+LE2fAvzfHuv+HX5+1/MIcFCafrojzS8AXwBemF7/BfDeNB3AWWn6YuDjPdaxgezBWxPrOAR4PdkdmvOAF5PdfX1sj7JdAHxoqvIA/6XXev3nv+6/iQf2mJXtpxGxHEDSG4FPSfpFsgD/R8qeMvo82SNkD+/x/l8F/ndEPAMQEZ3PrJ94yNfdZAG122bguvSQrM9HxNYeaU4kC9Kbs0ercCA/f0jY82RBHeDTHevr9Day58IQEXuBJyW9Gbg5Inancn8OOIH850bllcdsUg76VjkR8Q1JLwNeTtaafTlZy/9nkh4FDujxNjH542X3pP976VHnI+JraafyDuB6SX8SEZ/q8fnrI+LCIkUokGbiM3sZZ9+u1+7yTlkes6m4T98qR9JryLpAfkTWDbIrBfxfAV6Zkv2EbCi6CV8BflPSi9JnHDqN9b0yreMTZE9HnBiX9Wep9Q/ZQ8FOk3TYxOen90H2O5p44uF/Av6hx2puB96f3jsi6WDga8Cpkl6k7Gmq7wa+DvwAOEzSfElzgXcWKEb392HWk1sJVhUHSproVhGwNiL2ppOoX1A2OPxEnz8R8SNJ/5hOeH4pIn5X0nJgi6TngFsofvXNW4HflfQz4GlSNwxwDXCfpG9FxFmS/oBsNLMXkD0l8wPA/wN2A0dLupvsfESvE6kfBK6RdDZZC/396Yjmr8melgjwyYi4B0DSpWSjLX1vosw57gTWpe/woxGxIe8N1k5+yqbZLEl6OiJeXHY+zIpw946ZWYu4pW9m1iJu6ZuZtYiDvplZizjom5m1iIO+mVmLOOibmbXI/we2Uoa6k1stqQAAAABJRU5ErkJggg==\n", 123 | "text/plain": [ 124 | "
" 125 | ] 126 | }, 127 | "metadata": {}, 128 | "output_type": "display_data" 129 | } 130 | ], 131 | "source": [ 132 | "%matplotlib inline\n", 133 | "import matplotlib.pyplot as plt\n", 134 | "\n", 135 | "\n", 136 | "fig = plt.figure()\n", 137 | "ax1 = fig.add_subplot(111)\n", 138 | "ax2 = ax1.twiny()\n", 139 | "\n", 140 | "\n", 141 | "ax1.scatter(range(len(collected_lr)), collected_lr)\n", 142 | "ax1.set_xlabel('Batch step count')\n", 143 | "\n", 144 | "ax2.scatter([lr / iterations_per_epoch \n", 145 | " for lr in range(len(collected_lr))], collected_lr)\n", 146 | "ax2.set_xlabel('Epoch count')\n", 147 | "\n", 148 | "plt.show()" 149 | ] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "metadata": {}, 154 | "source": [ 155 | "## Example: `triangular2` learning rate" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": 45, 161 | "metadata": {}, 162 | "outputs": [ 163 | { 164 | "data": { 165 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEjCAYAAADe/dHWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X2UXHWd5/H3x24TsJWnKBxPEkw0cTUoEzRGHMVFGTSgO4EZ2ITBlXE4sj5wVpZh3DA6POTICM5RGI/MjChZARkTFwV7xiiyPPg0GtJIeIiINjErbThwNAgS5KHDd/+4v9aiUtV1u7uqbtW9n9c5dfrWvb+q+v2qqj/3V7/7pIjAzMyq4TlFV8DMzLrHoW9mViEOfTOzCnHom5lViEPfzKxCHPpmZhXi0Le+JGm3pC01tzVtfO4Fku5u1/PNoB5LJR1bdD2sXAaLroDZNP0uIpYWXYkOWwosAzYWXRErD/f0rVQkbZd0kaRb021Rmv8SSTdKujP9PTjNP0jStZLuSLc/Tk81IOlzkrZK+pakvRu8VsPHSjpT0t3pdkaa96xfD5LOknRemr6lps4/lXSEpFnAWmBV+iWzqpPvm1WHQ9/61d51wzu1ofhoRCwHPgNckuZ9BrgyIg4FrgY+neZ/Gvh2RPwR8Bpga5q/GLg0Ig4BfgP8eYM67PFYSa8F3gO8HjgceK+kw3K0ZzDV+Qzg3Ih4CjgH2BARSyNiQ47nMGvJwzvWryYb3vlSzd+L0/QbgD9L01cBn0jTbwXeDRARu4FHJO0P/DwitqQytwELGrxOo8e+Cbg2InYBSPoqcAQw3KI9X23xWmZt4Z6+lVE0mW5WppEna6Z3k7+DpCbzx3n2/9teTV5vKq9lNmUOfSujVTV/f5Cm/wNYnaZPBr6Xpm8E3g8gaUDSPlN4nUaP/Q5wnKTnSRoCjge+CzwIHChpjqTZwDtzPP9vgRdMoT5mLTn0rV/Vj+lfWLNstqRNwIeA/5nm/Q/gPZLuBP5bWkb6+xZJd5ENrRwyhTrs8diI+BHwBeBWYBPw+Yi4PSKeJtswuwn4d+AnOZ7/ZmCJN+RaO8mnVrYykbQdWBYRvyq6Lma9yD19M7MKcU/fzKxC3NM3M6uQ0oS+pBWS7pU02s7zsPQSSeskPVR3ZOcBkm6Q9LP0d/8i69hukuZLulnSPeno2A+l+aVtt6S90tG5d6Q2n5/mL5S0KbV5QzpqtzTSHlC3S/r3dL/s7d0u6a60oX4kzev497oUoS9pALgUOAZYApwkaUmxteqILwAr6uatAW6MiMVkuxCWbYU3Dvx1RLyS7AjXD6bPtsztfhJ4azrSdymwQtLhwEXAxanNDwOnFljHTvgQcE/N/bK3F+At6YjrZel+x7/XpQh9YDkwGhHb0uHr64GVBdep7SLiO8DOutkrgSvS9BXAcV2tVIdFxANpN0gi4rdkoTCXErc7Mo+lu89NtyA7AviaNL9UbZY0D3gH8Pl0X5S4vZPo+Pe6LKE/F7i/5v5YmlcFB0XEA5AFJHBgwfXpGEkLgMPI9nUvdbvTUMcW4CHgBuA+4DcRMZ6KlO07fgnwYeCZdH8O5W4vZCvyb0m6TdJpaV7Hv9dlOdy70aHv3i2pRCQ9H/gKcEZEPJp1BMsrnctnqaT9gGuBVzYq1t1adYakdwIPRcRtko6cmN2gaCnaW+ONEbFD0oHADZLyHLA3Y2Xp6Y8B82vuzwN2FFSXbntQ0osB0t+HCq5P20l6LlngXx0REycmK327ASLiN8AtZNsz9pM00VEr03f8jcCfpgPr1pMN61xCedsLQETsSH8fIluxL6cL3+uyhP5mYHHa2j+L7Bwrrc5qWBbDwClp+hTgawXWpe3S2O7lwD0R8amaRaVtt6QXpR4+ys7j/ydk2zJuBk5IxUrT5og4OyLmRcQCsv/dmyLiZEraXgBJQ5JeMDENvA24my58r0tzcJayy8pdAgwA6yLigoKr1HaSvgQcCbyQ7ARe5wLXAV8GDgZ+AZwYEfUbe/tWOlXxd4G7+MN479+SjeuXst2SDiXbiDdA1jH7ckSslfRSsp7wAcDtwLsi4snmz9R/0vDOWRHxzjK3N7Xt2nR3EPjXiLhA0hw6/L0uTeibmVlrZRneMTOzHBz6ZmYV4tA3M6sQh76ZWYWULvRrjmyrhKq1F9zmqnCbO6N0oQ9U7YtStfaC21wVbnMHlDH0zcysiZ7bT/85z3lO7L333tN+/Pj4OIODZTmlUGtVay+4zVXhNk/N448/HhHRsiPfc+/o3nvvza5du4quhplZX5H0uzzlPLxjZlYhDn0zswpx6JuZVYhD38ysQhz6ZmYV4tA3M6sQh76ZWYU49M3MKsShb2ZWIQ59M7MKceibmVWIQ9/MrEIc+mZmFeLQNzOrEIe+mVmF5Ap9SSsk3StpVNKaBstnS9qQlm+StCDNP1nSlprbM5KWtrcJZmaWV8srZ0kaAH4KHA2MAZuBkyLixzVlPgAcGhHvk7QaOD4iVtU9z6uBr0XESyd7vaGhofBFVMzMpkbS4xEx1Kpcnp7+cmA0IrZFxFPAemBlXZmVwBVp+hrgKEmqK3MS8KUcr2dmZh2SJ/TnAvfX3B9L8xqWiYhx4BFgTl2ZVTQJfUmnSRqRNDI+Pp6n3mZmNg15Qr++xw5QPyY0aRlJrwcej4i7G71ARFwWEcsiYlnVLoRsZtZNeUJ/DJhfc38esKNZGUmDwL7Azprlq/HQjplZ4fKE/mZgsaSFkmaRBfhwXZlh4JQ0fQJwU6QtxJKeA5xIti3AzMwK1HIsJSLGJZ0OXA8MAOsiYquktcBIRAwDlwNXSRol6+GvrnmKNwNjEbGt/dU3M7OpaLnLZrd5l00zs6lr5y6bZmZWEg59M7MKceibmVWIQ9/MrEIc+mZmFeLQNzOrEIe+mVmFOPTNzCrEoW9mViEOfTOzCnHom5lViEPfzKxCHPpmZhXi0DczqxCHvplZhTj0zcwqJNdVyCWtAP6R7MpZn4+IC+uWzwauBF4L/BpYFRHb07JDgc8C+wDPAK+LiCfa1YBuOfTcb/Lok7t/f3+f2QPcef6KAmtkZjZ1LXv6kgaAS4FjgCXASZKW1BU7FXg4IhYBFwMXpccOAl8E3hcRhwBHAk+3rfZdUh/4AI8+uZtDz/1mQTUyM5uePMM7y4HRiNgWEU+RXeB8ZV2ZlcAVafoa4ChJAt4G3BkRdwBExK8jYjd9pj7wW803M+tVeUJ/LnB/zf2xNK9hmYgYBx4B5gAvB0LS9ZJ+JOnDjV5A0mmSRiSNjI+PT7UNHfXR6+4qugpmZm2TJ/TVYF791dSblRkE3gScnP4eL+moPQpGXBYRyyJi2eBgrs0MXfPFH/5i0uXX3f7LLtXEzGzm8oT+GDC/5v48YEezMmkcf19gZ5r/7Yj4VUQ8DmwEXjPTSveSMzZsKboKZma55Qn9zcBiSQslzQJWA8N1ZYaBU9L0CcBNERHA9cChkp6XVgb/Gfhxe6puZmZT1XIsJSLGJZ1OFuADwLqI2CppLTASEcPA5cBVkkbJevir02MflvQpshVHABsj4usdaouZmbWgrEPeO4aGhmLXrl1FV+P3FqxpvY7afuE7ulATM7PmJD0eEUOtyvmI3EnkCXwzs37i0G8DrxzMrF849M3MKsShb2ZWIQ59M7MKceg34XF6Mysjh36beCVhZv3AoW9mViEOfTOzCnHom5lViEO/AY/Pm1lZOfTbyCsLM+t1Dn0zswpx6JuZVYhD38ysQhz6dU7+3A+KroKZWcc49Ot8/76dky5vdcEUrzTMrJflCn1JKyTdK2lU0poGy2dL2pCWb5K0IM1fIOl3krak27+0t/q9p9VKw8ysSC2vkStpALgUOBoYAzZLGo6I2gucnwo8HBGLJK0GLgJWpWX3RcTSNtfbzMymIU9PfzkwGhHbIuIpYD2wsq7MSuCKNH0NcJQkta+aZmbWDnlCfy5wf839sTSvYZmIGAceAeakZQsl3S7p25KOaPQCkk6TNCJpZHx8fEoN6KZLVvkHi5n1tzyh36jHHjnLPAAcHBGHAWcC/yppnz0KRlwWEcsiYtngYMsRp45pdUTtcYdl6zqHv5n1qzyhPwbMr7k/D9jRrIykQWBfYGdEPBkRvwaIiNuA+4CXz7TSRZsI/2Z8OgYz61V5Qn8zsFjSQkmzgNXAcF2ZYeCUNH0CcFNEhKQXpQ3BSHopsBjY1p6qm5nZVLUcS4mIcUmnA9cDA8C6iNgqaS0wEhHDwOXAVZJGgZ1kKwaANwNrJY0Du4H3RYT3aTQzK0iuAfSI2AhsrJt3Ts30E8CJDR73FeArM6xjT1h84FDRVTAzmzEfkZu0Goe/4cwjn3XfKwEz60cO/WmqXwnU88ZcM+tFDn0zswpx6JuZVYhD38ysQhz6tB5/b3U6ZTOzfuHQn4FWKwNvzDWzXuPQNzOrEIe+mVmFOPTNzCqk8qF/9KdumXS5N+KaWZlUPvR/9tCuGT2+1Urh9RfcMKPnNzNrp8qHfqc9+Nuniq6CmdnvOfTNzCrEoW9mViGVDv3rbv/lpMu9EdfMyiZX6EtaIeleSaOS1jRYPlvShrR8k6QFdcsPlvSYpLPaU+32OGPDlrY8j1cOZtYvWoZ+usbtpcAxwBLgJElL6oqdCjwcEYuAi4GL6pZfDHxj5tXtTz4dg5n1ijw9/eXAaERsi4ingPXAyroyK4Er0vQ1wFGSBCDpOLKLoW9tT5XNzGy68oT+XOD+mvtjaV7DMhExDjwCzJE0BPwv4PzJXkDSaZJGJI2Mj4/nrbuZmU1RntBXg3mRs8z5wMUR8dhkLxARl0XEsohYNjiY61rtHffGlx1QdBXMzNouT8KOAfNr7s8DdjQpMyZpENgX2Am8HjhB0ieA/YBnJD0REZ+Zcc1nqNU4+9XvfcOUnm/xgUMzPrrXzKzT8vT0NwOLJS2UNAtYDQzXlRkGTknTJwA3ReaIiFgQEQuAS4C/74XA7wRfKN3M+kHLnn5EjEs6HbgeGADWRcRWSWuBkYgYBi4HrpI0StbDX93JSpuZ2fTkGkCPiI3Axrp559RMPwGc2OI5zptG/czMrI0qfUSumVnVVDL0fSF0M6uqSoZ+p/hC6WbW6xz6ZmYV0htHQlnHNfuV4aEss2pxT78CJhtW8pCTWbVULvRf8ZGNky53z9fMyqxyof/E7vrTBrVXP14o/aPX3VV0FcysSyoX+kXr9oXS8wzffPGHv+hCTcysFzj0zcwqxKFvgDfomlVFpUK/ahdCd5CbWb1KhX67LoTeSquVR6uVj5lZp1Qq9HtFN1Y+0+nl+5eBWfk59M3MKsShb8/i3r5ZuTn0kzJdCN3BbWbN5Ap9SSsk3StpVNKaBstnS9qQlm+StCDNXy5pS7rdIen49lY/v3ZfCL2Vdx1+cFufz8ysHVqGvqQB4FLgGGAJcJKkJXXFTgUejohFwMXARWn+3cCyiFgKrAA+K6kSZ/b82HGvnnR5p3rjh577zRk/h38pmJVXnp7+cmA0IrZFxFPAemBlXZmVwBVp+hrgKEmKiMcjYjzN3wvo7IlvjEef3F10Fcysh+UJ/bnA/TX3x9K8hmVSyD8CzAGQ9HpJW4G7gPfVrAR+T9JpkkYkjYyP77HY2izPQWg+lsCsnPKEvhrMq++xNy0TEZsi4hDgdcDZkvbao2DEZRGxLCKWDQ52f/RnsFHt+1CeYZm8Rx1360A2M+uuPKE/BsyvuT8P2NGsTBqz3xfYWVsgIu4BdgGvmm5lp6tVGI5+vDOnX9hroCRrEzMrjTyhvxlYLGmhpFnAamC4rswwcEqaPgG4KSIiPWYQQNJLgP8EbG9LzfvATy44dtLl3d5gWtvLz9Pj9wZds/JpOZYSEeOSTgeuBwaAdRGxVdJaYCQihoHLgaskjZL18Fenh78JWCPpaeAZ4AMR8atONKTqHNBmlkeuAfSI2AhsrJt3Ts30E8CJDR53FXDVDOtoHfKuww/2BVTMKsZH5JbAdDfgtjqWIO9zm1n/KH3otwqtsp1D38xsMqUP/aK1Wqks7EJPerJ9iPKs9Bad7d6+WVk49As200OU8wy//HyGv2bGfRy1WWk49M3MKsSh38fynCohz/CN99k3q45Sh/7Jn/vBpMv7fSOuT5VgZlNV6tD//n07WxfqglYrl1Yrp27o9xWgmeVT6tDvF9NZObXz5GrtfE0z620OfTOzCqnEVayq6JJVS6f8mO0XvqNlb37Bmq8XNhTUrG4emjLLr7I9/emEYq/IM8xy3GH117npb5O12cNOZvmVNvRbBUG3Q7FfVjL7zB4ougp7cKibtU9pQ7/XtFrJ5A22PKdtmMlwx53nr2hZppshnPe1vGIwy8eh32d8RoTmHPxmrTn0bQ95fil89Lq7Ol6P6YS4g99scrlCX9IKSfdKGpW0psHy2ZI2pOWbJC1I84+WdJuku9Lft7a3+tNz0AtmFV2FaSli3/xmOn3xlZmEdzfOXGrWr1qGvqQB4FLgGGAJcJKkJXXFTgUejohFwMXARWn+r4D/EhGvJruGbleuotUqMDZ95OhuVGMPvlB6PjPtrXsIzKy5PD395cBoRGyLiKeA9cDKujIrgSvS9DXAUZIUEbdHxI40fyuwl6TZ7ah4P+r0hdLb2csv6iRseU4il4eHecwayxP6c4H7a+6PpXkNy0TEOPAIMKeuzJ8Dt0fEk/UvIOk0SSOSRsbHx/PWvVKqEmLtPIlcVd4zs6nIE/qNxiTqf0FPWkbSIWRDPv+90QtExGURsSwilg0O+iDhXtHtI107EdIOfrNnyxP6Y8D8mvvzgB3NykgaBPYFdqb784BrgXdHxH0zrXAV9dIG3HrtCtWpPM/2C9/hUy+YTVOe0N8MLJa0UNIsYDUwXFdmmGxDLcAJwE0REZL2A74OnB0R329XpSfjC6H3n6kGfief36zsWoZ+GqM/HbgeuAf4ckRslbRW0p+mYpcDcySNAmcCE7t1ng4sAv5O0pZ0O7DtregjrUJrOgHVyV1Qe+mqWvV1mcoKwMFvlsm1n35EbIyIl0fEyyLigjTvnIgYTtNPRMSJEbEoIpZHxLY0/2MRMRQRS2tuD3WuOeWTJ6yK2gW1Hbq5x5KD38xH5FqB2jWs4yE7s/y8q0wPO/Tcb7Ys043A68R59js9jj/Z6/biRm+vuKxbStXTP/pTt0y6vN/+sR59cnfRVShc3s+sl4d5Fp399VwrzXYdmGY2mVKF/s8e2lV0FXLphwul12t3b7sTejH4F6z5OuM5zwtxxoYt3u5gHVeq0C+L79+3s6f3zW8mT507PazTK+/JgjWte/edeKxZKw5965qixvGb6VSwFnHAmlleDv0+VUSPtlv77M+0bUUN83Sih+5ev7VbZUK/V372V1W3g6vbwd/p9jn4rV1KE/r99k/RryuhN77sgCk/pqhhnak81ys+snFar9HNnrh7/dYO3k+/DxW5wrj6vW+Y0j77RY/jv+vwg3Nd5euJ3VO/9MpMAjjPsQ+TvW4R34F+3LnA9uTQt46Zyn7nnQqLjx336tyXdswbpu06dcRMg7/2uTrho9fdNeXLYta2xyuA3qSI3rq43NDQUOzaNfX97fvx7JrT+YcX8PMeaEurug+K3Pund6NN7frFMdPefbefd6p64RgKmx5Jj0fEUMtyVQj9xQcOccOZR86gVp0xnX+wXvnnaWc4dKtNMwn+bpwYrqjLZfbyRnbLr1Kh34+9fMiOvP3+fTun9JheaUu7gqJXr85VW69u98K78Xq9skG4l4/H6JX/tbwc+jV6+cMrekPnTPTShdzz6mbYzaR9nXhveyXom5np96Go6zr0iryh7w25VohLVi0t5HVnsvF0Kq/RrueY6R4+vR70tfJuBC66Tc1ev1dXBvXc0y9YP/f0oX9/OvfTBsuiQ86KMdXvUlt7+pJWAP8IDACfj4gL65bPBq4EXgv8GlgVEdslzQGuAV4HfCEiTp9SK+z3ig7JduqFtrS7F9zJNs20199OzdrZC3Urm04dj9Ey9CUNAJcCRwNjwGZJwxHx45pipwIPR8QiSauBi4BVwBPA3wGvSreu64WAsT8o4+fRrTYVNVyTp33t2vBtnZenp78cGJ247q2k9cBKoDb0VwLnpelrgM9IUkTsAr4naVH7qlwuef6Rixr/zmMqQdRrgT/TEC3ypHe9vF3CK4Delif05wL319wfA17frExEjEt6BJgD/CpPJSSdBpwGMGvWrDwPqZTjDptbdBVKa7rBX/QKrBO9/k60qVdXAO0+DqOf5Al9NZhXv/U3T5mmIuIy4DLINuTmfZz1hn1mD7S8tGPRITmZfv210o5efzfbU8QKYKaX2yzjyiBP6I8B82vuzwN2NCkzJmkQ2BeY2lFHM9DskP/BRquiPtNLIdPMneevqMRFv3u1HVPt9fdCO9q9AuhUm4pcGXSqTS132Uwh/lPgKOCXwGbgLyJia02ZDwKvjoj3pQ25fxYR/7Vm+V8Cy/LsvTPd0zAsOvvZ1yIdFIx+vPgvd16NvkS98M85Ff3ehn7f/xr6vw39vPs1TO+I73Zp6xG5ko4FLiHbZXNdRFwgaS0wEhHDkvYCrgIOI+vhr67Z8Lsd2AeYBfwGeFvdnj/PMt3QNzOrskqdhsHMrOryhn5prpxlZmatOfTNzCrEoW9mViEOfTOzCnHom5lViEPfzKxCHPpmZhXi0DczqxCHvplZhTj0zcwqxKFvZlYhDn0zswpx6JuZVYhD38ysQhz6ZmYV4tA3M6uQXKEvaYWkeyWNSlrTYPlsSRvS8k2SFtQsOzvNv1fS29tXdTMzm6qWoS9pALgUOAZYApwkaUldsVOBhyNiEXAxcFF67BJgNXAIsAL4p/R8ZmZWgDw9/eXAaERsi4ingPXAyroyK4Er0vQ1wFGSlOavj4gnI+LnwGh6PjMzK0Ce0J8L3F9zfyzNa1gmIsaBR4A5OR+LpNMkjUgaGR8fz197MzObkjyhrwbz6q+m3qxMnscSEZdFxLKIWDY4OJijSmZmNh15Qn8MmF9zfx6wo1kZSYPAvsDOnI81M7MuyRP6m4HFkhZKmkW2YXa4rswwcEqaPgG4KSIizV+d9u5ZCCwGbm1P1c3MbKpajqVExLik04HrgQFgXURslbQWGImIYeBy4CpJo2Q9/NXpsVslfRn4MTAOfDAidneoLWZm1oKyDnnvGBoail27dhVdDTOzviLp8YgYalXOR+SamVWIQ9/MrEIc+mZmFeLQNzOrEIe+mVmFOPTNzCrEoW9mViEOfTOzCnHom5lViEPfzKxCHPpmZhXi0DczqxCHvplZhTj0zcwqpOdOrSzpGeB3M3iKQbJz91dF1doLbnNVuM1Ts3dEtOzI91zoz5SkkYhYVnQ9uqVq7QW3uSrc5s7w8I6ZWYU49M3MKqSMoX9Z0RXosqq1F9zmqnCbO6B0Y/pmZtZcGXv6ZmbWhEPfzKxCShP6klZIulfSqKQ1RddnpiRtl3SXpC2SRtK8AyTdIOln6e/+ab4kfTq1/U5Jr6l5nlNS+Z9JOqWo9jQiaZ2khyTdXTOvbW2U9Nr0Ho6mx6q7LdxTkzafJ+mX6bPeIunYmmVnp/rfK+ntNfMbft8lLZS0Kb0XGyTN6l7r9iRpvqSbJd0jaaukD6X5pf2cJ2lzb3zOEdH3N2AAuA94KTALuANYUnS9Ztim7cAL6+Z9AliTptcAF6XpY4FvAAIOBzal+QcA29Lf/dP0/kW3raY9bwZeA9zdiTYCtwJvSI/5BnBMj7b5POCsBmWXpO/ybGBh+o4PTPZ9B74MrE7T/wK8v+D2vhh4TZp+AfDT1K7Sfs6TtLknPuey9PSXA6MRsS0ingLWAysLrlMnrASuSNNXAMfVzL8yMj8E9pP0YuDtwA0RsTMiHgZuAFZ0u9LNRMR3gJ11s9vSxrRsn4j4QWT/GVfWPFdhmrS5mZXA+oh4MiJ+DoySfdcbft9TD/etwDXp8bXvXyEi4oGI+FGa/i1wDzCXEn/Ok7S5ma5+zmUJ/bnA/TX3x5j8Te4HAXxL0m2STkvzDoqIByD7YgEHpvnN2t+P70u72jg3TdfP71Wnp+GMdRNDHUy9zXOA30TEeN38niBpAXAYsImKfM51bYYe+JzLEvqNxvD6fV/UN0bEa4BjgA9KevMkZZu1v0zvy1Tb2E9t/2fgZcBS4AHgk2l+ados6fnAV4AzIuLRyYo2mFeWNvfE51yW0B8D5tfcnwfsKKgubRERO9Lfh4BryX7qPZh+zpL+PpSKN2t/P74v7WrjWJqun99zIuLBiNgdEc8AnyP7rGHqbf4V2XDIYN38Qkl6Lln4XR0RX02zS/05N2pzr3zOZQn9zcDitEV7FrAaGC64TtMmaUjSCyamgbcBd5O1aWKvhVOAr6XpYeDdac+Hw4FH0k/m64G3Sdo//ZR8W5rXy9rSxrTst5IOT2Og7655rp4yEX7J8WSfNWRtXi1ptqSFwGKyjZYNv+9pTPtm4IT0+Nr3rxDpvb8cuCciPlWzqLSfc7M298znXORW7nbeyLb6/5Rsa/dHiq7PDNvyUrIt9XcAWyfaQzaWdyPws/T3gDRfwKWp7XcBy2qe66/INgyNAu8pum117fwS2c/cp8l6Nae2s43AsvSPdR/wGdIR6D3Y5qtSm+5MAfDimvIfSfW/l5q9Upp939N359b0XvwfYHbB7X0T2dDDncCWdDu2zJ/zJG3uic/Zp2EwM6uQsgzvmJlZDg59M7MKceibmVWIQ9/MrEIc+mZmFeLQt54gaXc68+Adkn4k6Y9blN9P0gdyPO8tkqZ1oWlJfzudxxVB0tLaszaaNePQt17xu4hYGhF/BJwNfLxF+f2AlqE/Q30T+mSH9jv0rSWHvvWifYCHITt/iaQbU+//LkkTZ0+9EHhZ+nXwD6nsh1OZOyRdWPN8J0q6VdJPJR1R/2KSXizpO+m57pZ0RHr83mne1ancu9LzbJH0WUkDaf5jkj6Z6nijpBc1eI2DJF2b6nbHxC8ZSWem17xb0hlp3gI9+3z7Z0k6L03fIumi2vakozXXAqtS3VbN7O23Uiv6CEXffIsIgN1kRy7+BHgEeG2aP0h26lyAF5IdgShgAc8+J/0xwH8Az0v3J47wvAX4ZJo+9CDnAAACYUlEQVQ+Fvi/DV77r/nDUc8DwAvS9GM1ZV4J/Bvw3HT/n4B3p+kATk7T5wCfafAaG8hOvDXxGvsCryU7QnMIeD7Z0deHNWjbWcB5k7UH+MtGr+ubb/W3iRP2mBXtdxGxFEDSG4ArJb2KLOD/XtlZRp8hO4XsQQ0e/yfA/46IxwEiovac9RMn+bqNLFDrbQbWpZNkXRcRWxqUOYospDdnp1Zhb/5wkrBnyEId4Is1r1frrWTnhSEidgOPSHoTcG1E7Ert/ipwBK3PG9WqPWZNOfSt50TEDyS9EHgRWW/2RWQ9/6clbQf2avAw0fz0sk+mv7tp8J2PiO+klco7gKsk/UNEXNng+a+IiLPzNCFHmYnnbGScZw+91rd30vaYTcZj+tZzJL2CbAjk12TDIA+lwH8L8JJU7Ldkl6Kb8C3gryQ9Lz3HAVN4vZek1/gc2dkRJ67L+nTq/UN2UrATJB048fzpcZD9H02c8fAvgO81eJkbgfenxw5I2gf4DnCcpOcpO5vq8cB3gQeBAyXNkTQbeGeOZtS/H2YNuZdgvWJvSRPDKgJOiYjdaSPqvym7OPzEmD8R8WtJ308bPL8REX8jaSkwIukpYCP59745EvgbSU8Dj5GGYYDLgDsl/SgiTpb0UbKrmT2H7CyZHwT+H7ALOETSbWTbIxptSP0QcJmkU8l66O9Pv2i+QHa2RIDPR8TtAJLWkl1t6ecTbW7hZmBNeg8/HhEbWj3Aqsln2TSbIUmPRcTzi66HWR4e3jEzqxD39M3MKsQ9fTOzCnHom5lViEPfzKxCHPpmZhXi0Dczq5D/D0piNszzHkfdAAAAAElFTkSuQmCC\n", 166 | "text/plain": [ 167 | "
" 168 | ] 169 | }, 170 | "metadata": {}, 171 | "output_type": "display_data" 172 | } 173 | ], 174 | "source": [ 175 | "collected_lr = []\n", 176 | "\n", 177 | "optimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n", 178 | "scheduler = CyclicalLearningRate(optimizer,\n", 179 | " step_size=step_size,\n", 180 | " max_lr=0.06,\n", 181 | " mode='triangular2')\n", 182 | "for epoch in range(num_epochs):\n", 183 | " for batch in range(iterations_per_epoch):\n", 184 | " # train(...)\n", 185 | " # validate(...)\n", 186 | " # note that the scheduler should be called\n", 187 | " # after each batch (not only after each epoch)\n", 188 | " scheduler.step()\n", 189 | " \n", 190 | " collected_lr.append(optimizer.param_groups[0]['lr'])\n", 191 | " \n", 192 | " \n", 193 | "fig = plt.figure()\n", 194 | "ax1 = fig.add_subplot(111)\n", 195 | "ax2 = ax1.twiny()\n", 196 | "\n", 197 | "ax1.scatter(range(len(collected_lr)), collected_lr)\n", 198 | "ax1.set_xlabel('Batch step count')\n", 199 | "\n", 200 | "ax2.scatter([lr / iterations_per_epoch \n", 201 | " for lr in range(len(collected_lr))], collected_lr)\n", 202 | "ax2.set_xlabel('Epoch count')\n", 203 | "\n", 204 | "plt.show()" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": {}, 210 | "source": [ 211 | "## Example: `exp_range` learning rate" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 50, 217 | "metadata": {}, 218 | "outputs": [ 219 | { 220 | "data": { 221 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEjCAYAAADe/dHWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+0XWV95/H3x1wT8CooUVwuQBNNnDa0NGgasYqjUmxAp8EpLMLomGlZZbSyRsbSTqjWH1mlSrsKrC7ptFGYIjoSB4WmJRYZftT+0EiQ8CMicomMxLjIUhAhCHjjd/7Yz5XDybn37HPPPmf/+rzWuuvus89zznmec/b57OfsH89WRGBmZu3wrLIrYGZm4+PQNzNrEYe+mVmLOPTNzFrEoW9m1iIOfTOzFnHoWy1J2i9pR8ffhgKfe4mku4p6viHqsVLSyWXXw5plouwKmM3TTyJiZdmVGLGVwCpga9kVseZwT98aRdL9ki6Q9PX0tyzNf5mkGyTdkf6/NM1/saSrJd2e/n4tPdUCSZ+UtFPSlyUd3OO1ej5W0vsl3ZX+zknznvHrQdK5kj6Spm/uqPO3JR0vaSGwETg9/ZI5fZTvm7WHQ9/q6uCuzTudofjjiFgNfAK4OM37BPDpiDgG+Czwl2n+XwL/FBG/ArwK2JnmLwcuiYijgR8Bv9WjDgc8VtKrgd8GXgMcB/yupGNztGci1fkc4MMR8RTwIWBzRKyMiM05nsOsL2/esbqaa/PO5zr+X5SmXwv8xzR9BfBnafrNwLsAImI/8IikFwDfiYgdqcytwJIer9Prsa8Hro6IfQCSvggcD2zp054v9nkts0K4p29NFLNMz1amlyc7pveTv4OkWeZP88zv20GzvN4gr2U2MIe+NdHpHf+/mqb/DViXpt8B/EuavgF4D4CkBZIOGeB1ej32K8Apkp4jaRJ4O/DPwIPA4ZIWS1oEvC3H8z8KPG+A+pj15dC3uurepv/xjvsWSdoGvA/472nefwN+W9IdwH9O95H+v0nSnWSbVo4eoA4HPDYivgH8LfB1YBvwqYi4LSJ+SrZjdhvwD8C3cjz/TcAK78i1IslDK1uTSLofWBURPyi7LmZV5J6+mVmLuKdvZtYi7umbmbVIY0Jf0hpJ90iaKnIcliqRdJmkvV1ndh4m6XpJ96b/LyizjkWTdJSkmyTdnc6OfV+a39h2SzoonZ17e2rzR9P8pZK2pTZvTmftNkY6Auo2Sf+Qbje9vfdLujPtqN+e5o18uW5E6EtaAFwCnASsAM6QtKLcWo3E3wJruuZtAG6IiOVkhxA2bYU3Dfx+RPwi2Rmu702fbZPb/STw5nSm70pgjaTjgAuAi1KbHwbOLLGOo/A+4O6O201vL8Cb0hnXq9LtkS/XjQh9YDUwFRG70unrVwJrS65T4SLiK8BDXbPXApen6cuBU8ZaqRGLiO+nwyCJiEfJQuEIGtzuyDyWbj47/QXZGcBXpfmNarOkI4G3Ap9Kt0WD2zuHkS/XTQn9I4AHOm7vTvPa4MUR8X3IAhI4vOT6jIykJcCxZMe6N7rdaVPHDmAvcD1wH/CjiJhORZq2jF8M/CHws3R7Mc1uL2Qr8i9LulXSWWneyJfrppzu3evUdx+W1CCSngt8ATgnIn6cdQSbK43ls1LS84GrgV/sVWy8tRoNSW8D9kbErZLeODO7R9FGtLfD6yJij6TDgesl5Tlhb2hN6envBo7quH0ksKekuozbg5JeApD+7y25PoWT9GyywP9sRMwMTNb4dgNExI+Am8n2Zzxf0kxHrUnL+OuA30wn1l1JtlnnYprbXgAiYk/6v5dsxb6aMSzXTQn9W4DlaW//QrIxVvqNatgUW4D1aXo98Hcl1qVwadvupcDdEXFhx12NbbekF6UePsrG8f91sn0ZNwGnpmKNaXNEnBcRR0bEErLv7o0R8Q4a2l4ASZOSnjczDbwFuIsxLNeNOTlL2WXlLgYWAJdFxPklV6lwkj4HvBF4IdkAXh8GrgE+D7wU+C5wWkR07+ytrTRU8T8Dd/L09t4/Ituu38h2SzqGbCfeArKO2ecjYqOkl5P1hA8DbgPeGRFPzv5M9ZM275wbEW9rcntT265ONyeA/x0R50tazIiX68aEvpmZ9deUzTtmZpaDQ9/MrEUc+mZmLeLQNzNrkcaFfseZba3QtvaC29wWbvNoNC70gbYtKG1rL7jNbeE2j0ATQ9/MzGZRueP0n/WsZ8XBBx8878dPT08zMdGUIYX6a1t7wW1uC7d5MI8//nhERN+OfOXe0YMPPph9+/aVXQ0zs1qR9JM85bx5x8ysRRz6ZmYtkiv0+11/VtKidA3LqXRNyyVp/jvS9R9n/n4maWWxTTAzs7z67shN15/9NnAi2bj1twBnRMQ3O8r8HnBMRLxb0jrg7RFxetfz/DLwdxHx8rleb3JyMrxN38xsMJIej4jJfuXy9PTzXH+287qOVwEn6MBLG50BfC7H65mZ2YjkCf0815/9eZl0TctHyK5x2el0HPpmZqXKc8hmnmtVzllG0muAxyPirp4vkJ16fBbAwoULc1TJzMzmI09PP8/1Z39eJl3T8lCg82ov65ijlx8RmyJiVUSsatvJGGZm45Qn9PNcf7bzuo6nkl3jMgAkPQs4jWxfgJmZlahvtzoipiWdDVzH09ef3SlpI7A9IraQXbj6CklTZD38dR1P8QZgd0TsKr76ZmY2iMqNveNDNs3MBlfkIZtmZtYQDn0zsxZx6JuZtYhD38ysRRz6ZmYt4tA3M2sRh76ZWYs49M3MWsShb2bWIg59M7MWceibmbWIQ9/MrEUc+mZmLeLQNzNrEYe+mVmLOPTNzFrEoW9m1iIOfTOzFnHom5m1SK7Ql7RG0j2SpiRt6HH/Ikmb0/3bJC3puO8YSV+VtFPSnZIOKq76ZmY2iL6hL2kBcAlwErACOEPSiq5iZwIPR8Qy4CLggvTYCeAzwLsj4mjgjcBPC6u9mZkNJE9PfzUwFRG7IuIp4EpgbVeZtcDlafoq4ARJAt4C3BERtwNExA8jYn8xVTczs0HlCf0jgAc6bu9O83qWiYhp4BFgMfBKICRdJ+kbkv6w1wtIOkvSdknbp6enB22DmZnlNJGjjHrMi5xlJoDXA78KPA7cIOnWiLjhGQUjNgGbACYnJ7uf28zMCpKnp78bOKrj9pHAntnKpO34hwIPpfn/FBE/iIjHga3Aq4attJmZzU+e0L8FWC5pqaSFwDpgS1eZLcD6NH0qcGNEBHAdcIyk56SVwb8HvllM1c3MbFB9N+9ExLSks8kCfAFwWUTslLQR2B4RW4BLgSskTZH18Nelxz4s6UKyFUcAWyPi2hG1xczM+lDWIa+OycnJ2LdvX9nVMDOrFUmPR8Rkv3I+I9fMrEUc+mZmLeLQNzNrEYe+mVmL5Dk5y4AlGw486Oj+j7+1hJqYmc2fe/o59Ar8ueabmVWVQ7+Pa277XtlVMDMrjEO/j3M27yi7CmZmhXHoD8mbeMysThz6ZmYt4tA3M2sRh/4cvOnGzJrGoW9m1iIO/QJ88Jo7y66CmVkuDv0CfOZr3y27CmZmuTj0zcxaxKE/C+/ENbMmcuibmbVIrtCXtEbSPZKmJG3ocf8iSZvT/dskLUnzl0j6iaQd6e+vi62+mZkNom/oS1oAXAKcBKwAzpC0oqvYmcDDEbEMuAi4oOO++yJiZfp7d0H1rhxvDjKzOsjT018NTEXEroh4CrgSWNtVZi1weZq+CjhBkoqr5nideOHNZVfBzGwk8oT+EcADHbd3p3k9y0TENPAIsDjdt1TSbZL+SdLxvV5A0lmStkvaPj09PVADRuHevfvKroKZ2UjkuXJWrx575CzzfeClEfFDSa8GrpF0dET8+BkFIzYBmwAmJye7n9vMzAqSp6e/Gziq4/aRwJ7ZykiaAA4FHoqIJyPihwARcStwH/DKYSttZmbzkyf0bwGWS1oqaSGwDtjSVWYLsD5NnwrcGBEh6UVpRzCSXg4sB3YVU/Xq8b4AM6u6vpt3ImJa0tnAdcAC4LKI2ClpI7A9IrYAlwJXSJoCHiJbMQC8AdgoaRrYD7w7Ih4aRUOKMsxRON4XYGZVl2ebPhGxFdjaNe9DHdNPAKf1eNwXgC8MWUcrwNIN1x6wIwbg/o+/dex1MbPy+IzcFpgt8MHnF5i1jUO/BXw4lJnNcOgPyJtDzKzOHPod+m3qyBP43lxiZlXm0G+4PCshr6jM2sOhb2bWIg59M7MWcejPwyGLFpRdBTOzeXHoJ4PsxL3jo2uGeq5xqUo9zKw6HPoGeAVh1hYOfTOzFnHom5m1iEOf/kMiLz98cjwVKZA315hZLw59+g+JfP3733jAvH5n577m/OuHqVIpvKIwaz6H/og8+OhTZVfBzOwADv0G+uA1d5ZdBTOrKId+A33ma98tuwpmVlG5rpzVZMvOG35kzSY58cKbe+7DqIJe+xza9vmYDStXT1/SGkn3SJqStKHH/YskbU73b5O0pOv+l0p6TNK5xVS7ONNDXGGkiYFT1ev8zraT2TufzQbTN/QlLQAuAU4CVgBnSFrRVexM4OGIWAZcBFzQdf9FwJeGr269lBFI/X65NJGD3yy/PD391cBUROyKiKeAK4G1XWXWApen6auAEyQJQNIpwC5gZzFVtrn0++XSxF8nZpZfntA/Anig4/buNK9nmYiYBh4BFkuaBP4H8NG5XkDSWZK2S9o+PT2dt+7WEu7JmxUnT+irx7zu/uRsZT4KXBQRj831AhGxKSJWRcSqiYnx7Vsu4vKIddSvXQ5Zs+bKE/q7gaM6bh8J7JmtjKQJ4FDgIeA1wJ9Juh84B/gjSWcPWedKed0rDiu7Cj/X5rBuc9vNBpEn9G8BlktaKmkhsA7Y0lVmC7A+TZ8K3BiZ4yNiSUQsAS4G/jQiPlFQ3Svhs7/72jnvr1IY1fGXS5XeP7Mm6LstJSKmU+/8OmABcFlE7JS0EdgeEVuAS4ErJE2R9fDXjbLSZmY2P7k2oEfEVmBr17wPdUw/AZzW5zk+Mo/6WUmWbLi2dr8MfuEDW/nW+SeXXQ2zSmvtMAxN24k76GaQOrRv0DY9sX+IM+3MWqK1oV+kXocuVU0dQt7MRs+hX4Dv+BBIM6sJh77N6prbvlfaa893RekVrNncHPoNMN+g67fJ55zNO+b1vGZWXa0cWrlpO3H7aVp76szDQ1vZ3NO3ymnqJhoPD21V4NAviMezqY4qvtdVrJO1k0O/5oYNk4tPXznS5x+FJm4OqeL7bM3k0G+4fgF5yrHdo2SXy+FnNlqtC/227cRtqyqtPKpUF7PWhb7V28xKuUpDWhfFKwcbB4d+gca9M3dcITGu1znxwptzl+03pHVVOMitahz6DZZ3U1VVNmndu3df2VUwazyHvtXGoCunOvay61hnq5dWhb534jZL1T8vB7hVUatCv0nGHSijfj0HpNl4OPQL1q/3uXRM4TZoL7jqveb51q+slck7PvnVeT+26ivAJRuuPeDP6iNX6EtaI+keSVOSNvS4f5Gkzen+bZKWpPmrJe1If7dLenux1a8fX9upHf71vofKrsJIePyg+usb+pIWAJcAJwErgDMkregqdibwcEQsAy4CLkjz7wJWRcRKYA3wN5JaObKnzW6YwKj6L5QmcbA3Q56e/mpgKiJ2RcRTwJXA2q4ya4HL0/RVwAmSFBGPR8R0mn8QJXZ0j/nwP855f53CY9l55Xz5BjmOvkh1uBxlpyLCsY4BW9ZyaYPJE/pHAA903N6d5vUsk0L+EWAxgKTXSNoJ3Am8u2Ml8HOSzpK0XdL26ekD7i7Ej5/cP5LnLcN0n1XnfFdg/R5X1nH0/S5H2U8VA7SJK7J+y6VVQ57Q77V8dn+8s5aJiG0RcTTwq8B5kg46oGDEpohYFRGrJibqv/WnKjtz66CIQK7TrzTI6jvsisxsvvKE/m7gqI7bRwJ7ZiuTttkfCjxjT1ZE3A3sA35pvpVtCneImqvIXxVV+YUySD2qUmebXZ7QvwVYLmmppIXAOmBLV5ktwPo0fSpwY0REeswEgKSXAf8OuL+QmrfUB6+5s+wqjFVRvfgqhtEhixaUXQVrob6hn7bBnw1cB9wNfD4idkraKOk3U7FLgcWSpoD3AzOHdb4euF3SDuBq4Pci4gdFN6KffsdM12nzwGe+9t057x+2Lb4C2PzleW863987Prqmb/lhjvcvwnw+by8j1ZZrA3pEbAW2ds37UMf0E8BpPR53BXDFkHUcWlOPma67IsPh/o+/tZFh42XXilb/vaYV1S+Elmy4tla/MMrQhvfnnce9tO+vt7I0cSUKvdvVhmVthodhsNYYZYgNumlnxp+c8st9H1fHo72qusLwGcUO/Vrpt2D2u8h5Xu887qVD1SOPUXzJmtpbK+NorzaF4Iy2tLn1oV9UUFZBURc5z9P7HLWmXQ5xrhVSU1dWVQvRqtWnLI0P/X4fdFFBacUa1eUQR/HFH0eYjDOwmhiOrzn/+lzlmtj2bo0P/TL58MfeRtnugxbUbYCDesr76+Sa27434prk8+CjT5Vdhcpw6NdEv6B88fMWFvp6E32ys6orrG+df/JYX6/fQH6QLyCrsolnkJ3Geep8zuYdw1SnEIMuq1Vdtovi0G+IbR84sdDnm/pYeSE06gAs8qzmcQ7kN44wyrPTuCorKJufVof+8sMny66ClaCqx8XXUZ4VQJk95/m+dpN7+40O/X4f3PXvf+N4KmI/N44v07h6ovM9Nr+IsqNQdHusmhod+lVQxM7cqvY6fKz9+FVxWeh3XgeUU+9hX7OK73URHPoNMKqgbHIAN+kXRxGG6eVX4byObk0N7CI49G1sxvlFHHXgjqotdVpRtEETVx4OfbOaKTqI8jxfv0OCq7RDN+/rtHUF29jQ7/fB1+UD/4UPbO1fqERFD488TqMMobosX3kVfUhw2dT1fy5N6+03NvSrZJiduU/sn/vI6VGHS9PCqwijDoE873lRnYFxr7RH/d7lff6ZaxTnvVZx2RezKZJD38aijN5Sk1dY/ToDRWra+9jdnjxDdzTpYjYOfauEsoJlFNccLqIt47h+7ijanueEx1F1AOb7vHmH7mjKZp5coS9pjaR7JE1J2tDj/kWSNqf7t0lakuafKOlWSXem/28utvpWBXX+Mgx6du642prn+rnD1iVP2wddgZV1wmPeFdhs7WnT2fl9Q1/SAuAS4CRgBXCGpBVdxc4EHo6IZcBFwAVp/g+A/xARvwysZ0zXy23KhdCrsjN62Ncpc6VQl8+6bYreRj7s0Bp5V1Z17uDMyNPTXw1MRcSuiHgKuBJY21VmLXB5mr4KOEGSIuK2iNiT5u8EDpK0qIiKz6WK2988zPLsih4hdFTGPUzBKFdYo2xLnscV+R0t6hDNJl1QaS55Qv8I4IGO27vTvJ5lImIaeARY3FXmt4DbIuLJ7heQdJak7ZK2T09P5627NUTZhwPWeYVb57pXTd4LKtX9Pc8T+r12bXcfOjBnGUlHk23y+a+9XiAiNkXEqohYNTExkaNKVjVV/iLUZTPYuIzjF8u4Dt8s+kSsunyGw8gT+ruBozpuHwnsma2MpAngUOChdPtI4GrgXRFx37AVbot+C/M4ju7oNN8vQ5VXBoMoqx1tCKFRG8Vmmzov13lC/xZguaSlkhYC64AtXWW2kO2oBTgVuDEiQtLzgWuB8yLiX4uq9DDyjAg4Kv22Xee9jifkO7qjDqoSanX+Eg9S96qNfzRMffI+dtDrYFdlmRyVvqGfttGfDVwH3A18PiJ2Stoo6TdTsUuBxZKmgPcDM4d1ng0sA/5Y0o70d3jhrejQb0Eoc0TAftuufR3P+mpSULgt+dS1o5DrOP2I2BoRr4yIV0TE+WnehyJiS5p+IiJOi4hlEbE6Inal+X8SEZMRsbLjb+/ommNl6v4SVOlLMcyXv+x2VGF4g1GaT92rMBRGkZZsuPaAv1HxGbkVVNVxPor+IlTtUM06B2ceZVwZq8xfDeN47VHujB7V8ujQr6B+xzA35ed32YdqFmUcn8frXnHYyF+jTMvOq95+ibqsNAbl0B8zn6RVnvl8iavyeXz2d1/bt8xcdS3z+rd5nne64PHjxtkxqtvF1xsV+sd8+B/nvL8pPeQy9RuRcGZBrkpYDmoUg5BZcca9XOXNjEF+qZStUaH/4yf3l12Fxss7ImE/VV0BDzqGyzjbMd/XqkIgjXNndBnL1qC/VMrsFDUq9NugqmFZF4O8f3X8tdKrznkCqQ7LVdVPkMtbv7KXK4e+Fa4KPUurnmF6+9fc9r3CXqMuRtUWh34J6r5g9ht7vOidcuNW5QtrDxqcZe7ALdI5m3eU+vpF9farsGy1JvTrsGDXxbAXyij7s2j6yU5VlmfMqO73ftgLpBSlzteV6NSY0K/KG2o2anmP2a9iL38+Y0bl2bme5zq34zJsFo36M2lM6LdB2T1ke1qZn8Wwx+zXwUz9T7zw5lzlizqqrJ9RjzY7juXKoV+S6vRL5meUV1Uah3EPTW1PG2QZuHfvvr5lyhw5dzZVXuk69EvynYqEX1vVfWjqIlaeVVkBzyZvcI575NxBd+pWqZcPLQn9ibp3q61Sqh6WdVDUe1j1z6JqgQ8NCf1+x4VPfazaC0YeVV+4rV68PA2nzu9fI0K/7seF19WgC37VvihVq8+g2l7/stvf73yVvMbdjkaEfl2VvdDa4JrwmVXp8MY6G/Z8FShn07ND36zG5rMSGtfhjXnU/SiwYetRxqbnXKEvaY2keyRNSdrQ4/5Fkjan+7dJWpLmL5Z0k6THJH2i2KpbnVS1b1mV8LD2KWvZ6xv6khYAlwAnASuAMySt6Cp2JvBwRCwDLgIuSPOfAP4YOLewGg+oCV/qKrchb92acIhqlT+HvKrYBu8bGq88Pf3VwFRE7IqIp4ArgbVdZdYCl6fpq4ATJCki9kXEv5CFv5mNQN1Cx8r9zPKE/hHAAx23d6d5PctExDTwCLA4byUknSVpu6Tt09PTeR/WCP7Clq8tn0EVz1ydkfczqOpnNUi9Lj595Qhr0l+e0O+1Obb7IMk8ZWYVEZsiYlVErJqYmMj7MKuJqn5RB1H1NuSp37jPXG2bvMvIKcd295nHK0/o7waO6rh9JLBntjKSJoBDgYeKqGDbVT1soB51tOrrtxyV3UMuQhW+K3lC/xZguaSlkhYC64AtXWW2AOvT9KnAjRExtlOmZnsjq/AG51H3+kP92zBXPd2G8ZmtnhefvrL0HnIedfgMlCebJZ0MXAwsAC6LiPMlbQS2R8QWSQcBVwDHkvXw10XErvTY+4FDgIXAj4C3RMQ3Z3utycnJ2Lev/8h6Zmb2NEmPR0Tf04Rzhf44OfTNzAaXN/R9Rq6ZWYs49M3MWsShb2bWIg59M7MWceibmbWIQ9/MrEUc+mZmLeLQNzNrEYe+mVmLOPTNzFrEoW9m1iIOfTOzFnHom5m1iEPfzKxFHPpmZi3i0DczaxGHvplZizj0zcxaJFfoS1oj6R5JU5I29Lh/kaTN6f5tkpZ03Hdemn+PpN8orupmZjaovqEvaQFwCXASsAI4Q9KKrmJnAg9HxDLgIuCC9NgVwDrgaGAN8Ffp+czMrAR5evqrgamI2BURTwFXAmu7yqwFLk/TVwEnSFKaf2VEPBkR3wGm0vOZmVkJ8oT+EcADHbd3p3k9y0TENPAIsDjnY83MbEwmcpRRj3mRs0yexyLpLOAsgIULF+aokpmZzUeenv5u4KiO20cCe2YrI2kCOBR4KOdjiYhNEbEqIlZNTORZD5mZ2XzkCf1bgOWSlkpaSLZjdktXmS3A+jR9KnBjRESavy4d3bMUWA58vZiqm5nZoPp2qyNiWtLZwHXAAuCyiNgpaSOwPSK2AJcCV0iaIuvhr0uP3Snp88A3gWngvRGxf0RtMTOzPpR1yKtjcnIy9u3bV3Y1zMxqRdLjETHZr5zPyDUzaxGHvplZizj0zcxaxKFvZtYiDn0zsxZx6JuZtYhD38ysRRz6ZmYt4tA3M2sRh76ZWYs49M3MWsShb2bWIg59M7MWceibmbWIQ9/MrEUqN56+pJ8BPxniKSbILtjSFm1rL7jNbeE2D+bgiOjbka9c6A9L0vaIWFV2Pcalbe0Ft7kt3ObR8OYdM7MWceibmbVIE0N/U9kVGLO2tRfc5rZwm0egcdv0zcxsdk3s6ZuZ2Swc+mZmLdKY0Je0RtI9kqYkbSi7PsOSdL+kOyXtkLQ9zTtM0vWS7k3/X5DmS9JfprbfIelVHc+zPpW/V9L6strTi6TLJO2VdFfHvMLaKOnV6T2cSo/VeFt4oFna/BFJ30uf9Q5JJ3fcd16q/z2SfqNjfs/lXdJSSdvSe7FZ0sLxte5Ako6SdJOkuyXtlPS+NL+xn/Mcba7G5xwRtf8DFgD3AS8HFgK3AyvKrteQbbofeGHXvD8DNqTpDcAFafpk4EuAgOOAbWn+YcCu9P8FafoFZbetoz1vAF4F3DWKNgJfB16bHvMl4KSKtvkjwLk9yq5Iy/IiYGlaxhfMtbwDnwfWpem/Bt5TcntfArwqTT8P+HZqV2M/5znaXInPuSk9/dXAVETsioingCuBtSXXaRTWApen6cuBUzrmfzoyXwOeL+klwG8A10fEQxHxMHA9sGbclZ5NRHwFeKhrdiFtTPcdEhFfjeyb8emO5yrNLG2ezVrgyoh4MiK+A0yRLes9l/fUw30zcFV6fOf7V4qI+H5EfCNNPwrcDRxBgz/nOdo8m7F+zk0J/SOABzpu72buN7kOAviypFslnZXmvTgivg/ZggUcnubP1v46vi9FtfGINN09v6rOTpszLpvZ1MHgbV4M/CgiprvmV4KkJcCxwDZa8jl3tRkq8Dk3JfR7bcOr+7Gor4uIVwEnAe+V9IY5ys7W/ia9L4O2sU5t/5/AK4CVwPeBv0jzG9NmSc8FvgCcExE/nqtoj3lNaXMlPuemhP5u4KiO20cCe0qqSyEiYk/6vxe4muyn3oPp5yzp/95UfLb21/F9KaqNu9N09/zKiYgHI2J/RPwM+CTZZw2Dt/kHZJtDJrrml0rSs8nC77MR8cU0u9Gfc682V+Vzbkro3wIsT3u0FwLrgC0l12neJE1Ket7MNPDJIAAjAAAElUlEQVQW4C6yNs0ctbAe+Ls0vQV4Vzry4TjgkfST+TrgLZJekH5KviXNq7JC2pjue1TScWkb6Ls6nqtSZsIveTvZZw1Zm9dJWiRpKbCcbKdlz+U9bdO+CTg1Pb7z/StFeu8vBe6OiAs77mrs5zxbmyvzOZe5l7vIP7K9/t8m29v9gbLrM2RbXk62p/52YOdMe8i25d0A3Jv+H5bmC7gktf1OYFXHc/0O2Y6hKeC3y25bVzs/R/Yz96dkvZozi2wjsCp9se4DPkE6A72Cbb4itemOFAAv6Sj/gVT/e+g4KmW25T0tO19P78X/ARaV3N7Xk216uAPYkf5ObvLnPEebK/E5exgGM7MWacrmHTMzy8Ghb2bWIg59M7MWceibmbWIQ9/MrEUc+lYJkvankQdvl/QNSb/Wp/zzJf1ejue9WdK8LjQt6Y/m87gySFrZOWqj2Wwc+lYVP4mIlRHxK8B5wMf6lH8+0Df0h1Sb0Cc7td+hb3059K2KDgEehmz8Ekk3pN7/nZJmRk/9OPCK9Ovgz1PZP0xlbpf08Y7nO03S1yV9W9Lx3S8m6SWSvpKe6y5Jx6fHH5zmfTaVe2d6nh2S/kbSgjT/MUl/kep4g6QX9XiNF0u6OtXt9plfMpLen17zLknnpHlL9Mzx9s+V9JE0fbOkCzrbk87W3Aicnup2+nBvvzVa2Wco+s9/EQGwn+zMxW8BjwCvTvMnyIbOBXgh2RmIApbwzDHpTwL+DXhOuj1zhufNwF+k6ZOB/9vjtX+fp896XgA8L00/1lHmF4G/B56dbv8V8K40HcA70vSHgE/0eI3NZANvzbzGocCryc7QnASeS3b29bE92nYu8JG52gP8l16v6z//df/NDNhjVrafRMRKAEmvBT4t6ZfIAv5PlY0y+jOyIWRf3OPxvw78r4h4HCAiOsesnxnk61ayQO12C3BZGiTrmojY0aPMCWQhfUs2tAoH8/QgYT8jC3WAz3S8Xqc3k40LQ0TsBx6R9Hrg6ojYl9r9ReB4+o8b1a89ZrNy6FvlRMRXJb0QeBFZb/ZFZD3/n0q6Hziox8PE7MPLPpn+76fHMh8RX0krlbcCV0j684j4dI/nvzwizsvThBxlZp6zl2meuem1u71ztsdsLt6mb5Uj6RfINoH8kGwzyN4U+G8CXpaKPUp2KboZXwZ+R9Jz0nMcNsDrvSy9xifJRkecuS7rT1PvH7JBwU6VdPjM86fHQfY9mhnx8D8B/9LjZW4A3pMeu0DSIcBXgFMkPUfZaKpvB/4ZeBA4XNJiSYuAt+VoRvf7YdaTewlWFQdLmtmsImB9ROxPO1H/XtnF4We2+RMRP5T0r2mH55ci4g8krQS2S3oK2Er+o2/eCPyBpJ8Cj5E2wwCbgDskfSMi3iHpg2RXM3sW2SiZ7wX+H7APOFrSrWT7I3rtSH0fsEnSmWQ99PekXzR/SzZaIsCnIuI2AEkbya629J2ZNvdxE7AhvYcfi4jN/R5g7eRRNs2GJOmxiHhu2fUwy8Obd8zMWsQ9fTOzFnFP38ysRRz6ZmYt4tA3M2sRh76ZWYs49M3MWuT/A9YErIScqn/WAAAAAElFTkSuQmCC\n", 222 | "text/plain": [ 223 | "
" 224 | ] 225 | }, 226 | "metadata": {}, 227 | "output_type": "display_data" 228 | } 229 | ], 230 | "source": [ 231 | "collected_lr = []\n", 232 | "\n", 233 | "optimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n", 234 | "scheduler = CyclicalLearningRate(optimizer,\n", 235 | " step_size=step_size,\n", 236 | " max_lr=0.06,\n", 237 | " mode='exp_range',\n", 238 | " gamma=0.99995)\n", 239 | "for epoch in range(num_epochs):\n", 240 | " for batch in range(iterations_per_epoch):\n", 241 | " # train(...)\n", 242 | " # validate(...)\n", 243 | " # note that the scheduler should be called\n", 244 | " # after each batch (not only after each epoch)\n", 245 | " scheduler.step()\n", 246 | " \n", 247 | " collected_lr.append(optimizer.param_groups[0]['lr'])\n", 248 | " \n", 249 | " \n", 250 | "fig = plt.figure()\n", 251 | "ax1 = fig.add_subplot(111)\n", 252 | "ax2 = ax1.twiny()\n", 253 | "\n", 254 | "ax1.scatter(range(len(collected_lr)), collected_lr)\n", 255 | "ax1.set_xlabel('Batch step count')\n", 256 | "\n", 257 | "ax2.scatter([lr / iterations_per_epoch \n", 258 | " for lr in range(len(collected_lr))], collected_lr)\n", 259 | "ax2.set_xlabel('Epoch count')\n", 260 | "\n", 261 | "plt.show()" 262 | ] 263 | } 264 | ], 265 | "metadata": { 266 | "kernelspec": { 267 | "display_name": "Python 3", 268 | "language": "python", 269 | "name": "python3" 270 | }, 271 | "language_info": { 272 | "codemirror_mode": { 273 | "name": "ipython", 274 | "version": 3 275 | }, 276 | "file_extension": ".py", 277 | "mimetype": "text/x-python", 278 | "name": "python", 279 | "nbconvert_exporter": "python", 280 | "pygments_lexer": "ipython3", 281 | "version": "3.6.6" 282 | } 283 | }, 284 | "nbformat": 4, 285 | "nbformat_minor": 2 286 | } 287 | --------------------------------------------------------------------------------