├── .autoenv.zsh
├── .github
└── workflows
│ └── ci-testing.yml
├── .gitignore
├── LICENSE
├── README.md
├── configs
├── data
│ ├── cifar.yaml
│ └── mnist.yaml
├── defaults.yaml
├── model
│ ├── autoencoder.yaml
│ └── classifier.yaml
└── optim
│ ├── adam.yaml
│ └── sgd.yaml
├── environment.yaml
├── main.py
├── project
├── __init__.py
├── data
│ ├── __init__.py
│ ├── cifar.py
│ └── mnist.py
└── model
│ ├── __init__.py
│ ├── autoencoder.py
│ ├── classifier.py
│ └── lit_image_classifier.py
├── setup.cfg
├── setup.py
└── tests
├── __init__.py
├── requirements.txt
└── test_classifier.py
/.autoenv.zsh:
--------------------------------------------------------------------------------
1 | # Install https://github.com/Tarrasch/zsh-autoenv
2 | # to automatically execute these command when cd into this project
3 | autostash HYDRA_FULL_ERROR=1
4 | conda activate project
5 |
--------------------------------------------------------------------------------
/.github/workflows/ci-testing.yml:
--------------------------------------------------------------------------------
1 | name: CI testing
2 |
3 | # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
4 | on:
5 | # Trigger the workflow on push or pull request, but only for the master branch
6 | push:
7 | branches:
8 | - master
9 | pull_request:
10 | branches:
11 | - master
12 |
13 | jobs:
14 | pytest:
15 |
16 | runs-on: ${{ matrix.os }}
17 | strategy:
18 | fail-fast: false
19 | matrix:
20 | os: [ubuntu-20.04, macOS-10.15, windows-2019]
21 | python-version: [3.7]
22 |
23 | # Timeout: https://stackoverflow.com/a/59076067/4521646
24 | timeout-minutes: 35
25 |
26 | steps:
27 | - uses: actions/checkout@v2
28 | - name: Set up Python ${{ matrix.python-version }}
29 | uses: actions/setup-python@v2
30 | with:
31 | python-version: ${{ matrix.python-version }}
32 |
33 | # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
34 | - name: Setup macOS
35 | if: runner.os == 'macOS'
36 | run: |
37 | brew install libomp # https://github.com/pytorch/pytorch/issues/20030
38 |
39 | # Note: This uses an internal pip API and may not always work
40 | # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
41 | - name: Get pip cache
42 | id: pip-cache
43 | run: |
44 | python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
45 |
46 | - name: Cache pip
47 | uses: actions/cache@v2
48 | with:
49 | path: ${{ steps.pip-cache.outputs.dir }}
50 | key: ${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
51 | restore-keys: |
52 | ${{ runner.os }}-py${{ matrix.python-version }}-
53 |
54 | - name: Install dependencies
55 | run: |
56 | pip install --requirement requirements.txt --upgrade --quiet --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --use-feature=2020-resolver
57 | pip install --requirement tests/requirements.txt --quiet --use-feature=2020-resolver
58 | python --version
59 | pip --version
60 | pip list
61 | shell: bash
62 |
63 | - name: Tests
64 | run: |
65 | coverage run --source project -m py.test project tests -v --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml
66 |
67 | - name: Statistics
68 | if: success()
69 | run: |
70 | coverage report
71 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | MANIFEST
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .coverage.*
43 | .cache
44 | nosetests.xml
45 | coverage.xml
46 | *.cover
47 | .hypothesis/
48 | .pytest_cache/
49 |
50 | # Translations
51 | *.mo
52 | *.pot
53 |
54 | # Django stuff:
55 | *.log
56 | local_settings.py
57 | db.sqlite3
58 |
59 | # Flask stuff:
60 | instance/
61 | .webassets-cache
62 |
63 | # Scrapy stuff:
64 | .scrapy
65 |
66 | # Sphinx documentation
67 | docs/_build/
68 |
69 | # PyBuilder
70 | target/
71 |
72 | # Jupyter Notebook
73 | .ipynb_checkpoints
74 |
75 | # pyenv
76 | .python-version
77 |
78 | # celery beat schedule file
79 | celerybeat-schedule
80 |
81 | # SageMath parsed files
82 | *.sage.py
83 |
84 | # Environments
85 | .env
86 | .venv
87 | env/
88 | venv/
89 | ENV/
90 | env.bak/
91 | venv.bak/
92 |
93 | # Spyder project settings
94 | .spyderproject
95 | .spyproject
96 |
97 | # Rope project settings
98 | .ropeproject
99 |
100 | # mkdocs documentation
101 | /site
102 |
103 | # mypy
104 | .mypy_cache/
105 |
106 | # IDEs
107 | .idea
108 | .vscode
109 |
110 | # Misc
111 | .DS_Store
112 | .tags
113 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Deep Learning project template
2 | Use this template to rapidly bootstrap a DL project:
3 |
4 | - Write code in [Pytorch Lightning](https://www.pytorchlightning.ai/)'s `LightningModule` and `LightningDataModule`.
5 | - Run code from composable `yaml` configurations with [Hydra](https://hydra.cc/).
6 | - Manage packages in `environment.yaml` with [conda](https://docs.conda.io/projects/conda/en/latest/glossary.html#miniconda-glossary).
7 | - Log and visualize metrics + hyperparameters with [Tensorboard](https://tensorboard.dev/).
8 | - Sane default with best/good practices only where it makes sense for small-scale research-style project.
9 |
10 | Have an issue, found a bug, know a better practice? Feel free to open an issue, pull request or discussion thread. All contribution welcome.
11 |
12 | I hope to maintaining this repo with better deep learning engineering practices as they evolve.
13 |
14 | ## Quick start
15 |
16 |
18 |
19 | ### 0. Clone this template
20 | ```bash
21 | # clone project or create a new one from GitHub's template
22 | git clone https://github.com/lkhphuc/lightning-hydra-template new-project
23 | cd new-project
24 | rm -rf .git
25 | git init # Start of a new git history
26 | ```
27 |
28 | ### 1. Add project's info
29 | - Edit [`setup.py`](setup.py) and add relevant information.
30 | - Rename the directory `project/` to the your project name.
31 |
32 | ### 2. Create environment and install dependencies
33 | - Name your environment and add packages in [`environment.yaml`](environment.yaml), then create/update the environment with:
34 | ```bash
35 | # Run this command every time you update environment.yaml
36 | conda env update -f environment.yaml
37 | ```
38 |
39 | ### 3. Create Pytorch Lightning modules
40 | - `LightningModule`s are organized under [`project/model/`](project/model/).
41 | - `LightningDataModule`s are organized under [`project/data/`](project/data/).
42 |
43 | Each Lightning module should be in one separate file, while each file can contain all the relevant `nn.Module`s for that model.
44 |
45 | ### 4. Create Hydra configs
46 | Each `.py` file has its own corresponding `.yaml` file, such as `project/model/autoencoder.py` and `configs/model/autoencoder.yaml`.
47 |
48 | All `yaml` files are stored under `configs/` and the structure of this folder should be identical to the structure of the `project/`.
49 | ```bash
50 | $ tree project $ tree configs
51 | project configs
52 | ├── __init__.py ├── defaults.yaml
53 | ├── data ├── data
54 | │ ├── cifar.py │ ├── cifar.yaml
55 | │ └── mnist.py │ └── mnist.yaml
56 | └── model ├── model
57 | ├── autoencoder.py │ ├── autoencoder.yaml
58 | ├── classifier.py │ └── classifier.yaml
59 | └── optim
60 | ├── adam.yaml
61 | └── sgd.yaml
62 | ```
63 | [`configs/defaults.yaml`](configs/defaults.yaml) contains all the defaults modules and arguments, including that for the `Trainer()`.
64 |
65 |
66 | ### 5. Run
67 | ```bash
68 | # This will run with all the default arguments
69 | python main.py
70 | # Override defaults from command line
71 | python main.py model=autoencoder data=cifar trainer.gpus=8
72 | ```
73 |
82 |
83 | ### Entry points
84 | The launching point of the project is [`main.py`](main.py) located in the root directory.
85 | The `main()` function takes in a `DictConfig` object, which is prepared by `hydra` based on the `yaml` files and command line arguments provided at runtime.
86 |
87 | This is achieved by decorating the script `main()` function with `hydra.main()`, which requires a path to all the configs and a default `.yaml` file as follow:
88 | ```python
89 | @hydra.main(config_path="configs", config_name="defaults")
90 | def main(cfg: DictConfig) -> None: ...
91 | ```
92 | This allow us to define multiple entry points for different functionalities with different defaults, such as `train.py`, `ensemble.py`, `test.py`, etc.
93 |
94 |
95 | ### Dynamically instantiate modules
96 | We will [use Hydra to instantiate objects](https://hydra.cc/docs/patterns/instantiate_objects/overview).
97 | This allow us to use the same entry point (`main.py` above) to dynamically combine different models and data modules.
98 | Given a [`configs/defaults.yaml`](configs/defaults.yaml) file contains:
99 | ```yaml
100 | defaults:
101 | - data: mnist # Path to sub-config, can also omit the .yaml extension
102 | - model: classifier.yaml # full path for ease of navigation (e.g vim cursor in path, press gf)
103 | ```
104 |
105 | Different modules can be instantiated for each run by supplying a different set of configuration:
106 | ```bash
107 | # Using default
108 | $ python main.py
109 |
110 | # The default is equivalent to
111 | $ python main.py model=classifier data=mnist
112 |
113 | # Override a default module
114 | $ python main.py model=autoencoder
115 | $ python main.py data=cifar
116 |
117 | # Override multiple default modules and arguments
118 | $ python main.py model=autoencoder data=cifar trainer.gpus=4
119 | ```
120 |
121 | In python, the module will be instantiated by a line, for example `data_module = hydra.utils.instantiate(cfg.data)`.
122 |
123 | `cfg.data` is a `DictConfig` object created by `hydra` at runtime, and is stored in a config file, for example [`configs/data/mnist.yaml`](configs/data/mnist.yaml):
124 | ```yaml
125 | name: mnist
126 |
127 | # _target_ class to instantiate
128 | _target_: project.data.MNISTDataModule
129 | # Argument to feed into __init__() of target module
130 | data_dir: ~/datasets/MNIST/ # Use absolute path
131 | batch_size: 4
132 | num_workers: 2
133 |
134 | # Can also define arbitrary info specific to this module
135 | input_dim: 784
136 | output_dim: 10
137 | ```
138 | and the _target_: `project.data.MNISTDataModule` to be instantiated is:
139 | ```python
140 | class MNISTDataModule(pl.LightningDataModule):
141 | def __init__(self, data_dir: str = "",
142 | batch_size: int = 32,
143 | num_workers: int = 8,
144 | **kwargs): ...
145 | # kwargs is used to handle arguments in the DictConfig but not used for init
146 | ```
147 |
148 | ### Directory management
149 | Since `hydra` manages our entry point and command line arguments, it also manages the output directory of each run.
150 | We can easily customize the output directory to suit our project via [`defaults.yaml`](configs/defaults.yaml)
151 | ```yaml
152 | hydra:
153 | run:
154 | # Configure output dir of each experiment programmatically from the arguments
155 | # Example "outputs/mnist/classifier/baseline/2021-03-10-141516"
156 | dir: outputs/${data.name}/${model.name}/${experiment}/${now:%Y-%m-%d_%H%M%S}
157 | ```
158 | and tell `TensorBoardLogger()` to use the current working directory without adding anything:
159 | ```python
160 | tensorboard = pl.loggers.TensorBoardLogger(".", "", "")
161 | ```
162 |
163 |
170 |
171 | ### `LightningModule` and `LightningDataModule`
172 | #### Be explicit about input arguments
173 | Each modules should be self-contained and self-explanatory, to maximize reusability, even across projects.
174 | - **Don't** do this:
175 | ```python
176 | class LitAutoEncoder(pl.LightningModule):
177 | def __init__(self, cfg, **kwargs):
178 | super().__init__()
179 | self.cfg = cfg
180 | ```
181 | You will not like it when having to track down the config file every time just to remember what are the input arguments, their types and default values.
182 |
183 | - Do this instead:
184 | ```python
185 | class LitAutoEncoder(pl.LightningModule):
186 | def __init__(self,
187 | input_dim: int, output_dim: int, hidden_dim: int = 64,
188 | optim_encoder=None, optim_decoder=None,
189 | **kwargs):
190 | super().__init__()
191 | self.save_hyperparameters()
192 | # Later all input arguments can be accessed anywhere by
193 | self.hparams.input_dim
194 | # Use this to avoid boilderplate code such as
195 | self.input_dim = input_dim
196 | self.output_dim = output_dim
197 | ```
198 |
199 |
200 | Also see Pytorch Lightning's [official style guide](https://pytorch-lightning.readthedocs.io/en/latest/starter/style_guide.html).
201 |
202 | ### Tensorboard
203 | - Use forward slash `/` in naming metrics to group it together.
204 | - Don't: `loss_val`, `loss_train`
205 | - Do: `loss/val`, `loss_train`
206 | - Group metrics by type, not on what data it was evaluate with:
207 | - Don't: `val/loss`, `val/accuracy`, `train/loss`, `train/acc`
208 | - Do: `loss/val`, `loss/train`, `accuracy/val`, `accuracy/train`
209 | 
210 | - Log computation graph of `LightningModule` by:
211 | - Define `self.example_input_array` in your module's `__init__()`
212 | - Enable in TensorBoard with `TensorBoard(log_graph=True)`
213 | 
214 | - [Proper loggin](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html#logging-hyperparameters) of hyper-parameters and metrics
215 | 
216 |
217 |
218 | ### Hydra
219 |
220 | #### Script is for one run, launcher is for multiple run
221 | Hydra serves two intertwined purposes, configuration management and script launcher.
222 | These two purposes are dealt with jointly because each run can potentially has a different set of configs.
223 |
224 | This provides a nice separation of concerns, in which the python scripts only focus on the functionalities of individual run, while the `hydra` command line will orchestrate multiple runs.
225 | With this separation, it's easy to use Hydra's [sweeper](https://hydra.cc/docs/plugins/ax_sweeper) to do hyperparameters search, or [launcher](https://hydra.cc/docs/plugins/submitit_launcher) to run experiments on SLURM cluster or cloud.
226 |
227 | #### Provide absolute path in config
228 | To provide path into program, it's best to provide an absolute path for both local or cloud storage (start with `~`, `/`, `s3://`).
229 |
230 | That way you don't have litter your code with `hydra.utils.get_original_cwd()` to convert relative path, and therefore retaining the flexibility to use your module outside of `hydra`-managed entry points.
231 |
232 | #### Naming experiments
233 | Use `hydra` to created a hierarchical structure for experiments output based on configurations of each run, by setting the `configs/defaults.yaml` with
234 | ```
235 | dir: outputs/${data.name}/${model.name}/${experiment}/${now:%Y-%m-%d_%H%M%S}
236 | ```
237 |
238 | - `${data.name}/${model.name}` will be dynamically determined from config object. They are preferably nested by the order of least frequently changed.
239 | - `${experiment}` is a string briefly describe the purpose of the experiment
240 | - `${now:%Y-%m-%d_%H%M%S}` will insert the time of run, serves as a unique identifier for runs differ only in minor hyperparameters such as learning rate.
241 |
242 | Example output:`outputs/mnist/classifier/baseline/2021-03-10-141516`.
243 |
244 |
245 |
252 |
253 | ### Debug
254 |
255 | - Drop into a debugger anywhere in your code with a single line `import pdb; pdb.set_trace()`.
256 | - Use `ipdb` or [pudb](github.com/inducer/pudb) for nicer debugging experience, for example `import pudb; pudb.set_trace()`
257 | - Or just use `breakpoint()` for Python 3.7 or above. Set `PYTHONBREAKPOINT` environment variable to make `breakpoint()` use `ipdb` or `pudb`, for example `PYTHONBREAKPOINT=pudb.set_trace`.
258 | - Post mortem debugging by running script with `ipython --pdb`. It opens a debugger and drop you right into when and where an Exception is raised.
259 | ```bash
260 | $ ipython --pdb main.py -- model=autoencoder
261 | ```
262 | This is super helpful to inspect the variables values when it fails, without having to put a breakpoint and then run the script again, which can takes a long time to start for deep learning model.
263 | - Use `fast_dev_run` of PytorchLightning, and checkout the entire [debugging tutorial](https://pytorch-lightning.readthedocs.io/en/stable/common/debugging.html).
264 |
265 | ### Colored Logs
266 |
267 | It's 2021 already, don't squint at your 4K HDR Quantum dot monitor to find a line from the black & white log.
268 | `pip install hydra-colorlog` and edit `defaults.yaml` to colorize your log file:
269 | ```yaml
270 | defaults:
271 | - override hydra/job_logging: colorlog
272 | - override hydra/hydra_logging: colorlog
273 | ```
274 | This will colorize any python logger you created anywhere with:
275 | ```python
276 | import logging
277 | logger = logging.getLogger(__name__)
278 | logger.info("My log")
279 | ```
280 |
281 | Alternative: [loguru](https://github.com/Delgan/loguru), [coloredlogs](https://github.com/xolox/python-coloredlogs).
282 |
283 | ### Auto activate conda environment and export variables
284 |
285 | [Zsh-autoenv](https://github.com/Tarrasch/zsh-autoenv) will auto source the content of `.autoenv.zsh` when you `cd` into a folder contains that file.
286 | Say goodbye to activate conda or export a bunch of variables for every new terminal:
287 | ```bash
288 | conda activate project
289 | HYDRA_FULL_ERROR=1
290 | PYTHON_BREAKPOINT=pudb.set_trace
291 | ```
292 |
293 | Alternative: https://github.com/direnv/direnv, https://github.com/cxreg/smartcd, https://github.com/kennethreitz/autoenv
294 |
295 | Click to expand/collapse
17 | Click to expand/collapse
81 | Click to expand/collapse
169 | Click to expand/collapse
251 |