├── .deepsource.toml ├── .flake8 ├── .gitattributes ├── .github └── workflows │ ├── ci.yml │ └── codesee-arch-diagram.yml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── conf ├── augmentation │ ├── augs2.yaml │ ├── basic_augs.yaml │ └── basic_augs1.yaml ├── callbacks │ └── callbacks.yaml ├── config.yaml ├── config_ner.yaml ├── datamodule │ ├── image_classification.yaml │ ├── melanoma_classification.yaml │ ├── mnist_image_classification.yaml │ └── ner.yaml ├── inference │ └── basic.yaml ├── logging │ ├── comet_ml.yaml │ ├── loggers.yaml │ └── wandb.yaml ├── loss │ ├── bce_logits.yaml │ ├── cross_entropy.yaml │ ├── cutmix.yaml │ ├── huber.yaml │ ├── mae.yaml │ ├── mixup.yaml │ ├── smoothl1loss.yaml │ └── ventilator.yaml ├── metric │ ├── accuracy.yaml │ ├── averageprecision.yaml │ ├── binary_accuracy.yaml │ ├── f1_score.yaml │ ├── f1_score_mine.yaml │ ├── mae.yaml │ ├── metric_manager.yaml │ ├── multiclass_accuracy.yaml │ ├── roc_auc.yaml │ └── ventilatormae.yaml ├── mnist_config.yaml ├── model │ ├── bilstm_crf_simple.yaml │ ├── complex_model.yaml │ ├── efficientnet_model.yaml │ └── simple_model.yaml ├── optimizer │ ├── adabelief.yaml │ ├── adabound.yaml │ ├── adadelta.yaml │ ├── adagrad.yaml │ ├── adam.yaml │ ├── adamax.yaml │ ├── adamw.yaml │ ├── adamwschedulefree.yaml │ ├── adan.yaml │ ├── asgd.yaml │ ├── lion.yaml │ ├── novograd.yaml │ ├── rangeradabelief.yaml │ ├── rmsprop.yaml │ ├── rprop.yaml │ ├── sgd.yaml │ └── sgdschedulefree.yaml ├── private │ └── default.yaml ├── scheduler │ ├── cosine.yaml │ ├── cosinewarm.yaml │ ├── cyclic.yaml │ ├── exponential.yaml │ ├── lambda.yaml │ ├── linearwithwarmup.yaml │ ├── multi_step_reg.yaml │ ├── onecycle.yaml │ ├── plateau.yaml │ └── step.yaml ├── trainer │ └── default_trainer.yaml └── training │ ├── default_training.yaml │ └── ner_training.yaml ├── contributing.md ├── docs ├── conf.py ├── index.rst └── requirements.txt ├── experiments_overview.py ├── mypy.ini ├── predict.py ├── pyproject.toml ├── requirements.txt ├── src ├── __init__.py ├── callbacks │ └── callbacks.py ├── datasets │ ├── __init__.py │ ├── collators.py │ ├── get_dataset.py │ ├── image_dataset.py │ ├── mnist_dataset.py │ └── text_dataset.py ├── lightning_classes │ ├── __init__.py │ ├── datamodule_imagenette.py │ ├── datamodule_melanoma.py │ ├── datamodule_mnist.py │ ├── datamodule_ner.py │ ├── lightning_image_classification.py │ └── lightning_ner.py ├── losses │ └── losses.py ├── metrics │ ├── f1_score.py │ └── ventilator_mae.py ├── models │ ├── decoders │ │ └── basic_decoder.py │ ├── encoders │ │ ├── basic_encoder.py │ │ ├── complex_backbone.py │ │ └── efficientnet_encoder.py │ ├── layers │ │ ├── activation_functions.py │ │ └── layers.py │ ├── pytorch_models_crf.py │ └── simple_model.py ├── optimizers │ └── adan.py ├── schedulers │ ├── cosine_schedule_with_warmup.py │ ├── cosine_schedule_with_warmup1.py │ ├── cosine_with_hard_restarts_schedule_with_warmup.py │ ├── lambdar.py │ └── linear_schedule_with_warmup.py └── utils │ ├── __init__.py │ ├── loggers.py │ ├── ml_utils.py │ ├── technical_utils.py │ ├── text_utils.py │ └── utils.py ├── tests ├── __init__.py ├── test_cfg.py ├── test_metrics.py ├── test_optimizers.py └── test_schedulers.py ├── train.py └── train_ner.py /.deepsource.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | test_patterns = ["*/tests/*"] 4 | 5 | [[analyzers]] 6 | name = "python" 7 | enabled = true 8 | 9 | [analyzers.meta] 10 | runtime_version = "3.x.x" 11 | 12 | max_line_length = 120 13 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = I001,I002,I004,I005,I101,I201,C101,C403,C901,F401,F403,S001,D100,D101,D102,D103,D104,D105,D106,D107,D200,D210,D205,D400,T001,W504,D202,E203,W503,B006,T002,T100,P103,C408,F841 3 | max-line-length = 120 4 | exclude = outputs/* 5 | max-complexity = 10 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | # some parts are taken from here: https://github.com/ternaus/iglovikov_helper_functions/blob/master/.github/workflows/ci.yml 3 | name: CI 4 | 5 | # Controls when the action will run. Triggers the workflow on push or pull request 6 | # events but only for the master branch 7 | on: 8 | push: 9 | branches: [ master ] 10 | pull_request: 11 | branches: [ master ] 12 | 13 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 14 | jobs: 15 | # This workflow contains a single job called "build" 16 | build: 17 | # The type of runner that the job will run on 18 | runs-on: ubuntu-latest 19 | 20 | # Steps represent a sequence of tasks that will be executed as part of the job 21 | steps: 22 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 23 | - uses: actions/checkout@v2 24 | - name: Set up Python 25 | uses: actions/setup-python@v2 26 | with: 27 | python-version: '3.9.15' 28 | - name: Cache pip 29 | uses: actions/cache@v1 30 | with: 31 | path: ~/.cache/pip # This path is specific to Ubuntu 32 | # Look to see if there is a cache hit for the corresponding requirements file 33 | key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} 34 | restore-keys: | 35 | ${{ runner.os }}-pip- 36 | ${{ runner.os }}- 37 | # You can test your matrix by printing the current Python version 38 | - name: Display Python version 39 | run: python -c "import sys; print(sys.version)" 40 | - name: Install dependencies 41 | run: | 42 | python -m pip install --upgrade pip 43 | pip install -r requirements.txt 44 | pip install black flake8 mypy pytest 45 | - name: Run black 46 | run: 47 | black --check . 48 | - name: Run flake8 49 | run: flake8 50 | - name: Run Mypy 51 | run: mypy --ignore-missing-imports --warn-no-return --warn-redundant-casts --disallow-incomplete-defs . 52 | - name: tests 53 | run: pytest 54 | -------------------------------------------------------------------------------- /.github/workflows/codesee-arch-diagram.yml: -------------------------------------------------------------------------------- 1 | # This workflow was added by CodeSee. Learn more at https://codesee.io/ 2 | # This is v2.0 of this workflow file 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request_target: 8 | types: [opened, synchronize, reopened] 9 | 10 | name: CodeSee 11 | 12 | permissions: read-all 13 | 14 | jobs: 15 | codesee: 16 | runs-on: ubuntu-latest 17 | continue-on-error: true 18 | name: Analyze the repo with CodeSee 19 | steps: 20 | - uses: Codesee-io/codesee-action@v2 21 | with: 22 | codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | .idea/ 6 | 7 | # private files 8 | conf/private/custom.yaml 9 | outputs/ 10 | notebooks/ 11 | subs/ 12 | data/ 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # pyenv 84 | .python-version 85 | 86 | # celery beat schedule file 87 | celerybeat-schedule 88 | 89 | # SageMath parsed files 90 | *.sage.py 91 | 92 | # Environments 93 | .env 94 | .venv 95 | env/ 96 | venv/ 97 | ENV/ 98 | env.bak/ 99 | venv.bak/ 100 | 101 | # Spyder project settings 102 | .spyderproject 103 | .spyproject 104 | 105 | # Rope project settings 106 | .ropeproject 107 | 108 | # mkdocs documentation 109 | /site 110 | 111 | # mypy 112 | .mypy_cache/ 113 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.3.0 4 | hooks: 5 | - id: check-yaml 6 | - id: end-of-file-fixer 7 | - id: trailing-whitespace 8 | - repo: https://github.com/psf/black 9 | rev: '24.8.0' 10 | hooks: 11 | - id: black 12 | args: [--config=pyproject.toml] 13 | - repo: https://github.com/pre-commit/mirrors-mypy 14 | rev: 26de894 15 | hooks: 16 | - id: mypy 17 | additional_dependencies: 18 | - 'numpy' 19 | args: [--ignore-missing-imports, --warn-no-return, --warn-redundant-casts, --disallow-incomplete-defs, --no-namespace-packages] 20 | - repo: https://github.com/PyCQA/flake8 21 | rev: '6.1.0' 22 | hooks: 23 | - id: flake8 24 | additional_dependencies: [ 25 | 'flake8-bugbear==23.9.16', 26 | 'flake8-coding==1.3.2', 27 | 'flake8-comprehensions==3.14.0', 28 | 'flake8-debugger==4.1.2', 29 | 'flake8-deprecated==2.1.0', 30 | 'flake8-docstrings==1.7.0', 31 | 'flake8-isort==6.1.0', 32 | 'flake8-pep3101==2.0.0', 33 | 'flake8-polyfill==1.0.2', 34 | 'flake8-print==5.0.0', 35 | 'flake8-quotes==3.3.2', 36 | 'flake8-string-format==0.3.0', 37 | ] 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Andrey Lukyanenko 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tempest 2 | 3 | [![DeepSource](https://static.deepsource.io/deepsource-badge-light-mini.svg)](https://deepsource.io/gh/Erlemar/pytorch_tempest/?ref=repository-badge) 4 | 5 | This repository has my pipeline for training neural nets. 6 | 7 | Main frameworks used: 8 | 9 | * [hydra](https://github.com/facebookresearch/hydra) 10 | * [pytorch-lightning](https://github.com/PyTorchLightning/pytorch-lightning) 11 | 12 | The main ideas of the pipeline: 13 | 14 | * all parameters and modules are defined in configs; 15 | * prepare configs beforehand for different optimizers/schedulers and so on, so it is easy to switch between them; 16 | * have templates for different deep learning tasks. Currently, image classification and named entity recognition are supported; 17 | 18 | Examples of running the pipeline: 19 | This will run training on MNIST (data will be downloaded): 20 | ```shell 21 | >>> python train.py --config-name mnist_config model.encoder.params.to_one_channel=True 22 | ``` 23 | 24 | Running on MPS (M1 macbook) 25 | ```shell 26 | python train.py --config-name mnist_config model.encoder.params.to_one_channel=True trainer.accelerator=mps +trainer.devices=1 optimizer=adan training.lr=0.001 27 | ``` 28 | Running on MPS (M1 macbook) with schedule free optimizer https://github.com/facebookresearch/schedule_free/tree/main 29 | ```shell 30 | python train.py --config-name mnist_config model.encoder.params.to_one_channel=True trainer.accelerator=mps trainer.devices=1 optimizer=adamwschedulefree training.lr=0.001 scheduler.params.patience=100 31 | 32 | ``` 33 | The default run: 34 | 35 | ```shell 36 | >>> python train.py 37 | ``` 38 | 39 | The default version of the pipeline is run on imagenette dataset. To do it, download the data from this repository: 40 | https://github.com/fastai/imagenette 41 | unzip it and define the path to it in conf/datamodule/image_classification.yaml path 42 | -------------------------------------------------------------------------------- /conf/augmentation/augs2.yaml: -------------------------------------------------------------------------------- 1 | 2 | train: 3 | augs: 4 | - class_name: albumentations.Resize 5 | params: 6 | height: 256 7 | width: 256 8 | - class_name: albumentations.RandomRotate90 9 | params: 10 | p: 0.5 11 | - class_name: albumentations.OneOf 12 | params: 13 | - class_name: albumentations.Rotate 14 | params: 15 | limit: 16 | - 0 17 | - 0 18 | p: 1.0 19 | - class_name: albumentations.Rotate 20 | params: 21 | limit: 22 | - 90 23 | - 90 24 | p: 1.0 25 | - class_name: albumentations.Rotate 26 | params: 27 | limit: 28 | - 180 29 | - 180 30 | p: 1.0 31 | - class_name: albumentations.Rotate 32 | params: 33 | limit: 34 | - 270 35 | - 270 36 | p: 1.0 37 | - class_name: albumentations.VerticalFlip 38 | params: 39 | p: 0.5 40 | - class_name: albumentations.Normalize 41 | params: 42 | p: 1.0 43 | - class_name: albumentations.pytorch.transforms.ToTensorV2 44 | params: 45 | p: 1.0 46 | valid: 47 | augs: 48 | - class_name: albumentations.Resize 49 | params: 50 | height: 256 51 | width: 256 52 | - class_name: albumentations.Normalize 53 | params: 54 | p: 1.0 55 | - class_name: albumentations.pytorch.transforms.ToTensorV2 56 | params: 57 | p: 1.0 58 | -------------------------------------------------------------------------------- /conf/augmentation/basic_augs.yaml: -------------------------------------------------------------------------------- 1 | 2 | train: 3 | augs: 4 | - class_name: albumentations.Flip 5 | params: 6 | p: 0.5 7 | - class_name: albumentations.ShiftScaleRotate 8 | params: 9 | shift_limit: 0.0625 10 | scale_limit: 0.1 11 | rotate_limit: 15 12 | - class_name: albumentations.Resize 13 | params: 14 | height: 128 15 | width: 128 16 | - class_name: albumentations.Normalize 17 | params: 18 | p: 1.0 19 | - class_name: albumentations.pytorch.transforms.ToTensorV2 20 | params: 21 | p: 1.0 22 | valid: 23 | augs: 24 | - class_name: albumentations.Resize 25 | params: 26 | height: 128 27 | width: 128 28 | - class_name: albumentations.Normalize 29 | params: 30 | p: 1.0 31 | - class_name: albumentations.pytorch.transforms.ToTensorV2 32 | params: 33 | p: 1.0 34 | -------------------------------------------------------------------------------- /conf/augmentation/basic_augs1.yaml: -------------------------------------------------------------------------------- 1 | 2 | train: 3 | augs: 4 | - class_name: albumentations.Flip 5 | params: 6 | p: 0.5 7 | - class_name: albumentations.ShiftScaleRotate 8 | params: 9 | shift_limit: 0.0625 10 | scale_limit: 0.1 11 | rotate_limit: 15 12 | - class_name: albumentations.Resize 13 | params: 14 | height: ${datamodule.main_image_size} 15 | width: ${datamodule.main_image_size} 16 | - class_name: albumentations.Normalize 17 | params: 18 | p: 1.0 19 | - class_name: albumentations.pytorch.transforms.ToTensorV2 20 | params: 21 | p: 1.0 22 | valid: 23 | augs: 24 | - class_name: albumentations.Resize 25 | params: 26 | height: ${datamodule.main_image_size} 27 | width: ${datamodule.main_image_size} 28 | - class_name: albumentations.Normalize 29 | params: 30 | p: 1.0 31 | - class_name: albumentations.pytorch.transforms.ToTensorV2 32 | params: 33 | p: 1.0 34 | -------------------------------------------------------------------------------- /conf/callbacks/callbacks.yaml: -------------------------------------------------------------------------------- 1 | 2 | early_stopping: 3 | class_name: pytorch_lightning.callbacks.EarlyStopping 4 | params: 5 | monitor: valid_${training.metric} 6 | patience: 5 7 | min_delta: 0.001 8 | verbose: True 9 | mode: ${training.mode} 10 | 11 | model_checkpoint: 12 | class_name: pytorch_lightning.callbacks.ModelCheckpoint 13 | params: 14 | monitor: valid_${training.metric} 15 | save_top_k: 3 16 | dirpath: saved_models 17 | filename: '{epoch}-{valid_${training.metric}:.6f}' 18 | mode: ${training.mode} 19 | 20 | other_callbacks: 21 | - class_name: pytorch_lightning.callbacks.LearningRateMonitor 22 | params: 23 | logging_interval: step 24 | - class_name: pytorch_lightning.callbacks.RichProgressBar 25 | params: 26 | leave: True 27 | - class_name: pytorch_lightning.callbacks.ModelSummary 28 | params: 29 | max_depth: 1 30 | -------------------------------------------------------------------------------- /conf/config.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - _self_ 3 | - augmentation: basic_augs1 4 | - callbacks: callbacks 5 | - datamodule: image_classification 6 | - inference: basic 7 | - logging: loggers 8 | - loss: cross_entropy 9 | - metric: accuracy 10 | - model: simple_model 11 | - optimizer: adamw 12 | - private: custom 13 | - scheduler: plateau 14 | - trainer: default_trainer 15 | - training: default_training 16 | - override hydra/hydra_logging: colorlog 17 | - override hydra/job_logging: colorlog 18 | 19 | general: 20 | logs_folder_name: logs 21 | workspace: erlemar 22 | project_name: dl_pipeline 23 | device: cuda 24 | log_code: True 25 | save_pytorch_model: True 26 | save_best: True 27 | convert_to_jit: False 28 | predict: True 29 | 30 | hydra: 31 | run: 32 | dir: outputs/${now:%Y-%m-%d_%H-%M-%S} 33 | sweep: 34 | dir: outputs/${now:%Y-%m-%d_%H-%M-%S} 35 | subdir: ${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.num}_${datamodule.fold_n} 36 | -------------------------------------------------------------------------------- /conf/config_ner.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - _self_ 3 | - datamodule: ner 4 | - trainer: default_trainer 5 | - training: ner_training 6 | - logging: loggers 7 | - optimizer: adamw 8 | - scheduler: plateau 9 | - model: bilstm_crf_simple 10 | - callbacks: callbacks 11 | - private: custom 12 | - metric: f1_score_mine 13 | - inference: basic 14 | - override hydra/hydra_logging: colorlog 15 | - override hydra/job_logging: colorlog 16 | 17 | general: 18 | logs_folder_name: logs 19 | workspace: erlemar 20 | project_name: dl_pipeline 21 | device: cuda 22 | log_code: True 23 | save_pytorch_model: True 24 | save_best: True 25 | 26 | hydra: 27 | run: 28 | dir: outputs/${now:%Y-%m-%d_%H-%M-%S} 29 | sweep: 30 | dir: outputs/${now:%Y-%m-%d_%H-%M-%S} 31 | subdir: ${hydra.job.override_dirname} 32 | -------------------------------------------------------------------------------- /conf/datamodule/image_classification.yaml: -------------------------------------------------------------------------------- 1 | path: f:/imagenette2-160 2 | n_folds: 3 3 | fold_n: 0 4 | class_name: src.datasets.image_dataset.ImageClassificationDataset 5 | data_module_name: src.lightning_classes.datamodule_imagenette.ImagenetteDataModule 6 | labels_to_ohe: False 7 | num_workers: 0 8 | batch_size: 256 9 | pin_memory: True 10 | main_image_size: 256 11 | collate_fn: 12 | mix_params: 13 | alpha: 0.4 14 | train_len: 1 15 | -------------------------------------------------------------------------------- /conf/datamodule/melanoma_classification.yaml: -------------------------------------------------------------------------------- 1 | 2 | path: f:/melanoma_merged/512x512-dataset-melanoma/512x512-dataset-melanoma/ 3 | n_folds: 3 4 | fold_n: 0 5 | class_name: src.datasets.image_dataset.ImageClassificationDataset 6 | data_module_name: src.lightning_classes.datamodule_melanoma.MelanomaDataModule 7 | train_path: f:/melanoma_merged/marking.csv 8 | labels_to_ohe: False 9 | num_workers: 0 10 | batch_size: 64 11 | pin_memory: True 12 | main_image_size: 16 13 | collate_fn: 14 | mix_params: 15 | alpha: 0.4 16 | -------------------------------------------------------------------------------- /conf/datamodule/mnist_image_classification.yaml: -------------------------------------------------------------------------------- 1 | 2 | data_module_name: src.lightning_classes.datamodule_mnist.MNISTDataModule 3 | -------------------------------------------------------------------------------- /conf/datamodule/ner.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: src.datasets.text_dataset.NerDataset 3 | params: 4 | use_bulio_tokens: True 5 | vectorizer_class_name: src.datasets.text_dataset.Vectorizer 6 | data_module_name: src.lightning_classes.datamodule_ner.NerDataModule 7 | pin_memory: False 8 | prepare_data: True 9 | folder_path: d:/DataScience/Python_projects/Current_projects/pytorch_tempest/data/ 10 | file_name: conll2003/train.txt 11 | tag_to_idx_from_labels: True 12 | word_to_idx_name: 13 | num_workers: 0 14 | batch_size: 512 15 | valid_size: 0.1 16 | embeddings_dim: 300 17 | embeddings_path: f:/crawl-300d-2M.pkl 18 | embeddings_type: fasttext 19 | -------------------------------------------------------------------------------- /conf/inference/basic.yaml: -------------------------------------------------------------------------------- 1 | 2 | run_name: "2020_05_16_15_43_39" 3 | mode: valid 4 | device: cpu 5 | -------------------------------------------------------------------------------- /conf/logging/comet_ml.yaml: -------------------------------------------------------------------------------- 1 | 2 | log: True 3 | loggers: 4 | - class_name: pytorch_lightning.loggers.CometLogger 5 | params: 6 | save_dir: ${general.logs_folder_name} 7 | workspace: ${general.workspace} 8 | project_name: ${general.project_name} 9 | api_key: ${private.comet_api} 10 | experiment_name: ${general.run_name} 11 | -------------------------------------------------------------------------------- /conf/logging/loggers.yaml: -------------------------------------------------------------------------------- 1 | 2 | log: True 3 | loggers: 4 | - class_name: pytorch_lightning.loggers.CSVLogger 5 | params: 6 | save_dir: logs/csv_log.csv 7 | # name: ${general.run_name} 8 | name: . 9 | # - class_name: src.utils.loggers.PrintLogger 10 | # params: 11 | # csv_path: 12 | 13 | - class_name: pytorch_lightning.loggers.TensorBoardLogger 14 | params: 15 | save_dir: . 16 | name: . 17 | 18 | # - class_name: pytorch_lightning.loggers.CometLogger 19 | # params: 20 | # save_dir: ${general.logs_folder_name} 21 | # workspace: ${general.workspace} 22 | # project_name: ${general.project_name} 23 | # api_key: ${private.comet_api} 24 | # experiment_name: ${general.run_name} 25 | -------------------------------------------------------------------------------- /conf/logging/wandb.yaml: -------------------------------------------------------------------------------- 1 | 2 | log: True 3 | loggers: 4 | - class_name: pytorch_lightning.loggers.WandbLogger 5 | params: 6 | save_dir: ${general.logs_folder_name} 7 | # workspace: ${general.workspace} 8 | project: ${general.project_name} 9 | # api_key: ${private.comet_api} 10 | name: ${general.run_name} 11 | -------------------------------------------------------------------------------- /conf/loss/bce_logits.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.nn.BCEWithLogitsLoss 3 | -------------------------------------------------------------------------------- /conf/loss/cross_entropy.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.nn.CrossEntropyLoss 3 | -------------------------------------------------------------------------------- /conf/loss/cutmix.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: src.losses.losses.MixupLoss 3 | -------------------------------------------------------------------------------- /conf/loss/huber.yaml: -------------------------------------------------------------------------------- 1 | class_name: torch.nn.HuberLoss 2 | params: 3 | delta: 1.0 4 | -------------------------------------------------------------------------------- /conf/loss/mae.yaml: -------------------------------------------------------------------------------- 1 | class_name: torch.nn.L1Loss 2 | -------------------------------------------------------------------------------- /conf/loss/mixup.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: src.losses.losses.MixupLoss 3 | -------------------------------------------------------------------------------- /conf/loss/smoothl1loss.yaml: -------------------------------------------------------------------------------- 1 | class_name: torch.nn.SmoothL1Loss 2 | params: 3 | beta: 0.1 4 | -------------------------------------------------------------------------------- /conf/loss/ventilator.yaml: -------------------------------------------------------------------------------- 1 | class_name: src.losses.losses.VentilatorLoss 2 | -------------------------------------------------------------------------------- /conf/metric/accuracy.yaml: -------------------------------------------------------------------------------- 1 | metric: 2 | class_name: torchmetrics.Accuracy 3 | metric_name: accuracy 4 | params: 5 | task: MULTICLASS 6 | threshold: 0.5 7 | num_classes: ${training.n_classes} 8 | top_k: 1 9 | -------------------------------------------------------------------------------- /conf/metric/averageprecision.yaml: -------------------------------------------------------------------------------- 1 | metric: 2 | class_name: torchmetrics.AveragePrecision 3 | metric_name: averageprecision 4 | params: 5 | num_classes: ${training.n_classes} 6 | -------------------------------------------------------------------------------- /conf/metric/binary_accuracy.yaml: -------------------------------------------------------------------------------- 1 | metric: 2 | class_name: torchmetrics.classification.BinaryAccuracy 3 | metric_name: accuracy 4 | params: 5 | threshold: 0.5 6 | -------------------------------------------------------------------------------- /conf/metric/f1_score.yaml: -------------------------------------------------------------------------------- 1 | metric: 2 | class_name: torchmetrics.F1 3 | metric_name: f1_score 4 | params: 5 | average: weighted 6 | num_classes: ${training.n_classes} 7 | -------------------------------------------------------------------------------- /conf/metric/f1_score_mine.yaml: -------------------------------------------------------------------------------- 1 | metric: 2 | class_name: src.metrics.f1_score.F1Score 3 | metric_name: f1_score 4 | params: 5 | average: weighted 6 | -------------------------------------------------------------------------------- /conf/metric/mae.yaml: -------------------------------------------------------------------------------- 1 | metric: 2 | class_name: torch.nn.L1Loss 3 | metric_name: mae 4 | -------------------------------------------------------------------------------- /conf/metric/metric_manager.yaml: -------------------------------------------------------------------------------- 1 | 2 | metric: 3 | class_name: torchmetrics.Accuracy 4 | metric_name: accuracy 5 | params: 6 | threshold: 0.5 7 | num_classes: ${training.n_classes} 8 | top_k: 1 9 | 10 | other_metrics: 11 | - class_name: torchmetrics.Accuracy 12 | metric_name: accuracy_07 13 | params: 14 | threshold: 0.7 15 | num_classes: ${training.n_classes} 16 | top_k: 2 17 | 18 | - class_name: torchmetrics.F1 19 | metric_name: f1_score 20 | params: 21 | average: weighted 22 | num_classes: ${training.n_classes} 23 | -------------------------------------------------------------------------------- /conf/metric/multiclass_accuracy.yaml: -------------------------------------------------------------------------------- 1 | metric: 2 | class_name: torchmetrics.classification.MulticlassAccuracy 3 | metric_name: accuracy 4 | params: 5 | num_classes: ${training.n_classes} 6 | top_k: 1 7 | -------------------------------------------------------------------------------- /conf/metric/roc_auc.yaml: -------------------------------------------------------------------------------- 1 | metric: 2 | class_name: torchmetrics.AUROC 3 | metric_name: roc_auc 4 | params: 5 | num_classes: ${training.n_classes} 6 | -------------------------------------------------------------------------------- /conf/metric/ventilatormae.yaml: -------------------------------------------------------------------------------- 1 | metric: 2 | class_name: src.metrics.ventilator_mae.VentilatorMAE 3 | metric_name: ventilator_mae 4 | -------------------------------------------------------------------------------- /conf/mnist_config.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - _self_ 3 | - datamodule: mnist_image_classification 4 | - trainer: default_trainer 5 | - training: default_training 6 | - logging: loggers 7 | - optimizer: adamw 8 | - scheduler: plateau 9 | - model: simple_model 10 | - callbacks: callbacks 11 | - private: default 12 | - augmentation: basic_augs 13 | - loss: cross_entropy 14 | - metric: accuracy 15 | - inference: basic 16 | - override hydra/hydra_logging: colorlog 17 | - override hydra/job_logging: colorlog 18 | 19 | general: 20 | logs_folder_name: logs 21 | workspace: erlemar 22 | project_name: dl_pipeline 23 | device: cuda 24 | log_code: True 25 | save_pytorch_model: True 26 | save_best: True 27 | convert_to_jit: False 28 | 29 | hydra: 30 | run: 31 | dir: outputs/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.override_dirname} 32 | sweep: 33 | dir: outputs/${now:%Y-%m-%d_%H-%M-%S} 34 | subdir: ${hydra.job.override_dirname} 35 | -------------------------------------------------------------------------------- /conf/model/bilstm_crf_simple.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: src.models.pytorch_models_crf.BiLSTMCRF 3 | params: 4 | hidden_dim: 128 5 | spatial_dropout: 0.2 6 | -------------------------------------------------------------------------------- /conf/model/complex_model.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: src.models.simple_model.Net 3 | params: 4 | encoder: 5 | class_name: src.models.encoders.complex_backbone.BackboneModeI 6 | params: 7 | pretrained: imagenet 8 | arch: resnet50 9 | advdrop: False 10 | return_only_last_output: True 11 | # source: timm 12 | # n_layers: -2 13 | # freeze: False 14 | # to_one_channel: False 15 | # freeze_until_layer: 16 | decoder: 17 | class_name: src.models.decoders.basic_decoder.BasicDecoder 18 | params: 19 | pool_output_size: 1 20 | n_classes: ${training.n_classes} 21 | -------------------------------------------------------------------------------- /conf/model/efficientnet_model.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: src.models.simple_model.Net 3 | params: 4 | encoder: 5 | class_name: src.models.encoders.efficientnet_encoder.EfficientNetEncoder 6 | params: 7 | arch: efficientnet-b1 8 | freeze: True 9 | freeze_until_layer: 10 | decoder: 11 | class_name: src.models.decoders.basic_decoder.BasicDecoder 12 | params: 13 | pool_output_size: 2 14 | n_classes: 10 15 | -------------------------------------------------------------------------------- /conf/model/simple_model.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: src.models.simple_model.Net 3 | params: 4 | encoder: 5 | class_name: src.models.encoders.basic_encoder.BasicEncoder 6 | params: 7 | pretrained: imagenet 8 | arch: resnet50 9 | source: timm 10 | n_layers: -2 11 | freeze: False 12 | to_one_channel: False 13 | freeze_until_layer: 14 | decoder: 15 | class_name: src.models.decoders.basic_decoder.BasicDecoder 16 | params: 17 | pool_output_size: 1 18 | n_classes: ${training.n_classes} 19 | -------------------------------------------------------------------------------- /conf/optimizer/adabelief.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: adabelief_pytorch.AdaBelief 3 | params: 4 | lr: ${training.lr} 5 | eps: 1e-16 6 | weight_decouple: True 7 | rectify: True 8 | -------------------------------------------------------------------------------- /conf/optimizer/adabound.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: adabound.AdaBound 3 | params: 4 | lr: ${training.lr} 5 | final_lr: 0.1 6 | -------------------------------------------------------------------------------- /conf/optimizer/adadelta.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.Adadelta 3 | params: 4 | lr: ${training.lr} 5 | -------------------------------------------------------------------------------- /conf/optimizer/adagrad.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.Adagrad 3 | params: 4 | lr: ${training.lr} 5 | -------------------------------------------------------------------------------- /conf/optimizer/adam.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.Adam 3 | params: 4 | lr: ${training.lr} 5 | -------------------------------------------------------------------------------- /conf/optimizer/adamax.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.Adamax 3 | params: 4 | lr: ${training.lr} 5 | -------------------------------------------------------------------------------- /conf/optimizer/adamw.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.AdamW 3 | params: 4 | lr: ${training.lr} 5 | # decoder_lr: 0.01 6 | weight_decay: 0.001 7 | -------------------------------------------------------------------------------- /conf/optimizer/adamwschedulefree.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: schedulefree.AdamWScheduleFree 3 | params: 4 | lr: ${training.lr} 5 | weight_decay: 0 6 | warmup_steps: 0 7 | r: 0 8 | -------------------------------------------------------------------------------- /conf/optimizer/adan.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: src.optimizers.adan.Adan 3 | params: 4 | lr: ${training.lr} 5 | # decoder_lr: 0.01 6 | weight_decay: 0.02 7 | -------------------------------------------------------------------------------- /conf/optimizer/asgd.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.ASGD 3 | params: 4 | lr: ${training.lr} 5 | -------------------------------------------------------------------------------- /conf/optimizer/lion.yaml: -------------------------------------------------------------------------------- 1 | class_name: lion_pytorch.Lion 2 | params: 3 | lr: ${training.lr} 4 | weight_decay: 0.0001 5 | betas: [0.9, 0.99] 6 | -------------------------------------------------------------------------------- /conf/optimizer/novograd.yaml: -------------------------------------------------------------------------------- 1 | class_name: torch_optimizer.NovoGrad 2 | params: 3 | lr: ${training.lr} 4 | eps: 1e-8 5 | weight_decay: 0 6 | grad_averaging: False 7 | amsgrad: False 8 | -------------------------------------------------------------------------------- /conf/optimizer/rangeradabelief.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: ranger_adabelief.RangerAdaBelief 3 | params: 4 | lr: ${training.lr} 5 | eps: 1e-16 6 | -------------------------------------------------------------------------------- /conf/optimizer/rmsprop.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.RMSprop 3 | params: 4 | lr: ${training.lr} 5 | -------------------------------------------------------------------------------- /conf/optimizer/rprop.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.Rprop 3 | params: 4 | lr: ${training.lr} 5 | -------------------------------------------------------------------------------- /conf/optimizer/sgd.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.SGD 3 | params: 4 | lr: ${training.lr} 5 | momentum: 0.9 6 | weight_decay: 0.0005 7 | -------------------------------------------------------------------------------- /conf/optimizer/sgdschedulefree.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: schedulefree.SGDScheduleFree 3 | params: 4 | lr: ${training.lr} 5 | weight_decay: 0 6 | warmup_steps: 0 7 | r: 0 8 | momentum: 0.9 9 | weight_lr_power: 2 10 | -------------------------------------------------------------------------------- /conf/private/default.yaml: -------------------------------------------------------------------------------- 1 | 2 | some_api_key: 3 | -------------------------------------------------------------------------------- /conf/scheduler/cosine.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.lr_scheduler.CosineAnnealingLR 3 | step: epoch 4 | monitor: train_${training.metric} 5 | params: 6 | T_max: 10 7 | -------------------------------------------------------------------------------- /conf/scheduler/cosinewarm.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.lr_scheduler.CosineAnnealingWarmRestarts 3 | step: step 4 | monitor: train_${training.metric} 5 | params: 6 | T_0: 150 7 | T_mult: 3 8 | eta_min: 0 9 | -------------------------------------------------------------------------------- /conf/scheduler/cyclic.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.lr_scheduler.CyclicLR 3 | step: step 4 | monitor: train_${training.metric} 5 | params: 6 | base_lr: ${training.lr} 7 | max_lr: 0.1 8 | -------------------------------------------------------------------------------- /conf/scheduler/exponential.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.lr_scheduler.ExponentialLR 3 | step: epoch 4 | monitor: train_${training.metric} 5 | params: 6 | gamma: 0.2 7 | -------------------------------------------------------------------------------- /conf/scheduler/lambda.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: src.schedulers.lambdar.LambdaLRConfig 3 | step: epoch 4 | monitor: train_${training.metric} 5 | params: 6 | lr_lambda: 'lambda epoch: epoch // 30' 7 | # lr_lambda: 'lambda epoch: 1. / (1. + 0.05 * epoch)' 8 | last_epoch: -1 9 | -------------------------------------------------------------------------------- /conf/scheduler/linearwithwarmup.yaml: -------------------------------------------------------------------------------- 1 | class_name: src.schedulers.linear_schedule_with_warmup.LinearScheduleWithWarmupConfig 2 | step: step 3 | monitor: train_${training.metric} 4 | params: 5 | warmup_prop: 0 6 | last_epoch: -1 7 | epochs: ${trainer.max_epochs} 8 | train_len: ${datamodule.train_len} 9 | n_folds: ${datamodule.n_folds} 10 | -------------------------------------------------------------------------------- /conf/scheduler/multi_step_reg.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.lr_scheduler.MultiStepLR 3 | step: epoch 4 | monitor: train_${training.metric} 5 | params: 6 | milestones: [300, 600, 900, 1500] 7 | gamma: 0.5 8 | -------------------------------------------------------------------------------- /conf/scheduler/onecycle.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.lr_scheduler.OneCycleLR 3 | step: step 4 | monitor: train_${training.metric} 5 | params: 6 | total_steps: 500 7 | max_lr: 0.01 8 | last_epoch: -1 9 | -------------------------------------------------------------------------------- /conf/scheduler/plateau.yaml: -------------------------------------------------------------------------------- 1 | class_name: torch.optim.lr_scheduler.ReduceLROnPlateau 2 | step: epoch 3 | monitor: train_${training.metric} 4 | params: 5 | mode: ${training.mode} 6 | factor: 0.1 7 | patience: 1 8 | -------------------------------------------------------------------------------- /conf/scheduler/step.yaml: -------------------------------------------------------------------------------- 1 | 2 | class_name: torch.optim.lr_scheduler.StepLR 3 | step: epoch 4 | monitor: train_${training.metric} 5 | params: 6 | step_size: 10 7 | gamma: 0.9 8 | last_epoch: -1 9 | -------------------------------------------------------------------------------- /conf/trainer/default_trainer.yaml: -------------------------------------------------------------------------------- 1 | 2 | devices: 1 3 | accelerator: dp 4 | num_nodes: 1 5 | accumulate_grad_batches: 1 6 | profiler: False 7 | max_epochs: 50 8 | log_every_n_steps: 100 9 | num_sanity_val_steps: 0 10 | -------------------------------------------------------------------------------- /conf/training/default_training.yaml: -------------------------------------------------------------------------------- 1 | lr: 0.0001 2 | metric: ${metric.metric.metric_name}_epoch 3 | mode: "max" 4 | seed: 666 5 | debug: True 6 | n_classes: 10 7 | lightning_module_name: src.lightning_classes.lightning_image_classification.LitImageClassification 8 | -------------------------------------------------------------------------------- /conf/training/ner_training.yaml: -------------------------------------------------------------------------------- 1 | 2 | lr: 0.1 3 | metric: ${metric.metric.metric_name} 4 | mode: "max" 5 | seed: 666 6 | debug: False 7 | lightning_module_name: src.lightning_classes.lightning_ner.LitNER 8 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | ### Contributing 2 | 3 | If you want to submit a bug or have a feature request create an issue at https://github.com/Erlemar/pytorch_tempest/issues 4 | 5 | Contributing is done using pull requests (direct commits into master branch are disabled). 6 | 7 | ## To create a pull request: 8 | 1. Fork the repository. 9 | 2. Clone it. 10 | 3. Install pre-commit hook, initialize it and install requirements: 11 | 12 | ```shell 13 | pip install pre-commit 14 | pip install -r requirements.txt 15 | pre-commit install 16 | ``` 17 | 18 | 4. Make changes to the code. 19 | 5. Run tests: 20 | 21 | ```shell 22 | pytest 23 | ``` 24 | 25 | 6. Push code to your forked repo and create a pull request 26 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | import datetime 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import os 16 | import sys 17 | 18 | sys.path.insert(0, os.path.abspath('.')) 19 | sys.path.insert(0, os.path.abspath('../')) 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = 'pytorch_tempest' 24 | author = 'Andrey Lukyanenko' 25 | copyright = f'{datetime.datetime.now().year}, {author}' 26 | 27 | # The full version, including alpha/beta/rc tags 28 | release = '0.0.1' 29 | 30 | # -- General configuration --------------------------------------------------- 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 34 | # ones. 35 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.mathjax', 'sphinx.ext.napoleon'] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # List of patterns, relative to source directory, that match files and 41 | # directories to ignore when looking for source files. 42 | # This pattern also affects html_static_path and html_extra_path. 43 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 44 | napoleon_google_docstring = True 45 | 46 | # -- Options for HTML output ------------------------------------------------- 47 | 48 | # The theme to use for HTML and HTML Help pages. See the documentation for 49 | # a list of builtin themes. 50 | # 51 | html_theme = 'sphinx_rtd_theme' 52 | 53 | # Add any paths that contain custom static files (such as style sheets) here, 54 | # relative to this directory. They are copied after the builtin static files, 55 | # so a file named 'default.css' will overwrite the builtin 'default.css'. 56 | html_static_path = ['_static'] 57 | master_doc = 'index' 58 | source_suffix = '.rst' 59 | 60 | # version = get_version() 61 | # # The full version, including alpha/beta/rc tags. 62 | # release = version 63 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pytorch_tempest documentation master file, created by 2 | sphinx-quickstart on Mon Sep 7 14:18:37 2020. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pytorch_tempest's documentation! 7 | =========================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 4 11 | :caption: Contents: 12 | 13 | 14 | 15 | Indices and tables 16 | ================== 17 | 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx-autobuild 2 | sphinx==3.2.1 3 | sphinx_rtd_theme 4 | -------------------------------------------------------------------------------- /experiments_overview.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import warnings 4 | 5 | import comet_ml 6 | import pandas as pd 7 | from hydra import initialize, compose 8 | from omegaconf import DictConfig 9 | 10 | warnings.filterwarnings('ignore') 11 | logging.basicConfig(level=logging.INFO) 12 | 13 | 14 | def show_scores(local_cfg: DictConfig, metric: str = 'main_score') -> None: 15 | comet_api = comet_ml.api.API(local_cfg.private.comet_api) 16 | 17 | experiments = comet_api.get(f'{local_cfg.general.workspace}/{local_cfg.general.project_name}') 18 | 19 | experiment_results = [] 20 | for experiment in experiments: 21 | scores = experiment.get_metrics('main_score') 22 | if len(scores) > 10: 23 | best_score = experiment.get_metrics_summary(metric)['valueMin'] 24 | experiment_results.append((experiment.name, best_score)) 25 | 26 | scores = pd.DataFrame(experiment_results, columns=['id', 'score']) 27 | scores = scores.sort_values('score') 28 | logging.info(scores.head(10)) 29 | scores.to_csv('saved_objects/scores.csv', index=False) 30 | 31 | 32 | if __name__ == '__main__': 33 | parser = argparse.ArgumentParser(description='See experiment results for M5') 34 | parser.add_argument('--config_dir', help='main config dir', type=str, default='conf/') 35 | parser.add_argument('--main_config', help='main config', type=str, default='config.yaml') 36 | parser.add_argument('--metric', help='main config', type=str, default='main_score') 37 | args = parser.parse_args() 38 | 39 | initialize(config_path=args.config_dir) 40 | 41 | cfg = compose(config_name=args.main_config) 42 | 43 | show_scores(cfg, args.metric) 44 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | plugins = numpy.typing.mypy_plugin 3 | ignore_missing_imports = True 4 | -------------------------------------------------------------------------------- /predict.py: -------------------------------------------------------------------------------- 1 | # import argparse 2 | # import glob 3 | # 4 | # from hydra import experimental 5 | # import numpy as np 6 | # import pandas as pd 7 | # import torch 8 | # import yaml 9 | # from omegaconf import DictConfig, OmegaConf 10 | # 11 | # from src.datasets.get_dataset import get_test_dataset 12 | # from src.utils.utils import set_seed 13 | # 14 | # 15 | # def make_prediction(cfg: DictConfig) -> None: 16 | # """ 17 | # Run pytorch-lightning model inference 18 | # 19 | # Args: 20 | # cfg: hydra config 21 | # 22 | # Returns: 23 | # None 24 | # """ 25 | # set_seed(cfg.training.seed) 26 | # model_names = glob.glob(f'outputs/{cfg.inference.run_name}/saved_models/*') 27 | # 28 | # test_dataset = get_test_dataset(cfg) 29 | # loader = torch.utils.data.DataLoader( 30 | # test_dataset, batch_size=cfg.data.batch_size, num_workers=cfg.data.num_workers, shuffle=False 31 | # ) 32 | # sub = pd.read_csv(cfg.data.submission_path) 33 | # 34 | # y_pred = np.zeros((len(test_dataset), len(model_names))) 35 | # device = cfg.data.device 36 | # 37 | # for j, model_name in enumerate(model_names): 38 | # 39 | # lit_model = LitMelanoma.load_from_checkpoint(checkpoint_path=model_name, cfg=cfg) 40 | # 41 | # model = lit_model.model 42 | # 43 | # model.to(device) 44 | # model.eval() 45 | # 46 | # with torch.no_grad(): 47 | # 48 | # for ind, (img, _) in enumerate(loader): 49 | # logits, _ = model(img, _) 50 | # y_pred[ind * cfg.data.batch_size : (ind + 1) * cfg.data.batch_size, j] = ( 51 | # torch.sigmoid(logits).cpu().detach().numpy().reshape(-1) 52 | # ) 53 | # 54 | # sub['target'] = y_pred.mean(1) 55 | # sub.to_csv(f'subs/{cfg.inference.run_name}_{cfg.inference.mode}.csv', index=False) 56 | # 57 | # 58 | # if __name__ == '__main__': 59 | # parser = argparse.ArgumentParser(description='Inference in Melanoma competition') 60 | # parser.add_argument('--run_name', help='folder_name', type=str, default='2020_06_21_04_53_55') 61 | # parser.add_argument('--mode', help='valid or test', type=str, default='test') 62 | # args = parser.parse_args() 63 | # 64 | # experimental.initialize(config_dir='conf', strict=True) 65 | # inference_cfg = experimental.compose(config_file='config.yaml') 66 | # inference_cfg['inference']['run_name'] = args.run_name 67 | # inference_cfg['inference']['mode'] = args.mode 68 | # print(inference_cfg.inference.run_name) 69 | # path = f'outputs/{inference_cfg.inference.run_name}/.hydra/config.yaml' 70 | # 71 | # with open(path) as cfg: 72 | # cfg_yaml = yaml.safe_load(cfg) 73 | # 74 | # cfg_yaml['inference'] = inference_cfg['inference'] 75 | # cfg = OmegaConf.create(cfg_yaml) 76 | # make_prediction(cfg) 77 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 119 3 | skip-string-normalization = true 4 | target-version = ['py35', 'py36', 'py37', 'py38'] 5 | include = '\.pyi?$' 6 | exclude = ''' 7 | /( 8 | \.eggs 9 | | \.git 10 | | \.hg 11 | | \.mypy_cache 12 | | \.tox 13 | | \.venv 14 | | _build 15 | | buck-out 16 | | build 17 | | dist 18 | | outputs 19 | )/ 20 | ''' 21 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | albumentations==1.3.1 2 | comet_ml==3.34.1 3 | pretrainedmodels==0.7.4 4 | efficientnet_pytorch==0.7.1 5 | torch==2.4.0 6 | omegaconf==2.3.0 7 | pytorch_toolbelt==0.6.3 8 | hydra_core==1.3.2 9 | hydra_colorlog==1.2.0 10 | pytest==7.4.2 11 | numpy==1.26.1 12 | pandas==2.1.1 13 | torchvision==0.19.0 14 | pytorch_lightning==2.1.0 15 | Pillow==10.1.0 16 | scikit_learn==1.3.1 17 | adabound==0.0.5 18 | adabelief-pytorch==0.2.1 19 | ranger-adabelief==0.1.0 20 | timm==0.9.8 21 | torch-optimizer==0.3.0 22 | lion-pytorch==0.2.3 23 | wandb==0.15.12 24 | einops==0.7.0 25 | rich==13.6.0 26 | torchmetrics==1.2.0 27 | schedulefree==1.2.7 28 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Erlemar/pytorch_tempest/9dc97efa14db2c6e4dfd6b65fd3214a81069afd0/src/__init__.py -------------------------------------------------------------------------------- /src/callbacks/callbacks.py: -------------------------------------------------------------------------------- 1 | import pytorch_lightning as pl 2 | 3 | 4 | class MetricsCallback(pl.Callback): 5 | """ 6 | PyTorch Lightning metric callback. 7 | 8 | Use it like this: 9 | >>> metrics_callback = MetricsCallback() 10 | 11 | >>> trainer = pl.Trainer(callbacks=[metrics_callback]) 12 | >>> trainer.fit(model) 13 | 14 | >>> print(metrics_callback.metrics) 15 | 16 | """ 17 | 18 | def __init__(self): 19 | super().__init__() 20 | self.metrics = [] 21 | 22 | def on_validation_end(self, trainer, pl_module): 23 | self.metrics.append(trainer.callback_metrics) 24 | 25 | 26 | # class StagerCallback(pl.Callback): 27 | # def on_epoch_start(self, trainer, pl_module): 28 | # pl_module.configure_optimizers() 29 | -------------------------------------------------------------------------------- /src/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Erlemar/pytorch_tempest/9dc97efa14db2c6e4dfd6b65fd3214a81069afd0/src/datasets/__init__.py -------------------------------------------------------------------------------- /src/datasets/collators.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Union 2 | 3 | import numpy as np 4 | import torch 5 | from torch.utils.data.dataloader import default_collate 6 | 7 | 8 | def mixup(batch: Dict[str, torch.Tensor], alpha: float) -> Dict[str, Union[torch.Tensor, float]]: 9 | image = batch['image'] 10 | target = batch['target'] 11 | indices = torch.randperm(image.shape[0]) 12 | shuffled_data = image[indices] 13 | shuffled_target = target[indices] 14 | # TODO compare sampling from numpy and pytorch. from torch.distributions import beta 15 | lam = np.random.beta(alpha, alpha) 16 | image = image * lam + shuffled_data * (1 - lam) 17 | 18 | return {'image': image, 'target': target, 'shuffled_target': shuffled_target, 'lam': lam} 19 | 20 | 21 | def cutmix(batch: Dict[str, torch.Tensor], alpha: float) -> Dict[str, Union[torch.Tensor, float]]: 22 | image = batch['image'] 23 | target = batch['target'] 24 | 25 | indices = torch.randperm(image.size(0)) 26 | shuffled_data = image[indices] 27 | shuffled_target = target[indices] 28 | 29 | lam = np.random.beta(alpha, alpha) 30 | 31 | image_h, image_w = image.shape[2:] 32 | cx = np.random.uniform(0, image_w) 33 | cy = np.random.uniform(0, image_h) 34 | w = image_w * np.sqrt(1 - lam) 35 | h = image_h * np.sqrt(1 - lam) 36 | x0 = int(np.round(max(cx - w / 2, 0))) 37 | x1 = int(np.round(min(cx + w / 2, image_w))) 38 | y0 = int(np.round(max(cy - h / 2, 0))) 39 | y1 = int(np.round(min(cy + h / 2, image_h))) 40 | 41 | image[:, :, y0:y1, x0:x1] = shuffled_data[:, :, y0:y1, x0:x1] 42 | 43 | return {'image': image, 'target': target, 'shuffled_target': shuffled_target, 'lam': lam} 44 | 45 | 46 | class MixupCollator: 47 | """ 48 | Mixup Collator 49 | 50 | This is a modified version of code from: 51 | https://github.com/hysts/pytorch_image_classification/blob/master/pytorch_image_classification/collators/mixup.py 52 | 53 | """ 54 | 55 | def __init__(self, alpha: float = 0.4): 56 | self.alpha = alpha 57 | 58 | def __call__(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: # type: ignore 59 | batch = default_collate(batch) # type: ignore 60 | batch = mixup(batch, self.alpha) # type: ignore 61 | return batch 62 | 63 | 64 | class CutMixCollator: 65 | """ 66 | Cutmix Collator 67 | 68 | This is a modified version of code from: 69 | https://github.com/hysts/pytorch_image_classification/blob/master/pytorch_image_classification/collators/cutmix.py 70 | 71 | """ 72 | 73 | def __init__(self, alpha: float = 0.4): 74 | self.alpha = alpha 75 | 76 | def __call__(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: # type: ignore 77 | batch = default_collate(batch) # type: ignore 78 | batch = cutmix(batch, self.alpha) # type: ignore 79 | return batch 80 | -------------------------------------------------------------------------------- /src/datasets/get_dataset.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | import albumentations as A 4 | import omegaconf 5 | 6 | from src.utils.technical_utils import load_obj 7 | 8 | 9 | def load_augs(cfg: Dict) -> A.Compose: 10 | """ 11 | Load albumentations 12 | 13 | Args: 14 | cfg: 15 | 16 | Returns: 17 | compose object 18 | """ 19 | augs = [] 20 | for a in cfg: 21 | if a['class_name'] == 'albumentations.OneOf': 22 | small_augs = [] 23 | for small_aug in a['params']: 24 | # yaml can't contain tuples, so we need to convert manually 25 | params = { 26 | k: (v if not type(v) is omegaconf.listconfig.ListConfig else tuple(v)) 27 | for k, v in small_aug['params'].items() 28 | } 29 | aug = load_obj(small_aug['class_name'])(**params) 30 | small_augs.append(aug) 31 | aug = load_obj(a['class_name'])(small_augs) 32 | augs.append(aug) 33 | 34 | else: 35 | params = { 36 | k: (v if not type(v) is omegaconf.listconfig.ListConfig else tuple(v)) for k, v in a['params'].items() 37 | } 38 | aug = load_obj(a['class_name'])(**params) 39 | augs.append(aug) 40 | 41 | return A.Compose(augs) 42 | -------------------------------------------------------------------------------- /src/datasets/image_dataset.py: -------------------------------------------------------------------------------- 1 | from typing import List, Dict, Optional 2 | 3 | import cv2 4 | import numpy as np 5 | import numpy.typing as npt 6 | from albumentations.core.composition import Compose 7 | from torch.utils.data import Dataset 8 | 9 | 10 | class ImageClassificationDataset(Dataset): 11 | def __init__( 12 | self, 13 | image_names: List, 14 | transforms: Compose, 15 | labels: Optional[List[int]], 16 | img_path: str = '', 17 | mode: str = 'train', 18 | labels_to_ohe: bool = False, 19 | n_classes: int = 1, 20 | ): 21 | """ 22 | Image classification dataset. 23 | 24 | Args: 25 | df: dataframe with image id and bboxes 26 | mode: train/val/test 27 | img_path: path to images 28 | transforms: albumentations 29 | """ 30 | 31 | self.mode = mode 32 | self.transforms = transforms 33 | self.img_path = img_path 34 | self.image_names = image_names 35 | # TODO rename labels and targets into one name 36 | if labels is not None: 37 | if not labels_to_ohe: 38 | self.labels = np.array(labels) 39 | else: 40 | self.labels = np.zeros((len(labels), n_classes)) 41 | self.labels[np.arange(len(labels)), np.array(labels)] = 1 42 | 43 | def __getitem__(self, idx: int) -> Dict[str, npt.ArrayLike]: 44 | image_path = self.img_path + self.image_names[idx] 45 | image = cv2.imread(f'{image_path}', cv2.IMREAD_COLOR) 46 | image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) 47 | if image is None: 48 | raise FileNotFoundError(image_path) 49 | target = self.labels[idx] 50 | 51 | img = self.transforms(image=image)['image'] 52 | sample = {'image': img, 'target': np.array(target).astype('int64')} 53 | 54 | return sample 55 | 56 | def __len__(self) -> int: 57 | return len(self.image_names) 58 | -------------------------------------------------------------------------------- /src/datasets/mnist_dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | from torchvision.datasets import MNIST 4 | 5 | 6 | class MnistDataset(MNIST): 7 | def __getitem__(self, index): 8 | """ 9 | Args: 10 | index (int): Index 11 | 12 | Returns: 13 | tuple: (image, target) where target is index of the target class. 14 | """ 15 | img, target = self.data[index], int(self.targets[index]) 16 | 17 | # doing this so that it is consistent with all other datasets 18 | # to return a PIL Image 19 | img = Image.fromarray(img.numpy(), mode='L') 20 | 21 | if self.transform is not None: 22 | img = self.transform(img) 23 | 24 | if self.target_transform is not None: 25 | target = self.target_transform(target) 26 | 27 | return {'image': img, 'target': np.array(target).astype('int64')} 28 | -------------------------------------------------------------------------------- /src/datasets/text_dataset.py: -------------------------------------------------------------------------------- 1 | from typing import List, Dict, Tuple 2 | 3 | import numpy as np 4 | import numpy.typing as npt 5 | import torch 6 | from gensim.models import FastText 7 | from omegaconf import DictConfig 8 | from torch import nn 9 | from torch.utils.data import Dataset 10 | 11 | from src.utils.text_utils import pad_sequences, build_matrix 12 | 13 | 14 | class NerDataset(Dataset): 15 | def __init__(self, ner_data: List, word_to_idx: Dict, cfg: DictConfig, tag_to_idx: Dict, preload: bool = True): 16 | """ 17 | Prepare data for wheat competition. 18 | Args: 19 | ner_data: data 20 | word_to_idx: mapping of words do indexes 21 | cfg: config with parameters 22 | tag_to_idx: mapping of tags do indexes 23 | """ 24 | self.data_len = len(ner_data) 25 | self.cfg = cfg 26 | self.preload = preload 27 | if preload: 28 | self.tokens = np.array( 29 | [[word_to_idx[w] if w in word_to_idx.keys() else 1 for w in line['text']] for line in ner_data] 30 | ) 31 | self.labels = np.array([[tag_to_idx[w] for w in line['labels']] for line in ner_data]) 32 | else: 33 | self.ner_data = ner_data 34 | self.cfg = cfg 35 | self.word_to_idx = word_to_idx 36 | self.tag_to_idx = tag_to_idx 37 | 38 | def __getitem__(self, idx: int) -> Tuple[npt.ArrayLike, int, npt.ArrayLike]: 39 | if self.preload: 40 | return self.tokens[idx], len(self.tokens[idx]), self.labels[idx] 41 | 42 | else: 43 | line = self.ner_data[idx] 44 | tokens = [self.word_to_idx[w] if w in self.word_to_idx.keys() else 1 for w in line['text']] 45 | if self.cfg.dataset.params.use_bulio_tokens: 46 | labels = [self.tag_to_idx[w] for w in line['labels']] 47 | else: 48 | labels = [self.tag_to_idx[w] for w in line['labels_flat']] 49 | return np.array(tokens), len(tokens), np.array(labels) 50 | 51 | def __len__(self) -> int: 52 | return self.data_len 53 | 54 | 55 | class Collator: 56 | def __init__(self, test=False, percentile=100, pad_value=0): 57 | self.test = test 58 | self.percentile = percentile 59 | self.pad_value = pad_value 60 | 61 | def __call__(self, batch): 62 | tokens, lens, labels = zip(*batch) 63 | lens = np.array(lens) 64 | 65 | max_len = min(int(np.percentile(lens, self.percentile)), 100) 66 | 67 | tokens = torch.tensor( 68 | pad_sequences(tokens, maxlen=max_len, padding='post', value=self.pad_value), dtype=torch.long 69 | ) 70 | lens = torch.tensor([min(i, max_len) for i in lens], dtype=torch.long) 71 | labels = torch.tensor( 72 | pad_sequences(labels, maxlen=max_len, padding='post', value=self.pad_value), dtype=torch.long 73 | ) 74 | 75 | return tokens, lens, labels 76 | 77 | 78 | class Vectorizer(nn.Module): 79 | """ 80 | Transform tokens to embeddings 81 | """ 82 | 83 | def __init__(self, word_to_idx: Dict, embeddings_path: str, embeddings_type: str, embeddings_dim: int = 100): 84 | super(Vectorizer, self).__init__() 85 | self.weights_matrix, _, _ = build_matrix( 86 | word_to_idx, embeddings_path, embeddings_type, max_features=len(word_to_idx), embed_size=embeddings_dim 87 | ) 88 | self.weights_matrix = torch.tensor(self.weights_matrix, dtype=torch.float32) 89 | self.embedding = nn.Embedding.from_pretrained(self.weights_matrix) 90 | self.embedding.weight.requires_grad = False 91 | 92 | def forward(self, x: torch.LongTensor) -> torch.Tensor: 93 | embed = self.embedding(x) 94 | return embed 95 | 96 | 97 | class InferenceVectorizer: 98 | """ 99 | Transform tokens to embeddings 100 | """ 101 | 102 | def __init__(self, embeddings_path: str): 103 | self.fasttext = FastText.load(embeddings_path) 104 | 105 | def __call__(self, claim): 106 | splited_claim = claim.split() 107 | with torch.no_grad(): 108 | data_tensor = torch.tensor([self.fasttext[token] for token in splited_claim]).unsqueeze(0) 109 | length_tensor = torch.tensor(len(splited_claim)).unsqueeze(0) 110 | 111 | return data_tensor, length_tensor 112 | -------------------------------------------------------------------------------- /src/lightning_classes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Erlemar/pytorch_tempest/9dc97efa14db2c6e4dfd6b65fd3214a81069afd0/src/lightning_classes/__init__.py -------------------------------------------------------------------------------- /src/lightning_classes/datamodule_imagenette.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | from typing import Dict 4 | 5 | import pytorch_lightning as pl 6 | import torch 7 | from omegaconf import DictConfig 8 | 9 | from src.datasets.get_dataset import load_augs 10 | from src.utils.technical_utils import load_obj 11 | 12 | 13 | class ImagenetteDataModule(pl.LightningDataModule): 14 | def __init__(self, cfg: DictConfig): 15 | super().__init__() 16 | self.cfg = cfg 17 | 18 | def prepare_data(self): 19 | pass 20 | 21 | def setup(self, stage=None): 22 | mapping_dict = { 23 | 'n01440764': 0, 24 | 'n02102040': 1, 25 | 'n02979186': 2, 26 | 'n03000684': 3, 27 | 'n03028079': 4, 28 | 'n03394916': 5, 29 | 'n03417042': 6, 30 | 'n03425413': 7, 31 | 'n03445777': 8, 32 | 'n03888257': 9, 33 | } 34 | train_labels = [] 35 | train_images = [] 36 | for folder in glob.glob(f'{self.cfg.datamodule.path}/train/*'): 37 | class_name = os.path.basename(os.path.normpath(folder)) 38 | for filename in glob.glob(f'{folder}/*'): 39 | train_labels.append(mapping_dict[class_name]) 40 | train_images.append(filename) 41 | 42 | val_labels = [] 43 | val_images = [] 44 | 45 | for folder in glob.glob(f'{self.cfg.datamodule.path}/val/*'): 46 | class_name = os.path.basename(os.path.normpath(folder)) 47 | for filename in glob.glob(f'{folder}/*'): 48 | val_labels.append(mapping_dict[class_name]) 49 | val_images.append(filename) 50 | 51 | if self.cfg.training.debug: 52 | train_labels = train_labels[:1000] 53 | train_images = train_images[:1000] 54 | val_labels = val_labels[:1000] 55 | val_images = val_images[:1000] 56 | 57 | # train dataset 58 | dataset_class = load_obj(self.cfg.datamodule.class_name) 59 | 60 | # initialize augmentations 61 | train_augs = load_augs(self.cfg['augmentation']['train']['augs']) 62 | valid_augs = load_augs(self.cfg['augmentation']['valid']['augs']) 63 | 64 | self.train_dataset = dataset_class( 65 | image_names=train_images, 66 | labels=train_labels, 67 | transforms=train_augs, 68 | mode='train', 69 | labels_to_ohe=self.cfg.datamodule.labels_to_ohe, 70 | n_classes=self.cfg.training.n_classes, 71 | ) 72 | self.valid_dataset = dataset_class( 73 | image_names=val_images, 74 | labels=val_labels, 75 | transforms=valid_augs, 76 | mode='valid', 77 | labels_to_ohe=self.cfg.datamodule.labels_to_ohe, 78 | n_classes=self.cfg.training.n_classes, 79 | ) 80 | 81 | def train_dataloader(self): 82 | train_loader = torch.utils.data.DataLoader( 83 | self.train_dataset, 84 | batch_size=self.cfg.datamodule.batch_size, 85 | num_workers=self.cfg.datamodule.num_workers, 86 | pin_memory=self.cfg.datamodule.pin_memory, 87 | shuffle=True, 88 | collate_fn=( 89 | load_obj(self.cfg.datamodule.collate_fn)(**self.cfg.datamodule.mix_params) 90 | if self.cfg.datamodule.collate_fn 91 | else None 92 | ), 93 | drop_last=True, 94 | ) 95 | return train_loader 96 | 97 | def val_dataloader(self): 98 | valid_loader = torch.utils.data.DataLoader( 99 | self.valid_dataset, 100 | batch_size=self.cfg.datamodule.batch_size, 101 | num_workers=self.cfg.datamodule.num_workers, 102 | pin_memory=self.cfg.datamodule.pin_memory, 103 | shuffle=False, 104 | collate_fn=None, 105 | drop_last=False, 106 | ) 107 | 108 | return valid_loader 109 | 110 | def test_dataloader(self): 111 | return None 112 | -------------------------------------------------------------------------------- /src/lightning_classes/datamodule_melanoma.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import pytorch_lightning as pl 3 | import torch 4 | from omegaconf import DictConfig 5 | from sklearn.model_selection import train_test_split 6 | 7 | from src.datasets.get_dataset import load_augs 8 | from src.utils.ml_utils import stratified_group_k_fold 9 | from src.utils.technical_utils import load_obj 10 | 11 | 12 | class MelanomaDataModule(pl.LightningDataModule): 13 | def __init__(self, cfg: DictConfig): 14 | super().__init__() 15 | self.cfg = cfg 16 | 17 | def prepare_data(self): 18 | pass 19 | 20 | def setup(self, stage=None): 21 | train = pd.read_csv(self.cfg.datamodule.train_path) 22 | train = train.rename(columns={'image_id': 'image_name'}) 23 | train['image_name'] = train['image_name'] + '.jpg' 24 | 25 | # for fast training 26 | if self.cfg.training.debug: 27 | train, valid = train_test_split(train, test_size=0.1, random_state=self.cfg.training.seed) 28 | train = train[:1000] 29 | valid = valid[:1000] 30 | 31 | else: 32 | folds = list( 33 | stratified_group_k_fold(y=train['target'], groups=train['patient_id'], k=self.cfg.datamodule.n_folds) 34 | ) 35 | train_idx, valid_idx = folds[self.cfg.datamodule.fold_n] 36 | 37 | valid = train.iloc[valid_idx] 38 | train = train.iloc[train_idx] 39 | 40 | # train dataset 41 | dataset_class = load_obj(self.cfg.datamodule.class_name) 42 | 43 | # initialize augmentations 44 | train_augs = load_augs(self.cfg['augmentation']['train']['augs']) 45 | valid_augs = load_augs(self.cfg['augmentation']['valid']['augs']) 46 | 47 | self.train_dataset = dataset_class( 48 | image_names=train['image_name'].values, 49 | transforms=train_augs, 50 | labels=train['target'].values, 51 | img_path=self.cfg.datamodule.path, 52 | mode='train', 53 | labels_to_ohe=False, 54 | n_classes=self.cfg.training.n_classes, 55 | ) 56 | self.valid_dataset = dataset_class( 57 | image_names=valid['image_name'].values, 58 | transforms=valid_augs, 59 | labels=valid['target'].values, 60 | img_path=self.cfg.datamodule.path, 61 | mode='valid', 62 | labels_to_ohe=False, 63 | n_classes=self.cfg.training.n_classes, 64 | ) 65 | 66 | def train_dataloader(self): 67 | train_loader = torch.utils.data.DataLoader( 68 | self.train_dataset, 69 | batch_size=self.cfg.datamodule.batch_size, 70 | num_workers=self.cfg.datamodule.num_workers, 71 | shuffle=True, 72 | ) 73 | return train_loader 74 | 75 | def val_dataloader(self): 76 | valid_loader = torch.utils.data.DataLoader( 77 | self.valid_dataset, 78 | batch_size=self.cfg.datamodule.batch_size, 79 | num_workers=self.cfg.datamodule.num_workers, 80 | shuffle=False, 81 | ) 82 | 83 | return valid_loader 84 | 85 | def test_dataloader(self): 86 | return None 87 | -------------------------------------------------------------------------------- /src/lightning_classes/datamodule_mnist.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | import pytorch_lightning as pl 4 | from omegaconf import DictConfig 5 | from torch.utils.data import random_split, DataLoader 6 | from torchvision import transforms 7 | from torchvision.datasets import MNIST 8 | 9 | from src.datasets.mnist_dataset import MnistDataset 10 | 11 | 12 | class MNISTDataModule(pl.LightningDataModule): 13 | def __init__(self, cfg: DictConfig, data_dir: str = './'): 14 | super().__init__() 15 | self.data_dir = data_dir 16 | self.cfg = cfg 17 | 18 | def prepare_data(self): 19 | # download 20 | MnistDataset(self.data_dir, train=True, download=True) 21 | MnistDataset(self.data_dir, train=False, download=True) 22 | 23 | def setup(self, stage=None): 24 | transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) 25 | 26 | # Assign train/val datasets for use in dataloaders 27 | if stage == 'fit' or stage is None: 28 | mnist_full = MnistDataset(self.data_dir, train=True, transform=transform) 29 | self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) 30 | if self.cfg.training.debug: 31 | self.mnist_train, _ = random_split(self.mnist_train, [1000, 54000]) 32 | self.mnist_val, _ = random_split(self.mnist_val, [1000, 4000]) 33 | 34 | if stage == 'test' or stage is None: 35 | self.mnist_test = MNIST(self.data_dir, train=False, transform=self.transform) 36 | 37 | def train_dataloader(self): 38 | return DataLoader(self.mnist_train, batch_size=128) 39 | 40 | def val_dataloader(self): 41 | return DataLoader(self.mnist_val, batch_size=128) 42 | 43 | def test_dataloader(self): 44 | return DataLoader(self.mnist_test, batch_size=32) 45 | -------------------------------------------------------------------------------- /src/lightning_classes/datamodule_ner.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Dict 3 | 4 | import pytorch_lightning as pl 5 | import torch 6 | from omegaconf import DictConfig 7 | from sklearn.model_selection import train_test_split 8 | 9 | from src.datasets.text_dataset import Collator 10 | from src.utils.technical_utils import load_obj 11 | from src.utils.text_utils import _generate_tag_to_idx, _generate_word_to_idx, get_vectorizer 12 | 13 | 14 | class NerDataModule(pl.LightningDataModule): 15 | def __init__(self, cfg: DictConfig): 16 | super().__init__() 17 | self.cfg = cfg 18 | 19 | def prepare_data(self): 20 | pass 21 | 22 | @staticmethod 23 | def load_sentences(filepath): 24 | """ 25 | Load sentences (separated by newlines) from dataset 26 | 27 | Parameters 28 | ---------- 29 | filepath : str 30 | path to corpus file 31 | 32 | Returns 33 | ------- 34 | List of sentences represented as dictionaries 35 | 36 | """ 37 | 38 | sentences, tok, ne = [], [], [] 39 | 40 | with open(filepath, 'r') as f: 41 | for line in f.readlines(): 42 | if line in [('-DOCSTART- -X- -X- O\n'), '\n']: 43 | # Sentence as a sequence of tokens, POS, chunk and NE tags 44 | if tok != []: 45 | sentence = {'text': [], 'labels': []} 46 | sentence['text'] = tok 47 | sentence['labels'] = ne 48 | 49 | # Once a sentence is processed append it to the list of sentences 50 | sentences.append(sentence) 51 | 52 | # Reset sentence information 53 | tok = [] 54 | ne = [] 55 | else: 56 | splitted_line = line.split(' ') 57 | 58 | # Append info for next word 59 | tok.append(splitted_line[0]) 60 | ne.append(splitted_line[3].strip('\n')) 61 | 62 | return sentences 63 | 64 | def setup(self, stage=None): 65 | # with open(f'{self.cfg.datamodule.folder_path}{self.cfg.datamodule.file_name}', 'r', encoding='utf-8') as f: 66 | ner_data = self.load_sentences(f'{self.cfg.datamodule.folder_path}{self.cfg.datamodule.file_name}') 67 | 68 | # generate tag_to_idx 69 | labels = [labels['labels'] for labels in ner_data] 70 | flat_labels = list({label for sublist in labels for label in sublist}) 71 | entities_names = sorted({label.split('-')[1] for label in flat_labels if label != 'O'}) 72 | if self.cfg.datamodule.tag_to_idx_from_labels: 73 | self.tag_to_idx = {v: i for i, v in enumerate({i for j in labels for i in j}) if v != 'O'} 74 | for special_tag in ['O', 'PAD']: 75 | self.tag_to_idx[special_tag] = len(self.tag_to_idx) 76 | else: 77 | self.tag_to_idx = _generate_tag_to_idx(self.cfg, entities_names) 78 | 79 | # load or generate word_to_idx 80 | if self.cfg.datamodule.word_to_idx_name: 81 | with open( 82 | f'{self.cfg.datamodule.folder_path}{self.cfg.datamodule.word_to_idx_name}', 'r', encoding='utf-8' 83 | ) as f: 84 | self.word_to_idx = json.load(f) 85 | else: 86 | self.word_to_idx = _generate_word_to_idx(ner_data) 87 | 88 | train_data, valid_data = train_test_split( 89 | ner_data, random_state=self.cfg.training.seed, test_size=self.cfg.datamodule.valid_size 90 | ) 91 | 92 | dataset_class = load_obj(self.cfg.datamodule.class_name) 93 | 94 | self.train_dataset = dataset_class( 95 | ner_data=train_data, cfg=self.cfg, word_to_idx=self.word_to_idx, tag_to_idx=self.tag_to_idx 96 | ) 97 | self.valid_dataset = dataset_class( 98 | ner_data=valid_data, cfg=self.cfg, word_to_idx=self.word_to_idx, tag_to_idx=self.tag_to_idx 99 | ) 100 | 101 | self._vectorizer = get_vectorizer(self.cfg, self.word_to_idx) 102 | self.collate = Collator(percentile=100, pad_value=self.tag_to_idx['PAD']) 103 | 104 | def train_dataloader(self): 105 | train_loader = torch.utils.data.DataLoader( 106 | self.train_dataset, 107 | batch_size=self.cfg.datamodule.batch_size, 108 | num_workers=self.cfg.datamodule.num_workers, 109 | pin_memory=self.cfg.datamodule.pin_memory, 110 | collate_fn=self.collate, 111 | shuffle=True, 112 | ) 113 | return train_loader 114 | 115 | def val_dataloader(self): 116 | valid_loader = torch.utils.data.DataLoader( 117 | self.valid_dataset, 118 | batch_size=self.cfg.datamodule.batch_size, 119 | num_workers=self.cfg.datamodule.num_workers, 120 | pin_memory=self.cfg.datamodule.pin_memory, 121 | collate_fn=self.collate, 122 | shuffle=False, 123 | ) 124 | 125 | return valid_loader 126 | 127 | def test_dataloader(self): 128 | return None 129 | -------------------------------------------------------------------------------- /src/lightning_classes/lightning_image_classification.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Union 2 | 3 | import pytorch_lightning as pl 4 | import torch 5 | from omegaconf import DictConfig 6 | 7 | from src.utils.technical_utils import load_obj 8 | 9 | 10 | class LitImageClassification(pl.LightningModule): 11 | def __init__(self, cfg: DictConfig): 12 | super(LitImageClassification, self).__init__() 13 | self.cfg = cfg 14 | self.model = load_obj(cfg.model.class_name)(cfg=cfg) 15 | if 'params' in self.cfg.loss: 16 | self.loss = load_obj(cfg.loss.class_name)(**self.cfg.loss.params) 17 | else: 18 | self.loss = load_obj(cfg.loss.class_name)() 19 | self.metrics = torch.nn.ModuleDict( 20 | { 21 | self.cfg.metric.metric.metric_name: load_obj(self.cfg.metric.metric.class_name)( 22 | **cfg.metric.metric.params 23 | ) 24 | } 25 | ) 26 | if 'other_metrics' in self.cfg.metric.keys(): 27 | for metric in self.cfg.metric.other_metrics: 28 | self.metrics.update({metric.metric_name: load_obj(metric.class_name)(**metric.params)}) 29 | 30 | def forward(self, x, *args, **kwargs): 31 | return self.model(x) 32 | 33 | def configure_optimizers(self): 34 | if 'decoder_lr' in self.cfg.optimizer.params.keys(): 35 | params = [ 36 | {'params': self.model.decoder.parameters(), 'lr': self.cfg.optimizer.params.lr}, 37 | {'params': self.model.encoder.parameters(), 'lr': self.cfg.optimizer.params.decoder_lr}, 38 | ] 39 | optimizer = load_obj(self.cfg.optimizer.class_name)(params) 40 | 41 | else: 42 | optimizer = load_obj(self.cfg.optimizer.class_name)(self.model.parameters(), **self.cfg.optimizer.params) 43 | scheduler = load_obj(self.cfg.scheduler.class_name)(optimizer, **self.cfg.scheduler.params) 44 | 45 | return ( 46 | [optimizer], 47 | [{'scheduler': scheduler, 'interval': self.cfg.scheduler.step, 'monitor': self.cfg.scheduler.monitor}], 48 | ) 49 | 50 | def training_step(self, batch, *args, **kwargs): # type: ignore 51 | image = batch['image'] 52 | logits = self(image) 53 | 54 | target = batch['target'] 55 | shuffled_target = batch.get('shuffled_target') 56 | lam = batch.get('lam') 57 | if shuffled_target is not None: 58 | loss = self.loss(logits, (target, shuffled_target, lam)).view(1) 59 | else: 60 | loss = self.loss(logits, target) 61 | self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) 62 | 63 | for metric in self.metrics: 64 | score = self.metrics[metric](logits, target) 65 | self.log(f'train_{metric}', score, on_step=True, on_epoch=True, prog_bar=True, logger=True) 66 | return loss 67 | 68 | def validation_step(self, batch, *args, **kwargs): # type: ignore 69 | image = batch['image'] 70 | logits = self(image) 71 | 72 | target = batch['target'] 73 | shuffled_target = batch.get('shuffled_target') 74 | lam = batch.get('lam') 75 | if shuffled_target is not None: 76 | loss = self.loss(logits, (target, shuffled_target, lam), train=False).view(1) 77 | else: 78 | loss = self.loss(logits, target) 79 | 80 | self.log('valid_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) 81 | for metric in self.metrics: 82 | score = self.metrics[metric](logits, target) 83 | self.log(f'valid_{metric}', score, on_step=True, on_epoch=True, prog_bar=True, logger=True) 84 | -------------------------------------------------------------------------------- /src/lightning_classes/lightning_ner.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | import numpy as np 4 | import pytorch_lightning as pl 5 | import torch 6 | from omegaconf import DictConfig 7 | 8 | from src.utils.technical_utils import load_obj 9 | 10 | 11 | class LitNER(pl.LightningModule): 12 | def __init__(self, cfg: DictConfig, tag_to_idx: Dict): 13 | super(LitNER, self).__init__() 14 | self.cfg = cfg 15 | self.tag_to_idx = tag_to_idx 16 | self.model = load_obj(cfg.model.class_name)( 17 | embeddings_dim=cfg.datamodule.embeddings_dim, tag_to_idx=tag_to_idx, **cfg.model.params 18 | ) 19 | self.metrics = torch.nn.ModuleDict( 20 | { 21 | self.cfg.metric.metric.metric_name: load_obj(self.cfg.metric.metric.class_name)( 22 | **cfg.metric.metric.params 23 | ) 24 | } 25 | ) 26 | if 'other_metrics' in self.cfg.metric.keys(): 27 | for metric in self.cfg.metric.other_metrics: 28 | self.metrics.update({metric.metric_name: load_obj(metric.class_name)(**metric.params)}) 29 | 30 | def forward(self, x, lens, *args, **kwargs): 31 | return self.model(x, lens) 32 | 33 | def configure_optimizers(self): 34 | optimizer = load_obj(self.cfg.optimizer.class_name)(self.model.parameters(), **self.cfg.optimizer.params) 35 | scheduler = load_obj(self.cfg.scheduler.class_name)(optimizer, **self.cfg.scheduler.params) 36 | 37 | return ( 38 | [optimizer], 39 | [{'scheduler': scheduler, 'interval': self.cfg.scheduler.step, 'monitor': self.cfg.scheduler.monitor}], 40 | ) 41 | 42 | def training_step(self, batch, batch_idx): 43 | embeds, lens, labels = batch 44 | # transform tokens to embeddings 45 | embeds = self._vectorizer(embeds) 46 | score, tag_seq, loss = self.model(embeds, lens, labels) 47 | labels = labels.flatten() 48 | labels = labels[labels != self.tag_to_idx['PAD']] 49 | 50 | self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) 51 | for metric in self.metrics: 52 | score = self.metrics[metric](tag_seq, labels) 53 | self.log(f'train_{metric}', score, on_step=True, on_epoch=True, prog_bar=True, logger=True) 54 | return loss 55 | 56 | def validation_step(self, batch, batch_idx): 57 | embeds, lens, labels = batch 58 | embeds = self._vectorizer(embeds) 59 | score, tag_seq, loss = self.model(embeds, lens, labels) 60 | labels = labels.flatten() 61 | labels = labels[labels != self.tag_to_idx['PAD']] 62 | 63 | self.log('valid_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True) 64 | for metric in self.metrics: 65 | score = self.metrics[metric](tag_seq, labels) 66 | self.log(f'valid_{metric}', score, on_step=True, on_epoch=True, prog_bar=True, logger=True) 67 | -------------------------------------------------------------------------------- /src/losses/losses.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | import torch 4 | import torch.functional as F 5 | from torch import nn 6 | 7 | 8 | class VentilatorLoss(nn.Module): 9 | """ 10 | Directly optimizes the competition metric (kaggle ventilator) 11 | """ 12 | 13 | def __call__(self, preds, y, u_out): 14 | w = 1 - u_out 15 | mae = w * (y - preds).abs() 16 | mae = mae.sum(-1) / w.sum(-1) 17 | 18 | return mae 19 | 20 | 21 | class MAE(nn.Module): 22 | def __call__(self, preds, y, u_out): 23 | # print(preds.shape, y.shape) 24 | return torch.nn.L1Loss(preds, y).mean() 25 | 26 | 27 | class DenseCrossEntropy(nn.Module): 28 | # Taken from: https://www.kaggle.com/pestipeti/plant-pathology-2020-pytorch 29 | def __init__(self): 30 | super(DenseCrossEntropy, self).__init__() 31 | 32 | def forward(self, logits, labels): 33 | logits = logits.float() 34 | labels = labels.float() 35 | 36 | logprobs = F.log_softmax(logits, dim=-1) 37 | 38 | loss = -labels * logprobs 39 | loss = loss.sum(-1) 40 | 41 | return loss.mean() 42 | 43 | 44 | class CutMixLoss: 45 | # https://github.com/hysts/pytorch_image_classification/blob/master/pytorch_image_classification/losses/cutmix.py 46 | def __init__(self, reduction: str = 'mean'): 47 | self.criterion = nn.CrossEntropyLoss(reduction=reduction) 48 | 49 | def __call__( 50 | self, predictions: torch.Tensor, targets: Tuple[torch.Tensor, torch.Tensor, float], train: bool = True 51 | ) -> torch.Tensor: 52 | if train: 53 | targets1, targets2, lam = targets 54 | loss = lam * self.criterion(predictions, targets1) + (1 - lam) * self.criterion(predictions, targets2) 55 | else: 56 | loss = self.criterion(predictions, targets) 57 | return loss 58 | 59 | 60 | class MixupLoss: 61 | # https://github.com/hysts/pytorch_image_classification/blob/master/pytorch_image_classification/losses/mixup.py 62 | def __init__(self, reduction: str = 'mean'): 63 | self.criterion = nn.CrossEntropyLoss(reduction=reduction) 64 | 65 | def __call__( 66 | self, predictions: torch.Tensor, targets: Tuple[torch.Tensor, torch.Tensor, float], train: bool = True 67 | ) -> torch.Tensor: 68 | if train: 69 | targets1, targets2, lam = targets 70 | loss = lam * self.criterion(predictions, targets1) + (1 - lam) * self.criterion(predictions, targets2) 71 | else: 72 | loss = self.criterion(predictions, targets) 73 | return loss 74 | -------------------------------------------------------------------------------- /src/metrics/f1_score.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | import torch 4 | 5 | 6 | class F1Score: 7 | """ 8 | Class for f1 calculation in Pytorch. 9 | """ 10 | 11 | def __init__(self, average: str = 'weighted'): 12 | """ 13 | Init. 14 | 15 | Args: 16 | average: averaging method 17 | """ 18 | self.average = average 19 | if average not in [None, 'micro', 'macro', 'weighted']: 20 | raise ValueError('Wrong value of average parameter') 21 | 22 | @staticmethod 23 | def calc_f1_micro(predictions: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: 24 | """ 25 | Calculate f1 micro. 26 | 27 | Args: 28 | predictions: tensor with predictions 29 | labels: tensor with original labels 30 | 31 | Returns: 32 | f1 score 33 | """ 34 | true_positive = torch.eq(labels, predictions).sum().float() 35 | f1_score = torch.div(true_positive, len(labels)) 36 | return f1_score 37 | 38 | @staticmethod 39 | def calc_f1_count_for_label( 40 | predictions: torch.Tensor, labels: torch.Tensor, label_id: int 41 | ) -> Tuple[torch.Tensor, torch.Tensor]: 42 | """ 43 | Calculate f1 and true count for the label 44 | 45 | Args: 46 | predictions: tensor with predictions 47 | labels: tensor with original labels 48 | label_id: id of current label 49 | 50 | Returns: 51 | f1 score and true count for label 52 | """ 53 | # label count 54 | true_count = torch.eq(labels, label_id).sum() 55 | 56 | # true positives: labels equal to prediction and to label_id 57 | true_positive = torch.logical_and(torch.eq(labels, predictions), torch.eq(labels, label_id)).sum().float() 58 | # precision for label 59 | precision = torch.div(true_positive, torch.eq(predictions, label_id).sum().float()) 60 | # replace nan values with 0 61 | precision = torch.where(torch.isnan(precision), torch.zeros_like(precision).type_as(true_positive), precision) 62 | 63 | # recall for label 64 | recall = torch.div(true_positive, true_count) 65 | # f1 66 | f1 = 2 * precision * recall / (precision + recall) 67 | # replace nan values with 0 68 | f1 = torch.where(torch.isnan(f1), torch.zeros_like(f1).type_as(true_positive), f1) 69 | return f1, true_count 70 | 71 | def __call__(self, predictions: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: 72 | """ 73 | Calculate f1 score based on averaging method defined in init. 74 | 75 | Args: 76 | predictions: tensor with predictions 77 | labels: tensor with original labels 78 | 79 | Returns: 80 | f1 score 81 | """ 82 | 83 | # simpler calculation for micro 84 | if self.average == 'micro': 85 | return self.calc_f1_micro(predictions, labels) 86 | 87 | f1_score = torch.tensor(0.0).type_as(predictions).float() 88 | for label_id in labels.unique(): 89 | f1, true_count = self.calc_f1_count_for_label(predictions, labels, label_id) 90 | 91 | if self.average == 'weighted': 92 | f1_score += f1 * true_count 93 | elif self.average == 'macro': 94 | f1_score += f1 95 | 96 | if self.average == 'weighted': 97 | f1_score = torch.div(f1_score, len(labels)) 98 | elif self.average == 'macro': 99 | f1_score = torch.div(f1_score, len(labels.unique())) 100 | 101 | return f1_score 102 | -------------------------------------------------------------------------------- /src/metrics/ventilator_mae.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | 5 | class VentilatorMAE(nn.Module): 6 | """ 7 | Class for f1 calculation in Pytorch. 8 | """ 9 | 10 | def __init__(self): 11 | """ 12 | Init. 13 | 14 | Args: 15 | average: averaging method 16 | """ 17 | super().__init__() 18 | 19 | def __call__(self, predictions: torch.Tensor, labels: torch.Tensor, u_out: torch.Tensor) -> torch.Tensor: 20 | """ 21 | Calculate f1 score based on averaging method defined in init. 22 | 23 | Args: 24 | predictions: tensor with predictions 25 | labels: tensor with original labels 26 | 27 | Returns: 28 | f1 score 29 | """ 30 | weights = 1 - u_out 31 | mae = (labels - predictions).abs() * weights 32 | return mae.sum(-1) / weights.sum(-1) 33 | -------------------------------------------------------------------------------- /src/models/decoders/basic_decoder.py: -------------------------------------------------------------------------------- 1 | import torch.nn.functional as F 2 | from pytorch_toolbelt.modules.activations import Mish 3 | from torch import nn 4 | 5 | from src.models.layers.layers import RMSNorm, GeM 6 | 7 | 8 | class BasicDecoder(nn.Module): 9 | def __init__(self, pool_output_size: int = 2, n_classes: int = 1, output_dimension: int = 512) -> None: 10 | """ 11 | Initialize Decoder. 12 | 13 | Args: 14 | pool_output_size: the size of the result feature map after adaptive pooling layer 15 | n_classes: n classes to output 16 | output_dimension: output dimension of encoder 17 | """ 18 | super().__init__() 19 | self.pool = nn.AdaptiveAvgPool2d(output_size=pool_output_size) 20 | self.fc = nn.Linear(output_dimension * pool_output_size * pool_output_size, n_classes) 21 | 22 | def forward(self, x): 23 | x = self.pool(x) 24 | output = self.fc(x.view(x.size()[0], -1)) 25 | 26 | return output 27 | 28 | 29 | class LightHead(nn.Module): 30 | def __init__(self, in_features, num_classes): 31 | super().__init__() 32 | 33 | self.norm = RMSNorm(in_features) 34 | self.fc = nn.Linear(in_features, num_classes) 35 | 36 | def forward(self, x): 37 | x = self.norm(x) 38 | x = self.fc(x) 39 | return x 40 | 41 | 42 | class CnnHead(nn.Module): 43 | def __init__(self, in_features, num_classes, dropout=0.5): 44 | super().__init__() 45 | 46 | self.dropout = dropout 47 | # 48 | # if MEMORY_EFFICIENT: 49 | # MishClass = MemoryEfficientMish 50 | # else: 51 | MishClass = Mish 52 | 53 | self.mish1 = MishClass() 54 | self.conv = nn.Conv2d(in_features, in_features, 1, 1) 55 | self.norm = nn.BatchNorm2d(in_features) 56 | self.pool = GeM() 57 | 58 | self.fc1 = nn.Linear(in_features, in_features // 2) 59 | self.mish2 = MishClass() 60 | self.rms_norm = RMSNorm(in_features // 2) 61 | self.fc2 = nn.Linear(in_features // 2, num_classes) 62 | 63 | def forward(self, x): 64 | x = self.mish1(x) 65 | x = self.conv(x) 66 | x = self.norm(x) 67 | x = self.pool(x) 68 | x = x.view(x.size(0), -1) 69 | x = F.dropout(x, p=self.dropout) 70 | x = self.fc1(x) 71 | 72 | x = self.mish2(x) 73 | x = self.rms_norm(x) 74 | x = F.dropout(x, p=self.dropout / 2) 75 | x = self.fc2(x) 76 | return x 77 | 78 | 79 | class HeavyHead(nn.Module): 80 | def __init__(self, in_features, num_classes): 81 | super().__init__() 82 | # if MEMORY_EFFICIENT: 83 | # MishClass = MemoryEfficientMish 84 | # else: 85 | MishClass = Mish 86 | 87 | self.bn1 = RMSNorm(in_features) 88 | self.fc1 = nn.Linear(in_features, in_features // 2) 89 | self.bn2 = RMSNorm(in_features // 2) 90 | self.mish = MishClass() 91 | self.fc2 = nn.Linear(in_features // 2, num_classes) 92 | 93 | def forward(self, x): 94 | x = self.bn1(x) 95 | x = self.fc1(x) 96 | x = self.mish(x) 97 | x = self.bn2(x) 98 | x = self.fc2(x) 99 | return x 100 | -------------------------------------------------------------------------------- /src/models/encoders/basic_encoder.py: -------------------------------------------------------------------------------- 1 | from typing import Union, Optional 2 | 3 | import pretrainedmodels 4 | import timm 5 | import torch 6 | import torchvision 7 | from efficientnet_pytorch import EfficientNet 8 | from torch import nn 9 | 10 | from src.utils.ml_utils import freeze_until 11 | 12 | 13 | class BasicEncoder(nn.Module): 14 | def __init__( 15 | self, 16 | arch: str = 'resnet18', 17 | source: str = 'pretrainedmodels', 18 | pretrained: Union[str, bool] = 'imagenet', 19 | n_layers: int = -2, 20 | freeze: bool = False, 21 | to_one_channel: bool = False, 22 | freeze_until_layer: Optional[str] = None, 23 | ) -> None: 24 | """ 25 | Initialize Encoder. 26 | 27 | Args: 28 | num_classes: the number of target classes, the size of the last layer's output 29 | arch: the name of the architecture form pretrainedmodels 30 | pretrained: the mode for pretrained model from pretrainedmodels 31 | n_layers: number of layers to keep 32 | freeze: to freeze model 33 | freeze_until: freeze until this layer. If None, then freeze all layers 34 | """ 35 | super().__init__() 36 | if source == 'pretrainedmodels': 37 | net = pretrainedmodels.__dict__[arch](pretrained=pretrained) 38 | self.output_dimension = list(net.children())[-1].in_features 39 | elif source == 'torchvision': 40 | net = torchvision.models.__dict__[arch](pretrained=pretrained) 41 | self.output_dimension = list(net.children())[-1].in_features 42 | elif source == 'timm': 43 | net = timm.create_model(arch, pretrained=pretrained) 44 | self.output_dimension = net.fc.in_features 45 | if source == 'efficientnet': 46 | net = EfficientNet.from_pretrained(arch) 47 | self.output_dimension = net._fc.in_features 48 | 49 | if freeze: 50 | freeze_until(net, freeze_until_layer) 51 | 52 | layers = list(net.children())[:n_layers] 53 | if to_one_channel: 54 | # https://www.kaggle.com/c/bengaliai-cv19/discussion/130311#745589 55 | # saving the weights of the first conv in w 56 | w = layers[0].weight 57 | # creating new Conv2d to accept 1 channel 58 | layers[0] = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False) 59 | # substituting weights of newly created Conv2d with w from but we have to take mean 60 | # to go from 3 channel to 1 61 | layers[0].weight = nn.Parameter(torch.mean(w, dim=1, keepdim=True)) 62 | 63 | self.layers = nn.Sequential(*layers) 64 | 65 | def forward(self, x): 66 | output = self.layers(x) 67 | 68 | return output 69 | -------------------------------------------------------------------------------- /src/models/encoders/complex_backbone.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Union 2 | 3 | from efficientnet_pytorch import EfficientNet 4 | from torch import nn 5 | 6 | 7 | class BackboneModeI(nn.Module): 8 | """ 9 | By phalanx @ZFPhalanx 10 | 11 | """ 12 | 13 | def __init__(self, arch: str, pretrain: bool = True, advdrop: bool = False, return_only_last_output: bool = True): 14 | super().__init__() 15 | self.arch = arch 16 | self.encoder = nn.ModuleList() 17 | self.model = None 18 | self.return_only_last_output = return_only_last_output 19 | 20 | if 'resnext' in arch or 'resnet' in arch or arch.startswith('resnest'): 21 | pretrain_value: Optional[Union[bool, str]] = None 22 | if arch.startswith('se'): 23 | pretrain_value = 'imagenet' if pretrain else None 24 | else: 25 | pretrain_value = pretrain 26 | model = eval(arch)(pretrain=pretrain_value) 27 | if arch.startswith('se'): 28 | self.encoder.append(nn.Sequential(model.layer0, model.layer1)) 29 | else: 30 | self.encoder.append(nn.Sequential(*(list(model.children())[:4]), model.layer1)) 31 | 32 | self.encoder.append(model.layer2) 33 | self.encoder.append(model.layer3) 34 | self.encoder.append(model.layer4) 35 | 36 | elif arch.startswith('densenet'): 37 | model = eval(arch)(pretrain=pretrain).features 38 | transition1 = list(model.transition1.children()) 39 | transition2 = list(model.transition2.children()) 40 | transition3 = list(model.transition3.children()) 41 | self.encoder.append(nn.Sequential(*(list(model.children())[:4]), model.denseblock1, *transition1[:2])) 42 | self.encoder.append(nn.Sequential(*transition1[2:], model.denseblock2, *transition2[:2])) 43 | self.encoder.append(nn.Sequential(*transition2[2:], model.denseblock3, *transition3[:2])) 44 | self.encoder.append(nn.Sequential(*transition3[2:], model.denseblock4, nn.ReLU(True))) 45 | 46 | elif arch.startswith('efficientnet'): 47 | if pretrain: 48 | self.model = EfficientNet.from_pretrained(arch, advprop=advdrop) 49 | else: 50 | self.model = EfficientNet.from_pretrained(arch) 51 | # self.indexes = EFFICIENT_BLOCK_INDEXES(backbone_name) 52 | del self.model._avg_pooling, self.model._dropout, self.model._fc 53 | 54 | def forward(self, x): 55 | outputs = [] 56 | if 'efficientnet' in self.arch: 57 | x = self.model._swish(self.model._bn0(self.model._conv_stem(x))) 58 | for idx, block in enumerate(self.model._blocks): 59 | drop_connect_rate = self.model._global_params.drop_connect_rate 60 | if drop_connect_rate: 61 | drop_connect_rate *= float(idx) / len(self.model._blocks) 62 | x = block(x, drop_connect_rate=drop_connect_rate) 63 | # TODO use indexes[0] for high resolution image 64 | # if idx in self.indexes[1:]: 65 | if idx > 1: 66 | outputs.append(x) 67 | x = self.model._swish(self.model._bn1(self.model._conv_head(x))) 68 | outputs.append(x) 69 | else: 70 | for e in self.encoder: 71 | x = e(x) 72 | outputs.append(x) 73 | 74 | if self.return_only_last_output: 75 | outputs = outputs[-1] 76 | 77 | return outputs 78 | -------------------------------------------------------------------------------- /src/models/encoders/efficientnet_encoder.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from efficientnet_pytorch import EfficientNet 4 | from torch import nn 5 | 6 | from src.utils.ml_utils import freeze_until 7 | 8 | 9 | class EfficientNetEncoder(nn.Module): 10 | def __init__( 11 | self, arch: str = 'efficientnet-b0', freeze: bool = False, freeze_until_layer: Optional[str] = None 12 | ) -> None: 13 | """ 14 | Initialize Encoder. 15 | 16 | Args: 17 | num_classes: the number of target classes, the size of the last layer's output 18 | arch: the name of the architecture form pretrainedmodels 19 | pretrained: the mode for pretrained model from pretrainedmodels 20 | n_layers: number of layers to keep 21 | freeze: to freeze model 22 | freeze_until: freeze until this layer. If None, then freeze all layers 23 | """ 24 | super().__init__() 25 | 26 | self.net = EfficientNet.from_pretrained(arch) 27 | self.output_dimension = self.net._fc.in_features 28 | if freeze: 29 | freeze_until(self.net, freeze_until_layer) 30 | 31 | def forward(self, x): 32 | # TODO compare this and complex backbone 33 | output = self.net.extract_features(x) 34 | 35 | return output 36 | -------------------------------------------------------------------------------- /src/models/layers/activation_functions.py: -------------------------------------------------------------------------------- 1 | # import torch 2 | # from efficientnet_pytorch.model import MemoryEfficientSwish 3 | # from torch import nn 4 | # import torch.nn.functional as F 5 | # 6 | # class MishFunction(torch.autograd.Function): 7 | # @staticmethod 8 | # def forward(ctx, x): 9 | # ctx.save_for_backward(x) 10 | # return x * torch.tanh(F.softplus(x)) # x * tanh(ln(1 + exp(x))) 11 | # 12 | # @staticmethod 13 | # def backward(ctx, grad_output): 14 | # x = ctx.saved_variables[0] 15 | # sigmoid = torch.sigmoid(x) 16 | # tanh_sp = torch.tanh(F.softplus(x)) 17 | # return grad_output * (tanh_sp + x * sigmoid * (1 - tanh_sp * tanh_sp)) 18 | # 19 | # class MemoryEfficientMish(nn.Module): 20 | # def forward(self, x): 21 | # return MishFunction.apply(x) 22 | # 23 | # class Mish(nn.Module): 24 | # def __init__(self): 25 | # super().__init__() 26 | # 27 | # def forward(self, x): 28 | # return x * (torch.tanh(F.softplus(x))) 29 | -------------------------------------------------------------------------------- /src/models/layers/layers.py: -------------------------------------------------------------------------------- 1 | import einops 2 | import torch 3 | import torch.functional as F 4 | from torch import nn 5 | from torch.nn.parameter import Parameter 6 | 7 | 8 | class Flatten(nn.Module): 9 | @staticmethod 10 | def forward(x): 11 | return x.view(x.size()[0], -1) 12 | 13 | 14 | def lme_pool(x, alpha=1.0): # log-mean-exp pool 15 | """ 16 | Pooling lme. 17 | alpha -> approximates maxpool, alpha -> 0 approximates mean pool 18 | Args: 19 | x: 20 | alpha: 21 | 22 | Returns: 23 | result of pooling 24 | """ 25 | T = x.shape[1] 26 | return 1 / alpha * torch.log(1 / T * torch.exp(alpha * x).sum(1)) 27 | 28 | 29 | def gem(x, p=3, eps=1e-6): 30 | return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p) 31 | 32 | 33 | class GeM(nn.Module): 34 | def __init__(self, p=3, eps=1e-6): 35 | super(GeM, self).__init__() 36 | self.p = Parameter(torch.ones(1) * p) 37 | self.eps = eps 38 | 39 | def forward(self, x): 40 | return gem(x, p=self.p, eps=self.eps) 41 | 42 | def __repr__(self): 43 | return f'{self.__class__.__name__}(p={self.p.data.tolist()[0]:.4f}, eps=self.eps)' 44 | 45 | 46 | class AdaptiveConcatPool2d(nn.Module): 47 | def __init__(self, sz=None): 48 | super().__init__() 49 | sz = sz or (1, 1) 50 | self.ap = nn.AdaptiveAvgPool2d(sz) 51 | self.mp = nn.AdaptiveMaxPool2d(sz) 52 | 53 | def forward(self, x): 54 | return torch.cat([self.mp(x), self.ap(x)], 1) 55 | 56 | 57 | class RMSNorm(nn.Module): 58 | """An implementation of RMS Normalization. 59 | 60 | # https://catalyst-team.github.io/catalyst/_modules/catalyst/contrib/nn/modules/rms_norm.html#RMSNorm 61 | """ 62 | 63 | def __init__(self, dimension: int, epsilon: float = 1e-8, is_bias: bool = False): 64 | """ 65 | Args: 66 | dimension (int): the dimension of the layer output to normalize 67 | epsilon (float): an epsilon to prevent dividing by zero 68 | in case the layer has zero variance. (default = 1e-8) 69 | is_bias (bool): a boolean value whether to include bias term 70 | while normalization 71 | """ 72 | super().__init__() 73 | self.dimension = dimension 74 | self.epsilon = epsilon 75 | self.is_bias = is_bias 76 | self.scale = nn.Parameter(torch.ones(self.dimension)) 77 | if self.is_bias: 78 | self.bias = nn.Parameter(torch.zeros(self.dimension)) 79 | 80 | def forward(self, x: torch.Tensor) -> torch.Tensor: 81 | x_std = torch.sqrt(torch.mean(x**2, -1, keepdim=True)) 82 | x_norm = x / (x_std + self.epsilon) 83 | if self.is_bias: 84 | return self.scale * x_norm + self.bias 85 | return self.scale * x_norm 86 | 87 | 88 | class SpatialDropout(nn.Module): 89 | """ 90 | Spatial Dropout drops a certain percentage of dimensions from each word vector in the training sample 91 | implementation: https://discuss.pytorch.org/t/spatial-dropout-in-pytorch/21400 92 | explanation: https://www.kaggle.com/c/quora-insincere-questions-classification/discussion/76883 93 | """ 94 | 95 | def __init__(self, p: float): 96 | super(SpatialDropout, self).__init__() 97 | self.spatial_dropout = nn.Dropout2d(p=p) 98 | 99 | def forward(self, x: torch.Tensor) -> torch.Tensor: 100 | x = x.permute(0, 2, 1) # convert to [batch, channels, time] 101 | x = self.spatial_dropout(x) 102 | x = x.permute(0, 2, 1) # back to [batch, time, channels] 103 | return x 104 | 105 | 106 | def softmax_windows(self, x): 107 | """ 108 | https://towardsdatascience.com/swap-softmax-weighted-average-pooling-70977a69791b 109 | Take image `x`, with dimension (height, width). 110 | Convert to (2, 2, height*width/4): tile image into 2x2 blocks 111 | Take softmax over each of these blocks 112 | Convert softmax'd image back to (height, width) 113 | 114 | Usage: 115 | self.pool = nn.AvgPool2d(2, 2) 116 | x = self.pool(self.softmax_windows(a) * a) 117 | """ 118 | x_strided = einops.rearrange(x, 'b c (h hs) (w ws) -> b c h w (hs ws)', hs=2, ws=2) 119 | x_softmax_windows = F.softmax(x_strided, dim=-1) 120 | x_strided_softmax = einops.rearrange(x_softmax_windows, 'b c h w (hs ws) -> b c (h hs) (w ws)', hs=2, ws=2) 121 | 122 | return x_strided_softmax 123 | -------------------------------------------------------------------------------- /src/models/pytorch_models_crf.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple, Dict, Optional 2 | 3 | import torch 4 | from torch import nn 5 | from torchcrf import CRF 6 | 7 | from src.models.layers.layers import SpatialDropout 8 | 9 | 10 | class BiLSTMCRF(nn.Module): 11 | """ 12 | New model without nn.Embedding layer 13 | """ 14 | 15 | def __init__( 16 | self, tag_to_idx: Dict, embeddings_dim: int = 100, hidden_dim: int = 4, spatial_dropout: float = 0.2 17 | ): # type: ignore 18 | super(BiLSTMCRF, self).__init__() 19 | self.embedding_dim = embeddings_dim 20 | self.hidden_dim = hidden_dim 21 | self.tag_to_idx = tag_to_idx 22 | self.tagset_size = len(tag_to_idx.values()) 23 | self.crf = CRF(self.tagset_size, batch_first=True) 24 | self.embedding_dropout = SpatialDropout(spatial_dropout) 25 | 26 | self.lstm = nn.LSTM(embeddings_dim, hidden_dim // 2, num_layers=1, bidirectional=True, batch_first=True) 27 | # Maps the output of the LSTM into tag space. 28 | self.hidden2tag = nn.Linear(hidden_dim, hidden_dim // 2) 29 | self.hidden2tag2 = nn.Linear(hidden_dim // 2, self.tagset_size) 30 | 31 | def _get_lstm_features(self, embeds: torch.Tensor, lens: torch.Tensor) -> torch.Tensor: 32 | """ 33 | LSTM forward 34 | 35 | Args: 36 | embeds: batch with embeddings 37 | lens: lengths of sequences 38 | """ 39 | embeds = self.embedding_dropout(embeds) 40 | packed_embeds = torch.nn.utils.rnn.pack_padded_sequence( 41 | embeds, lens.cpu(), batch_first=True, enforce_sorted=False 42 | ) 43 | lstm_out, self.hidden = self.lstm(packed_embeds) 44 | lstm_out, _ = torch.nn.utils.rnn.pad_packed_sequence(lstm_out, batch_first=True) 45 | lstm_feats = self.hidden2tag2(self.hidden2tag(lstm_out.reshape(embeds.shape[0], -1, self.hidden_dim))) 46 | return lstm_feats 47 | 48 | def forward( 49 | self, embeds: torch.Tensor, lens: torch.Tensor, tags: Optional[torch.Tensor] = None 50 | ) -> Tuple[int, torch.Tensor, torch.Tensor]: 51 | """ 52 | Forward 53 | 54 | Args: 55 | embeds: batch with embeddings 56 | lens: lengths of sequences 57 | tags: list of tags (optional) 58 | """ 59 | lstm_feats = self._get_lstm_features(embeds, lens) 60 | if tags is not None: 61 | mask = tags != self.tag_to_idx['PAD'] 62 | loss = self.crf(lstm_feats, tags, mask=mask) 63 | else: 64 | loss = 0 65 | if tags is not None: 66 | mask = torch.tensor(mask) 67 | tag_seq = self.crf.decode(emissions=lstm_feats, mask=mask) 68 | else: 69 | tag_seq = self.crf.decode(lstm_feats) 70 | score = 0 71 | tag_seq = torch.tensor([i for j in tag_seq for i in j]).type_as(embeds) 72 | 73 | return score, tag_seq, -loss 74 | -------------------------------------------------------------------------------- /src/models/simple_model.py: -------------------------------------------------------------------------------- 1 | from omegaconf import DictConfig 2 | from torch import nn 3 | 4 | from src.utils.technical_utils import load_obj 5 | 6 | 7 | class Net(nn.Module): 8 | def __init__(self, cfg: DictConfig) -> None: 9 | """ 10 | Model class. 11 | 12 | Args: 13 | cfg: main config 14 | """ 15 | super().__init__() 16 | self.encoder = load_obj(cfg.model.encoder.class_name)(**cfg.model.encoder.params) 17 | self.decoder = load_obj(cfg.model.decoder.class_name)( 18 | output_dimension=self.encoder.output_dimension, **cfg.model.decoder.params 19 | ) 20 | 21 | def forward(self, x): 22 | out = self.encoder(x) 23 | logits = self.decoder(out) 24 | return logits 25 | -------------------------------------------------------------------------------- /src/optimizers/adan.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Garena Online Private Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the 'License'); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an 'AS IS' BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Code taken from hre: https://github.com/sail-sg/Adan/blob/main/adan.py and updated to work with pytorch-lightning 16 | 17 | 18 | import math 19 | 20 | import torch 21 | from torch.optim.optimizer import Optimizer 22 | 23 | 24 | class Adan(Optimizer): 25 | """ 26 | Implements a pytorch variant of Adan 27 | Adan was proposed in 28 | Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models[J]. 29 | arXiv preprint arXiv:2208.06677, 2022. 30 | https://arxiv.org/abs/2208.06677 31 | Arguments: 32 | params (iterable): iterable of parameters to optimize or dicts defining parameter groups. 33 | lr (float, optional): learning rate. (default: 1e-3) 34 | betas (Tuple[float, float, flot], optional): coefficients used for computing 35 | running averages of gradient and its norm. (default: (0.98, 0.92, 0.99)) 36 | eps (float, optional): term added to the denominator to improve 37 | numerical stability. (default: 1e-8) 38 | weight_decay (float, optional): decoupled weight decay (L2 penalty) (default: 0) 39 | max_grad_norm (float, optional): value used to clip 40 | global grad norm (default: 0.0 no clip) 41 | no_prox (bool): how to perform the decoupled weight decay (default: False) 42 | """ 43 | 44 | def __init__( 45 | self, params, lr=1e-3, betas=(0.98, 0.92, 0.99), eps=1e-8, weight_decay=0.0, max_grad_norm=0.0, no_prox=False 46 | ): 47 | if not 0.0 <= max_grad_norm: 48 | raise ValueError(f'Invalid Max grad norm: {max_grad_norm}') 49 | if not 0.0 <= lr: 50 | raise ValueError(f'Invalid learning rate: {lr}') 51 | if not 0.0 <= eps: 52 | raise ValueError(f'Invalid epsilon value: {eps}') 53 | if not 0.0 <= betas[0] < 1.0: 54 | raise ValueError(f'Invalid beta parameter at index 0: {betas[0]}') 55 | if not 0.0 <= betas[1] < 1.0: 56 | raise ValueError(f'Invalid beta parameter at index 1: {betas[1]}') 57 | if not 0.0 <= betas[2] < 1.0: 58 | raise ValueError(f'Invalid beta parameter at index 2: {betas[2]}') 59 | defaults = dict( 60 | lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, max_grad_norm=max_grad_norm, no_prox=no_prox 61 | ) 62 | super(Adan, self).__init__(params, defaults) 63 | 64 | def __setstate__(self, state): 65 | super(Adan, self).__setstate__(state) 66 | for group in self.param_groups: 67 | group.setdefault('no_prox', False) 68 | 69 | @torch.no_grad() 70 | def restart_opt(self): 71 | for group in self.param_groups: 72 | group['step'] = 0 73 | for p in group['params']: 74 | if p.requires_grad: 75 | state = self.state[p] 76 | # State initialization 77 | 78 | # Exponential moving average of gradient values 79 | state['exp_avg'] = torch.zeros_like(p) 80 | # Exponential moving average of squared gradient values 81 | state['exp_avg_sq'] = torch.zeros_like(p) 82 | # Exponential moving average of gradient difference 83 | state['exp_avg_diff'] = torch.zeros_like(p) 84 | 85 | @torch.no_grad() 86 | def step(self, closure=None): 87 | """ 88 | Perform a single optimization step. 89 | """ 90 | if closure is not None: 91 | with torch.enable_grad(): 92 | loss = closure() 93 | if self.defaults['max_grad_norm'] > 0: 94 | device = self.param_groups[0]['params'][0].device 95 | global_grad_norm = torch.zeros(1, device=device) 96 | 97 | max_grad_norm = torch.tensor(self.defaults['max_grad_norm'], device=device) 98 | for group in self.param_groups: 99 | for p in group['params']: 100 | if p.grad is not None: 101 | grad = p.grad 102 | global_grad_norm.add_(grad.pow(2).sum()) 103 | 104 | global_grad_norm = torch.sqrt(global_grad_norm) 105 | 106 | clip_global_grad_norm = torch.clamp(max_grad_norm / (global_grad_norm + group['eps']), max=1.0) 107 | else: 108 | clip_global_grad_norm = 1.0 109 | 110 | for group in self.param_groups: 111 | beta1, beta2, beta3 = group['betas'] 112 | # assume same step across group now to simplify things 113 | # per parameter step can be easily support by making it tensor, or pass list into kernel 114 | if 'step' in group: 115 | group['step'] += 1 116 | else: 117 | group['step'] = 1 118 | 119 | bias_correction1 = 1.0 - beta1 ** group['step'] 120 | 121 | bias_correction2 = 1.0 - beta2 ** group['step'] 122 | 123 | bias_correction3 = 1.0 - beta3 ** group['step'] 124 | 125 | for p in group['params']: 126 | if p.grad is None: 127 | continue 128 | 129 | state = self.state[p] 130 | if len(state) == 0: 131 | state['exp_avg'] = torch.zeros_like(p) 132 | state['exp_avg_sq'] = torch.zeros_like(p) 133 | state['exp_avg_diff'] = torch.zeros_like(p) 134 | 135 | grad = p.grad.mul_(clip_global_grad_norm) 136 | if 'pre_grad' not in state or group['step'] == 1: 137 | state['pre_grad'] = grad 138 | 139 | copy_grad = grad.clone() 140 | 141 | exp_avg, exp_avg_sq, exp_avg_diff = state['exp_avg'], state['exp_avg_sq'], state['exp_avg_diff'] 142 | diff = grad - state['pre_grad'] 143 | 144 | update = grad + beta2 * diff 145 | exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) # m_t 146 | exp_avg_diff.mul_(beta2).add_(diff, alpha=1 - beta2) # diff_t 147 | exp_avg_sq.mul_(beta3).addcmul_(update, update, value=1 - beta3) # n_t 148 | 149 | denom = ((exp_avg_sq).sqrt() / math.sqrt(bias_correction3)).add_(group['eps']) 150 | update = ((exp_avg / bias_correction1 + beta2 * exp_avg_diff / bias_correction2)).div_(denom) 151 | 152 | if group['no_prox']: 153 | p.data.mul_(1 - group['lr'] * group['weight_decay']) 154 | p.add_(update, alpha=-group['lr']) 155 | else: 156 | p.add_(update, alpha=-group['lr']) 157 | p.data.div_(1 + group['lr'] * group['weight_decay']) 158 | 159 | state['pre_grad'] = copy_grad 160 | -------------------------------------------------------------------------------- /src/schedulers/cosine_schedule_with_warmup.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch.optim.lr_scheduler import LambdaLR 5 | 6 | 7 | class CosineScheduleWithWarmupConfig(LambdaLR): 8 | """ 9 | Inherit LambdaLR so that it can be defined in config. 10 | https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#LambdaLR 11 | https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_with_hard_restarts_schedule_with_warmup 12 | """ 13 | 14 | def __init__( 15 | self, 16 | optimizer: torch.optim.Optimizer, 17 | num_warmup_steps: int, 18 | num_training_steps: int, 19 | num_cycles: float = 0.5, 20 | last_epoch: int = -1, 21 | ) -> None: 22 | def lr_lambda(current_step): 23 | if current_step < num_warmup_steps: 24 | return float(current_step) / float(max(1, num_warmup_steps)) 25 | progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) 26 | return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) 27 | 28 | super().__init__(optimizer, lr_lambda, last_epoch) 29 | -------------------------------------------------------------------------------- /src/schedulers/cosine_schedule_with_warmup1.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch.optim.lr_scheduler import LambdaLR 5 | 6 | 7 | class CosineScheduleWithWarmupConfig(LambdaLR): 8 | """ 9 | Inherit LambdaLR so that it can be defined in config. 10 | https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#LambdaLR 11 | https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_with_hard_restarts_schedule_with_warmup 12 | """ 13 | 14 | def __init__( 15 | self, 16 | optimizer: torch.optim.Optimizer, 17 | num_cycles: float = 0.5, 18 | warmup_prop: float = 0.1, 19 | last_epoch: int = -1, 20 | epochs: int = 200, 21 | train_len: int = 6036000, 22 | n_folds: int = 5, 23 | ) -> None: 24 | len_train = train_len * (n_folds - 1) // n_folds 25 | self.num_warmup_steps = int(warmup_prop * epochs * len_train) 26 | self.num_training_steps = int(epochs * len_train) 27 | 28 | def lr_lambda(current_step): 29 | if current_step < self.num_warmup_steps: 30 | return float(current_step) / float(max(1, self.num_warmup_steps)) 31 | progress = float(current_step - self.num_warmup_steps) / float( 32 | max(1, self.num_training_steps - self.num_warmup_steps) 33 | ) 34 | return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) 35 | 36 | super().__init__(optimizer, lr_lambda, last_epoch) 37 | -------------------------------------------------------------------------------- /src/schedulers/cosine_with_hard_restarts_schedule_with_warmup.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch.optim.lr_scheduler import LambdaLR 5 | 6 | 7 | class CosineWithHardRestartsScheduleWithWarmupConfig(LambdaLR): 8 | """ 9 | Inherit LambdaLR so that it can be defined in config. 10 | https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#LambdaLR 11 | https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_with_hard_restarts_schedule_with_warmup 12 | """ 13 | 14 | def __init__( 15 | self, 16 | optimizer: torch.optim.Optimizer, 17 | num_warmup_steps: int, 18 | num_training_steps: int, 19 | num_cycles: int = 1, 20 | last_epoch: int = -1, 21 | ) -> None: 22 | def lr_lambda(current_step): 23 | if current_step < num_warmup_steps: 24 | return float(current_step) / float(max(1, num_warmup_steps)) 25 | progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) 26 | if progress >= 1.0: 27 | return 0.0 28 | return max(0.0, 0.5 * (1.0 + math.cos(math.pi * ((float(num_cycles) * progress) % 1.0)))) 29 | 30 | super().__init__(optimizer, lr_lambda, last_epoch) 31 | -------------------------------------------------------------------------------- /src/schedulers/lambdar.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | import torch 4 | from torch.optim.lr_scheduler import LambdaLR 5 | 6 | 7 | class LambdaLRConfig(LambdaLR): 8 | """ 9 | Inherit LambdaLR so that it can be defined in config. 10 | https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#LambdaLR 11 | """ 12 | 13 | def __init__(self, optimizer: torch.optim.Optimizer, lr_lambda: str, last_epoch: int = -1) -> None: 14 | lr_lambda_: Callable = eval(lr_lambda) 15 | self.optimizer = optimizer 16 | 17 | if not isinstance(lr_lambda_, list) and not isinstance(lr_lambda_, tuple): 18 | self.lr_lambdas = [lr_lambda_] * len(optimizer.param_groups) 19 | else: 20 | if len(lr_lambda_) != len(optimizer.param_groups): 21 | raise ValueError(f'Expected {len(optimizer.param_groups)} lr_lambdas, but got {len(lr_lambda_)}') 22 | self.lr_lambdas = list(lr_lambda_) 23 | self.last_epoch = last_epoch 24 | super().__init__(optimizer, lr_lambda_, last_epoch) 25 | -------------------------------------------------------------------------------- /src/schedulers/linear_schedule_with_warmup.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch.optim.lr_scheduler import LambdaLR 5 | 6 | 7 | class LinearScheduleWithWarmupConfig(LambdaLR): 8 | """ 9 | Inherit LambdaLR so that it can be defined in config. 10 | https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#LambdaLR 11 | https://huggingface.co/transformers/_modules/transformers/optimization.html#get_linear_schedule_with_warmup 12 | """ 13 | 14 | def __init__( 15 | self, 16 | optimizer: torch.optim.Optimizer, 17 | warmup_prop: int, 18 | last_epoch: int = -1, 19 | epochs: int = 200, 20 | train_len: int = 6036000, 21 | n_folds: int = 5, 22 | ) -> None: 23 | len_train = train_len * (n_folds - 1) // n_folds 24 | self.num_warmup_steps = int(warmup_prop * epochs * len_train) 25 | self.num_training_steps = int(epochs * len_train) 26 | 27 | def lr_lambda(current_step: int) -> float: 28 | if current_step < self.num_warmup_steps: 29 | return float(current_step) / float(max(1, self.num_warmup_steps)) 30 | return max( 31 | 0.0, 32 | float(self.num_training_steps - current_step) 33 | / float(max(1, self.num_training_steps - self.num_warmup_steps)), 34 | ) 35 | 36 | super().__init__(optimizer, lr_lambda, last_epoch) 37 | -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Erlemar/pytorch_tempest/9dc97efa14db2c6e4dfd6b65fd3214a81069afd0/src/utils/__init__.py -------------------------------------------------------------------------------- /src/utils/loggers.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import csv 3 | from typing import Dict, Optional, Union, Any 4 | 5 | # 6 | # class CsvLogger(LightningLoggerBase): 7 | # # TODO Check if works 8 | # @property 9 | # def experiment(self) -> Any: 10 | # pass 11 | # 12 | # def log_hyperparams(self, params: argparse.Namespace) -> Any: 13 | # pass 14 | # 15 | # @property 16 | # def version(self) -> Union[int, str]: 17 | # pass 18 | # 19 | # def __init__(self, csv_path: str = 'csv_log.csv'): 20 | # # TODO: add folder to filename 21 | # super().__init__() 22 | # self.csv_path = csv_path 23 | # 24 | # def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> Any: 25 | # if step == 0: 26 | # header = ['step'] + list(metrics.keys()) 27 | # with open(self.csv_path, 'w') as f: 28 | # writer = csv.writer(f) 29 | # writer.writerow(header) 30 | # if step: 31 | # fields = [float(step)] + list(metrics.values()) 32 | # else: 33 | # fields = list(metrics.values()) 34 | # with open(self.csv_path, 'a') as f: 35 | # writer = csv.writer(f) 36 | # writer.writerow(fields) 37 | # 38 | # @property 39 | # def name(self): 40 | # return 'CsvLogger' 41 | # 42 | # 43 | # class PrintLogger(LightningLoggerBase): 44 | # # TODO Check if works 45 | # @property 46 | # def experiment(self) -> Any: 47 | # pass 48 | # 49 | # def log_hyperparams(self, params: argparse.Namespace) -> Any: 50 | # pass 51 | # 52 | # @property 53 | # def version(self) -> Union[int, str]: 54 | # pass 55 | # 56 | # def __init__(self, csv_path: str = 'csv_log.csv'): 57 | # # TODO: add folder to filename 58 | # super().__init__() 59 | # self.csv_path = csv_path 60 | # 61 | # def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> Any: 62 | # if step == 0: 63 | # header = ['step'] + list(metrics.keys()) 64 | # print(header) 65 | # if step: 66 | # fields = [float(step)] + list(metrics.values()) 67 | # else: 68 | # fields = list(metrics.values()) 69 | # print(fields) 70 | # 71 | # @property 72 | # def name(self): 73 | # return 'PrintLogger' 74 | -------------------------------------------------------------------------------- /src/utils/ml_utils.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import random 3 | from typing import Any, Optional 4 | 5 | import numpy as np 6 | 7 | 8 | def format_prediction_string(boxes, scores): 9 | pred_strings = [] 10 | for s, b in zip(scores, boxes.astype(int)): 11 | pred_strings.append(f'{s:.4f} {b[0]} {b[1]} {b[2] - b[0]} {b[3] - b[1]}') 12 | 13 | return ' '.join(pred_strings) 14 | 15 | 16 | def freeze_until(net: Any, param_name: Optional[str]) -> None: 17 | """ 18 | Freeze net until param_name 19 | 20 | https://opendatascience.slack.com/archives/CGK4KQBHD/p1588373239292300?thread_ts=1588105223.275700&cid=CGK4KQBHD 21 | 22 | Args: 23 | net: 24 | param_name: 25 | 26 | """ 27 | found_name = False 28 | for name, params in net.named_parameters(): 29 | if name == param_name: 30 | found_name = True 31 | params.requires_grad = found_name 32 | 33 | 34 | def stratified_group_k_fold(y, groups, k, seed=None): 35 | """ 36 | Stratify by groups. 37 | 38 | https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation 39 | """ 40 | labels_num = np.max(y) + 1 41 | y_counts_per_group = collections.defaultdict(lambda: np.zeros(labels_num)) 42 | y_distr = collections.Counter() 43 | for label, g in zip(y, groups): 44 | y_counts_per_group[g][label] += 1 45 | y_distr[label] += 1 46 | 47 | y_counts_per_fold = collections.defaultdict(lambda: np.zeros(labels_num)) 48 | groups_per_fold = collections.defaultdict(set) 49 | 50 | def eval_y_counts_per_fold(y_counts, fold): 51 | y_counts_per_fold[fold] += y_counts 52 | std_per_label = [] 53 | for label in range(labels_num): 54 | label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)]) 55 | std_per_label.append(label_std) 56 | y_counts_per_fold[fold] -= y_counts 57 | return np.mean(std_per_label) 58 | 59 | groups_and_y_counts = list(y_counts_per_group.items()) 60 | random.Random(seed).shuffle(groups_and_y_counts) 61 | 62 | for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])): 63 | best_fold = None 64 | min_eval = None 65 | for i in range(k): 66 | fold_eval = eval_y_counts_per_fold(y_counts, i) 67 | if min_eval is None or fold_eval < min_eval: 68 | min_eval = fold_eval 69 | best_fold = i 70 | y_counts_per_fold[best_fold] += y_counts 71 | groups_per_fold[best_fold].add(g) 72 | 73 | all_groups = set(groups) 74 | for i in range(k): 75 | train_groups = all_groups - groups_per_fold[i] 76 | test_groups = groups_per_fold[i] 77 | 78 | train_indices = [i for i, g in enumerate(groups) if g in train_groups] 79 | test_indices = [i for i, g in enumerate(groups) if g in test_groups] 80 | 81 | yield train_indices, test_indices 82 | 83 | 84 | def collate_fn(batch): 85 | return tuple(zip(*batch)) 86 | -------------------------------------------------------------------------------- /src/utils/technical_utils.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import importlib 3 | from itertools import product 4 | from typing import Any, Dict, Generator 5 | 6 | import torch 7 | from omegaconf import DictConfig, OmegaConf 8 | from torch import nn 9 | 10 | 11 | def load_obj(obj_path: str, default_obj_path: str = '') -> Any: 12 | """ 13 | Extract an object from a given path. 14 | https://github.com/quantumblacklabs/kedro/blob/9809bd7ca0556531fa4a2fc02d5b2dc26cf8fa97/kedro/utils.py 15 | Args: 16 | obj_path: Path to an object to be extracted, including the object name. 17 | default_obj_path: Default object path. 18 | Returns: 19 | Extracted object. 20 | Raises: 21 | AttributeError: When the object does not have the given named attribute. 22 | """ 23 | obj_path_list = obj_path.rsplit('.', 1) 24 | obj_path = obj_path_list.pop(0) if len(obj_path_list) > 1 else default_obj_path 25 | obj_name = obj_path_list[0] 26 | module_obj = importlib.import_module(obj_path) 27 | if not hasattr(module_obj, obj_name): 28 | raise AttributeError(f'Object `{obj_name}` cannot be loaded from `{obj_path}`.') 29 | return getattr(module_obj, obj_name) 30 | 31 | 32 | def convert_to_jit(model: nn.Module, save_name: str, cfg: DictConfig) -> None: 33 | input_shape = (1, 3, cfg.datamodule.main_image_size, cfg.datamodule.main_image_size) 34 | input_shape1 = 1 35 | out_path = f'saved_models/{save_name}_jit.pt' 36 | model.eval() 37 | 38 | device = next(model.parameters()).device 39 | input_tensor = torch.ones(input_shape).float().to(device) 40 | input_tensor1 = torch.ones(input_shape1, dtype=torch.long).to(device) 41 | traced_model = torch.jit.trace(model, (input_tensor, input_tensor1)) 42 | torch.jit.save(traced_model, out_path) 43 | 44 | 45 | def product_dict(**kwargs: Dict) -> Generator: 46 | """ 47 | Convert dict with lists in values into lists of all combinations 48 | 49 | This is necessary to convert config with experiment values 50 | into format usable by hydra 51 | Args: 52 | **kwargs: 53 | 54 | Returns: 55 | list of lists 56 | 57 | --- 58 | Example: 59 | >>> list_dict = {'a': [1, 2], 'b': [2, 3]} 60 | >>> list(product_dict(**list_dict)) 61 | >>> [['a=1', 'b=2'], ['a=1', 'b=3'], ['a=2', 'b=2'], ['a=2', 'b=3']] 62 | 63 | """ 64 | keys = kwargs.keys() 65 | vals = kwargs.values() 66 | for instance in product(*vals): 67 | zip_list = list(zip(keys, instance)) 68 | yield [f'{i}={j}' for i, j in zip_list] 69 | 70 | 71 | def config_to_hydra_dict(cfg: DictConfig) -> Dict: 72 | """ 73 | Convert config into dict with lists of values, where key is full name of parameter 74 | 75 | This fuction is used to get key names which can be used in hydra. 76 | 77 | Args: 78 | cfg: 79 | 80 | Returns: 81 | converted dict 82 | 83 | """ 84 | experiment_dict = {} 85 | for k, v in cfg.items(): 86 | for k1, v1 in v.items(): 87 | experiment_dict[f'{k!r}.{k1!r}'] = v1 88 | 89 | return experiment_dict 90 | 91 | 92 | def flatten_omegaconf(d, sep='_'): 93 | d = OmegaConf.to_container(d) 94 | 95 | obj = collections.OrderedDict() 96 | 97 | def recurse(t, parent_key=''): 98 | if isinstance(t, list): 99 | for i, _ in enumerate(t): 100 | recurse(t[i], parent_key + sep + str(i) if parent_key else str(i)) 101 | elif isinstance(t, dict): 102 | for k, v in t.items(): 103 | recurse(v, parent_key + sep + k if parent_key else k) 104 | else: 105 | obj[parent_key] = t 106 | 107 | recurse(d) 108 | obj = {k: v for k, v in obj.items() if isinstance(v, (int, float))} 109 | 110 | return obj 111 | -------------------------------------------------------------------------------- /src/utils/text_utils.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | from collections import Counter 3 | from typing import List, Union, Dict, Tuple, Any, Optional 4 | 5 | import numpy as np 6 | import numpy.typing as npt 7 | import torch.nn as nn 8 | from gensim.models import FastText 9 | from omegaconf import DictConfig 10 | 11 | from src.utils.technical_utils import load_obj 12 | 13 | 14 | def _generate_tag_to_idx(cfg: DictConfig, entities_names: List) -> dict: 15 | """ 16 | Generate tag-idx vocab 17 | Args: 18 | cfg: config 19 | entities_names: list of train entities 20 | Returns: 21 | tag-idx vocab 22 | """ 23 | tag_to_idx: Dict = {} 24 | for ind, entity in enumerate(entities_names): 25 | if cfg.datamodule.params.use_bulio_tokens: 26 | tag_to_idx[f'B-{entity}'] = len(tag_to_idx) 27 | tag_to_idx[f'I-{entity}'] = len(tag_to_idx) 28 | tag_to_idx[f'L-{entity}'] = len(tag_to_idx) 29 | tag_to_idx[f'U-{entity}'] = len(tag_to_idx) 30 | else: 31 | tag_to_idx[entity] = ind + 1 32 | for special_tag in ['O', 'PAD']: 33 | tag_to_idx[special_tag] = len(tag_to_idx) 34 | 35 | return tag_to_idx 36 | 37 | 38 | def _generate_word_to_idx( 39 | data: List, 40 | use_pad_token: bool = False, 41 | use_unk_token: bool = False, 42 | min_words: Union[int, float] = 0.0, 43 | max_words: Union[int, float] = 1.0, 44 | ) -> Dict[str, int]: 45 | """ 46 | Generate word-idx vocab 47 | Args: 48 | data: ner dataset 49 | use_pad_token: pad_token flag 50 | use_pad_token: pad_token flag 51 | min_words: min word count 52 | max_words: max word count 53 | Returns: 54 | word-idx vocab 55 | """ 56 | all_words = [] 57 | if use_pad_token: 58 | all_words.append('') 59 | if use_unk_token: 60 | all_words.append('') 61 | 62 | for tokens in data: 63 | all_words.extend(tokens['text']) 64 | count = Counter(all_words).most_common() 65 | max_count = count[0][1] 66 | if isinstance(min_words, float): 67 | min_words = max_count * min_words 68 | if isinstance(max_words, float): 69 | max_words = max_count * max_words 70 | 71 | all_words = [w[0] for w in count if max_words >= w[1] >= min_words] 72 | word_to_idx = dict(zip(all_words, range(len(all_words)))) 73 | 74 | return word_to_idx 75 | 76 | 77 | def get_vectorizer(cfg: DictConfig, word_to_idx: Dict) -> nn.Module: 78 | """ 79 | Get model 80 | 81 | Args: 82 | word_to_idx: 83 | cfg: config 84 | 85 | Returns: 86 | initialized model 87 | """ 88 | 89 | vectorizer = load_obj(cfg.datamodule.vectorizer_class_name) 90 | vectorizer = vectorizer( 91 | word_to_idx=word_to_idx, 92 | embeddings_path=cfg.datamodule.embeddings_path, 93 | embeddings_type=cfg.datamodule.embeddings_type, 94 | embeddings_dim=cfg.datamodule.embeddings_dim, 95 | ) 96 | 97 | return vectorizer 98 | 99 | 100 | def get_word_to_idx(datasets: List) -> Dict[str, int]: 101 | """ 102 | Get dictionary with words and indexes 103 | Args: 104 | datasets: 105 | 106 | Returns: 107 | Dict with words and indexes 108 | """ 109 | word_to_idx: Dict[str, int] = {'пропущено': 1} 110 | for dataset in datasets: 111 | for claim in dataset: 112 | for word in claim['text']: 113 | if word not in word_to_idx: 114 | word_to_idx[word] = len(word_to_idx) + 1 115 | 116 | return word_to_idx 117 | 118 | 119 | def get_coefs(word: str, *arr: npt.ArrayLike) -> Tuple[str, npt.ArrayLike]: 120 | """ 121 | Get word and coefficient from line in embeddings 122 | Args: 123 | word: 124 | *arr: 125 | 126 | Returns: 127 | word and array with values 128 | """ 129 | return word, np.asarray(arr, dtype='float32') 130 | 131 | 132 | def load_embeddings(embedding_path: str, embedding_type: str = 'fasttext') -> Union[Dict, Any]: 133 | """ 134 | Load embeddings into dictionary 135 | Args: 136 | embedding_path: path to embeddings 137 | embedding_type: type of pretrained embeddings ('word2vec', 'glove', 'fasttext') 138 | Returns: 139 | loaded embeddings 140 | """ 141 | if 'pkl' in embedding_path: 142 | with open(embedding_path, 'rb') as vec_f: 143 | return pickle.load(vec_f) 144 | elif embedding_type == 'word2vec': 145 | with open(embedding_path, 'r', encoding='utf-8', errors='ignore') as f: 146 | next(f) 147 | return dict(get_coefs(*line.strip().split(' ')) for line in f) 148 | elif embedding_type == 'glove': 149 | with open(embedding_path, 'r', encoding='utf-8', errors='ignore') as f: 150 | return dict(get_coefs(*line.strip().split(' ')) for line in f) 151 | elif embedding_type == 'fasttext': 152 | return FastText.load(embedding_path) 153 | else: 154 | return None 155 | 156 | 157 | def get_vector(embedding_type: str, embedding_index: dict, word: str) -> Optional[npt.ArrayLike]: 158 | """ 159 | Return vector in relation to embedding_type parameter 160 | Args: 161 | embedding_type: type of pretrained embeddings ('word2vec', 'glove', 'fasttext') 162 | embedding_index: dict of vectors 163 | word: keyword 164 | Returns: 165 | word vector 166 | """ 167 | emb = None 168 | 169 | if embedding_type in ['word2vec', 'glove']: 170 | emb = embedding_index.get(word) 171 | elif embedding_type == 'fasttext' and type(embedding_index) is FastText: 172 | emb = embedding_index[word] 173 | elif embedding_type == 'fasttext' and type(embedding_index) is dict: 174 | emb = embedding_index.get(word) 175 | return emb 176 | 177 | 178 | def build_matrix( 179 | word_dict: Dict, 180 | embedding_path: str = '', 181 | embeddings_type: str = 'fasttext', 182 | max_features: int = 100000, 183 | embed_size: int = 300, 184 | ) -> Tuple[npt.ArrayLike, int, List]: 185 | """ 186 | Create embedding matrix 187 | 188 | Args: 189 | embedding_path: path to embeddings 190 | embeddings_type: type of pretrained embeddings ('word2vec', 'glove', 'fasttext') 191 | word_dict: tokenizer 192 | max_features: max features to use 193 | embed_size: size of embeddings 194 | 195 | Returns: 196 | embedding matrix, number of of words and the list of not found words 197 | """ 198 | if embeddings_type not in ['word2vec', 'glove', 'fasttext']: 199 | raise ValueError('Unacceptable embedding type.\nPermissible values: word2vec, glove, fasttext') 200 | embedding_index = load_embeddings(embedding_path, embeddings_type) 201 | nb_words = min(max_features, len(word_dict)) 202 | if embeddings_type in ['word2vec', 'glove']: 203 | embed_size = embed_size if embed_size is not None else len(list(embedding_index.values())[0]) 204 | all_embs = np.stack(embedding_index.values()) # type: ignore 205 | emb_mean, emb_std = all_embs.mean(), all_embs.std() 206 | embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size)) 207 | elif embeddings_type == 'fasttext': 208 | embedding_matrix = np.zeros((nb_words, embed_size)) 209 | 210 | unknown_words = [] 211 | for word, i in word_dict.items(): 212 | key = word 213 | embedding_vector = get_vector(embeddings_type, embedding_index, word) 214 | if embedding_vector is not None: 215 | embedding_matrix[i] = embedding_vector 216 | continue 217 | embedding_vector = get_vector(embeddings_type, embedding_index, word.lower()) 218 | if embedding_vector is not None: 219 | embedding_matrix[i] = embedding_vector 220 | continue 221 | embedding_vector = get_vector(embeddings_type, embedding_index, word.upper()) 222 | if embedding_vector is not None: 223 | embedding_matrix[i] = embedding_vector 224 | continue 225 | embedding_vector = get_vector(embeddings_type, embedding_index, word.capitalize()) 226 | if embedding_vector is not None: 227 | embedding_matrix[i] = embedding_vector 228 | continue 229 | unknown_words.append(key) 230 | return embedding_matrix, nb_words, unknown_words 231 | 232 | 233 | def pad_sequences( 234 | sequences: List, 235 | maxlen: Optional[int], 236 | dtype: str = 'int32', 237 | padding: str = 'post', 238 | truncating: str = 'post', 239 | value: int = 0, 240 | ) -> npt.ArrayLike: 241 | """Pad sequences to the same length. 242 | from Keras 243 | 244 | This function transforms a list of 245 | `num_samples` sequences (lists of integers) 246 | into a 2D Numpy array of shape `(num_samples, num_timesteps)`. 247 | `num_timesteps` is either the `maxlen` argument if provided, 248 | or the length of the longest sequence otherwise. 249 | 250 | Sequences that are shorter than `num_timesteps` 251 | are padded with `value` at the end. 252 | 253 | Sequences longer than `num_timesteps` are truncated 254 | so that they fit the desired length. 255 | The position where padding or truncation happens is determined by 256 | the arguments `padding` and `truncating`, respectively. 257 | 258 | Pre-padding is the default. 259 | 260 | # Arguments 261 | sequences: List of lists, where each element is a sequence. 262 | maxlen: Int, maximum length of all sequences. 263 | dtype: Type of the output sequences. 264 | To pad sequences with variable length strings, you can use `object`. 265 | padding: String, 'pre' or 'post': 266 | pad either before or after each sequence. 267 | truncating: String, 'pre' or 'post': 268 | remove values from sequences larger than 269 | `maxlen`, either at the beginning or at the end of the sequences. 270 | value: Float or String, padding value. 271 | 272 | # Returns 273 | x: Numpy array with shape `(len(sequences), maxlen)` 274 | 275 | # Raises 276 | ValueError: In case of invalid values for `truncating` or `padding`, 277 | or in case of invalid shape for a `sequences` entry. 278 | """ 279 | if not hasattr(sequences, '__len__'): 280 | raise ValueError('`sequences` must be iterable.') 281 | num_samples = len(sequences) 282 | 283 | lengths = [] 284 | for x in sequences: 285 | try: 286 | lengths.append(len(x)) 287 | except TypeError: 288 | raise ValueError('`sequences` must be a list of iterables. ' 'Found non-iterable: ' + str(x)) 289 | 290 | if maxlen is None: 291 | maxlen = np.max(lengths) 292 | 293 | # take the sample shape from the first non empty sequence 294 | # checking for consistency in the main loop below. 295 | sample_shape: Tuple[int, ...] = () 296 | for s in sequences: 297 | if len(s) > 0: 298 | sample_shape = np.asarray(s).shape[1:] 299 | break 300 | 301 | x = np.full((num_samples, maxlen) + sample_shape, value, dtype=dtype) 302 | for idx, s in enumerate(sequences): 303 | if not len(s): 304 | continue # empty list/array was found 305 | if truncating == 'pre': 306 | trunc = s[-maxlen:] 307 | elif truncating == 'post': 308 | trunc = s[:maxlen] 309 | else: 310 | raise ValueError(f'Truncating type "{truncating}" ' 'not understood') 311 | 312 | # check `trunc` has expected shape 313 | trunc = np.asarray(trunc, dtype=dtype) 314 | if trunc.shape[1:] != sample_shape: 315 | raise ValueError( 316 | f'Shape of sample {trunc.shape[1:]} of sequence at position {idx}' 317 | f'is different from expected shape {sample_shape}' 318 | ) 319 | 320 | if padding == 'post': 321 | x[idx, : len(trunc)] = trunc 322 | elif padding == 'pre': 323 | x[idx, -len(trunc) :] = trunc 324 | else: 325 | raise ValueError(f'Padding type "{padding}" not understood') 326 | return x 327 | -------------------------------------------------------------------------------- /src/utils/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import shutil 4 | 5 | import hydra 6 | import numpy as np 7 | import torch 8 | import logging 9 | 10 | 11 | def set_seed(seed: int = 666, precision: int = 10) -> None: 12 | np.random.seed(seed) 13 | random.seed(seed) 14 | os.environ['PYTHONHASHSEED'] = str(seed) 15 | torch.backends.cudnn.benchmark = False 16 | torch.backends.cudnn.deterministic = True 17 | torch.manual_seed(seed) 18 | torch.cuda.manual_seed(seed) 19 | torch.set_printoptions(precision=precision) 20 | 21 | 22 | def save_useful_info() -> None: 23 | logging.info(hydra.utils.get_original_cwd()) 24 | logging.info(os.getcwd()) 25 | shutil.copytree( 26 | os.path.join(hydra.utils.get_original_cwd(), 'src'), 27 | os.path.join(hydra.utils.get_original_cwd(), f'{os.getcwd()}/code/src'), 28 | ) 29 | shutil.copy2( 30 | os.path.join(hydra.utils.get_original_cwd(), 'train.py'), 31 | os.path.join(hydra.utils.get_original_cwd(), os.getcwd(), 'code'), 32 | ) 33 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Erlemar/pytorch_tempest/9dc97efa14db2c6e4dfd6b65fd3214a81069afd0/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_cfg.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from hydra import compose, initialize 5 | from omegaconf import DictConfig 6 | 7 | config_files = [f.split('.')[0] for f in os.listdir('conf') if 'yaml' in f] 8 | 9 | 10 | @pytest.mark.parametrize('config_name', config_files) 11 | def test_cfg(config_name: str) -> None: 12 | with initialize(config_path='../conf'): 13 | cfg = compose(config_name=config_name, overrides=['private=default']) 14 | assert isinstance(cfg, DictConfig) 15 | -------------------------------------------------------------------------------- /tests/test_metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | import torch 4 | from sklearn.metrics import f1_score 5 | 6 | from src.metrics.f1_score import F1Score 7 | from src.utils.utils import set_seed 8 | 9 | 10 | @torch.no_grad() 11 | @pytest.mark.parametrize('average', ['micro', 'macro', 'weighted']) 12 | def test_f1score_metric(average: str) -> None: 13 | set_seed(42) 14 | labels = torch.randint(1, 10, (4096, 100)).flatten() 15 | predictions = torch.randint(1, 10, (4096, 100)).flatten() 16 | labels_numpy = labels.numpy() 17 | predictions_numpy = predictions.numpy() 18 | f1_metric = F1Score(average) 19 | my_pred = f1_metric(predictions, labels) 20 | 21 | f1_pred = f1_score(labels_numpy, predictions_numpy, average=average) 22 | 23 | assert np.isclose(my_pred.item(), f1_pred.item()) 24 | -------------------------------------------------------------------------------- /tests/test_optimizers.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | import torch 5 | from hydra import compose, initialize 6 | 7 | from src.utils.technical_utils import load_obj 8 | 9 | 10 | @pytest.mark.parametrize('opt_name', os.listdir('conf/optimizer')) 11 | def test_optimizers(opt_name: str) -> None: 12 | optimizer_name = opt_name.split('.')[0] 13 | with initialize(config_path='../conf'): 14 | cfg = compose(config_name='config', overrides=[f'optimizer={optimizer_name}', 'private=default']) 15 | load_obj(cfg.optimizer.class_name)(torch.nn.Linear(1, 1).parameters(), **cfg.optimizer.params) 16 | -------------------------------------------------------------------------------- /tests/test_schedulers.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | import torch 5 | from hydra import compose, initialize 6 | 7 | from src.utils.technical_utils import load_obj 8 | 9 | 10 | @pytest.mark.parametrize('sch_name', os.listdir('conf/scheduler')) 11 | def test_schedulers(sch_name: str) -> None: 12 | scheduler_name = sch_name.split('.')[0] 13 | with initialize(config_path='../conf'): 14 | cfg = compose( 15 | config_name='config', overrides=[f'scheduler={scheduler_name}', 'optimizer=sgd', 'private=default'] 16 | ) 17 | optimizer = load_obj(cfg.optimizer.class_name)(torch.nn.Linear(1, 1).parameters(), **cfg.optimizer.params) 18 | load_obj(cfg.scheduler.class_name)(optimizer, **cfg.scheduler.params) 19 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import warnings 4 | from pathlib import Path 5 | 6 | import hydra 7 | import pytorch_lightning as pl 8 | import torch 9 | from omegaconf import DictConfig, OmegaConf 10 | from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping 11 | 12 | from src.utils.technical_utils import load_obj, convert_to_jit 13 | from src.utils.utils import set_seed, save_useful_info 14 | 15 | warnings.filterwarnings('ignore') 16 | logging.basicConfig(level=logging.INFO) 17 | 18 | 19 | def run(cfg: DictConfig) -> None: 20 | """ 21 | Run pytorch-lightning model 22 | 23 | 24 | Args: 25 | cfg: hydra config 26 | 27 | """ 28 | set_seed(cfg.training.seed) 29 | run_name = os.path.basename(os.getcwd()) 30 | cfg.callbacks.model_checkpoint.params.dirpath = Path( 31 | os.getcwd(), cfg.callbacks.model_checkpoint.params.dirpath 32 | ).as_posix() 33 | callbacks = [] 34 | for callback in cfg.callbacks.other_callbacks: 35 | if callback.params: 36 | callback_instance = load_obj(callback.class_name)(**callback.params) 37 | else: 38 | callback_instance = load_obj(callback.class_name)() 39 | callbacks.append(callback_instance) 40 | 41 | loggers = [] 42 | if cfg.logging.log: 43 | for logger in cfg.logging.loggers: 44 | if 'experiment_name' in logger.params.keys(): 45 | logger.params['experiment_name'] = run_name 46 | elif 'name' in logger.params.keys(): 47 | logger.params['name'] = run_name 48 | 49 | loggers.append(load_obj(logger.class_name)(**logger.params)) 50 | 51 | callbacks.append(EarlyStopping(**cfg.callbacks.early_stopping.params)) 52 | callbacks.append(ModelCheckpoint(**cfg.callbacks.model_checkpoint.params)) 53 | 54 | trainer = pl.Trainer( 55 | logger=loggers, 56 | callbacks=callbacks, 57 | **cfg.trainer, 58 | ) 59 | 60 | model = load_obj(cfg.training.lightning_module_name)(cfg=cfg) 61 | dm = load_obj(cfg.datamodule.data_module_name)(cfg=cfg) 62 | trainer.fit(model, dm) 63 | 64 | if cfg.general.save_pytorch_model and cfg.general.save_best: 65 | if os.path.exists(trainer.checkpoint_callback.best_model_path): # type: ignore 66 | best_path = trainer.checkpoint_callback.best_model_path # type: ignore 67 | # extract file name without folder 68 | save_name = os.path.basename(os.path.normpath(best_path)) 69 | logging.info(f'{save_name = }') 70 | model = load_obj(cfg.training.lightning_module_name).load_from_checkpoint(best_path, cfg=cfg, strict=False) 71 | model_name = Path( 72 | cfg.callbacks.model_checkpoint.params.dirpath, f'best_{save_name}'.replace('.ckpt', '.pth') 73 | ).as_posix() 74 | torch.save(model.model.state_dict(), model_name) 75 | else: 76 | os.makedirs('saved_models', exist_ok=True) 77 | model_name = 'saved_models/last.pth' 78 | torch.save(model.model.state_dict(), model_name) 79 | 80 | if cfg.general.convert_to_jit and os.path.exists(trainer.checkpoint_callback.best_model_path): # type: ignore 81 | best_path = trainer.checkpoint_callback.best_model_path # type: ignore 82 | save_name = os.path.basename(os.path.normpath(best_path)) 83 | convert_to_jit(model, save_name, cfg) 84 | 85 | logging.info(f'{run_name = }') 86 | 87 | 88 | @hydra.main(config_path='conf', config_name='config') 89 | def run_model(cfg: DictConfig) -> None: 90 | os.makedirs('logs', exist_ok=True) 91 | logging.info(OmegaConf.to_yaml(cfg)) 92 | if cfg.general.log_code: 93 | save_useful_info() 94 | run(cfg) 95 | 96 | 97 | if __name__ == '__main__': 98 | run_model() 99 | -------------------------------------------------------------------------------- /train_ner.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import warnings 4 | from pathlib import Path 5 | 6 | import hydra 7 | import pytorch_lightning as pl 8 | import torch 9 | from omegaconf import DictConfig, OmegaConf 10 | from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping 11 | 12 | from src.utils.technical_utils import load_obj 13 | from src.utils.utils import set_seed, save_useful_info 14 | 15 | logging.basicConfig(level=logging.INFO) 16 | warnings.filterwarnings('ignore') 17 | 18 | 19 | def run(cfg: DictConfig) -> None: 20 | """ 21 | Run pytorch-lightning model 22 | 23 | Args: 24 | new_dir: 25 | cfg: hydra config 26 | 27 | """ 28 | set_seed(cfg.training.seed) 29 | run_name = os.path.basename(os.getcwd()) 30 | 31 | cfg.callbacks.model_checkpoint.params.dirpath = Path( 32 | os.getcwd(), cfg.callbacks.model_checkpoint.params.dirpath 33 | ).as_posix() 34 | callbacks = [] 35 | for callback in cfg.callbacks.other_callbacks: 36 | if callback.params: 37 | callback_instance = load_obj(callback.class_name)(**callback.params) 38 | else: 39 | callback_instance = load_obj(callback.class_name)() 40 | callbacks.append(callback_instance) 41 | 42 | loggers = [] 43 | if cfg.logging.log: 44 | for logger in cfg.logging.loggers: 45 | if 'experiment_name' in logger.params.keys(): 46 | logger.params['experiment_name'] = run_name 47 | loggers.append(load_obj(logger.class_name)(**logger.params)) 48 | 49 | callbacks.append(EarlyStopping(**cfg.callbacks.early_stopping.params)) 50 | callbacks.append(ModelCheckpoint(**cfg.callbacks.model_checkpoint.params)) 51 | 52 | trainer = pl.Trainer( 53 | logger=loggers, 54 | callbacks=callbacks, 55 | **cfg.trainer, 56 | ) 57 | 58 | dm = load_obj(cfg.datamodule.data_module_name)(cfg=cfg) 59 | dm.setup() 60 | model = load_obj(cfg.training.lightning_module_name)(cfg=cfg, tag_to_idx=dm.tag_to_idx) 61 | model._vectorizer = dm._vectorizer 62 | trainer.fit(model, dm) 63 | 64 | if cfg.general.save_pytorch_model: 65 | if cfg.general.save_best: 66 | best_path = trainer.checkpoint_callback.best_model_path # type: ignore 67 | # extract file name without folder 68 | save_name = os.path.basename(os.path.normpath(best_path)) 69 | model = model.load_from_checkpoint(best_path, cfg=cfg, tag_to_idx=dm.tag_to_idx, strict=False) 70 | model_name = Path( 71 | cfg.callbacks.model_checkpoint.params.dirpath, f'best_{save_name}'.replace('.ckpt', '.pth') 72 | ).as_posix() 73 | torch.save(model.model.state_dict(), model_name) 74 | else: 75 | os.makedirs('saved_models', exist_ok=True) 76 | model_name = 'saved_models/last.pth' 77 | torch.save(model.model.state_dict(), model_name) 78 | 79 | 80 | @hydra.main(config_path='conf', config_name='config_ner') 81 | def run_model(cfg: DictConfig) -> None: 82 | os.makedirs('logs', exist_ok=True) 83 | logging.info(OmegaConf.to_yaml(cfg)) 84 | if cfg.general.log_code: 85 | save_useful_info() 86 | run(cfg) 87 | 88 | 89 | if __name__ == '__main__': 90 | run_model() 91 | --------------------------------------------------------------------------------