├── assets └── banner.png ├── v1 ├── requirements.txt ├── eval.ipynb ├── sweep.py └── src.py ├── requirements.txt ├── v0 ├── requirements.txt ├── eval.py ├── notes │ ├── sweep_060423.md │ ├── compare_submissions.py │ ├── sweep_040423.md │ ├── sweep_010423.md │ ├── sweep_020423.md │ └── sub12.csv ├── sweep.ipynb ├── eval.ipynb ├── sweep.py └── src.py ├── scripts ├── encode_dataset.py ├── pixel_stats_3.yaml ├── pixel_stats_1.yaml ├── pixel_stats_2.yaml ├── split_dataset.py └── pixel_stats.py ├── eval.py ├── LICENSE ├── eval.ipynb ├── README.md ├── .gitignore ├── sweep.py └── src.py /assets/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hu-po/ashenvenus/HEAD/assets/banner.png -------------------------------------------------------------------------------- /v1/requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.13.0 2 | tensorboardx==2.6 3 | hyperopt==0.2.7 4 | scipy==1.9.3 5 | opencv-python==4.6.0.66 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.13.0 2 | tensorboardx==2.6 3 | hyperopt==0.2.7 4 | scipy==1.9.3 5 | opencv-python==4.6.0.66 6 | polars==0.17.2 -------------------------------------------------------------------------------- /v0/requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.9.1 2 | torchvision==0.10.1 3 | matplotlib==3.4.3 4 | tensorboard==2.6.0 5 | tensorboardX==2.4 6 | hyperopt==0.2.5 -------------------------------------------------------------------------------- /v0/eval.py: -------------------------------------------------------------------------------- 1 | from src import eval_from_episode_dir 2 | 3 | eval_from_episode_dir( 4 | episode_dir='/home/tren/dev/ashenvenus/output/6b230419', 5 | output_dir='/home/tren/dev/ashenvenus/preds', 6 | quantize=False, 7 | device='gpu', 8 | resize_ratio=1.0, 9 | batch_size=64, 10 | ) -------------------------------------------------------------------------------- /scripts/encode_dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | Encode dataset so we can train longer/faster. 3 | """ 4 | 5 | import polars as pl 6 | from segment_anything import sam_model_registry 7 | 8 | 9 | # Load encoder 10 | device = get_device(device) 11 | sam_model = sam_model_registry[model](checkpoint=weights_filepath) 12 | 13 | # For each directory 14 | 15 | # Load dataset 16 | 17 | # Go through N items of dataset 18 | 19 | # Encode each item 20 | 21 | # Save encoded dataset -------------------------------------------------------------------------------- /v0/notes/sweep_060423.md: -------------------------------------------------------------------------------- 1 | Copy over files from linux computers to windows computer: 2 | 3 | ``` 4 | scp -r tren@192.168.1.30:/home/tren/dev/ashenvenus/output/* C:\Users\ook\Documents\dev\ashenvenus\output\ 5 | scp -r oop@192.168.1.34:/home/oop/dev/ashenvenus/output/* C:\Users\ook\Documents\dev\ashenvenus\output\ 6 | ``` 7 | 8 | Great runs 9 | 10 | (c39ac15b|82934db2|7a7a91cf|29d55a5f|77b1f3dc) 11 | 12 | Okay runs 13 | 14 | (49cee3f6|e40f6408|97e70391|73885935) 15 | 16 | Learnings 17 | 18 | -------------------------------------------------------------------------------- /scripts/pixel_stats_3.yaml: -------------------------------------------------------------------------------- 1 | bg: 2 | count: 51194732500 3 | max: 1.0 4 | mean: 0.21197298048064112 5 | min: 0.0 6 | std: 0.1832872246950865 7 | ink: 8 | count: 7204350000 9 | max: 1.0 10 | mean: 0.2130055943131447 11 | min: 0.0 12 | std: 0.18590407781302928 13 | mask: 14 | count: 58398962500 15 | max: 1.0 16 | mean: 0.21210011525079608 17 | min: 0.0 18 | std: 0.1836130627989769 19 | raw: 20 | count: 0 21 | max: .nan 22 | mean: .nan 23 | min: .nan 24 | std: .nan 25 | -------------------------------------------------------------------------------- /scripts/pixel_stats_1.yaml: -------------------------------------------------------------------------------- 1 | bg: 2 | count: 52384670000 3 | max: 1.0 4 | mean: 0.19238760709762573 5 | min: 0.0 6 | std: 0.17632672891020776 7 | ink: 8 | count: 11801235000 9 | max: 1.0 10 | mean: 0.19464799194596707 11 | min: 0.0 12 | std: 0.1828428265452385 13 | mask: 14 | count: 64185905000 15 | max: 1.0 16 | mean: 0.1928032992966473 17 | min: 0.0 18 | std: 0.17754690028727055 19 | raw: 20 | count: 0 21 | max: .nan 22 | mean: .nan 23 | min: .nan 24 | std: .nan 25 | -------------------------------------------------------------------------------- /scripts/pixel_stats_2.yaml: -------------------------------------------------------------------------------- 1 | bg: 2 | count: 169671680000 3 | max: 1.0 4 | mean: 0.20369526762515305 5 | min: 0.0 6 | std: 0.17836732387542725 7 | ink: 8 | count: 36664595000 9 | max: 1.0 10 | mean: 0.20485440285876394 11 | min: 0.0 12 | std: 0.1808986598998308 13 | mask: 14 | count: 206297625000 15 | max: 1.0 16 | mean: 0.20393421886488794 17 | min: 0.0 18 | std: 0.1788088697195053 19 | raw: 20 | count: 0 21 | max: .nan 22 | mean: .nan 23 | min: .nan 24 | std: .nan 25 | -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- 1 | from src import eval_from_episode_dir 2 | 3 | eval_from_episode_dir( 4 | eval_dir = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\data\\split\\valid", 5 | episode_dir = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\output\\34fdec50", 6 | output_dir = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\output\\pred_34fdec50", 7 | weights_filename = "model.pth", 8 | device='gpu', 9 | eval_on = '1', 10 | threshold = 0.3, 11 | max_num_samples_eval = 80000, 12 | max_time_hours = 0.5, 13 | batch_size = 6, 14 | log_images = False, 15 | save_pred_img = True, 16 | save_submit_csv = False, 17 | save_histograms = False, 18 | ) -------------------------------------------------------------------------------- /v0/notes/compare_submissions.py: -------------------------------------------------------------------------------- 1 | from src import save_rle_as_image 2 | 3 | sub_16_filepath="C:\\Users\\ook\\Documents\\dev\\ashenvenus\\logs\\sub16.csv" 4 | sub_12_filepath="C:\\Users\\ook\\Documents\\dev\\ashenvenus\\logs\\sub12.csv" 5 | sub_11_filepath="C:\\Users\\ook\\Documents\\dev\\ashenvenus\\logs\\sub11.csv" 6 | output_dir="C:\\Users\\ook\\Documents\\dev\\ashenvenus\\logs" 7 | 8 | a_size = (2727, 6330) 9 | b_size = (5454, 6330) 10 | 11 | save_rle_as_image(sub_16_filepath, output_dir, 'a', a_size, '16') 12 | save_rle_as_image(sub_16_filepath, output_dir, 'b', b_size, '16') 13 | save_rle_as_image(sub_12_filepath, output_dir, 'a', a_size, '12') 14 | save_rle_as_image(sub_12_filepath, output_dir, 'b', b_size, '12') 15 | save_rle_as_image(sub_11_filepath, output_dir, 'a', a_size, '11') 16 | save_rle_as_image(sub_11_filepath, output_dir, 'b', b_size, '11') -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Hugo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /v0/notes/sweep_040423.md: -------------------------------------------------------------------------------- 1 | Copy over files from linux computers to windows computer: 2 | 3 | ``` 4 | scp -r tren@192.168.1.30:/home/tren/dev/ashenvenus/output/* C:\Users\ook\Documents\dev\ashenvenus\output\ 5 | scp -r oop@192.168.1.34:/home/oop/dev/ashenvenus/output/* C:\Users\ook\Documents\dev\ashenvenus\output\ 6 | ``` 7 | 8 | Great runs 9 | 10 | (c39ac15b|82934db2|7a7a91cf|29d55a5f|77b1f3dc) 11 | 12 | Okay runs 13 | 14 | (49cee3f6|e40f6408|97e70391|73885935) 15 | 16 | Learnings 17 | 18 | - `num_samples` - 120k is better than 60k which is better than 8k. The question here is how big can you get before blowing up memory. Bigger is always better. 19 | - `model` - Bigger model is better, resnext50_32x4d actually does better then convnext_small 20 | - `freeze` - Not freezing actually works better but I think its because its overfitting to the train data and since the test data is the train data it works better. 21 | - `num_epochs` and `max_time_hours` longer are better obviously. 22 | - `relu` does consistently better than `gelu`, but high scores exist with both 23 | - `321` works better than `123` probably due to catastrophic forgetting and testing on train. -------------------------------------------------------------------------------- /eval.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# WHEN SUBMITTING FOR KAGGLE\n", 10 | "# COPY PASTE src.py DIRECTLY HERE\n", 11 | "# THEN MAKE SURE HPARAMS MATCH BELOW\n", 12 | "from src import eval" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "eval(\n", 22 | " output_dir='/kaggle/working',\n", 23 | " eval_dir='/kaggle/input/vesuvius-challenge-ink-detection/test',\n", 24 | " save_pred_img=False,\n", 25 | " save_submit_csv=True,\n", 26 | " # Evaluation\n", 27 | " batch_size=1,\n", 28 | " num_samples_eval=100,\n", 29 | " points_per_side=32,\n", 30 | " pred_iou_thresh=0.86,\n", 31 | " stability_score_thresh=0.92,\n", 32 | " crop_n_layers=1,\n", 33 | " crop_n_points_downscale_factor=2,\n", 34 | " min_mask_region_area=100,\n", 35 | " # Model\n", 36 | " model='vit_b',\n", 37 | " weights_filepath='/kaggle/input/29d55a5f/model.pth',\n", 38 | " # Dataset\n", 39 | " crop_size=(3, 68, 68),\n", 40 | " label_size=(1024, 1024),\n", 41 | " points_per_crop=8,\n", 42 | " avg_depth=27.,\n", 43 | " std_depth=10.,\n", 44 | ")\n" 45 | ] 46 | } 47 | ], 48 | "metadata": { 49 | "kernelspec": { 50 | "display_name": "base", 51 | "language": "python", 52 | "name": "python3" 53 | }, 54 | "language_info": { 55 | "codemirror_mode": { 56 | "name": "ipython", 57 | "version": 3 58 | }, 59 | "file_extension": ".py", 60 | "mimetype": "text/x-python", 61 | "name": "python", 62 | "nbconvert_exporter": "python", 63 | "pygments_lexer": "ipython3", 64 | "version": "3.9.16" 65 | }, 66 | "orig_nbformat": 4 67 | }, 68 | "nbformat": 4, 69 | "nbformat_minor": 2 70 | } 71 | -------------------------------------------------------------------------------- /v1/eval.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# WHEN SUBMITTING FOR KAGGLE\n", 10 | "# COPY PASTE src.py DIRECTLY HERE\n", 11 | "# THEN MAKE SURE HPARAMS MATCH BELOW\n", 12 | "from src import eval" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "eval(\n", 22 | " output_dir='/kaggle/working',\n", 23 | " eval_dir='/kaggle/input/vesuvius-challenge-ink-detection/test',\n", 24 | " save_pred_img=False,\n", 25 | " save_submit_csv=True,\n", 26 | " # Evaluation\n", 27 | " batch_size=1,\n", 28 | " num_samples_eval=100,\n", 29 | " points_per_side=32,\n", 30 | " pred_iou_thresh=0.86,\n", 31 | " stability_score_thresh=0.92,\n", 32 | " crop_n_layers=1,\n", 33 | " crop_n_points_downscale_factor=2,\n", 34 | " min_mask_region_area=100,\n", 35 | " # Model\n", 36 | " model='vit_b',\n", 37 | " weights_filepath='/kaggle/input/29d55a5f/model.pth',\n", 38 | " # Dataset\n", 39 | " crop_size=(3, 68, 68),\n", 40 | " label_size=(1024, 1024),\n", 41 | " points_per_crop=8,\n", 42 | " avg_depth=27.,\n", 43 | " std_depth=10.,\n", 44 | ")\n" 45 | ] 46 | } 47 | ], 48 | "metadata": { 49 | "kernelspec": { 50 | "display_name": "base", 51 | "language": "python", 52 | "name": "python3" 53 | }, 54 | "language_info": { 55 | "codemirror_mode": { 56 | "name": "ipython", 57 | "version": 3 58 | }, 59 | "file_extension": ".py", 60 | "mimetype": "text/x-python", 61 | "name": "python", 62 | "nbconvert_exporter": "python", 63 | "pygments_lexer": "ipython3", 64 | "version": "3.9.16" 65 | }, 66 | "orig_nbformat": 4 67 | }, 68 | "nbformat": 4, 69 | "nbformat_minor": 2 70 | } 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🌋 AshenVenus 🌋 2 | 3 | This repo is an open source entry to the [2023 Scroll Prize](https://scrollprize.org/) aka _The Vesuvius Challenge_. The name for this repo comes from [Venus](https://en.wikipedia.org/wiki/Venus_(mythology)#Iconography), who was the patron saint of [Pompeii](https://en.wikipedia.org/wiki/Pompeii), the city covered in Ash by the volcano Vesuvius. 4 | 5 | ![roman village at the foot of a large erupting volcano, ancient mosaic fresco, apocalypse, fantasy digital art, roman columns villa --v 5 --ar 2:1](assets/banner.png) 6 | 7 | ### Setup 8 | 9 | Dependencies: 10 | 11 | - [Segment Anything](https://github.com/facebookresearch/segment-anything) 12 | - HyperOpt 13 | - PyTorch 14 | - Tensorboard 15 | 16 | ``` 17 | pip install -r requirements.txt 18 | ``` 19 | 20 | ### Training 21 | 22 | To run a hyperparameter sweep use: 23 | 24 | ``` 25 | python sweep.py 26 | ``` 27 | 28 | To run a saved model use the evaluation notebook `eval.ipynb`. Follow the copy paste instructions to submit to Kaggle. 29 | 30 | ## YouTube 31 | 32 | This repo was built live on YouTube, you can find the playlist here: 33 | 34 | [![IMAGE_ALT](https://img.youtube.com/vi/J63V5n5OwMA/0.jpg)](https://youtube.com/playlist?list=PLwq2F0NejwX5Hc80-ExN9JfnbMAHR7HAn) 35 | 36 | ## Sources 37 | 38 | Various sources used for reference: 39 | 40 | - [PyTorch Pre-trained Models](https://pytorch.org/vision/main/models.html) 41 | - [Pretrained ViT Multiscale Vision Transformers](https://arxiv.org/pdf/2104.11227.pdf) 42 | - [Pretrained Video Transformer (SWIN)](https://github.com/pytorch/vision/blob/main/torchvision/models/video/swin_transformer.py) 43 | - [A ConvNet for the 2020s](https://arxiv.org/pdf/2201.03545.pdf) 44 | - [Adapting Pre-trained Vision Transformers from 2D to 3D through Weight Inflation Improves Medical Image Segmentation](https://proceedings.mlr.press/v193/zhang22a/zhang22a.pdf) 45 | - [Segment Anything](https://github.com/facebookresearch/segment-anything) 46 | 47 | 48 | ## Citation 49 | 50 | ``` 51 | @misc{ashenvenus-vesuvius-challenge-2023, 52 | title={AshenVenus: Open Source Entry to the 2023 Scroll Prize}, 53 | author={Hugo Ponte}, 54 | year={2023}, 55 | url={https://github.com/hu-po/ashenvenus} 56 | } 57 | ``` 58 | -------------------------------------------------------------------------------- /v0/sweep.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import shutil\n", 10 | "\n", 11 | "import numpy as np\n", 12 | "from hyperopt import fmin, tpe\n", 13 | "\n", 14 | "from src import sweep_episode\n", 15 | "from sweep import search_space\n", 16 | "\n", 17 | "# Override the search space with the test config\n", 18 | "search_space['train_dir'] = \"C:\\\\Users\\\\ook\\\\Documents\\\\dev\\\\ashenvenus\\\\data\\\\split_train\"\n", 19 | "search_space['valid_dir'] = \"C:\\\\Users\\\\ook\\\\Documents\\\\dev\\\\ashenvenus\\\\data\\\\split_valid\"\n", 20 | "search_space['output_dir'] = \"C:\\\\Users\\\\ook\\\\Documents\\\\dev\\\\ashenvenus\\\\output\"\n", 21 | "search_space['batch_size'] = 64\n", 22 | "search_space['resize_ratio'] = 0.3\n", 23 | "search_space['epochs'] = 64\n", 24 | "\n", 25 | "# # Test Config\n", 26 | "# print('\\n\\n Running in TEST mode \\n\\n')\n", 27 | "# search_space['interpolation'] = 'nearest'\n", 28 | "# search_space['curriculum'] = '1'\n", 29 | "# search_space['num_samples_train'] = 64\n", 30 | "# search_space['num_samples_valid'] = 64\n", 31 | "# search_space['num_epochs'] = 2\n", 32 | "# search_space['slice_depth'] = 4\n", 33 | "\n", 34 | "# Clean output dir\n", 35 | "# shutil.rmtree(search_space['output_dir'], ignore_errors=True)\n", 36 | "\n", 37 | "# Run the optimization\n", 38 | "best = fmin(\n", 39 | " sweep_episode,\n", 40 | " space=search_space,\n", 41 | " algo=tpe.suggest,\n", 42 | " max_evals=100,\n", 43 | " rstate=np.random.Generator(np.random.PCG64(42)),\n", 44 | ")" 45 | ] 46 | } 47 | ], 48 | "metadata": { 49 | "kernelspec": { 50 | "display_name": "base", 51 | "language": "python", 52 | "name": "python3" 53 | }, 54 | "language_info": { 55 | "codemirror_mode": { 56 | "name": "ipython", 57 | "version": 3 58 | }, 59 | "file_extension": ".py", 60 | "mimetype": "text/x-python", 61 | "name": "python", 62 | "nbconvert_exporter": "python", 63 | "pygments_lexer": "ipython3", 64 | "version": "3.9.16" 65 | }, 66 | "orig_nbformat": 4 67 | }, 68 | "nbformat": 4, 69 | "nbformat_minor": 2 70 | } 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Data folder 2 | data/ 3 | train/ 4 | test/ 5 | output* 6 | sweeps* 7 | models* 8 | 9 | # Model images 10 | model_graph* 11 | 12 | # Submissions 13 | submission.csv 14 | 15 | # Model weights 16 | *.pth 17 | 18 | # Byte-compiled / optimized / DLL files 19 | __pycache__/ 20 | *.py[cod] 21 | *$py.class 22 | 23 | # C extensions 24 | *.so 25 | 26 | # Distribution / packaging 27 | .Python 28 | build/ 29 | develop-eggs/ 30 | dist/ 31 | downloads/ 32 | eggs/ 33 | .eggs/ 34 | lib/ 35 | lib64/ 36 | parts/ 37 | sdist/ 38 | var/ 39 | wheels/ 40 | pip-wheel-metadata/ 41 | share/python-wheels/ 42 | *.egg-info/ 43 | .installed.cfg 44 | *.egg 45 | MANIFEST 46 | 47 | # PyInstaller 48 | # Usually these files are written by a python script from a template 49 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 50 | *.manifest 51 | *.spec 52 | 53 | # Installer logs 54 | pip-log.txt 55 | pip-delete-this-directory.txt 56 | 57 | # Unit test / coverage reports 58 | htmlcov/ 59 | .tox/ 60 | .nox/ 61 | .coverage 62 | .coverage.* 63 | .cache 64 | nosetests.xml 65 | coverage.xml 66 | *.cover 67 | *.py,cover 68 | .hypothesis/ 69 | .pytest_cache/ 70 | 71 | # Translations 72 | *.mo 73 | *.pot 74 | 75 | # Django stuff: 76 | *.log 77 | local_settings.py 78 | db.sqlite3 79 | db.sqlite3-journal 80 | 81 | # Flask stuff: 82 | instance/ 83 | .webassets-cache 84 | 85 | # Scrapy stuff: 86 | .scrapy 87 | 88 | # Sphinx documentation 89 | docs/_build/ 90 | 91 | # PyBuilder 92 | target/ 93 | 94 | # Jupyter Notebook 95 | .ipynb_checkpoints 96 | 97 | # IPython 98 | profile_default/ 99 | ipython_config.py 100 | 101 | # pyenv 102 | .python-version 103 | 104 | # pipenv 105 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 106 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 107 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 108 | # install all needed dependencies. 109 | #Pipfile.lock 110 | 111 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 112 | __pypackages__/ 113 | 114 | # Celery stuff 115 | celerybeat-schedule 116 | celerybeat.pid 117 | 118 | # SageMath parsed files 119 | *.sage.py 120 | 121 | # Environments 122 | .env 123 | .venv 124 | env/ 125 | venv/ 126 | ENV/ 127 | env.bak/ 128 | venv.bak/ 129 | 130 | # Spyder project settings 131 | .spyderproject 132 | .spyproject 133 | 134 | # Rope project settings 135 | .ropeproject 136 | 137 | # mkdocs documentation 138 | /site 139 | 140 | # mypy 141 | .mypy_cache/ 142 | .dmypy.json 143 | dmypy.json 144 | 145 | # Pyre type checker 146 | .pyre/ 147 | -------------------------------------------------------------------------------- /v0/eval.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stderr", 10 | "output_type": "stream", 11 | "text": [ 12 | " 63%|██████▎ | 2130/3379 [04:38<03:30, 5.93it/s, Eval a]" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "# WHEN SUBMITTING FOR KAGGLE\n", 18 | "# COPY PASTE src.py DIRECTLY HERE\n", 19 | "# THEN MAKE SURE HPARAMS MATCH\n", 20 | "from src import eval, eval_from_episode_dir\n", 21 | "\n", 22 | "eval_from_episode_dir(\n", 23 | " # episode_dir = \"C:\\\\Users\\\\ook\\\\Documents\\\\dev\\\\ashenvenus\\\\sweep_060423\\\\9b6e540b\",\n", 24 | " episode_dir = \"C:\\\\Users\\\\ook\\\\Documents\\\\dev\\\\ashenvenus\\\\sweep_060423\\\\722d6d87\",\n", 25 | " output_dir = \"C:\\\\Users\\\\ook\\\\Documents\\\\dev\\\\ashenvenus\\\\output\",\n", 26 | " weights_filename = 'model_best_dice.1.valid.pth',\n", 27 | " # weights_filename = 'model_best_dice.2.valid.pth',\n", 28 | " # weights_filename = 'model_best_dice.3.valid.pth',\n", 29 | " quantize=False,\n", 30 | " device='gpu',\n", 31 | " save_pred_img=True,\n", 32 | " save_submit_csv=False,\n", 33 | " save_histograms=True,\n", 34 | " batch_size=128,\n", 35 | ")" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "eval(\n", 45 | " train_dir = '/kaggle/input/vesuvius-challenge-ink-detection/train',\n", 46 | " output_dir = '/kaggle/working',\n", 47 | " eval_dir = '/kaggle/input/vesuvius-challenge-ink-detection/test',\n", 48 | " weights_filepath = '/kaggle/input/29d55a5f/model.pth',\n", 49 | " \n", 50 | " batch_size = 1028,\n", 51 | " curriculum = '321',\n", 52 | " freeze = False,\n", 53 | " image_augs = False,\n", 54 | " lr = 0.0009151398565001475,\n", 55 | " lr_gamma = 0.98,\n", 56 | " max_time_hours = 8,\n", 57 | " model = 'resnext50_32x4d',\n", 58 | " num_epochs = 16,\n", 59 | " num_samples = 60000,\n", 60 | " num_workers = 0,\n", 61 | " optimizer = 'adam',\n", 62 | " patch_size_x = 64,\n", 63 | " patch_size_y = 64,\n", 64 | " postproc_kernel = 6,\n", 65 | " resize_ratio = 0.08,\n", 66 | " slice_depth = 65,\n", 67 | " threshold = 0.5,\n", 68 | "\n", 69 | " save_pred_img=False,\n", 70 | " save_submit_csv=True,\n", 71 | ")" 72 | ] 73 | } 74 | ], 75 | "metadata": { 76 | "kernelspec": { 77 | "display_name": "base", 78 | "language": "python", 79 | "name": "python3" 80 | }, 81 | "language_info": { 82 | "codemirror_mode": { 83 | "name": "ipython", 84 | "version": 3 85 | }, 86 | "file_extension": ".py", 87 | "mimetype": "text/x-python", 88 | "name": "python", 89 | "nbconvert_exporter": "python", 90 | "pygments_lexer": "ipython3", 91 | "version": "3.9.16" 92 | }, 93 | "orig_nbformat": 4 94 | }, 95 | "nbformat": 4, 96 | "nbformat_minor": 2 97 | } 98 | -------------------------------------------------------------------------------- /v0/notes/sweep_010423.md: -------------------------------------------------------------------------------- 1 | # Best Runs 2 | 3 | 4 | 79845d77_curriculum_123_freeze_False_lr_0.00010660475052758971_lr_gamma_None_num_samples_100000_model_convnext_tiny_optimizer_adam_resize_ratio_0.08_ 5 | 6 | a3a3aaba_curriculum_1_freeze_True_lr_0.00031480211542209374_lr_gamma_0.9_num_samples_100000_model_resnext50_32x4d_optimizer_adam_resize_ratio_0.08_ 7 | 8 | 40308516_curriculum_321_freeze_False_lr_0.006968837167492869_lr_gamma_0.9_num_samples_50000_model_simplenet_optimizer_adam_resize_ratio_0.08_ 9 | 10 | 95dd19b3_curriculum_13_freeze_False_lr_0.0050250476046940435_lr_gamma_None_num_samples_100000_model_resnext50_32x4d_optimizer_adam_resize_ratio_0.08_ 11 | 12 | ## Using GPT-4 13 | 14 | Below are 4 hyperparameter configurations for the best performing runs of my computer vision model. Can you help me choose the best setting for hyperparameters, explain your reasoning. 15 | 16 | Best 17 | 18 | ``` 19 | {'batch_size': 1028, 20 | 'curriculum': '123', 21 | 'eval_dir': 'data/test', 22 | 'freeze': False, 23 | 'image_augs': False, 24 | 'lr': 0.00010660475052758971, 25 | 'lr_gamma': None, 26 | 'num_samples': 100000, 27 | 'model': 'convnext_tiny', 28 | 'num_epochs': 12, 29 | 'num_workers': 1, 30 | 'optimizer': 'adam', 31 | 'output_dir': 'output', 32 | 'patch_size_x': 64, 33 | 'patch_size_y': 64, 34 | 'resize_ratio': 0.08, 35 | 'slice_depth': 65, 36 | 'train_dir': 'data/train'} 37 | ``` 38 | 39 | Great 40 | 41 | ``` 42 | {'batch_size': 512, 43 | 'curriculum': '1', 44 | 'eval_dir': 'data/test', 45 | 'freeze': True, 46 | 'image_augs': True, 47 | 'lr': 0.00031480211542209374, 48 | 'lr_gamma': 0.9, 49 | 'num_samples': 100000, 50 | 'model': 'resnext50_32x4d', 51 | 'num_epochs': 6, 52 | 'num_workers': 1, 53 | 'optimizer': 'adam', 54 | 'output_dir': 'output', 55 | 'patch_size_x': 64, 56 | 'patch_size_y': 64, 57 | 'resize_ratio': 0.08, 58 | 'slice_depth': 65, 59 | 'train_dir': 'data/train'} 60 | ``` 61 | 62 | Good 63 | 64 | ``` 65 | {'batch_size': 1028, 66 | 'curriculum': '321', 67 | 'eval_dir': 'data/test', 68 | 'freeze': False, 69 | 'image_augs': False, 70 | 'lr': 0.006968837167492869, 71 | 'lr_gamma': 0.9, 72 | 'num_samples': 50000, 73 | 'model': 'simplenet', 74 | 'num_epochs': 12, 75 | 'num_workers': 1, 76 | 'optimizer': 'adam', 77 | 'output_dir': 'output', 78 | 'patch_size_x': 64, 79 | 'patch_size_y': 64, 80 | 'resize_ratio': 0.08, 81 | 'slice_depth': 65, 82 | 'train_dir': 'data/train'} 83 | ``` 84 | 85 | Okay 86 | 87 | ``` 88 | {'batch_size': 1028, 89 | 'curriculum': '13', 90 | 'eval_dir': 'data/test', 91 | 'freeze': False, 92 | 'image_augs': True, 93 | 'lr': 0.0050250476046940435, 94 | 'lr_gamma': None, 95 | 'num_samples': 100000, 96 | 'model': 'resnext50_32x4d', 97 | 'num_epochs': 12, 98 | 'num_workers': 1, 99 | 'optimizer': 'adam', 100 | 'output_dir': 'output', 101 | 'patch_size_x': 64, 102 | 'patch_size_y': 64, 103 | 'resize_ratio': 0.08, 104 | 'slice_depth': 65, 105 | 'train_dir': 'data/train'} 106 | ``` 107 | 108 | #### Answer 109 | 110 | best_config = { 111 | 'batch_size': 1028, 112 | 'curriculum': '123', 113 | 'eval_dir': 'data/test', 114 | 'freeze': False, 115 | 'image_augs': True, 116 | 'lr': 0.00010660475052758971, 117 | 'lr_gamma': None, 118 | 'num_samples': 100000, 119 | 'model': 'convnext_tiny', 120 | 'num_epochs': 12, 121 | 'num_workers': 1, 122 | 'optimizer': 'adam', 123 | 'output_dir': 'output', 124 | 'patch_size_x': 64, 125 | 'patch_size_y': 64, 126 | 'resize_ratio': 0.08, 127 | 'slice_depth': 65, 128 | 'train_dir': 'data/train' 129 | } 130 | -------------------------------------------------------------------------------- /v0/notes/sweep_020423.md: -------------------------------------------------------------------------------- 1 | Copy over files from linux computers to windows computer: 2 | 3 | ``` 4 | scp -r tren@192.168.1.30:/home/tren/dev/ashenvenus/output/* C:\Users\ook\Documents\dev\ashenvenus\output\ 5 | scp -r oop@192.168.1.34:/home/oop/dev/ashenvenus/output/* C:\Users\ook\Documents\dev\ashenvenus\output\ 6 | ``` 7 | 8 | ## Best Runs 9 | 10 | 0c2d7701_curriculum_321_freeze_False_image_augs_False_lr_0.0007381201935115487_lr_gamma_0.9_num_samples_60000_model_simplenet_optimizer_adam_resize_ratio_0.08_use_gelu_True_ 11 | 12 | 34f7adb7_curriculum_13_freeze_False_image_augs_False_lr_0.00010448739241324328_lr_gamma_0.98_num_samples_60000_model_convnext_tiny_optimizer_adam_resize_ratio_0.08_use_gelu_True_ 13 | 14 | c8eabb8b_curriculum_123_freeze_True_image_augs_True_lr_0.00012661275858523555_lr_gamma_None_num_samples_60000_model_convnext_tiny_optimizer_adam_resize_ratio_0.08_use_gelu_False_ 15 | 16 | dd3fcfe8_curriculum_321_freeze_True_image_augs_False_lr_5.351981930205677e-05_lr_gamma_0.98_num_samples_60000_model_resnext50_32x4d_optimizer_adam_resize_ratio_0.08_use_gelu_False_ 17 | 18 | REGEX `(0c2d7701|34f7adb7|c8eabb8b|dd3fcfe8)` 19 | 20 | Hyperparams 21 | 22 | ``` 23 | {'batch_size': 512, 24 | 'curriculum': '321', 25 | 'eval_dir': 'data/test', 26 | 'freeze': False, 27 | 'image_augs': False, 28 | 'lr': 0.0007381201935115487, 29 | 'lr_gamma': 0.9, 30 | 'num_samples': 60000, 31 | 'model': 'simplenet', 32 | 'num_epochs': 16, 33 | 'num_workers': 0, 34 | 'optimizer': 'adam', 35 | 'output_dir': 'output', 36 | 'patch_size_x': 64, 37 | 'patch_size_y': 64, 38 | 'resize_ratio': 0.08, 39 | 'slice_depth': 65, 40 | 'train_dir': 'data/train', 41 | 'use_gelu': True} 42 | ``` 43 | 44 | ``` 45 | {'batch_size': 512, 46 | 'curriculum': '13', 47 | 'eval_dir': 'data/test', 48 | 'freeze': False, 49 | 'image_augs': False, 50 | 'lr': 0.00010448739241324328, 51 | 'lr_gamma': 0.98, 52 | 'num_samples': 60000, 53 | 'model': 'convnext_tiny', 54 | 'num_epochs': 8, 55 | 'num_workers': 0, 56 | 'optimizer': 'adam', 57 | 'output_dir': 'output', 58 | 'patch_size_x': 64, 59 | 'patch_size_y': 64, 60 | 'resize_ratio': 0.08, 61 | 'slice_depth': 65, 62 | 'train_dir': 'data/train', 63 | 'use_gelu': True} 64 | ``` 65 | 66 | ``` 67 | {'batch_size': 512, 68 | 'curriculum': '123', 69 | 'eval_dir': 'data/test', 70 | 'freeze': True, 71 | 'image_augs': True, 72 | 'lr': 0.00012661275858523555, 73 | 'lr_gamma': None, 74 | 'num_samples': 60000, 75 | 'model': 'convnext_tiny', 76 | 'num_epochs': 16, 77 | 'num_workers': 0, 78 | 'optimizer': 'adam', 79 | 'output_dir': 'output', 80 | 'patch_size_x': 64, 81 | 'patch_size_y': 64, 82 | 'resize_ratio': 0.08, 83 | 'slice_depth': 65, 84 | 'train_dir': 'data/train', 85 | 'use_gelu': False} 86 | ``` 87 | 88 | ``` 89 | {'batch_size': 512, 90 | 'curriculum': '321', 91 | 'eval_dir': 'data/test', 92 | 'freeze': True, 93 | 'image_augs': False, 94 | 'lr': 5.351981930205677e-05, 95 | 'lr_gamma': 0.98, 96 | 'num_samples': 60000, 97 | 'model': 'resnext50_32x4d', 98 | 'num_epochs': 16, 99 | 'num_workers': 0, 100 | 'optimizer': 'adam', 101 | 'output_dir': 'output', 102 | 'patch_size_x': 64, 103 | 'patch_size_y': 64, 104 | 'resize_ratio': 0.08, 105 | 'slice_depth': 65, 106 | 'train_dir': 'data/train', 107 | 'use_gelu': False} 108 | ``` 109 | 110 | ### Conclusion 111 | 112 | - 'curriculum' - All 3 performs better, order doesn't seem to matter 113 | - 'model' - Doesn't seem to matter much, all models perform similarly, even simplenet 114 | - 'freeze' - Doesn't seem to matter much, which is odd 115 | - 'use_gelu' - Doesn't seem to matter much, maybe slight gain 116 | - 'image_augs' - Hurts a little more than helps, but doesn't matter much 117 | - 'lr_gamma' - Doesn't seem to matter much between 0.9 and None (1.0) 118 | - 'num_samples'- Larger is better, strongest predictor of score -------------------------------------------------------------------------------- /scripts/split_dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | Split dataset into Train and Validation, by cutting 3 | the image into a top half and bottom half. 4 | """ 5 | 6 | import os 7 | from PIL import Image 8 | from tqdm import tqdm 9 | 10 | image_mask_filename='mask.png' 11 | image_labels_filename='inklabels.png' 12 | image_ir_filename = 'ir.png' 13 | slices_dir_filename='surface_volume' 14 | 15 | 16 | num_slices = 65 17 | split = 0.85 18 | 19 | target_dir = '/home/tren/dev/ashenvenus/data/train' 20 | output_dir_train = '/home/tren/dev/ashenvenus/data/split_train' 21 | output_dir_valid = '/home/tren/dev/ashenvenus/data/split_valid' 22 | 23 | # target_dir = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\data\\train" 24 | # output_dir_train = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\data\\split_train" 25 | # output_dir_valid = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\data\\split_valid" 26 | 27 | # Baseline is to use image mask to create guess submission 28 | for dataset in os.listdir(target_dir): 29 | dataset_filepath = os.path.join(target_dir, dataset) 30 | dataset_train_filepath = os.path.join(output_dir_train, dataset) 31 | os.makedirs(dataset_train_filepath, exist_ok=True) 32 | dataset_valid_filepath = os.path.join(output_dir_valid, dataset) 33 | os.makedirs(dataset_valid_filepath, exist_ok=True) 34 | 35 | # Open IR image 36 | _image_ir_filepath = os.path.join(dataset_filepath, image_ir_filename) 37 | _image_ir_train_filepath = os.path.join(dataset_train_filepath, image_ir_filename) 38 | _image_ir_valid_filepath = os.path.join(dataset_valid_filepath, image_ir_filename) 39 | _ir_img = Image.open(_image_ir_filepath).convert("L") 40 | 41 | # Split into train and val 42 | width, height = _ir_img.size 43 | _ir_img_train = _ir_img.crop((0, 0, width, int(height * split))) 44 | _ir_img_train.save(_image_ir_train_filepath) 45 | _ir_img_val = _ir_img.crop((0, int(height * split), width, height)) 46 | _ir_img_val.save(_image_ir_valid_filepath) 47 | 48 | continue 49 | 50 | # Open Mask image 51 | _image_mask_filepath = os.path.join(dataset_filepath, image_mask_filename) 52 | _image_mask_train_filepath = os.path.join(dataset_train_filepath, image_mask_filename) 53 | _image_mask_valid_filepath = os.path.join(dataset_valid_filepath, image_mask_filename) 54 | _mask_img = Image.open(_image_mask_filepath).convert("1") 55 | 56 | # Split into train and val 57 | width, height = _mask_img.size 58 | _mask_img_train = _mask_img.crop((0, 0, width, int(height * split))) 59 | _mask_img_train.save(_image_mask_train_filepath) 60 | _mask_img_val = _mask_img.crop((0, int(height * split), width, height)) 61 | _mask_img_val.save(_image_mask_valid_filepath) 62 | 63 | # Open Labels image 64 | _image_labels_filepath = os.path.join(dataset_filepath, image_labels_filename) 65 | _image_labels_train_filepath = os.path.join(dataset_train_filepath, image_labels_filename) 66 | _image_labels_valid_filepath = os.path.join(dataset_valid_filepath, image_labels_filename) 67 | _labels_img = Image.open(_image_labels_filepath).convert("1") 68 | 69 | # Split into train and val 70 | width, height = _labels_img.size 71 | _labels_img_train = _labels_img.crop((0, 0, width, int(height * split))) 72 | _labels_img_train.save(_image_labels_train_filepath) 73 | _labels_img_val = _labels_img.crop((0, int(height * split), width, height)) 74 | _labels_img_val.save(_image_labels_valid_filepath) 75 | 76 | # Open Slices 77 | _slice_dir = os.path.join(dataset_filepath, slices_dir_filename) 78 | _slice_train_dir = os.path.join(dataset_train_filepath, slices_dir_filename) 79 | os.makedirs(_slice_train_dir, exist_ok=True) 80 | _slice_valid_dir = os.path.join(dataset_valid_filepath, slices_dir_filename) 81 | os.makedirs(_slice_valid_dir, exist_ok=True) 82 | 83 | for i in tqdm(range(num_slices), postfix='converting slices'): 84 | _slice_filepath = os.path.join(_slice_dir, f"{i:02d}.tif") 85 | _slice_img = Image.open(_slice_filepath) 86 | 87 | # Split into train and val 88 | width, height = _slice_img.size 89 | _slice_img_train = _slice_img.crop((0, 0, width, int(height * split))) 90 | _slice_img_train.save(os.path.join(_slice_train_dir, f"{i:02d}.tif")) 91 | _slice_img_val = _slice_img.crop((0, int(height * split), width, height)) 92 | _slice_img_val.save(os.path.join(_slice_valid_dir, f"{i:02d}.tif")) -------------------------------------------------------------------------------- /v0/sweep.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import shutil 3 | 4 | import numpy as np 5 | from hyperopt import fmin, hp, tpe 6 | 7 | from src import sweep_episode 8 | 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument('--batch_size', type=int, default=2) 11 | parser.add_argument('--seed', type=int, default=0) 12 | parser.add_argument('--resize', type=float, default=None) 13 | 14 | # Define the search space 15 | search_space = { 16 | 'output_dir': 'output', 17 | 'train_dir': 'data/split_train', 18 | 'valid_dir': 'data/split_valid', 19 | 'curriculum': hp.choice('curriculum', [ 20 | # All 3 performs better, order doesn't seem to matter 21 | # '1', 22 | # '2', 23 | # '3', 24 | # '13', 25 | # '32', 26 | '123', 27 | # '321', 28 | ]), 29 | 'model': hp.choice('model', [ 30 | # 'convnext_tiny', 31 | 'convnext_small', 32 | 'convnext_base', 33 | # 'convnext_large', 34 | 'resnext50_32x4d', 35 | # 'resnext101_32x8d', 36 | # 'resnext101_64x4d', 37 | # ViTs are picky about input size 38 | # 'vit_b_32', 39 | # 'vit_l_32', 40 | # 'vit_h_14', 41 | ]), 42 | 'freeze': hp.choice('freeze', [ 43 | # Doesn't seem to matter much, which is odd 44 | True, 45 | # False, 46 | ]), 47 | 'image_augs': hp.choice('image_augs', [ 48 | # Sweep 060423 - Mostly helps a little 49 | # Sweep 040423 - Hurts a little more than helps, but doesn't matter much 50 | True, 51 | # False, 52 | ]), 53 | 'optimizer': hp.choice('optimizer', [ 54 | 'adam', 55 | # 'sgd', # Garbo 56 | ]), 57 | 'weight_decay': hp.choice('weight_decay', [ 58 | # Hard to tell, good runs exist with and without 59 | 1e-4, 60 | 1e-3, 61 | ]), 62 | 'lr_gamma': hp.choice('lr_gamma', [ 63 | # Doesn't seem to matter much between 0.9 and None (1.0) 64 | # 0.1, # Garbo 65 | # 0.9, 66 | 0.98, 67 | # None, 68 | ]), 69 | 'num_workers': 0, 70 | 'resize_ratio': hp.choice('resize_ratio', [ 71 | # Below 0.1 it can't learn 72 | # Above 0.3 it takes hours to perform eval 73 | 0.2, 74 | ]), 75 | 'interpolation': hp.choice('interpolation', [ 76 | # Bicubic is best, but nearest is faster 77 | # 'nearest', 78 | 'bilinear', 79 | 'bicubic', 80 | ]), 81 | 'input_size': hp.choice('input_size', [ 82 | # Smaller resolutions actually work quite well 83 | # '224.224.65', 84 | '32.32.65', 85 | '64.64.65', 86 | # '128.64.65', 87 | ]), 88 | # Careful with small learning rates, less than 1e-5 is too small 89 | 'lr': hp.loguniform('lr',np.log(0.00001), np.log(0.01)), 90 | 'num_epochs': hp.choice('num_epochs', [8]), 91 | 'num_samples_train': hp.choice('num_samples_train', [ 92 | # Larger is better, strongest predictor of score 93 | 200000, 94 | # 120000, 95 | # 60000, 96 | # 4000, 97 | # 2000, 98 | ]), 99 | 'num_samples_valid': hp.choice('num_samples_valid', [ 100 | # Larger is more thorough, but takes more time 101 | 4000 102 | ]), 103 | 'max_time_hours': hp.choice('max_time_hours', [ 104 | 8, 105 | # 9, 106 | ]), 107 | 'threshold': hp.choice('threshold', [0.5]), 108 | 'postproc_kernel': hp.choice('postproc_kernel', [3]), 109 | } 110 | 111 | if __name__ == '__main__': 112 | args = parser.parse_args() 113 | if args.seed == 0: 114 | print('\n\n Running in TEST mode \n\n') 115 | search_space['interpolation'] = 'nearest' 116 | search_space['curriculum'] = '1' 117 | search_space['num_samples_train'] = 64 118 | search_space['num_samples_valid'] = 64 119 | search_space['num_epochs'] = 2 120 | search_space['batch_size'] = args.batch_size 121 | if args.resize: 122 | search_space['resize_ratio'] = args.resize 123 | 124 | # Clean output dir 125 | shutil.rmtree(search_space['output_dir'], ignore_errors=True) 126 | 127 | best = fmin( 128 | sweep_episode, 129 | space=search_space, 130 | algo=tpe.suggest, 131 | max_evals=100, 132 | rstate=np.random.Generator(np.random.PCG64(args.seed)), 133 | ) -------------------------------------------------------------------------------- /v1/sweep.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import shutil 4 | import pprint 5 | import uuid 6 | import yaml 7 | from tensorboardX import SummaryWriter 8 | from hyperopt import fmin, hp, tpe 9 | from src import train_valid 10 | 11 | if os.name == 'nt': 12 | print("Windows Computer Detected") 13 | ROOT_DIR = "C:\\Users\\ook\\Documents\\dev\\" 14 | DATA_DIR = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\data" 15 | MODEL_DIR = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\models" 16 | OUTPUT_DIR = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\output" 17 | else: 18 | print("Linux Computer Detected") 19 | ROOT_DIR = "/home/tren/dev/" 20 | DATA_DIR = "/home/tren/dev/ashenvenus/data" 21 | MODEL_DIR = "/home/tren/dev/ashenvenus/models" 22 | OUTPUT_DIR = "/home/tren/dev/ashenvenus/output" 23 | 24 | # Define the search space 25 | HYPERPARAMS = { 26 | 'train_dir_name' : 'split_train', 27 | 'valid_dir_name' : 'split_valid', 28 | # Model 29 | 'model_str': hp.choice('model_str', [ 30 | 'vit_b|sam_vit_b_01ec64.pth', 31 | # 'vit_h|sam_vit_h_4b8939.pth', 32 | # 'vit_l|sam_vit_l_0b3195.pth', 33 | ]), 34 | 35 | # Dataset 36 | 'curriculum': hp.choice('curriculum', [ 37 | # '1', # Depth of 1 - 40/45 38 | # '2', # Depth of 1 - 53/58 39 | # '3', # Depth of 1 - 48/53 40 | '123', 41 | ]), 42 | 'num_samples_train': hp.choice('num_samples_train', [ 43 | 16, 44 | ]), 45 | 'num_samples_valid': hp.choice('num_samples_valid', [ 46 | 16, 47 | ]), 48 | 'crop_size_str': hp.choice('crop_size_str', [ 49 | '3.1024.1024', # HACK: This cannot be changed for pretrained models 50 | ]), 51 | 'label_size_str': hp.choice('label_size_str', [ 52 | '256.256', # HACK: This cannot be changed for pretrained models 53 | ]), 54 | 55 | # Training 56 | 'batch_size' : 1, 57 | 'num_epochs': hp.choice('num_epochs', [32]), 58 | 'lr': hp.loguniform('lr',np.log(0.00001), np.log(0.0001)), 59 | 'wd': hp.choice('wd', [ 60 | 1e-4, 61 | 1e-3, 62 | ]), 63 | } 64 | 65 | def sweep_episode(hparams) -> float: 66 | 67 | # Print hyperparam dict with logging 68 | print(f"\n\n Starting EPISODE \n\n") 69 | print(f"\n\nHyperparams:\n\n{pprint.pformat(hparams)}\n\n") 70 | 71 | # Create output directory based on run_name 72 | run_name: str = str(uuid.uuid4())[:8] 73 | output_dir = os.path.join(OUTPUT_DIR, run_name) 74 | os.makedirs(output_dir, exist_ok=True) 75 | 76 | # Train and Validation directories 77 | train_dir = os.path.join(DATA_DIR, hparams['train_dir_name']) 78 | valid_dir = os.path.join(DATA_DIR, hparams['valid_dir_name']) 79 | 80 | # Save hyperparams to file with YAML 81 | with open(os.path.join(output_dir, 'hparams.yaml'), 'w') as f: 82 | yaml.dump(hparams, f) 83 | 84 | # HACK: Convert Hyperparam strings to correct format 85 | hparams['crop_size'] = [int(x) for x in hparams['crop_size_str'].split('.')] 86 | hparams['label_size'] = [int(x) for x in hparams['label_size_str'].split('.')] 87 | model, weights_filepath = hparams['model_str'].split('|') 88 | weights_filepath = os.path.join(MODEL_DIR, weights_filepath) 89 | 90 | try: 91 | writer = SummaryWriter(log_dir=output_dir) 92 | # Train and evaluate a TFLite model 93 | score_dict = train_valid( 94 | run_name =run_name, 95 | output_dir = output_dir, 96 | train_dir = train_dir, 97 | valid_dir = valid_dir, 98 | model=model, 99 | weights_filepath=weights_filepath, 100 | writer=writer, 101 | **hparams, 102 | ) 103 | # Only works without tb open 104 | # writer.add_hparams(hparams, score_dict) 105 | writer.close() 106 | # Score is average of all scores 107 | score = sum(score_dict.values()) / len(score_dict) 108 | except Exception as e: 109 | print(f"\n\n (ERROR) EPISODE FAILED (ERROR) \n\n") 110 | print(f"Potentially Bad Hyperparams:\n\n{pprint.pformat(hparams)}\n\n") 111 | raise e 112 | # print(e) 113 | # score = 0 114 | # Maximize score is minimize negative score 115 | return -score 116 | 117 | if __name__ == "__main__": 118 | 119 | # Clean output dir 120 | shutil.rmtree(OUTPUT_DIR, ignore_errors=True) 121 | 122 | best = fmin( 123 | sweep_episode, 124 | space=HYPERPARAMS, 125 | algo=tpe.suggest, 126 | max_evals=100, 127 | rstate=np.random.Generator(np.random.PCG64(42)), 128 | ) -------------------------------------------------------------------------------- /scripts/pixel_stats.py: -------------------------------------------------------------------------------- 1 | """ 2 | Get the pixel statistics of the data. 3 | """ 4 | 5 | import os 6 | 7 | import numpy as np 8 | import yaml 9 | import pprint 10 | from PIL import Image 11 | from tqdm import tqdm 12 | from matplotlib import pyplot as plt 13 | 14 | image_mask_filename = 'mask.png' 15 | image_labels_filename = 'inklabels.png' 16 | image_ir_filename = 'ir.png' 17 | slices_dir_filename = 'surface_volume' 18 | num_slices = 50 19 | 20 | # data_dir = "C:\\Users\\ook\\Documents\\dev\\ashenvenus\\data\\" 21 | data_dir = '/home/tren/dev/ashenvenus/data/split/' 22 | 23 | # Find all nested sub directories that contain an image mask image 24 | dataset_dirs = [] 25 | for root, dirs, files in os.walk(data_dir): 26 | if image_labels_filename in files and image_mask_filename in files: 27 | print(f"Found dataset: {root}") 28 | dataset_dirs.append(root) 29 | 30 | for dataset_dir in dataset_dirs: 31 | print(f"Processing dataset: {dataset_dir}") 32 | 33 | # Output will be a YAML file in the dataset directory 34 | # mask - only pixels in fragment mask 35 | # ink - only pixels within areas labeled as ink 36 | # bg - pixels in mask but that aren't in ink 37 | pixel_stats = { 38 | 'raw': {'min': [], 'max': [], 'mean': [], 'std': [], 'count': 0}, 39 | 'mask': {'min': [], 'max': [], 'mean': [], 'std': [], 'count': 0}, 40 | 'ink': {'min': [], 'max': [], 'mean': [], 'std': [], 'count': 0}, 41 | 'bg': {'min': [], 'max': [], 'mean': [], 'std': [], 'count': 0}, 42 | } 43 | 44 | # Open Mask Image 45 | _image_mask_filepath = os.path.join(dataset_dir, image_mask_filename) 46 | mask_img = Image.open(_image_mask_filepath).convert("L") 47 | mask = np.array(mask_img, dtype=np.uint8) 48 | 49 | # Open Labels image 50 | _image_labels_filepath = os.path.join(dataset_dir, image_labels_filename) 51 | labels_img = Image.open(_image_labels_filepath).convert("L") 52 | labels = np.array(labels_img, dtype=np.uint8) 53 | 54 | # Open Slices into numpy array 55 | fragment = np.zeros((num_slices, labels_img.height, 56 | labels_img.width), dtype=np.float32) 57 | _slice_dir = os.path.join(dataset_dir, slices_dir_filename) 58 | for i in tqdm(range(num_slices), postfix='converting slices'): 59 | _slice_filepath = os.path.join(_slice_dir, f"{i:02d}.tif") 60 | slice_img = Image.open(_slice_filepath).convert("F") 61 | fragment[i, :, :] = np.array(slice_img) / 65535.0 62 | 63 | # # Raw - Every Pixel in Fragment 64 | # pixel_stats['raw']['min'].append(float(fragment.min())) 65 | # pixel_stats['raw']['max'].append(float(fragment.max())) 66 | # pixel_stats['raw']['mean'].append(float(fragment.mean())) 67 | # pixel_stats['raw']['std'].append(float(fragment.std())) 68 | # pixel_stats['raw']['count'] += int(fragment.size) 69 | 70 | # Mask - Only Pixels in Mask 71 | fragment_mask = fragment[:, mask > 0] 72 | pixel_stats['mask']['min'].append(float(fragment_mask.min())) 73 | pixel_stats['mask']['max'].append(float(fragment_mask.max())) 74 | pixel_stats['mask']['mean'].append(float(fragment_mask.mean())) 75 | pixel_stats['mask']['std'].append(float(fragment_mask.std())) 76 | pixel_stats['mask']['count'] += int(fragment_mask.size) 77 | 78 | # Ink - Only Pixels labeled as ink 79 | fragment_ink = fragment[:, labels > 0] 80 | pixel_stats['ink']['min'].append(float(fragment_ink.min())) 81 | pixel_stats['ink']['max'].append(float(fragment_ink.max())) 82 | pixel_stats['ink']['mean'].append(float(fragment_ink.mean())) 83 | pixel_stats['ink']['std'].append(float(fragment_ink.std())) 84 | pixel_stats['ink']['count'] += int(fragment_ink.size) 85 | 86 | # Background - Pixels in Mask but that aren't Ink 87 | fragment_bg = fragment[:, (mask > 0) & (labels < 1)] 88 | pixel_stats['bg']['min'].append(float(fragment_bg.min())) 89 | pixel_stats['bg']['max'].append(float(fragment_bg.max())) 90 | pixel_stats['bg']['mean'].append(float(fragment_bg.mean())) 91 | pixel_stats['bg']['std'].append(float(fragment_bg.std())) 92 | pixel_stats['bg']['count'] += int(fragment_bg.size) 93 | 94 | # Calculate average statistics 95 | for key in pixel_stats.keys(): 96 | pixel_stats[key]['min'] = float(np.mean(pixel_stats[key]['min'])) 97 | pixel_stats[key]['max'] = float(np.mean(pixel_stats[key]['max'])) 98 | pixel_stats[key]['mean'] = float(np.mean(pixel_stats[key]['mean'])) 99 | pixel_stats[key]['std'] = float(np.mean(pixel_stats[key]['std'])) 100 | 101 | print(f"Pixel Stats: {pprint.pformat(pixel_stats)}") 102 | 103 | # Save stats to yaml file 104 | _pixel_stats_yaml_filepath = os.path.join(dataset_dir, 'pixel_stats.yaml') 105 | with open(_pixel_stats_yaml_filepath, 'w') as f: 106 | yaml.dump(pixel_stats, f) 107 | 108 | # Matplotlib Histogram of all pixels with lines for each type of fragment 109 | plt.figure(figsize=(10, 10)) 110 | plt.hist(fragment_mask.flatten(), bins=100, alpha=0.5, label='mask') 111 | plt.hist(fragment_bg.flatten(), bins=100, alpha=0.5, label='bg') 112 | plt.hist(fragment_ink.flatten(), bins=100, alpha=0.5, label='ink') 113 | plt.legend(loc='upper right') 114 | plt.title('Pixel Histogram') 115 | plt.xlabel('Pixel Value') 116 | plt.ylabel('Count') 117 | 118 | # Save to image 119 | _pixel_stats_png_filepath = os.path.join(dataset_dir, 'pixel_stats.png') 120 | plt.savefig(_pixel_stats_png_filepath) 121 | -------------------------------------------------------------------------------- /sweep.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import pprint 4 | import shutil 5 | import uuid 6 | 7 | import numpy as np 8 | import yaml 9 | from hyperopt import fmin, hp, tpe 10 | from tensorboardX import SummaryWriter 11 | 12 | from src import train_valid, eval_from_episode_dir 13 | 14 | if os.name == 'nt': 15 | print("Windows Computer Detected") 16 | DEFAULT_BATCH_SIZE = 5 17 | DEFAULT_SEED = 340 18 | ROOT_DIR = "C:\\Users\\ook\\Documents\\dev" 19 | DATA_DIR = os.path.join(ROOT_DIR, "ashenvenus\\data\\split") 20 | MODEL_DIR = os.path.join(ROOT_DIR, "ashenvenus\\models") 21 | OUTPUT_DIR = os.path.join(ROOT_DIR, "ashenvenus\\output") 22 | shutil.rmtree(OUTPUT_DIR, ignore_errors=True) 23 | else: 24 | if os.path.isdir("/home/tren"): 25 | print("Linux Computer 1 Detected") 26 | ROOT_DIR = "/home/tren/dev/" 27 | DEFAULT_BATCH_SIZE = 2 28 | DEFAULT_SEED = 7 29 | DATA_DIR = os.path.join(ROOT_DIR, "ashenvenus/data/split") 30 | MODEL_DIR = os.path.join(ROOT_DIR, "ashenvenus/models") 31 | OUTPUT_DIR = os.path.join(ROOT_DIR, "ashenvenus/output") 32 | elif os.path.isdir("/home/oop"): 33 | print("Linux Computer 2 Detected") 34 | ROOT_DIR = "/home/oop/dev/" 35 | DEFAULT_BATCH_SIZE = 3 36 | DEFAULT_SEED = 420 37 | DATA_DIR = os.path.join(ROOT_DIR, "ashenvenus/data/split") 38 | MODEL_DIR = os.path.join(ROOT_DIR, "ashenvenus/models") 39 | OUTPUT_DIR = os.path.join(ROOT_DIR, "ashenvenus/output") 40 | shutil.rmtree(OUTPUT_DIR, ignore_errors=True) 41 | 42 | parser = argparse.ArgumentParser() 43 | parser.add_argument('--seed', type=int, default=DEFAULT_SEED) 44 | parser.add_argument('--batch_size', type=int, default=DEFAULT_BATCH_SIZE) 45 | 46 | # Define the search space 47 | HYPERPARAMS = { 48 | 'train_dir_name' : 'train', 49 | 'valid_dir_name' : 'valid', 50 | 'eval_dir_name' : 'valid', 51 | # Model 52 | 'model_str': hp.choice('model_str', [ 53 | 'vit_b|sam_vit_b_01ec64.pth', 54 | # 'vit_h|sam_vit_h_4b8939.pth', 55 | # 'vit_l|sam_vit_l_0b3195.pth', 56 | ]), 57 | 'freeze': hp.choice('freeze', [ 58 | True, 59 | # False, # Uses up too much memory 60 | ]), 61 | "hidden_dim1" : hp.choice("hidden_dim1", [ 62 | 256, 63 | 128, 64 | 64, 65 | ]), 66 | "hidden_dim2" : hp.choice("hidden_dim2", [ 67 | 256, 68 | 128, 69 | 64, 70 | ]), 71 | "dropout_prob" : hp.choice("dropout_prob", [ 72 | 0.5, 73 | 0.2, 74 | 0, 75 | ]), 76 | # Dataset 77 | 'threshold': hp.choice('threshold', [ 78 | # 0.5, 79 | 0.2, 80 | # 0.1, 81 | ]), 82 | 'curriculum': hp.choice('curriculum', [ 83 | '1', # Depth of 1 - 40/45 84 | # '2', # Depth of 1 - 53/58 85 | # '3', # Depth of 1 - 48/53 86 | # '123', 87 | ]), 88 | 'num_samples_train': hp.choice('num_samples_train', [ 89 | # 2, 90 | # 2000, 91 | # 8000, 92 | 20000, 93 | # 200000, 94 | ]), 95 | 'num_samples_valid': hp.choice('num_samples_valid', [ 96 | # 2, 97 | 200, 98 | # 8000, 99 | ]), 100 | 'resize': hp.choice('resize', [ 101 | 1.0, # Universal Harmonics 102 | # 0.3, 103 | ]), 104 | 'pixel_norm': hp.choice('pixel_norm', [ 105 | "mask", 106 | "ink", 107 | "bg", 108 | ]), 109 | 'crop_size_str': hp.choice('crop_size_str', [ 110 | '256.256', # Universal Harmonics 111 | # '128.128', 112 | # '68.68', 113 | ]), 114 | 'max_depth': hp.choice('max_depth', [ 115 | 42, # Universal Harmonics 116 | ]), 117 | 'lr_sched': hp.choice('lr_sched', [ 118 | # 'cosine', 119 | # 'gamma', 120 | 'flat', 121 | ]), 122 | # Training 123 | 'seed': 0, 124 | 'batch_size' : 2, 125 | 'num_epochs': hp.choice('num_epochs', [ 126 | # 1, 127 | # 8, 128 | 16, 129 | ]), 130 | 'warmup_epochs': hp.choice('warmup_epochs', [ 131 | 0, 132 | 1, 133 | ]), 134 | 'lr': hp.loguniform('lr',np.log(0.0001), np.log(0.01)), 135 | 'wd': hp.choice('wd', [ 136 | 1e-4, 137 | 1e-3, 138 | 0, 139 | ]), 140 | } 141 | 142 | def sweep_episode(hparams) -> float: 143 | 144 | # Print hyperparam dict with logging 145 | print(f"\n\n Starting EPISODE \n\n") 146 | print(f"\n\nHyperparams:\n\n{pprint.pformat(hparams)}\n\n") 147 | 148 | # Create output directory based on run_name 149 | run_name: str = str(uuid.uuid4())[:8] 150 | output_dir = os.path.join(OUTPUT_DIR, run_name) 151 | os.makedirs(output_dir, exist_ok=True) 152 | 153 | # Train and Validation directories 154 | train_dir = os.path.join(DATA_DIR, hparams['train_dir_name']) 155 | valid_dir = os.path.join(DATA_DIR, hparams['valid_dir_name']) 156 | eval_dir = os.path.join(DATA_DIR, hparams['eval_dir_name']) 157 | 158 | # Save hyperparams to file with YAML 159 | with open(os.path.join(output_dir, 'hparams.yaml'), 'w') as f: 160 | yaml.dump(hparams, f) 161 | 162 | # HACK: Convert Hyperparam strings to correct format 163 | hparams['crop_size'] = [int(x) for x in hparams['crop_size_str'].split('.')] 164 | model, weights_filepath = hparams['model_str'].split('|') 165 | weights_filepath = os.path.join(MODEL_DIR, weights_filepath) 166 | 167 | try: 168 | writer = SummaryWriter(logdir=output_dir) 169 | # Train and evaluate a TFLite model 170 | score_dict = train_valid( 171 | run_name =run_name, 172 | output_dir = output_dir, 173 | train_dir = train_dir, 174 | valid_dir = valid_dir, 175 | model=model, 176 | weights_filepath=weights_filepath, 177 | writer=writer, 178 | **hparams, 179 | ) 180 | writer.add_hparams(hparams, score_dict) 181 | eval_from_episode_dir( 182 | eval_dir = eval_dir, 183 | episode_dir = output_dir, 184 | output_dir = output_dir, 185 | eval_on = hparams['curriculum'], 186 | max_num_samples_eval = 5000, 187 | max_time_hours = 0.1, 188 | log_images = False, 189 | save_pred_img = True, 190 | save_submit_csv = False, 191 | save_histograms = False, 192 | writer=writer, 193 | **hparams, 194 | ) 195 | writer.close() 196 | # Score is average of all scores 197 | score = sum(score_dict.values()) / len(score_dict) 198 | except Exception as e: 199 | print(f"\n\n (ERROR) EPISODE FAILED (ERROR) \n\n") 200 | print(f"Potentially Bad Hyperparams:\n\n{pprint.pformat(hparams)}\n\n") 201 | raise e 202 | # print(e) 203 | # score = 0 204 | # Maximize score is minimize negative score 205 | return -score 206 | 207 | if __name__ == "__main__": 208 | args = parser.parse_args() 209 | HYPERPARAMS['seed'] = args.seed 210 | HYPERPARAMS['batch_size'] = args.batch_size 211 | best = fmin( 212 | sweep_episode, 213 | space=HYPERPARAMS, 214 | algo=tpe.suggest, 215 | max_evals=100, 216 | rstate=np.random.Generator(np.random.PCG64(args.seed)), 217 | ) -------------------------------------------------------------------------------- /v1/src.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import gc 3 | import os 4 | import pprint 5 | from io import StringIO 6 | from typing import Dict, Tuple 7 | 8 | import cv2 9 | import numpy as np 10 | import PIL.Image as Image 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.functional as F 14 | import yaml 15 | from segment_anything import SamAutomaticMaskGenerator, sam_model_registry 16 | from segment_anything.modeling import (ImageEncoderViT, MaskDecoder, 17 | PromptEncoder, Sam) 18 | from segment_anything.utils.amg import (MaskData, batched_mask_to_box, 19 | build_point_grid) 20 | from torch.utils.data import DataLoader, Dataset 21 | from torchvision import transforms 22 | from tqdm import tqdm 23 | 24 | 25 | def get_device(device: str = None): 26 | if device == None or device == "gpu": 27 | if torch.cuda.is_available(): 28 | print("Using GPU") 29 | print("Clearing GPU memory") 30 | torch.cuda.empty_cache() 31 | gc.collect() 32 | return torch.device("cuda") 33 | print("Using CPU") 34 | return torch.device("cpu") 35 | 36 | 37 | def image_to_rle_fast(img): 38 | img[0] = 0 39 | img[-1] = 0 40 | runs = np.where(img[1:] != img[:-1])[0] + 2 41 | runs[1::2] = runs[1::2] - runs[:-1:2] 42 | f = StringIO() 43 | np.savetxt(f, runs.reshape(1, -1), delimiter=" ", fmt="%d") 44 | return f.getvalue().strip() 45 | 46 | 47 | def save_rle_as_image(rle_csv_path, output_dir, subtest_name, image_shape): 48 | with open(rle_csv_path, "r") as csvfile: 49 | csv_reader = csv.reader(csvfile) 50 | next(csv_reader) # Skip header 51 | for row in csv_reader: 52 | _subtest_name, rle_data = row 53 | if _subtest_name != subtest_name: 54 | continue 55 | rle_pairs = list(map(int, rle_data.split())) 56 | 57 | # Decode RLE data 58 | img = np.zeros(image_shape[0] * image_shape[1], dtype=np.uint8) 59 | for i in range(0, len(rle_pairs), 2): 60 | start = rle_pairs[i] - 1 61 | end = start + rle_pairs[i + 1] 62 | img[start:end] = 1 63 | 64 | # Reshape decoded image data to original shape 65 | img = img.reshape(image_shape) 66 | img = Image.fromarray(img * 255).convert("1") 67 | _image_filepath = os.path.join(output_dir, 68 | f"pred_{subtest_name}_rle.png") 69 | img.save(_image_filepath) 70 | 71 | 72 | def dice_score(preds, label, beta=0.5, epsilon=1e-6): 73 | preds = torch.sigmoid(preds) 74 | preds = preds.flatten() 75 | label = label.flatten() 76 | tp = preds[label == 1].sum() 77 | fp = preds[label == 0].sum() 78 | fn = label.sum() - tp 79 | p = tp / (tp + fp + epsilon) 80 | r = tp / (tp + fn + epsilon) 81 | _score = (1 + beta * beta) * (p * r) / (beta * beta * p + r + epsilon) 82 | return _score 83 | 84 | 85 | class FragmentDataset(Dataset): 86 | def __init__( 87 | self, 88 | # Directory containing the dataset 89 | data_dir: str, 90 | # Number of random crops to take from fragment volume 91 | dataset_size: int = 2, 92 | # Number of points to sample per crop 93 | points_per_crop: int = 4, 94 | # Filenames of the images we'll use 95 | image_mask_filename="mask.png", 96 | image_labels_filename="inklabels.png", 97 | slices_dir_filename="surface_volume", 98 | ir_image_filename="ir.png", 99 | # Expected slices per fragment 100 | crop_size: Tuple[int] = (3, 68, 68), 101 | label_size: Tuple[int] = (256, 256), 102 | # Depth in scan is a Clipped Normal distribution 103 | min_depth: int = 0, 104 | max_depth: int = 65, 105 | avg_depth: float = 27.0, 106 | std_depth: float = 10.0, 107 | # Training vs Testing mode 108 | train: bool = True, 109 | # Device to use 110 | device: str = "cuda", 111 | ): 112 | print(f"Making Dataset from {data_dir}") 113 | self.dataset_size = dataset_size 114 | self.points_per_crop = points_per_crop 115 | self.train = train 116 | self.device = device 117 | 118 | # Open Mask image 119 | _image_mask_filepath = os.path.join(data_dir, image_mask_filename) 120 | self.mask = np.array( 121 | cv2.imread(_image_mask_filepath, 122 | cv2.IMREAD_GRAYSCALE)).astype(np.uint8) 123 | # Image dimmensions (depth, height, width) 124 | self.original_size = self.mask.shape 125 | self.crop_size = crop_size 126 | self.label_size = label_size 127 | # Open Label image 128 | if self.train: 129 | _image_labels_filepath = os.path.join(data_dir, image_labels_filename) 130 | image_lables = cv2.imread(_image_labels_filepath, cv2.IMREAD_GRAYSCALE) 131 | self.labels = np.array(image_lables).astype(np.uint8) 132 | # Get connected components 133 | outputs = cv2.connectedComponentsWithStats(image_lables, 4, cv2.CV_32S) 134 | num_labels = outputs[0] 135 | labels = outputs[1] 136 | stats = outputs[2] 137 | centroids = outputs[3] 138 | # loop over the number of unique connected component labels 139 | for i in range(0, num_labels): 140 | # skip first component as it is the background which we do 141 | # not want to consider 142 | if i == 0: 143 | continue 144 | # extract the connected component statistics and centroid for 145 | # the current label 146 | x = stats[i, cv2.CC_STAT_LEFT] 147 | y = stats[i, cv2.CC_STAT_TOP] 148 | w = stats[i, cv2.CC_STAT_WIDTH] 149 | h = stats[i, cv2.CC_STAT_HEIGHT] 150 | area = stats[i, cv2.CC_STAT_AREA] 151 | (cX, cY) = centroids[i] 152 | # construct a mask for the current connected component by 153 | # finding a pixels in the labels array that have the current 154 | # connected component ID 155 | componentMask = (labels == i).astype("uint8") * 255 156 | 157 | # Slices 158 | self.slice_dir = os.path.join(data_dir, slices_dir_filename) 159 | 160 | # Sample random crops within the image 161 | self.indices = np.zeros((dataset_size, 2, 3), dtype=np.int64) 162 | for i in range(dataset_size): 163 | # Select a random starting point for the subvolume 164 | start_depth = int( 165 | np.clip(np.random.normal(avg_depth, std_depth), min_depth, 166 | max_depth)) 167 | start_height = np.random.randint( 168 | 0, self.original_size[0] - self.crop_size[1]) 169 | start_width = np.random.randint( 170 | 0, self.original_size[1] - self.crop_size[2]) 171 | self.indices[i, 0, :] = [start_depth, start_height, start_width] 172 | # End point is start point + crop size 173 | self.indices[i, 1, :] = [ 174 | start_depth + self.crop_size[0], 175 | start_height + self.crop_size[1], 176 | start_width + self.crop_size[2], 177 | ] 178 | 179 | # DEBUG: IR image 180 | _image_ir_filepath = os.path.join(data_dir, ir_image_filename) 181 | self.ir_image = np.array(cv2.imread(_image_ir_filepath)).astype( 182 | np.float32) / 255.0 183 | self.ir_image = np.transpose(self.ir_image, (2, 0, 1)) 184 | 185 | # Pixel stats for ir image, only for values inside mask 186 | self.pixel_mean = np.mean(self.ir_image[:, self.mask == 1]) 187 | self.pixel_std = np.std(self.ir_image[:, self.mask == 1]) 188 | 189 | # TODO: Pixel stats for points inside labels, outside labels for all slices 190 | # This might be better calculated beforehand and then loaded 191 | 192 | def __len__(self): 193 | return self.dataset_size 194 | 195 | def __getitem__(self, idx): 196 | # Start and End points for the crop in pixel space 197 | start = self.indices[idx, 0, :] 198 | end = self.indices[idx, 1, :] 199 | 200 | # Create a grid of points in the crop 201 | points = build_point_grid(self.points_per_crop) 202 | # Conver points into pixel space 203 | points[:, 0] = points[:, 0] * self.crop_size[1] 204 | points[:, 1] = points[:, 1] * self.crop_size[2] 205 | points = points.astype(np.int32) 206 | # Get the label for each point 207 | point_labels = np.zeros((self.points_per_crop**2), dtype=np.int32) 208 | for i, point in enumerate(points): 209 | point_labels[i] = self.labels[ 210 | point[0] + start[1], 211 | point[1] + start[2], 212 | ] 213 | # Points float32 (64, B, 2) 214 | # Point int32 Labels (64, B) 215 | points = torch.from_numpy(points).to(dtype=torch.float32, device=self.device) 216 | point_labels = torch.from_numpy(point_labels).to(dtype=torch.int32, device=self.device) 217 | 218 | # # Load the relevant slices and pack into image tensor 219 | # image = np.zeros(self.crop_size, dtype=np.float32) 220 | # for i, _depth in enumerate(range(start[0], end[0])): 221 | # _slice_filepath = os.path.join(self.slice_dir, f"{_depth:02d}.tif") 222 | # _slice = np.array(cv2.imread(_slice_filepath, cv2.IMREAD_GRAYSCALE)).astype(np.float32) 223 | # image[i, :, :] = _slice[start[1]: end[1], start[2]: end[2]] 224 | # image = torch.from_numpy(image).to(device=self.device) 225 | 226 | # TODO: Take a 3D Volume and show the top, back, left, right view of volume 227 | # Bias towards a longer height than width 228 | # Try to get the entire depth 229 | # Find some tiling of the width and height such that you get 1024x1024 230 | # 65x128x3 231 | 232 | # DEBUG: Use IR image instead of surface volume as toy problem 233 | image = self.ir_image[:, start[1]:end[1], start[2]:end[2]] 234 | image = torch.from_numpy(image).to(device=self.device) 235 | 236 | # Normalize image 237 | # image = (image - self.pixel_mean) / self.pixel_std 238 | 239 | if self.train: 240 | labels = self.labels[start[1]:end[1], start[2]:end[2]] 241 | low_res_labels = cv2.resize( 242 | labels.astype(np.uint8), 243 | self.label_size, 244 | interpolation=cv2.INTER_NEAREST, 245 | ) 246 | low_res_labels = torch.from_numpy(low_res_labels).to(dtype=torch.float32) 247 | low_res_labels = low_res_labels.unsqueeze(0).to(device=self.device) 248 | labels = torch.from_numpy(labels).unsqueeze(0).to(dtype=torch.float32) 249 | return image, points, point_labels, low_res_labels, labels 250 | else: 251 | return image 252 | 253 | class CombinedLoss(nn.Module): 254 | def __init__(self, alpha=0.5): 255 | super(CombinedLoss, self).__init__() 256 | self.alpha = alpha 257 | self.bce_with_logits_loss = nn.BCEWithLogitsLoss() 258 | self.mse_loss = nn.MSELoss() 259 | 260 | def binary_iou(self, mask1, mask2): 261 | intersection = (mask1 * mask2).sum(dim=(-1, -2)) 262 | union = torch.logical_or(mask1, mask2).sum(dim=(-1, -2)) 263 | iou = intersection.float() / union.float() 264 | return iou 265 | 266 | def forward(self, logits, gt_masks, predicted_iou): 267 | # Calculate pixel-wise binary cross-entropy loss 268 | bce_loss = self.bce_with_logits_loss(logits, gt_masks.float()) 269 | 270 | # Calculate predicted masks 271 | pred_masks = torch.sigmoid(logits) >= 0.5 272 | pred_masks = pred_masks.float() 273 | 274 | # Calculate actual IoU scores for each pair in the batch 275 | actual_iou = self.binary_iou(pred_masks, gt_masks) 276 | 277 | # Calculate the MSE loss between predicted and actual IoU scores 278 | iou_loss = self.mse_loss(predicted_iou, actual_iou) 279 | 280 | # Combine the two losses using a weighting factor alpha 281 | combined_loss = self.alpha * bce_loss + (1 - self.alpha) * iou_loss 282 | 283 | return combined_loss 284 | 285 | def train_valid( 286 | run_name: str = "testytest", 287 | output_dir: str = None, 288 | train_dir: str = None, 289 | valid_dir: str = None, 290 | # Model 291 | model: str = "vit_b", 292 | weights_filepath: str = "path/to/model.pth", 293 | save_model: bool = True, 294 | # Training 295 | device: str = None, 296 | num_samples_train: int = 2, 297 | num_samples_valid: int = 2, 298 | num_epochs: int = 2, 299 | batch_size: int = 1, 300 | optimizer: str = "adam", 301 | lr: float = 1e-5, 302 | wd: float = 1e-4, 303 | alpha: float = 0.5, 304 | writer=None, 305 | log_images: bool = False, 306 | # Dataset 307 | curriculum: str = "1", 308 | crop_size: Tuple[int] = (3, 68, 68), 309 | label_size: Tuple[int] = (1024, 1024), 310 | points_per_crop: int = 8, 311 | avg_depth: float = 27.0, 312 | std_depth: float = 10.0, 313 | **kwargs, 314 | ): 315 | # device = get_device(device) 316 | # device = "cpu" 317 | model = sam_model_registry[model](checkpoint=weights_filepath) 318 | # TODO: Which of these should be frozen? 319 | # for param in model.image_encoder.parameters(): 320 | # param.requires_grad = False 321 | # for param in model.prompt_encoder.parameters(): 322 | # param.requires_grad = False 323 | # for param in model.mask_decoder.parameters(): 324 | # param.requires_grad = False 325 | model.to(device=device) 326 | model.train() 327 | optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd) 328 | loss_fn = CombinedLoss(alpha=alpha) 329 | # TODO: Learning rate scheduler 330 | # TODO: Learning rate warmup 331 | 332 | train_step = 0 333 | best_score_dict: Dict[str, float] = {} 334 | for epoch in range(num_epochs): 335 | print(f"\n\n --- Epoch {epoch+1} of {num_epochs} --- \n\n") 336 | for phase, data_dir, num_samples in [("Train", train_dir, num_samples_train), ("Valid", valid_dir, num_samples_valid)]: 337 | for _dataset_id in curriculum: 338 | _dataset_filepath = os.path.join(data_dir, _dataset_id) 339 | print(f"{phase} on {_dataset_filepath} ...") 340 | _score_name = f"Dice/{phase}/{_dataset_id}" 341 | if _score_name not in best_score_dict: 342 | best_score_dict[_score_name] = 0 343 | _dataset = FragmentDataset( 344 | data_dir=_dataset_filepath, 345 | dataset_size=num_samples, 346 | points_per_crop=points_per_crop, 347 | crop_size=crop_size, 348 | label_size=label_size, 349 | avg_depth=avg_depth, 350 | std_depth=std_depth, 351 | train=True, 352 | device=device, 353 | ) 354 | _dataloader = DataLoader( 355 | dataset=_dataset, 356 | batch_size=batch_size, 357 | shuffle=True, 358 | # pin_memory=True, 359 | ) 360 | _loader = tqdm(_dataloader) 361 | score = 0 362 | for images, points, point_labels, low_res_labels, labels in _loader: 363 | train_step += 1 364 | if writer and log_images: 365 | writer.add_images(f"input-image/{phase}/{_dataset_id}", 366 | images, train_step) 367 | writer.add_images(f"input-label/{phase}/{_dataset_id}", 368 | labels * 255, train_step) 369 | # Plot point coordinates into a blank image of size images 370 | _point_coords = points.cpu().numpy() 371 | _point_labels = point_labels.cpu().numpy() 372 | _point_image = np.zeros( 373 | (1, 3, labels.shape[2], labels.shape[3]), 374 | dtype=np.uint8) 375 | _point_image[0, 2, :, :] = labels.cpu().numpy()[0, :, :, :] 376 | point_width = 4 377 | for i in range(_point_coords.shape[1]): 378 | _height = int(_point_coords[0, i, 0]) 379 | _width = int(_point_coords[0, i, 1]) 380 | if _point_labels[0, i] == 0: 381 | _point_image[0, 0, _height - 382 | point_width:_height + point_width, 383 | _width - point_width:_width + 384 | point_width] = 255 385 | else: 386 | _point_image[0, 1, _height - 387 | point_width:_height + point_width, 388 | _width - point_width:_width + 389 | point_width] = 255 390 | writer.add_images( 391 | f"input-points/{phase}/{_dataset_id}", 392 | _point_image, train_step) 393 | writer.flush() 394 | image_embeddings = model.image_encoder(images) 395 | # TODO: LoRAs around encoder and decoder? 396 | # Points float32 (B, 64, 2) 397 | # Point int32 Labels (B, 64) 398 | sparse_embeddings, dense_embeddings = model.prompt_encoder( 399 | points=(points, point_labels), 400 | # TODO: These boxes and labels might have to be per-point 401 | # boxes=batched_mask_to_box(labels), 402 | boxes = None, 403 | # masks=labels, 404 | masks = None, 405 | ) 406 | # sparse_embeddings float32 (64, 2, 256) 407 | # dense_embeddings float32 (64, 256, 64, 64) 408 | # image_embeddings float32 (B, 256, 64, 64) 409 | # TODO: Batch size over 1 fucks this up 410 | low_res_masks, iou_predictions = model.mask_decoder( 411 | image_embeddings=image_embeddings, 412 | image_pe=model.prompt_encoder.get_dense_pe(), 413 | sparse_prompt_embeddings=sparse_embeddings, 414 | dense_prompt_embeddings=dense_embeddings, 415 | multimask_output=False, 416 | ) 417 | if writer and log_images: 418 | writer.add_images( 419 | f"output.masks/{phase}/{_dataset_id}", 420 | low_res_masks, train_step) 421 | loss = loss_fn(low_res_masks, low_res_labels, 422 | iou_predictions) 423 | optimizer.zero_grad() 424 | loss.backward() 425 | optimizer.step() 426 | _loss_name = f"{loss_fn.__class__.__name__}/{phase}/{_dataset_id}" 427 | _loader.set_postfix_str(f"{_loss_name}: {loss.item():.4f}") 428 | if writer: 429 | writer.add_scalar(f"{_loss_name}", loss.item(), 430 | train_step) 431 | 432 | score += dice_score(low_res_masks, low_res_labels) 433 | score /= len(_dataloader) 434 | if writer: 435 | writer.add_scalar(f"Dice/{phase}", score, train_step) 436 | # Overwrite best score if it is better 437 | if score > best_score_dict[_score_name]: 438 | print(f"New best score! {score:.4f} ") 439 | print(f"(was {best_score_dict[_score_name]:.4f})") 440 | best_score_dict[_score_name] = score 441 | if save_model: 442 | _model_filepath = os.path.join( 443 | output_dir, 444 | f"model_{run_name}_best_{_dataset_id}.pth") 445 | print(f"Saving model to {_model_filepath}") 446 | torch.save(model.state_dict(), _model_filepath) 447 | # Flush writer every epoch 448 | writer.flush() 449 | writer.close() 450 | return best_score_dict 451 | 452 | 453 | def eval( 454 | output_dir: str = None, 455 | eval_dir: str = None, 456 | save_pred_img: bool = True, 457 | save_submit_csv: bool = False, 458 | # Evaluation 459 | batch_size: int = 1, 460 | num_samples_eval: int = 100, 461 | points_per_side=32, 462 | pred_iou_thresh=0.86, 463 | stability_score_thresh=0.92, 464 | crop_n_layers=1, 465 | crop_n_points_downscale_factor=2, 466 | min_mask_region_area=100, # Requires open-cv to run post-processing 467 | # Model 468 | model: str = "vit_b", 469 | weights_filepath: str = None, 470 | # Dataset 471 | crop_size: Tuple[int] = (3, 68, 68), 472 | label_size: Tuple[int] = (1024, 1024), 473 | points_per_crop: int = 8, 474 | avg_depth: float = 27.0, 475 | std_depth: float = 10.0, 476 | **kwargs, 477 | ): 478 | device = get_device(device) 479 | model = SamAutomaticMaskGenerator( 480 | model=sam_model_registry[model](checkpoint=weights_filepath), 481 | points_per_side=points_per_side, 482 | pred_iou_thresh=pred_iou_thresh, 483 | stability_score_thresh=stability_score_thresh, 484 | crop_n_layers=crop_n_layers, 485 | crop_n_points_downscale_factor=crop_n_points_downscale_factor, 486 | min_mask_region_area=min_mask_region_area, 487 | ) 488 | model.eval() 489 | model.to(device) 490 | 491 | if save_submit_csv: 492 | submission_filepath = os.path.join(output_dir, "submission.csv") 493 | with open(submission_filepath, "w") as f: 494 | # Write header 495 | f.write("Id,Predicted\n") 496 | 497 | # Baseline is to use image mask to create guess submission 498 | for dataset_id in os.listdir(eval_dir): 499 | print(f"Evaluating on {dataset_id}") 500 | _dataset_filepath = os.path.join(eval_dir, dataset_id) 501 | _dataset = FragmentDataset( 502 | data_dir=_dataset_filepath, 503 | dataset_size=num_samples_eval, 504 | points_per_crop=points_per_crop, 505 | crop_size=crop_size, 506 | label_size=label_size, 507 | avg_depth=avg_depth, 508 | std_depth=std_depth, 509 | train=False, 510 | device=device, 511 | ) 512 | _dataloader = DataLoader( 513 | dataset=_dataset, 514 | batch_size=batch_size, 515 | shuffle=False, 516 | # pin_memory=True, 517 | ) 518 | mask_data: MaskData = None 519 | _loader = tqdm(_dataloader) 520 | for img in _loader: 521 | 522 | # Get masks from image 523 | _mask_data: MaskData = model.generate(img) 524 | """ 525 | `segmentation` : the mask 526 | `area` : the area of the mask in pixels 527 | `bbox` : the boundary box of the mask in XYWH format 528 | `predicted_iou` : the model's own prediction for the quality of the mask 529 | `point_coords` : the sampled input point that generated this mask 530 | `stability_score` : an additional measure of mask quality 531 | `crop_box` : the crop of the image used to generate this mask in XYWH format 532 | """ 533 | # Filter the masks using NMS 534 | 535 | # Group all the predicted masks together 536 | if mask_data is None: 537 | mask_data = _mask_data 538 | else: 539 | mask_data.cat(_mask_data) 540 | 541 | # Convert masks to single image 542 | 543 | if save_pred_img: 544 | print("Saving prediction image...") 545 | _image_filepath = os.path.join(output_dir, 546 | f"pred_{dataset_id}.png") 547 | img.save(_image_filepath) 548 | 549 | if save_submit_csv: 550 | print("Saving submission csv...") 551 | img = np.array(img).flatten() 552 | inklabels_rle_fast = image_to_rle_fast(img) 553 | with open(submission_filepath, "a") as f: 554 | f.write(f"{dataset_id},{inklabels_rle_fast}\n") 555 | 556 | 557 | def eval_from_episode_dir( 558 | episode_dir: str = None, 559 | output_dir: str = None, 560 | weights_filename: str = "model.pth", 561 | hparams_filename: str = "hparams.yaml", 562 | **kwargs, 563 | ): 564 | # Get hyperparams from text file 565 | _hparams_filepath = os.path.join(episode_dir, hparams_filename) 566 | with open(_hparams_filepath, "r") as f: 567 | hparams = yaml.load(f, Loader=yaml.FullLoader) 568 | _weights_filepath = os.path.join(episode_dir, weights_filename) 569 | # Merge kwargs with hparams, kwargs takes precedence 570 | hparams = {**hparams, **kwargs} 571 | print(f"Hyperparams:\n{pprint.pformat(hparams)}\n") 572 | eval( 573 | output_dir=output_dir, 574 | weights_filepath=_weights_filepath, 575 | **hparams, 576 | ) 577 | -------------------------------------------------------------------------------- /v0/notes/sub12.csv: -------------------------------------------------------------------------------- 1 | Id,Predicted 2 | b,1401735 21 1402216 60 1408065 21 1408546 60 1414395 21 1414876 60 1420725 21 1421206 60 1427055 21 1427536 60 1433385 21 1433866 60 1439715 21 1440196 60 1446045 21 1446526 60 1452375 21 1452856 60 1458705 21 1459186 60 1465035 21 1465516 60 1471365 21 1471846 60 1477695 21 1478176 60 1484025 21 1484506 60 1490355 21 1490836 60 1496685 21 1497166 60 1503015 21 1503496 60 1509345 21 1509826 60 1515675 21 1516156 60 1522005 21 1522486 60 1527694 862 1528656 300 1534024 862 1534986 300 1540354 862 1541316 300 1546684 862 1547646 300 1553014 862 1553976 300 1559344 862 1560306 300 1565674 862 1566636 300 1572004 862 1572966 300 1578334 862 1579296 300 1584664 862 1585626 300 1590994 862 1591956 300 1597324 862 1598286 300 1603654 862 1604616 300 1609984 862 1610946 300 1616314 862 1617276 300 1622644 862 1623606 300 1628974 862 1629936 300 1635304 862 1636266 300 1641634 862 1642596 300 1647964 862 1648926 300 1654274 1322 1660604 1322 1666934 1322 1673264 1322 1679594 1322 1685924 1322 1692254 1322 1698584 1322 1704914 1322 1711244 1322 1717574 1322 1723904 1322 1730234 1322 1736564 1322 1742894 1322 1749224 1322 1755554 1322 1761884 1322 1768214 1322 1774544 1322 1780874 1322 1787204 1322 1793534 1322 1799864 1322 1806194 1322 1812524 1322 1818854 1322 1825184 1322 1831514 1322 1837844 1322 1844174 1322 1850504 1322 1856834 1322 1863164 1322 1869494 1322 1875824 1322 1882154 1322 1888484 1322 1894814 1322 1901144 1322 1907454 1342 1913784 1342 1920114 1342 1926444 1342 1932774 1342 1939104 1342 1945434 1342 1951764 1342 1958094 1342 1964424 1342 1970754 1342 1977084 1342 1983414 1342 1989744 1342 1996074 1342 2002404 1342 2008734 1342 2015064 1342 2021394 1342 2027704 1362 2034034 1462 2035518 39 2040364 1462 2041848 39 2046694 1462 2048178 39 2053024 1462 2054508 39 2059354 1462 2060838 39 2065684 1462 2067168 39 2072014 1462 2073498 39 2078344 1462 2079828 39 2084674 1462 2086158 39 2091004 1462 2092488 39 2097334 1462 2098818 39 2103664 1462 2105148 39 2109994 1462 2111478 39 2116324 1462 2117808 39 2122654 1462 2124138 39 2128984 1462 2130468 39 2135314 1462 2136798 39 2141644 1462 2143128 39 2147974 1462 2149458 39 2154304 1462 2155788 39 2160634 1523 2166984 1503 2173314 1503 2179644 1503 2185974 1503 2192304 1503 2198634 1503 2204964 1503 2211294 1503 2217624 1503 2223954 1503 2230284 1503 2236614 1503 2242944 1503 2249274 1503 2255604 1503 2261934 1503 2268264 1503 2274594 1503 2280924 1503 2287254 1523 2293584 1523 2299914 1523 2306244 1523 2312574 1523 2318904 1523 2325234 1523 2331564 1523 2337894 1523 2344224 1523 2350554 1523 2356884 1523 2363214 1523 2369544 1523 2375874 1523 2382204 1523 2388534 1523 2394864 1523 2401194 1523 2407524 1523 2413854 1543 2420184 1543 2426514 1543 2432844 1543 2439174 1543 2445504 1543 2451834 1543 2458164 1543 2464494 1543 2470824 1543 2477154 1543 2483484 1543 2489814 1543 2496144 1543 2502474 1543 2508804 1543 2515134 1543 2521464 1543 2527794 1543 2534124 1543 2540454 961 2541437 560 2546784 961 2547767 560 2553114 961 2554097 560 2559444 961 2560427 560 2565774 961 2566757 560 2572104 961 2573087 560 2578434 961 2579417 560 2584764 961 2585747 560 2591094 961 2592077 560 2597424 961 2598407 560 2603754 961 2604737 560 2610084 961 2611067 560 2616414 961 2617397 560 2622744 961 2623727 560 2629074 961 2630057 560 2635404 961 2636387 560 2641734 961 2642717 560 2648064 961 2649047 560 2654394 961 2655377 560 2660724 961 2661707 560 2667074 922 2668056 541 2673404 922 2674386 541 2679734 922 2680716 541 2686064 922 2687046 541 2692394 922 2693376 541 2698724 922 2699706 541 2705054 922 2706036 541 2711384 922 2712366 541 2717714 922 2718696 541 2724044 922 2725026 541 2730374 922 2731356 541 2736704 922 2737686 541 2743034 922 2744016 541 2749364 922 2750346 541 2755694 922 2756676 541 2762024 922 2763006 541 2768354 922 2769336 541 2774684 922 2775666 541 2781014 922 2781996 541 2787344 922 2788326 541 2793694 181 2794015 441 2794676 320 2795037 140 2800024 181 2800345 441 2801006 320 2801367 140 2806354 181 2806675 441 2807336 320 2807697 140 2812684 181 2813005 441 2813666 320 2814027 140 2819014 181 2819335 441 2819996 320 2820357 140 2825344 181 2825665 441 2826326 320 2826687 140 2831674 181 2831995 441 2832656 320 2833017 140 2838004 181 2838325 441 2838986 320 2839347 140 2844334 181 2844655 441 2845316 320 2845677 140 2850664 181 2850985 441 2851646 320 2852007 140 2856994 181 2857315 441 2857976 320 2858337 140 2863324 181 2863645 441 2864306 320 2864667 140 2869654 181 2869975 441 2870636 320 2870997 140 2875984 181 2876305 441 2876966 320 2877327 140 2882314 181 2882635 441 2883296 320 2883657 140 2888644 181 2888965 441 2889626 320 2889987 140 2894974 181 2895295 441 2895956 320 2896317 140 2901304 181 2901625 441 2902286 320 2902647 140 2907634 181 2907955 441 2908616 320 2908977 140 2913964 181 2914285 441 2914946 320 2915307 140 2920675 200 2921296 220 2921657 120 2927005 200 2927626 220 2927987 120 2933335 200 2933956 220 2934317 120 2939665 200 2940286 220 2940647 120 2945995 200 2946616 220 2946977 120 2952325 200 2952946 220 2953307 120 2958655 200 2959276 220 2959637 120 2964985 200 2965606 220 2965967 120 2971315 200 2971936 220 2972297 120 2977645 200 2978266 220 2978627 120 2983975 200 2984596 220 2984957 120 2990305 200 2990926 220 2991287 120 2996635 200 2997256 220 2997617 120 3002965 200 3003586 220 3003947 120 3009295 200 3009916 220 3010277 120 3015625 200 3016246 220 3016607 120 3021955 200 3022576 220 3022937 120 3028285 200 3028906 220 3029267 120 3034615 200 3035236 220 3035597 120 3040945 200 3041566 220 3041927 120 3047916 200 3048277 80 3054246 200 3054607 80 3060576 200 3060937 80 3066906 200 3067267 80 3073236 200 3073597 80 3079566 200 3079927 80 3085896 200 3086257 80 3092226 200 3092587 80 3098556 200 3098917 80 3104886 200 3105247 80 3111216 200 3111577 80 3117546 200 3117907 80 3123876 200 3124237 80 3130206 200 3130567 80 3136536 200 3136897 80 3142866 200 3143227 80 3149196 200 3149557 80 3155526 200 3155887 80 3161856 200 3162217 80 3168186 200 3168547 80 3174615 22 3180945 22 3187275 22 3193605 22 3199935 22 3206265 22 3212595 22 3218925 22 3225255 22 3231585 22 3237915 22 3244245 22 3250575 22 3256905 22 3263235 22 3269565 22 3275895 22 3282225 22 3288555 22 3294885 22 10022875 20 10029205 60 10035535 60 10041865 60 10048195 60 10054525 60 10060855 60 10067185 60 10073515 60 10079845 60 10086175 60 10092505 60 10098835 60 10105165 60 10111495 60 10117825 60 10124155 60 10130485 60 10136815 60 10143145 60 10149475 60 10155825 40 10162155 40 10168485 40 10174815 40 10181145 40 10187475 40 10193805 40 10200135 40 10206465 40 10212795 40 10219125 40 10225455 40 10231785 40 10238115 40 10244445 40 10250775 40 10257105 40 10263435 40 10269765 40 10276095 40 10282405 60 10288735 60 10295065 60 10301395 60 10307725 60 10314055 60 10320385 60 10326715 60 10333045 60 10339375 60 10345705 60 10352035 60 10358365 60 10364695 60 10371025 60 10377355 60 10383685 60 10390015 60 10396345 60 10402675 60 10409005 20 13454535 22 13460865 22 13467195 22 13473525 22 13479855 22 13486185 22 13492515 22 13498845 22 13505175 22 13511505 22 13517835 22 13524165 22 13530495 22 13536825 22 13543155 22 13549485 22 13555815 22 13562145 22 13568475 22 13574805 22 13581135 22 13587426 80 13593756 80 13600086 80 13606416 80 13612746 80 13619076 80 13625406 80 13631736 80 13638066 80 13644396 80 13650726 80 13657056 80 13663386 80 13669716 80 13676046 80 13682376 80 13688706 80 13695036 80 13701366 80 13707696 80 13714026 80 13720356 80 13726686 80 13733016 80 13739346 80 13745676 80 13752006 80 13758336 80 13764666 80 13770996 80 13777326 80 13783656 80 13789986 80 13796316 80 13802646 80 13808976 80 13815306 80 13821636 80 13827966 80 13834296 100 13840606 120 13846936 120 13853266 120 13859596 120 13865926 120 13872256 120 13878586 120 13884916 120 13891246 120 13897576 120 13903906 120 13910236 120 13916566 120 13922896 120 13929226 120 13935556 120 13941886 120 13948216 120 13954546 120 13960876 120 13967206 100 13973536 100 13979866 100 13986196 100 13992526 100 13998856 100 14005186 100 14011516 100 14017846 100 14024176 100 14030506 100 14036836 100 14043166 100 14049496 100 14055826 100 14062156 100 14068486 100 14074816 100 14081146 100 14087476 100 14093806 120 14100136 120 14106466 120 14112796 120 14119126 120 14125456 120 14131786 120 14138116 120 14144446 120 14150776 120 14157106 120 14163436 120 14169766 120 14176096 120 14182426 120 14188756 120 14195086 120 14201416 120 14207746 120 14213334 22 14214076 120 14219664 22 14220406 100 14225994 22 14226736 100 14232324 22 14233066 100 14238654 22 14239396 100 14244984 22 14245726 100 14251314 22 14252056 100 14257644 22 14258386 100 14263974 22 14264716 100 14270304 22 14271046 100 14276634 22 14277376 100 14282964 22 14283706 100 14289294 22 14290036 100 14295624 22 14296366 100 14301954 22 14302696 100 14308284 22 14309026 100 14314614 22 14315356 100 14320944 22 14321686 100 14327274 22 14328016 100 14333604 22 14334346 100 14339934 22 14340315 1 14340376 1 14340676 100 14346264 22 14346645 21 14346686 21 14347006 100 14352975 21 14353016 21 14353336 120 14359305 21 14359346 21 14359666 120 14365635 21 14365676 21 14365996 120 14371965 21 14372006 21 14372326 120 14378295 21 14378336 21 14378656 120 14384625 21 14384666 21 14384986 120 14390955 21 14390996 21 14391316 120 14397285 21 14397326 21 14397646 120 14403615 21 14403656 21 14403976 120 14409945 21 14409986 21 14410306 120 14416275 21 14416316 21 14416636 120 14422605 21 14422646 21 14422966 120 14428935 21 14428976 21 14429296 120 14435265 21 14435306 21 14435626 120 14441595 21 14441636 21 14441956 120 14447925 21 14447966 21 14448286 120 14454255 21 14454296 21 14454616 120 14460585 21 14460626 21 14460946 120 14466915 21 14466956 21 14467256 140 14473246 60 14473586 140 14479576 60 14479916 140 14485906 60 14486246 140 14492236 60 14492576 140 14498566 60 14498906 140 14504896 60 14505236 140 14511226 60 14511566 140 14517556 60 14517896 140 14523886 60 14524226 140 14530216 60 14530556 140 14536546 60 14536886 140 14542876 60 14543216 140 14549206 60 14549546 140 14555536 60 14555876 140 14561866 60 14562206 140 14568196 60 14568536 140 14574526 60 14574866 140 14580856 60 14581196 140 14587186 60 14587526 140 14593516 60 14593577 19 14593856 140 14599825 20 14599846 80 14600186 140 14606155 101 14606536 120 14612485 101 14612866 120 14618815 101 14619196 120 14625145 101 14625526 120 14631475 101 14631856 120 14637805 101 14638186 120 14644135 101 14644516 120 14650465 101 14650846 120 14656795 101 14657176 120 14663125 101 14663506 120 14669455 101 14669836 120 14675785 101 14676166 120 14682115 101 14682496 120 14688445 101 14688826 120 14694775 101 14695156 120 14701105 101 14701486 120 14707435 101 14707816 120 14713765 101 14714146 120 14719715 60 14720095 101 14720476 120 14726045 60 14726425 101 14726806 100 14732375 60 14732755 81 14733136 100 14738705 60 14739085 81 14739466 100 14745035 60 14745415 81 14745796 100 14751365 60 14751745 81 14752126 100 14757695 60 14758075 81 14758456 100 14764025 60 14764405 81 14764786 100 14770355 60 14770735 81 14771116 100 14776685 60 14777065 81 14777446 100 14783015 60 14783395 81 14783776 100 14789345 60 14789725 81 14790106 100 14795675 60 14796055 81 14796436 100 14802005 60 14802385 81 14802766 100 14808335 60 14808715 81 14809096 100 14814665 60 14815045 81 14815426 100 14820995 60 14821375 81 14821756 100 14827325 60 14827705 81 14828086 100 14833655 60 14834035 81 14834416 100 14839985 60 14840365 81 14840746 100 14846315 60 14846695 81 14847076 100 14852645 60 14853025 81 14853426 80 14859355 81 14859756 80 14865685 81 14866086 80 14872015 81 14872416 80 14878345 81 14878746 80 14884675 81 14885076 80 14891005 81 14891406 80 14897335 81 14897736 80 14903665 81 14904066 80 14909995 81 14910396 80 14916325 81 14916726 80 14922655 81 14923056 80 14928985 81 14929386 80 14935315 81 14935716 80 14941645 81 14942046 80 14947975 81 14948376 80 14954305 81 14954706 80 14960635 81 14961036 80 14966965 81 14967366 80 14973295 81 14973696 80 14979646 40 14980026 60 14985976 40 14986356 60 14992306 40 14992686 60 14998636 40 14999016 60 15004966 40 15005346 60 15011296 40 15011676 60 15017626 40 15018006 60 15023956 40 15024336 60 15030286 40 15030666 60 15036616 40 15036996 60 15042946 40 15043326 60 15049276 40 15049656 60 15055606 40 15055986 60 15061936 40 15062316 60 15068266 40 15068646 60 15074596 40 15074976 60 15080926 40 15081306 60 15087256 40 15087636 60 15093586 40 15093966 60 15099916 40 15100296 60 15106246 40 15112576 40 15118906 40 15125236 40 15131566 40 15137896 40 15144226 40 15150556 40 15156886 40 15163216 40 15169546 40 15175876 40 15182206 40 15188536 40 15194866 40 15201196 40 15207526 40 15213856 40 15220186 40 15226516 40 28175232 40 28181562 40 28187892 40 28194222 40 28200552 40 28206882 40 28213212 40 28219542 40 28225872 40 28232202 40 28238532 40 28244862 40 28251192 40 28257522 40 28263852 40 28270182 40 28276512 40 28282842 40 28289172 40 28295502 40 28301812 60 28308142 60 28314472 60 28320802 60 28327132 60 28333462 60 28339792 60 28346122 60 28352452 60 28358782 60 28365112 60 28371442 60 28377772 60 28384102 60 28390432 60 28396762 60 28403092 60 28409422 60 28415752 60 28422082 60 28428412 60 28434742 60 28441072 60 28447402 60 28453732 60 28460062 60 28466392 60 28472722 60 28479052 60 28485382 60 28491712 60 28498042 60 28504372 60 28510702 60 28517032 60 28523362 60 28529692 60 28536022 60 28542352 60 28548682 60 28555012 60 28561342 40 28567672 40 28574002 40 28580332 40 28586662 40 28592992 40 28599322 40 28605652 40 28611982 40 28618312 40 28624642 40 28630972 40 28637302 40 28643632 40 28649962 40 28656292 40 28662622 40 28668952 40 28675282 40 28681612 40 28687942 40 28694272 40 28700602 40 28706932 40 28713262 40 28719592 40 28725922 40 28732252 40 28738582 40 28744912 40 28751242 40 28757572 40 28763902 40 28770232 40 28776562 40 28782892 40 28789222 40 28795552 40 28801882 40 28808212 40 28814542 40 28820872 40 28827202 40 28833532 40 28839862 40 28846192 40 28852522 40 28858852 40 28865182 40 28871512 40 28877842 40 28884172 40 28890502 40 28896832 40 28903162 40 28909492 40 28915822 40 28922152 40 28928482 40 28934812 40 28941142 40 28947472 40 28953802 40 28960132 40 28966462 40 28972792 40 28979122 40 28985452 40 28991782 40 28998112 40 29004442 40 29010772 40 29017102 40 29023432 40 29029762 40 29036092 40 29042422 40 29048752 40 29055082 40 29061412 40 29067722 60 29074052 60 29080382 60 29086712 60 29093042 60 29099372 60 29105702 60 29112032 60 29118362 60 29124692 60 29131022 60 29137352 60 29143682 60 29150012 60 29156342 60 29162672 60 29169002 60 29175332 60 29181662 60 29187992 60 29194322 60 29200652 60 29206982 60 29213312 60 29219642 60 29225972 60 29232302 60 29238632 60 29244962 60 29251292 60 29257622 60 29263952 60 29270282 60 29276612 60 29282942 60 29289272 60 29295602 60 29301932 60 29308262 60 29314592 60 29320922 40 29327252 40 29333582 40 29339912 40 29346242 40 29352572 40 29358902 40 29365232 40 29371562 40 29377892 40 29384222 40 29390552 40 29396882 40 29403212 40 29409542 40 29415872 40 29422202 40 29428532 40 29434862 40 29441192 40 29447522 40 29453852 40 29460182 40 29466512 40 29472842 40 29479172 40 29485502 40 29491832 40 29498162 40 29504492 40 29510822 40 29517152 40 29523482 40 29529812 40 29536142 40 29542472 40 29548802 40 29555132 40 29561462 40 29567792 40 29574122 40 29580452 40 29586782 40 29593112 40 29599442 40 29605772 40 29612102 40 29618432 40 29624762 40 29631092 40 29637422 40 29643752 40 29650082 40 29656412 40 29662742 40 29669072 40 29675402 40 29681732 40 29688062 40 29694392 40 29700722 40 29703185 41 29707052 40 29709515 41 29713382 40 29715845 41 29719712 40 29722175 41 29726042 40 29728505 41 29732372 40 29734835 41 29738702 40 29741165 41 29745032 40 29747495 41 29751362 40 29753825 41 29757692 40 29760155 41 29764022 40 29766485 41 29770352 40 29772815 41 29776682 40 29779145 41 29783012 40 29785475 41 29789342 40 29791805 41 29795672 40 29798135 41 29802002 40 29804465 41 29808332 40 29810795 41 29814662 40 29817125 41 29820992 40 29823455 41 29829745 81 29836075 81 29842405 81 29848735 81 29855065 81 29861395 81 29867725 81 29874055 81 29880385 81 29886715 81 29893045 81 29899375 81 29905705 81 29912035 81 29918365 81 29924695 81 29931025 81 29937355 81 29943685 81 29950015 81 29956345 81 29962675 81 29969005 81 29975335 81 29981665 81 29987995 81 29994325 81 30000655 81 30006985 81 30013315 81 30019645 81 30025975 81 30032305 81 30038635 81 30044965 81 30051295 81 30057625 81 30063955 81 30070285 81 30076615 81 30082925 101 30089255 101 30095585 101 30101915 101 30108245 101 30114575 101 30120905 101 30127235 101 30133565 101 30139895 101 30146225 101 30152555 101 30158885 101 30165215 101 30171545 101 30177875 101 30184205 101 30190535 101 30196865 101 30203175 121 30209505 100 30215835 100 30222165 100 30228495 100 30234825 100 30241155 100 30247485 100 30253815 100 30260145 100 30266475 100 30272805 100 30279135 100 30285465 100 30291795 100 30298125 100 30304455 100 30310785 100 30317115 100 30323445 100 30329775 100 30336105 80 30342455 60 30348785 60 30355115 60 30361445 60 30367775 60 30374105 60 30380435 60 30386765 60 30393095 60 30399425 60 30405755 60 30412085 60 30418415 60 30424745 60 30431075 60 30437405 60 30443735 60 30450065 60 30456395 60 3 | a,2795958 80 2802288 80 2808618 80 2814948 80 2821278 80 2827608 80 2833938 80 2840268 80 2846598 80 2852928 80 2859258 80 2865588 80 2871918 80 2878248 80 2884578 80 2890908 80 2897238 80 2903568 80 2909898 80 2916228 80 2922498 140 2928828 140 2935158 140 2941488 140 2947818 140 2954148 140 2960478 140 2966808 140 2973138 140 2979468 140 2985798 140 2992128 140 2998458 140 3004788 140 3011118 140 3017448 140 3023778 140 3030108 140 3036438 140 3042768 120 3049078 140 3055408 140 3061738 140 3068068 140 3074398 140 3080728 140 3087058 140 3093388 140 3099718 140 3106048 140 3112378 140 3118708 140 3125038 140 3131368 140 3137698 140 3144028 140 3150358 140 3156688 140 3163018 140 3169348 140 3175658 180 3181988 180 3188318 180 3194648 180 3200978 180 3207308 180 3213638 180 3219968 180 3226298 180 3232628 180 3238958 180 3245288 180 3251618 180 3257948 180 3264278 180 3270608 180 3276938 180 3283268 180 3289598 180 3295928 180 3302258 140 3302418 20 3308588 140 3314918 140 3321248 140 3327578 140 3333908 140 3340238 140 3346568 140 3352898 140 3359228 140 3365558 140 3371888 140 3378218 140 3384548 140 3390878 140 3397208 140 3403538 140 3409868 140 3416198 140 3422528 140 3428838 140 3435168 140 3441498 140 3447828 140 3454158 140 3460488 140 3466818 140 3473148 140 3479478 140 3485808 140 3492138 140 3498468 140 3504798 140 3511128 140 3517458 140 3523788 140 3530118 140 3536448 140 3542778 140 3549108 140 3555438 120 3561768 120 3568098 120 3574428 120 3580758 120 3587088 120 3593418 120 3599748 120 3606078 120 3612408 120 3618738 120 3625068 120 3631398 120 3637728 120 3644058 120 3650388 120 3656718 120 3663048 120 3669378 120 3675708 120 3682097 22 3688427 22 3694757 22 3701087 22 3707417 22 3713747 22 3720077 22 3726407 22 3732737 22 3739067 22 3745397 22 3751727 22 3758057 22 3764387 22 3770717 22 3777047 22 3783377 22 3789707 22 3796037 22 3802367 22 3808697 22 3814167 60 3820497 60 3826827 60 3833157 60 3839487 60 3845817 60 3852147 60 3858477 60 3864807 60 3871137 60 3877467 60 3883797 60 3890127 60 3896457 60 3902787 60 3909117 60 3915447 60 3921777 60 3928107 60 3934437 60 3940747 120 3941167 40 3947077 120 3947497 40 3953407 120 3953827 40 3959737 120 3960157 40 3966067 120 3966487 40 3972397 120 3972817 40 3978727 120 3979147 40 3985057 120 3985477 40 3991387 120 3991807 40 3997717 120 3998137 40 4004047 120 4004467 40 4010377 120 4010797 40 4016707 120 4017127 40 4023037 120 4023457 40 4029367 120 4029787 40 4035697 120 4036117 40 4042027 120 4042447 40 4048357 120 4048777 40 4054687 120 4055107 40 4061017 120 4061437 40 4067326 141 4067767 40 4073656 141 4074097 40 4079986 141 4080427 40 4086316 141 4086757 40 4092646 141 4093087 40 4098976 141 4099417 40 4105306 141 4105747 40 4111636 141 4112077 40 4117966 141 4118407 40 4124296 141 4124737 40 4130626 141 4131067 40 4136956 141 4137397 40 4143286 141 4143727 40 4149616 141 4150057 40 4155946 141 4156387 40 4162276 141 4162717 40 4168606 141 4169047 40 4174936 141 4175377 40 4181266 141 4181707 40 4187596 141 4188038 39 4193886 181 4194326 21 4194368 39 4200216 181 4200656 21 4200698 39 4206546 181 4206986 21 4207028 39 4212876 181 4213316 21 4213358 39 4219206 181 4219646 21 4219688 39 4225536 181 4225976 21 4226018 39 4231866 181 4232306 21 4232348 39 4238196 181 4238636 21 4238678 39 4244526 181 4244966 21 4245008 39 4250856 181 4251296 21 4251338 39 4257186 181 4257626 21 4257668 39 4263516 181 4263956 21 4263998 39 4269846 181 4270286 21 4270328 39 4276176 181 4276616 21 4276658 39 4282506 181 4282946 21 4282988 39 4288836 181 4289276 21 4289318 39 4295166 181 4295606 21 4295648 39 4301496 181 4301936 21 4301978 39 4307826 181 4308266 21 4308308 39 4314156 181 4314596 21 4314638 39 4320106 40 4320466 201 4320926 21 4320968 20 4326436 40 4326796 201 4327256 21 4327297 21 4332766 40 4333126 201 4333586 21 4333627 21 4339096 40 4339456 201 4339916 21 4339957 21 4345426 40 4345786 201 4346246 21 4346287 21 4351756 40 4352116 201 4352576 21 4352617 21 4358086 40 4358446 201 4358906 21 4358947 21 4364416 40 4364776 201 4365236 21 4365277 21 4370746 40 4371106 201 4371566 21 4371607 21 4377076 40 4377436 201 4377896 21 4377937 21 4383406 40 4383766 201 4384226 21 4384267 21 4389736 40 4390096 201 4390556 21 4390597 21 4396066 40 4396426 201 4396886 21 4396927 21 4402396 40 4402756 201 4403216 21 4403257 21 4408726 40 4409086 201 4409546 21 4409587 21 4415056 40 4415416 201 4415876 21 4415917 21 4421386 40 4421746 201 4422206 21 4422247 21 4427716 40 4428076 201 4428536 21 4428577 21 4434046 40 4434406 201 4434866 21 4434907 21 4440376 40 4440736 201 4441196 21 4441237 21 4446646 140 4447046 221 4447526 21 4447567 21 4452976 140 4453376 221 4453856 21 4453897 21 4459306 140 4459706 221 4460186 21 4460227 21 4465636 140 4466036 221 4466516 21 4466557 21 4471966 140 4472366 221 4472846 21 4472887 21 4478296 140 4478696 221 4479176 21 4479217 21 4484626 140 4485026 221 4485506 21 4485547 21 4490956 140 4491356 221 4491836 21 4491877 21 4497286 140 4497686 221 4498166 21 4498207 21 4503616 140 4504016 221 4504496 21 4504537 21 4509946 140 4510346 221 4510826 21 4510867 21 4516276 140 4516676 221 4517156 21 4517197 21 4522606 140 4523006 221 4523486 21 4523527 21 4528936 140 4529336 221 4529816 21 4529857 21 4535266 140 4535666 221 4536146 21 4536187 21 4541596 140 4541996 221 4542476 21 4542517 21 4547926 140 4548326 221 4548806 21 4548847 21 4554256 140 4554656 221 4555136 21 4555177 21 4560586 140 4560986 221 4561466 21 4561507 21 4566916 140 4567316 221 4567796 21 4567837 21 4573186 220 4573646 221 4579516 220 4579976 221 4585846 220 4586306 221 4592176 220 4592636 221 4598506 220 4598966 221 4604836 220 4605296 221 4611166 220 4611626 221 4617496 220 4617956 221 4623826 220 4624286 221 4630156 220 4630616 221 4636486 220 4636946 221 4642816 220 4643276 221 4649146 220 4649606 221 4655476 220 4655936 221 4661806 220 4662266 221 4668136 220 4668596 221 4674466 220 4674926 221 4680796 220 4681256 221 4687126 220 4687586 221 4693456 220 4693916 241 4699705 321 4700206 281 4706035 321 4706536 281 4712365 321 4712866 281 4718695 321 4719196 281 4725025 321 4725526 281 4731355 321 4731856 281 4737685 321 4738186 281 4744015 321 4744516 281 4750345 321 4750846 281 4756675 321 4757176 281 4763005 321 4763506 281 4769335 321 4769836 281 4775665 321 4776166 281 4781995 321 4782496 281 4788325 321 4788826 281 4794655 321 4795156 281 4800985 321 4801486 281 4807315 321 4807816 281 4813645 321 4814146 281 4819975 321 4820476 281 4826205 441 4826806 281 4832535 441 4833136 261 4838865 441 4839466 261 4845195 441 4845796 261 4851525 441 4852126 261 4857855 441 4858456 261 4864185 441 4864786 261 4870515 441 4871116 261 4876845 441 4877446 261 4883175 441 4883776 261 4889505 441 4890106 261 4895835 441 4896436 261 4902165 441 4902766 261 4908495 441 4909096 261 4914825 441 4915426 261 4921155 441 4921756 261 4927485 441 4928086 261 4933815 441 4934416 261 4940145 441 4940746 261 4946475 441 4947076 261 4952725 541 4953406 241 4959055 541 4959736 241 4965385 541 4966066 241 4971715 541 4972396 241 4978045 541 4978726 241 4984375 541 4985056 241 4990705 541 4991386 241 4997035 541 4997716 241 5003365 541 5004046 241 5009695 541 5010376 241 5016025 541 5016706 241 5022355 541 5023036 241 5028685 541 5029366 241 5035015 541 5035696 241 5041345 541 5042026 241 5047675 541 5048356 241 5054005 541 5054686 241 5060335 541 5061016 241 5066665 541 5067346 241 5072995 541 5073676 241 5079185 681 5080026 201 5085515 681 5086356 201 5091845 681 5092686 201 5098175 681 5099016 201 5104505 681 5105346 201 5110835 681 5111676 201 5117165 681 5118006 201 5123495 681 5124336 201 5129825 681 5130666 201 5136155 681 5136996 201 5142485 681 5143326 201 5148815 681 5149656 201 5155145 681 5155986 201 5161475 681 5162316 201 5167805 681 5168646 201 5174135 681 5174976 201 5180465 681 5181306 201 5186795 681 5187636 201 5193125 681 5193966 201 5199455 681 5200296 201 5205725 741 5206626 201 5212055 741 5212956 201 5218385 741 5219286 201 5224715 741 5225616 201 5231045 741 5231946 201 5237375 741 5238276 201 5243705 741 5244606 201 5250035 741 5250936 201 5256365 741 5257266 201 5262695 741 5263596 201 5269025 741 5269926 201 5275355 741 5276256 201 5281685 741 5282586 201 5288015 741 5288916 201 5294345 741 5295246 201 5300675 741 5301576 201 5307005 741 5307906 201 5313335 741 5314236 201 5319665 741 5320566 201 5325995 741 5326896 201 5332285 761 5333266 141 5338615 761 5339596 141 5344945 761 5345926 141 5351275 761 5352256 141 5357605 761 5358586 141 5363935 761 5364916 141 5370265 761 5371246 141 5376595 761 5377576 141 5382925 761 5383906 141 5389255 761 5390236 141 5395585 761 5396566 141 5401915 761 5402896 141 5408245 761 5409226 141 5414575 761 5415556 141 5420905 761 5421886 141 5427235 761 5428216 141 5433565 761 5434546 141 5439895 761 5440876 141 5446225 761 5447206 141 5452555 761 5453536 141 5458865 781 5459926 21 5465195 781 5466256 21 5471525 781 5472586 21 5477855 781 5478916 21 5484185 781 5485246 21 5490515 781 5491576 21 5496845 781 5497906 21 5503175 781 5504236 21 5509505 781 5510566 21 5515835 781 5516896 21 5522165 781 5523226 21 5528495 781 5529556 21 5534825 781 5535886 21 5541155 781 5542216 21 5547485 781 5548546 21 5553815 781 5554876 21 5560145 781 5561206 21 5566475 781 5567536 21 5572805 781 5573866 21 5579135 781 5580196 21 5585425 801 5591755 801 5598085 801 5604415 801 5610745 801 5617075 801 5623405 801 5629735 801 5636065 801 5642395 801 5648725 801 5655055 801 5661385 801 5667715 801 5674045 801 5680375 801 5686705 801 5693035 801 5699365 801 5705695 801 5712005 801 5718335 801 5724665 801 5730995 801 5737325 801 5743655 801 5749985 801 5756315 801 5762645 801 5768975 801 5775305 801 5781635 801 5787965 801 5794295 801 5800625 801 5806955 801 5813285 801 5819615 801 5825945 801 5832275 801 5838605 561 5839206 180 5844935 561 5845536 180 5851265 561 5851866 180 5857595 561 5858196 180 5863925 561 5864526 180 5870255 561 5870856 180 5876585 561 5877186 180 5882915 561 5883516 180 5889245 561 5889846 180 5895575 561 5896176 180 5901905 561 5902506 180 5908235 561 5908836 180 5914565 561 5915166 180 5920895 561 5921496 180 5927225 561 5927826 180 5933555 561 5934156 180 5939885 561 5940486 180 5946215 561 5946816 180 5952545 561 5953146 180 5958875 561 5959476 180 5965185 280 5965665 21 5965825 22 5971515 280 5971995 21 5972155 22 5977845 280 5978325 21 5978485 22 5984175 280 5984655 21 5984815 22 5990505 280 5990985 21 5991145 22 5996835 280 5997315 21 5997475 22 6003165 280 6003645 21 6003805 22 6009495 280 6009975 21 6010135 22 6015825 280 6016305 21 6016465 22 6022155 280 6022635 21 6022795 22 6028485 280 6028965 21 6029125 22 6034815 280 6035295 21 6035455 22 6041145 280 6041625 21 6041785 22 6047475 280 6047955 21 6048115 22 6053805 280 6054285 21 6054445 22 6060135 280 6060615 21 6060775 22 6066465 280 6066945 21 6067105 22 6072795 280 6073275 21 6073435 22 6079125 280 6079605 21 6079765 22 6085455 280 6085935 21 6086095 22 6091765 280 6098095 280 6104425 280 6110755 280 6117085 280 6123415 280 6129745 280 6136075 280 6142405 280 6148735 280 6155065 280 6161395 280 6167725 280 6174055 280 6180385 280 6186715 280 6193045 280 6199375 280 6205705 280 6212035 280 6218365 280 6224695 240 6231025 240 6237355 240 6243685 240 6250015 240 6256345 240 6262675 240 6269005 240 6275335 240 6281665 240 6287995 240 6294325 240 6300655 240 6306985 240 6313315 240 6319645 240 6325975 240 6332305 240 6338635 240 6344965 240 6351275 240 6357605 240 6363935 240 6370265 240 6376595 240 6382925 240 6389255 240 6395585 240 6401915 240 6408245 240 6414575 240 6420905 240 6427235 240 6433565 240 6439895 240 6446225 240 6452555 240 6458885 240 6465215 240 6471545 240 6477875 220 6484205 220 6490535 220 6496865 220 6503195 220 6509525 220 6515855 220 6522185 220 6528515 220 6534845 220 6541175 220 6547505 220 6553835 220 6560165 220 6566495 220 6572825 220 6579155 220 6585485 220 6591815 220 6598145 220 6604455 220 6610785 220 6617115 220 6623445 220 6629775 220 6636105 220 6642435 220 6648765 220 6655095 220 6661425 220 6667755 220 6674085 220 6680415 220 6686745 220 6693075 220 6699405 220 6705735 220 6712065 220 6718395 220 6724725 220 6731055 180 6737385 180 6743715 180 6750045 180 6756375 180 6762705 180 6769035 180 6775365 180 6781695 180 6788025 180 6794355 180 6800685 180 6807015 180 6813345 180 6819675 180 6826005 180 6832335 180 6838665 180 6844995 180 6851325 180 6857655 160 6863985 160 6870315 160 6876645 160 6882975 160 6889305 160 6895635 160 6901965 160 6908295 160 6914625 160 6920955 160 6927285 160 6933615 160 6939945 160 6946275 160 6952605 160 6958935 160 6965265 160 6971595 160 6977925 160 6984255 120 6990585 120 6996915 120 7003245 120 7009575 120 7015905 120 7022235 120 7028565 120 7034895 120 7041225 120 7047555 120 7053885 120 7060215 120 7066545 120 7072875 120 7079205 120 7085535 120 7091865 120 7098195 120 7104525 120 7110855 80 7117185 80 7123515 80 7129845 80 7136175 80 7142505 80 7148835 80 7155165 80 7161495 80 7167825 80 7174155 80 7180485 80 7186815 80 7193145 80 7199475 80 7205805 80 7212135 80 7218465 80 7224795 80 7231124 81 7237454 22 7243784 22 7250114 22 7256444 22 7262774 22 7269104 22 7275434 22 7281764 22 7288094 22 7294424 22 7300754 22 7307084 22 7313414 22 7319744 22 7326074 22 7332404 22 7338734 22 7345064 22 7351394 22 7357724 22 7364054 1 11674164 20 11680454 60 11686784 60 11693114 60 11699444 60 11705774 60 11712104 60 11718434 60 11724764 60 11731094 60 11737424 60 11743754 60 11750084 60 11756414 60 11762744 60 11769074 60 11775404 60 11781734 60 11788064 60 11794394 60 11800724 60 11807054 60 11813384 40 11819714 40 11826044 40 11832374 40 11838704 40 11845034 40 11851364 40 11857694 40 11864024 40 11870354 40 11876684 40 11883014 40 11889344 40 11895674 40 11902004 40 11908334 40 11914664 40 11920994 40 11927324 40 11933654 40 11939984 40 11946314 40 11952644 40 11958974 40 11965304 40 11971634 40 11977964 40 11984294 40 11990624 40 11996954 40 12003284 40 12009614 40 12015944 40 12022274 40 12028604 40 12034934 40 12041264 40 12047594 40 12053903 61 12060233 41 12066563 41 12072893 41 12079223 41 12085553 41 12091883 41 12098213 41 12104543 41 12110873 41 12117203 41 12123533 41 12129863 41 12136193 41 12142523 41 12148853 41 12155183 41 12161513 41 12167843 41 12174173 41 12180503 41 12186833 21 4 | -------------------------------------------------------------------------------- /src.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import gc 3 | import math 4 | import os 5 | import sys 6 | import pprint 7 | import time 8 | from io import StringIO 9 | from typing import Dict, Tuple, Union 10 | 11 | import cv2 12 | import numpy as np 13 | import PIL.Image as Image 14 | import torch 15 | import torch.nn as nn 16 | import torch.nn.functional as F 17 | import yaml 18 | from PIL import Image, ImageFilter 19 | from segment_anything import SamAutomaticMaskGenerator, sam_model_registry 20 | from segment_anything.modeling import (ImageEncoderViT, MaskDecoder, 21 | PromptEncoder, Sam) 22 | from segment_anything.utils.amg import (MaskData, batched_mask_to_box, 23 | build_point_grid) 24 | from tensorboardX import SummaryWriter 25 | from torch.optim.lr_scheduler import LambdaLR 26 | from torch.utils.data import (DataLoader, Dataset, RandomSampler, 27 | SequentialSampler) 28 | from torchvision import transforms 29 | from tqdm import tqdm 30 | 31 | 32 | def get_device(device: str = None): 33 | if device == None or device == "gpu": 34 | if torch.cuda.is_available(): 35 | print("Using GPU") 36 | print("Clearing GPU memory") 37 | torch.cuda.empty_cache() 38 | gc.collect() 39 | return torch.device("cuda") 40 | print("Using CPU") 41 | return torch.device("cpu") 42 | 43 | 44 | def image_to_rle_fast(img): 45 | img[0] = 0 46 | img[-1] = 0 47 | runs = np.where(img[1:] != img[:-1])[0] + 2 48 | runs[1::2] = runs[1::2] - runs[:-1:2] 49 | f = StringIO() 50 | np.savetxt(f, runs.reshape(1, -1), delimiter=" ", fmt="%d") 51 | return f.getvalue().strip() 52 | 53 | 54 | def save_rle_as_image(rle_csv_path, output_dir, subtest_name, image_shape): 55 | with open(rle_csv_path, "r") as csvfile: 56 | csv_reader = csv.reader(csvfile) 57 | next(csv_reader) # Skip header 58 | for row in csv_reader: 59 | _subtest_name, rle_data = row 60 | if _subtest_name != subtest_name: 61 | continue 62 | rle_pairs = list(map(int, rle_data.split())) 63 | 64 | # Decode RLE data 65 | img = np.zeros(image_shape[0] * image_shape[1], dtype=np.uint8) 66 | for i in range(0, len(rle_pairs), 2): 67 | start = rle_pairs[i] - 1 68 | end = start + rle_pairs[i + 1] 69 | img[start:end] = 1 70 | 71 | # Reshape decoded image data to original shape 72 | img = img.reshape(image_shape) 73 | img = Image.fromarray(img * 255).convert("1") 74 | _image_filepath = os.path.join(output_dir, 75 | f"pred_{subtest_name}_rle.png") 76 | img.save(_image_filepath) 77 | 78 | 79 | def dice_score(preds, label, beta=0.5, epsilon=1e-6): 80 | preds = torch.sigmoid(preds) 81 | preds = preds.flatten() 82 | label = label.flatten() 83 | tp = preds[label == 1].sum() 84 | fp = preds[label == 0].sum() 85 | fn = label.sum() - tp 86 | p = tp / (tp + fp + epsilon) 87 | r = tp / (tp + fn + epsilon) 88 | _score = (1 + beta * beta) * (p * r) / (beta * beta * p + r + epsilon) 89 | return _score 90 | 91 | 92 | # Types of interpolation used in resizing 93 | INTERPOLATION_MODES = { 94 | 'bilinear': Image.BILINEAR, 95 | 'bicubic': Image.BICUBIC, 96 | 'nearest': Image.NEAREST, 97 | } 98 | 99 | class TiledDataset(Dataset): 100 | def __init__( 101 | self, 102 | # Directory containing the dataset 103 | data_dir: str, 104 | # Filenames of the images we'll use 105 | image_mask_filename="mask.png", 106 | image_labels_filename="inklabels.png", 107 | slices_dir_filename="surface_volume", 108 | ir_image_filename="ir.png", 109 | pixel_stat_filename="pixel_stats.yaml", 110 | resize: float = 1.0, 111 | interp: str = "bilinear", 112 | # Pixel normalization to use 113 | pixel_norm: str = "mask", 114 | # Expected slices per fragment 115 | crop_size: Tuple[int] = (256, 256), 116 | encoder_size: Tuple[int] = (1024, 1024), 117 | label_size: Tuple[int] = (8, 8), 118 | # Depth into scan to take label from 119 | min_depth: int = 0, 120 | max_depth: int = 42, 121 | # Training vs Testing mode 122 | train: bool = True, 123 | # Device to use 124 | device: str = "cuda", 125 | ): 126 | print(f"Making TiledDataset Dataset from {data_dir}") 127 | self.train = train 128 | self.device = device 129 | self.crop_size = crop_size 130 | self.crop_half_height = self.crop_size[0] // 2 131 | self.crop_half_width = self.crop_size[1] // 2 132 | self.encoder_size = encoder_size 133 | self.label_size = label_size 134 | self.label_half_height = self.label_size[0] // 2 135 | self.label_half_width = self.label_size[1] // 2 136 | self.interp = interp 137 | 138 | # Pixel stats for ir image, only for values inside mask 139 | _pixel_stats_filepath = os.path.join(data_dir, pixel_stat_filename) 140 | with open(_pixel_stats_filepath, "r") as f: 141 | pixel_stats = yaml.safe_load(f) 142 | self.pixel_mean = pixel_stats[pixel_norm]["mean"] 143 | self.pixel_std = pixel_stats[pixel_norm]["std"] 144 | 145 | # Open Mask image 146 | _image_mask_filepath = os.path.join(data_dir, image_mask_filename) 147 | mask_image = Image.open(_image_mask_filepath).convert("L") 148 | self.original_size = mask_image.height, mask_image.width 149 | self.resize = resize 150 | self.resized_size = [ 151 | int(self.original_size[0] * resize), 152 | int(self.original_size[1] * resize), 153 | ] 154 | if self.resize != 1.0: 155 | mask_image = mask_image.resize(self.resized_size[::-1], resample=INTERPOLATION_MODES[interp]) 156 | mask = np.array(mask_image, dtype=np.uint8) 157 | # Open Label image 158 | if self.train: 159 | _image_labels_filepath = os.path.join(data_dir,image_labels_filename) 160 | labels_image = Image.open(_image_labels_filepath).convert("L") 161 | if self.resize != 1.0: 162 | labels_image = labels_image.resize(self.resized_size[::-1], resample=INTERPOLATION_MODES[interp]) 163 | labels = np.array(labels_image, dtype=np.uint8) / 255.0 164 | 165 | # Open Slices into numpy array 166 | self.crop_depth = max_depth - min_depth 167 | fragment = np.zeros(( 168 | self.crop_depth, 169 | self.original_size[0], 170 | self.original_size[1], 171 | ), 172 | dtype=np.float32) 173 | _slice_dir = os.path.join(data_dir, slices_dir_filename) 174 | _loader = tqdm( 175 | range(min_depth, max_depth), 176 | postfix=f"Opening Slices", 177 | position=0, leave=True) 178 | for i in _loader: 179 | _slice_filepath = os.path.join(_slice_dir, f"{i:02d}.tif") 180 | slice_img = Image.open(_slice_filepath).convert("F") 181 | if self.resize != 1.0: 182 | slice_img = slice_img.resize(self.resized_size[::-1], resample=INTERPOLATION_MODES[interp]) 183 | fragment[i, :, :] = np.array(slice_img) / 65535.0 184 | 185 | if train: 186 | # Sample evenly from label and background 187 | _label_points = np.where(labels > 0) 188 | _bg_points = np.where((labels == 0) & (mask == 255)) 189 | _num_points = min(len(_label_points[0]), len(_bg_points[0])) 190 | _label_points = ( 191 | _label_points[0][:_num_points], 192 | _label_points[1][:_num_points], 193 | ) 194 | _bg_points = ( 195 | _bg_points[0][:_num_points], 196 | _bg_points[1][:_num_points], 197 | ) 198 | self.sample_points = ( 199 | np.concatenate((_label_points[0], _bg_points[0])), 200 | np.concatenate((_label_points[1], _bg_points[1])), 201 | ) 202 | else: 203 | self.sample_points = np.where(mask == 255) 204 | 205 | # Pad the fragment using crop size 206 | self.fragment = np.pad( 207 | fragment, 208 | ((0, 0), (self.crop_half_height, self.crop_half_height), 209 | (self.crop_half_width, self.crop_half_width)), 210 | mode='constant', constant_values=0., 211 | ) 212 | 213 | if train: 214 | # Pad the labels using label size 215 | self.labels = np.pad( 216 | labels, 217 | ((self.label_half_height, self.label_half_height), 218 | (self.label_half_width, self.label_half_width)), 219 | mode="constant", constant_values=0., 220 | ) 221 | 222 | def __len__(self): 223 | return len(self.sample_points[0]) 224 | 225 | def __getitem__(self, idx): 226 | crop_center_height = self.sample_points[0][idx] 227 | crop_center_width = self.sample_points[1][idx] 228 | 229 | # Account for padding 230 | crop_center_height_pad = self.crop_half_height + crop_center_height 231 | crop_center_width_pad = self.crop_half_width + crop_center_width 232 | # Crop the fragment 233 | crop = self.fragment[:, 234 | crop_center_height_pad - self.crop_half_height:crop_center_height_pad + self.crop_half_height, 235 | crop_center_width_pad - self.crop_half_width:crop_center_width_pad + self.crop_half_width, 236 | ] 237 | 238 | # Calculate tileing dimmensions (square image) 239 | n_tiles = int(np.ceil(self.crop_depth / 3.0)) 240 | n_rows = int(np.ceil(np.sqrt(n_tiles))) 241 | n_cols = int(np.ceil(n_tiles / n_rows)) 242 | 243 | # Initialize a larger array of 3xNxN 244 | tiled_image = np.zeros((3, n_rows * self.crop_size[0], n_cols * self.crop_size[1]), dtype=np.float32) 245 | for idx in range(n_tiles): 246 | row = idx // n_cols 247 | col = idx % n_cols 248 | # Fill in the 3xNxN array with the cropped slices 249 | tiled_image[:, row * self.crop_size[0]:(row + 1) * self.crop_size[0], col * self.crop_size[1]:(col + 1) * self.crop_size[1]] = crop[idx * 3:(idx + 1) * 3, :, :] 250 | 251 | # # Resize to encoder size 252 | if self.resize != 1.0 or \ 253 | tiled_image.shape[1] != self.encoder_size[0] \ 254 | or tiled_image.shape[2] != self.encoder_size[1]: 255 | # TODO: This just seems to fuck it up, loosing information 256 | tiled_image = tiled_image.transpose((1, 2, 0)) * 255.0 257 | tiled_image = Image.fromarray(tiled_image, 'RGB') 258 | tiled_image = tiled_image.resize(self.encoder_size[::-1], resample=INTERPOLATION_MODES[self.interp]) 259 | tiled_image = np.array(tiled_image, dtype=np.float32) / 255.0 260 | tiled_image = np.transpose(tiled_image, (2, 0, 1)) 261 | 262 | # Normalize image 263 | tiled_image = (tiled_image - self.pixel_mean) / self.pixel_std 264 | 265 | # Centerpoint of crop in pixel space 266 | centerpoint = (crop_center_height, crop_center_width) 267 | 268 | if self.train: 269 | # Account for padding 270 | crop_center_height_pad = crop_center_height + self.label_half_height 271 | crop_center_width_pad = crop_center_width + self.label_half_width 272 | return tiled_image, centerpoint, self.labels[ 273 | crop_center_height_pad - self.label_half_height : crop_center_height_pad + self.label_half_height, 274 | crop_center_width_pad - self.label_half_width : crop_center_width_pad + self.label_half_width, 275 | ] 276 | else: 277 | return tiled_image, centerpoint 278 | 279 | 280 | class ClassyModel(nn.Module): 281 | def __init__(self, 282 | image_encoder: nn.Module = None, 283 | label_size: Tuple[int] = (8, 8), 284 | num_channels=256, 285 | hidden_dim1=128, 286 | hidden_dim2=64, 287 | dropout_prob=0.2): 288 | super(ClassyModel, self).__init__() 289 | # Outputs a (batch_size, 256, 64, 64) 290 | self.image_encoder = image_encoder 291 | self.label_size = label_size 292 | 293 | # Add the classifier head 294 | self.classifier_head = nn.Sequential( 295 | nn.AdaptiveAvgPool2d((1, 1)), 296 | nn.Flatten(), 297 | nn.Linear(num_channels, hidden_dim1), 298 | nn.LayerNorm(hidden_dim1), 299 | nn.GELU(), 300 | nn.Dropout(dropout_prob), 301 | nn.Linear(hidden_dim1, hidden_dim2), 302 | nn.LayerNorm(hidden_dim2), 303 | nn.GELU(), 304 | nn.Dropout(dropout_prob), 305 | # Go from flat 1d hidden_dim (B, hidden_dim2) to label size (B, 8, 8) 306 | nn.Linear(hidden_dim2, label_size[0] * label_size[1]), 307 | ) 308 | 309 | def forward(self, x): 310 | x = self.image_encoder(x) 311 | x = self.classifier_head(x) 312 | # reshape output to labels size 313 | x = x.view(-1, self.label_size[0], self.label_size[1]) 314 | return x 315 | 316 | 317 | def warmup_cosine_annealing(epoch, total_epochs, warmup_epochs, eta_min=0, eta_max=1): 318 | if epoch < warmup_epochs: 319 | return (eta_max - eta_min) * (epoch / warmup_epochs) + eta_min 320 | else: 321 | T_cur = epoch - warmup_epochs 322 | T_max = total_epochs - warmup_epochs 323 | return eta_min + 0.5 * (eta_max - eta_min) * (1 + torch.cos(torch.tensor(T_cur / T_max * math.pi))) 324 | 325 | 326 | def warmup_gamma(epoch, total_epochs, warmup_epochs, gamma=0.1, eta_min=0, eta_max=1): 327 | if epoch < warmup_epochs: 328 | return (eta_max - eta_min) * (epoch / warmup_epochs) + eta_min 329 | else: 330 | T_cur = epoch - warmup_epochs 331 | T_max = total_epochs - warmup_epochs 332 | return eta_min + (eta_max - eta_min) * (gamma ** (T_cur / T_max)) 333 | 334 | 335 | def train_valid( 336 | run_name: str = "testytest", 337 | output_dir: str = None, 338 | train_dir: str = None, 339 | valid_dir: str = None, 340 | # Model 341 | model: str = "vit_b", 342 | weights_filepath: str = "path/to/model.pth", 343 | freeze: bool = True, 344 | save_model: bool = True, 345 | num_channels: int = 256, 346 | hidden_dim1: int = 128, 347 | hidden_dim2: int = 64, 348 | dropout_prob: int = 0.2, 349 | # Training 350 | device: str = None, 351 | num_samples_train: int = 2, 352 | num_samples_valid: int = 2, 353 | num_epochs: int = 2, 354 | warmup_epochs: int = 0, 355 | batch_size: int = 1, 356 | threshold: float = 0.3, 357 | optimizer: str = "adam", 358 | lr: float = 1e-5, 359 | lr_sched = "cosine", 360 | wd: float = 1e-4, 361 | writer=None, 362 | log_images: bool = False, 363 | # Dataset 364 | curriculum: str = "1", 365 | resize: float = 1.0, 366 | interp: str = "bilinear", 367 | pixel_norm: bool = "mask", 368 | crop_size: Tuple[int] = (3, 256, 256), 369 | label_size: Tuple[int] = (8, 8), 370 | min_depth: int = 0, 371 | max_depth: int = 60, 372 | **kwargs, 373 | ): 374 | device = get_device(device) 375 | # device = "cpu" 376 | sam_model = sam_model_registry[model](checkpoint=weights_filepath) 377 | model = ClassyModel( 378 | image_encoder=sam_model.image_encoder, 379 | label_size = label_size, 380 | num_channels = num_channels, 381 | hidden_dim1 = hidden_dim1, 382 | hidden_dim2 = hidden_dim2, 383 | dropout_prob = dropout_prob, 384 | ) 385 | model.train() 386 | if freeze: 387 | print("Freezing image encoder") 388 | for param in model.image_encoder.parameters(): 389 | param.requires_grad = False 390 | model.to(device=device) 391 | optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd) 392 | loss_fn = nn.BCEWithLogitsLoss() 393 | if lr_sched == "cosine": 394 | _f = lambda x: warmup_cosine_annealing(x, num_epochs, warmup_epochs, eta_min=0.1 * lr, eta_max=lr) 395 | elif lr_sched == "gamma": 396 | _f = lambda x: warmup_gamma(x, num_epochs, warmup_epochs, gamma=0.9, eta_min=0.1 * lr, eta_max=lr) 397 | else: 398 | _f = lambda x: lr 399 | lr_sched = LambdaLR(optimizer, _f) 400 | 401 | step = 0 402 | best_score_dict: Dict[str, float] = {} 403 | for epoch in range(num_epochs): 404 | print(f"\n\n --- Epoch {epoch+1} of {num_epochs} --- \n\n") 405 | for phase, data_dir, num_samples in [ 406 | ("Train", train_dir, num_samples_train), 407 | ("Valid", valid_dir, num_samples_valid), 408 | ]: 409 | for _dataset_id in curriculum: 410 | _dataset_filepath = os.path.join(data_dir, _dataset_id) 411 | _score_name = f"Dice/{phase}/{_dataset_id}" 412 | if _score_name not in best_score_dict: 413 | best_score_dict[_score_name] = 0 414 | _dataset = TiledDataset( 415 | data_dir=_dataset_filepath, 416 | crop_size=crop_size, 417 | label_size=label_size, 418 | resize=resize, 419 | interp=interp, 420 | pixel_norm=pixel_norm, 421 | min_depth=min_depth, 422 | max_depth=max_depth, 423 | train=True, 424 | device=device, 425 | ) 426 | _dataloader = DataLoader( 427 | dataset=_dataset, 428 | batch_size=batch_size, 429 | sampler = RandomSampler( 430 | _dataset, 431 | num_samples=num_samples, 432 | # Generator with constant seed for reproducibility during validation 433 | generator=torch.Generator().manual_seed(42) if phase == "Valid" else None, 434 | ), 435 | pin_memory=True, 436 | ) 437 | # TODO: prevent tqdm from printing on every iteration 438 | _loader = tqdm(_dataloader, postfix=f"{phase}/{_dataset_id}/", position=0, leave=True) 439 | score = 0 440 | print(f"{phase} on {_dataset_filepath} ...") 441 | for images, centerpoint, labels in _loader: 442 | step += 1 443 | if writer and log_images: 444 | writer.add_images(f"input-images/{phase}/{_dataset_id}", images, step) 445 | writer.add_images(f"input-labels/{phase}/{_dataset_id}", labels.to(dtype=torch.uint8).unsqueeze(1) * 255, step) 446 | images = images.to(dtype=torch.float32, device=device) 447 | labels = labels.to(dtype=torch.float32, device=device) 448 | preds = model(images) 449 | loss = loss_fn(preds, labels) 450 | optimizer.zero_grad() 451 | loss.backward() 452 | optimizer.step() 453 | _loss_name = f"{loss_fn.__class__.__name__}/{phase}/{_dataset_id}" 454 | _loader.set_postfix_str(f"{_loss_name}: {loss.item():.4f}") 455 | if writer: 456 | writer.add_scalar(_loss_name, loss.item(), step) 457 | with torch.no_grad(): 458 | preds = (torch.sigmoid(preds) > threshold).to(dtype=torch.uint8) 459 | score += dice_score(preds, labels).item() 460 | if writer and log_images: 461 | writer.add_images(f"output-preds/{phase}/{_dataset_id}", preds.unsqueeze(1) * 255, step) 462 | score /= len(_dataloader) 463 | if writer: 464 | writer.add_scalar(f"Dice/{phase}/{_dataset_id}", score, step) 465 | # Overwrite best score if it is better 466 | if score > best_score_dict[_score_name]: 467 | print(f"New best score! {score:.4f} ") 468 | print(f"(was {best_score_dict[_score_name]:.4f})") 469 | best_score_dict[_score_name] = score 470 | if save_model: 471 | _model_filepath = os.path.join( 472 | output_dir, 473 | f"model.pth") 474 | # f"model_{run_name}_best_{_dataset_id}.pth") 475 | print(f"Saving model to {_model_filepath}") 476 | torch.save(model.state_dict(), _model_filepath) 477 | # Flush ever batch 478 | writer.flush() 479 | return best_score_dict 480 | 481 | 482 | def eval( 483 | output_dir: str = None, 484 | eval_dir: str = None, 485 | # Model 486 | model: str = "vit_b", 487 | weights_filepath: str = "path/to/model.pth", 488 | num_channels: int = 256, 489 | hidden_dim1: int = 128, 490 | hidden_dim2: int = 64, 491 | dropout_prob: int = 0.2, 492 | # Evaluation 493 | device: str = None, 494 | batch_size: int = 2, 495 | threshold: float = 0.5, 496 | postproc_kernel: int = 3, 497 | log_images: bool = False, 498 | save_pred_img: bool = True, 499 | save_submit_csv: bool = False, 500 | save_histograms: bool = False, 501 | writer=None, 502 | max_time_hours: float = 0.5, 503 | # Dataset 504 | eval_on: str = '123', 505 | resize: float = 1.0, 506 | interp: str = "bilinear", 507 | pixel_norm: str = "mask", 508 | max_num_samples_eval: int = 1000, 509 | crop_size: Tuple[int] = (256, 256), 510 | label_size: Tuple[int] = (8, 8), 511 | min_depth: int = 0, 512 | max_depth: int = 60, 513 | **kwargs, 514 | ): 515 | print(f"Eval will run for maximum of {max_time_hours} hours (PER DATASET)") 516 | max_time_seconds = max_time_hours * 60 * 60 517 | 518 | device = get_device(device) 519 | sam_model = sam_model_registry[model](checkpoint=None) 520 | model = ClassyModel( 521 | image_encoder=sam_model.image_encoder, 522 | label_size = label_size, 523 | num_channels = num_channels, 524 | hidden_dim1 = hidden_dim1, 525 | hidden_dim2 = hidden_dim2, 526 | dropout_prob = dropout_prob, 527 | ) 528 | model = model.eval() 529 | with open(weights_filepath, "rb") as f: 530 | state_dict = torch.load(f) 531 | model.load_state_dict(state_dict) 532 | model.to(device=device) 533 | 534 | if save_submit_csv: 535 | submission_filepath = os.path.join(output_dir, 'submission.csv') 536 | with open(submission_filepath, 'w') as f: 537 | # Write header 538 | f.write("Id,Predicted\n") 539 | 540 | best_score_dict: Dict[str, float] = {} 541 | step = 0 542 | for _dataset_id in eval_on: 543 | _dataset_filepath = os.path.join(eval_dir, _dataset_id) 544 | print(f"Evaluate on {_dataset_filepath} ...") 545 | _score_name = f"Dice/Eval/{_dataset_id}" 546 | if _score_name not in best_score_dict: 547 | best_score_dict[_score_name] = 0 548 | _dataset = TiledDataset( 549 | data_dir=_dataset_filepath, 550 | crop_size=crop_size, 551 | label_size=label_size, 552 | resize=resize, 553 | interp=interp, 554 | pixel_norm=pixel_norm, 555 | min_depth=min_depth, 556 | max_depth=max_depth, 557 | train=False, 558 | device=device, 559 | ) 560 | _dataloader = DataLoader( 561 | dataset=_dataset, 562 | batch_size=batch_size, 563 | # sampler = SequentialSampler(_dataset), 564 | sampler = RandomSampler( 565 | _dataset, 566 | # TODO: Num samples just based on time, and average the predictions 567 | # thus making it iteratively better over time. You can maybe 568 | # condition the image on the previous prediction (aka the label is one of the tiles) 569 | # which you can emulate with the labels during training. 570 | # effectively doing a kind of pseudo-labeling, which is similar to the 571 | # original SAM approach. 572 | num_samples=max_num_samples_eval, 573 | # Generator with constant seed for reproducibility during eval 574 | generator=torch.Generator().manual_seed(42), 575 | ), 576 | pin_memory=True, 577 | ) 578 | 579 | # Make a blank prediction image 580 | pred_image = np.zeros(_dataset.resized_size, dtype=np.float32) 581 | # Pad prediction with label size 582 | label_size_half = (label_size[0] // 2, label_size[1] // 2) 583 | pred_image = np.pad(pred_image, 584 | ((label_size_half[0], label_size_half[0]), (label_size_half[1], label_size_half[1])), 585 | mode='constant', constant_values=0, 586 | ) 587 | 588 | time_start = time.time() 589 | time_elapsed = 0 590 | phase = "Eval" 591 | _loader = tqdm(_dataloader, postfix=f"{phase}/{_dataset_id}/", position=0, leave=True) 592 | for i, (images, centerpoint) in enumerate(_loader): 593 | step += 1 594 | if writer and log_images: 595 | writer.add_images(f"input-images/{phase}/{_dataset_id}", images, step) 596 | with torch.no_grad(): 597 | images = images.to(dtype=torch.float32, device=device) 598 | preds = model(images) 599 | preds = torch.sigmoid(preds) 600 | if writer and log_images: 601 | writer.add_images(f"output-preds/{phase}/{_dataset_id}", preds.unsqueeze(1) * 255, step) 602 | for i, pred in enumerate(preds): 603 | # centerpoint to padded pixel coords, and then size of label 604 | h_s = int(centerpoint[0][i].cpu() + label_size[0] // 2 - label_size[0] // 2) 605 | h_e = int(centerpoint[0][i].cpu() + label_size[0] // 2 + label_size[0] // 2) 606 | w_s = int(centerpoint[1][i].cpu() + label_size[1] // 2 - label_size[1] // 2) 607 | w_e = int(centerpoint[1][i].cpu() + label_size[1] // 2 + label_size[1] // 2) 608 | _existing_pred = pred_image[h_s:h_e, w_s:w_e] 609 | # Combine existing prediction with new prediction 610 | pred = pred.cpu().numpy() 611 | pred_image[h_s:h_e, w_s:w_e] = np.mean(np.stack([_existing_pred, pred]), axis=0) 612 | 613 | # Check if we have exceeded the time limit 614 | time_elapsed = time.time() - time_start 615 | if time_elapsed > max_time_seconds: 616 | print(f"Time limit exceeded for dataset {_dataset_id}") 617 | break 618 | 619 | # Remove padding from prediction 620 | pred_image = pred_image[label_size_half[0]:-label_size_half[0], label_size_half[1]:-label_size_half[1]] 621 | 622 | if writer is not None: 623 | print("Writing prediction image to TensorBoard...") 624 | writer.add_image(f'pred_{_dataset_id}', np.expand_dims(pred_image, axis=0) * 255, step) 625 | 626 | if save_histograms: 627 | # Save histogram of predictions as image 628 | print("Saving prediction histogram...") 629 | _num_bins = 100 630 | np_hist, _ = np.histogram(pred_image.flatten(), 631 | bins=_num_bins, 632 | range=(0, 1), 633 | density=True) 634 | np_hist = np_hist / np_hist.sum() 635 | hist = np.zeros((100, _num_bins), dtype=np.uint8) 636 | for bin in range(_num_bins): 637 | _height = int(np_hist[bin] * 100) 638 | hist[0:_height, bin] = 255 639 | hist_img = Image.fromarray(hist) 640 | _histogram_filepath = os.path.join(output_dir, f"pred_{_dataset_id}_hist.png") 641 | hist_img.save(_histogram_filepath) 642 | 643 | if writer is not None: 644 | print("Writing prediction histogram to TensorBoard...") 645 | writer.add_histogram(f'pred_{_dataset_id}', hist, step) 646 | 647 | # Resize pred_image to original size 648 | if resize != 1.0: 649 | img = Image.fromarray(pred_image * 255).convert('1') 650 | img = img.resize(( 651 | _dataset_id.original_size[0], 652 | _dataset_id.original_size[1], 653 | ), resample=INTERPOLATION_MODES[interp]) 654 | 655 | if save_pred_img: 656 | print("Saving prediction image...") 657 | img = Image.fromarray(pred_image * 255).convert('1') 658 | _image_filepath = os.path.join(output_dir, f"pred_{_dataset_id}.png") 659 | img.save(_image_filepath) 660 | 661 | # if postproc_kernel is not None: 662 | # print("Postprocessing...") 663 | # # Erosion then Dilation 664 | # img = img.filter(ImageFilter.MinFilter(postproc_kernel)) 665 | # img = img.filter(ImageFilter.MaxFilter(postproc_kernel)) 666 | 667 | # if save_pred_img: 668 | # print("Saving prediction image...") 669 | # _image_filepath = os.path.join(output_dir, f"pred_{_dataset_id}_post.png") 670 | # img.save(_image_filepath) 671 | 672 | if save_submit_csv: 673 | print("Saving submission csv...") 674 | # Convert image to binary using threshold 675 | # img_thresh = np.where(pred_image > threshold, 1, 0).astype(np.uint8) 676 | pred_image[pred_image > threshold] = 1 677 | pred_image[pred_image <= threshold] = 0 678 | inklabels_rle_fast = image_to_rle_fast(pred_image) 679 | with open(submission_filepath, 'a') as f: 680 | f.write(f"{_dataset_id},{inklabels_rle_fast}\n") 681 | 682 | if save_pred_img and save_submit_csv: 683 | save_rle_as_image(submission_filepath, output_dir, _dataset_id, 684 | pred_image.shape) 685 | 686 | 687 | def eval_from_episode_dir( 688 | episode_dir: str = None, 689 | eval_dir: str = None, 690 | output_dir: str = None, 691 | weights_filename: str = "model.pth", 692 | hparams_filename: str = "hparams.yaml", 693 | **kwargs, 694 | ): 695 | # Get hyperparams from text file 696 | _hparams_filepath = os.path.join(episode_dir, hparams_filename) 697 | with open(_hparams_filepath, "r") as f: 698 | hparams = yaml.load(f, Loader=yaml.FullLoader) 699 | _weights_filepath = os.path.join(episode_dir, weights_filename) 700 | # Merge kwargs with hparams, kwargs takes precedence 701 | hparams = {**hparams, **kwargs} 702 | print(f"Hyperparams:\n{pprint.pformat(hparams)}\n") 703 | # Make sure output dir exists 704 | os.makedirs(output_dir, exist_ok=True) 705 | eval( 706 | eval_dir=eval_dir, 707 | output_dir=output_dir, 708 | weights_filepath=_weights_filepath, 709 | **hparams, 710 | ) 711 | -------------------------------------------------------------------------------- /v0/src.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import gc 3 | import os 4 | import pprint 5 | import subprocess 6 | import time 7 | import uuid 8 | from io import StringIO 9 | from typing import Dict, Union 10 | 11 | import matplotlib.pyplot as plt 12 | import numpy as np 13 | import PIL.Image as Image 14 | import torch 15 | import torch.nn as nn 16 | import torch.nn.functional as F 17 | import torch.optim.lr_scheduler as lr_scheduler 18 | import torch.quantization as quantization 19 | import torch.utils.data as data 20 | import yaml 21 | from PIL import Image, ImageFilter 22 | from tensorboardX import SummaryWriter 23 | from torch.utils.data import DataLoader, RandomSampler, SequentialSampler 24 | from torchvision import models, transforms 25 | from tqdm import tqdm 26 | 27 | 28 | class PatchDataset(data.Dataset): 29 | 30 | def __init__( 31 | self, 32 | # Directory containing the datasets 33 | data_dir: str, 34 | # Expected slices per fragment 35 | slice_depth: int = 4, 36 | # Size of an individual patch 37 | patch_size_x: int = 1028, 38 | patch_size_y: int = 256, 39 | # Image resize ratio 40 | resize_ratio: float = 1.0, 41 | interpolation: str = 'bilinear', 42 | # Training vs Testing mode 43 | train: bool = True, 44 | # Type of interpolation to use when resizing 45 | # Filenames of the images we'll use 46 | image_mask_filename='mask.png', 47 | image_labels_filename='inklabels.png', 48 | slices_dir_filename='surface_volume', 49 | ): 50 | print(f"Creating CurriculumDataset for {data_dir}") 51 | # Train mode also loads the labels 52 | self.train = train 53 | # Resize ratio reduces the size of the image 54 | self.resize_ratio = resize_ratio 55 | # Data will be B x slice_depth x patch_size_x x patch_size_y 56 | self.patch_size_x = patch_size_x 57 | self.patch_size_y = patch_size_y 58 | self.patch_size_x_half = int(patch_size_x / 2) 59 | self.patch_size_y_half = int(patch_size_y / 2) 60 | self.slice_depth = slice_depth 61 | assert os.path.exists( 62 | data_dir), f"Data directory {data_dir} does not exist" 63 | # Open Mask image 64 | _image_mask_filepath = os.path.join(data_dir, image_mask_filename) 65 | _mask_img = Image.open(_image_mask_filepath).convert("1") 66 | # Get original size and resized size 67 | self.original_size = _mask_img.size 68 | self.resized_size = ( 69 | int(self.original_size[0] * self.resize_ratio), 70 | int(self.original_size[1] * self.resize_ratio), 71 | ) 72 | # Resize the mask 73 | # print(f"Mask original size: {original_size}") 74 | _mask_img = _mask_img.resize( 75 | self.resized_size, resample=INTERPOLATION_MODES[interpolation]) 76 | # print(f"Mask resized size: {_mask_img.size}") 77 | _mask = torch.from_numpy(np.array(_mask_img)).to(torch.bool) 78 | # print(f"Mask tensor shape: {_mask.shape}") 79 | # print(f"Mask tensor dtype: {_mask.dtype}") 80 | if train: 81 | _image_labels_filepath = os.path.join( 82 | data_dir, image_labels_filename) 83 | _labels_img = Image.open(_image_labels_filepath).convert("1") 84 | # print(f"Labels original size: {original_size}") 85 | _labels_img = _labels_img.resize( 86 | self.resized_size, resample=INTERPOLATION_MODES[interpolation]) 87 | # print(f"Labels resized size: {_labels_img.size}") 88 | self.labels = torch.from_numpy( 89 | np.array(_labels_img)).to(torch.bool) 90 | # print(f"Labels tensor shape: {self.labels.shape}") 91 | # print(f"Labels tensor dtype: {self.labels.dtype}") 92 | # Pre-allocate the entire fragment 93 | self.fragment = torch.zeros(( 94 | self.slice_depth, 95 | self.resized_size[1], 96 | self.resized_size[0], 97 | ), dtype=torch.float32 98 | ) 99 | # print(f"Fragment tensor shape: {self.fragment.shape}") 100 | # print(f"Fragment tensor dtype: {self.fragment.dtype}") 101 | # Open up slices 102 | _slice_dir = os.path.join(data_dir, slices_dir_filename) 103 | for i in tqdm(range(self.slice_depth), postfix='loading dataset'): 104 | _slice_filepath = os.path.join(_slice_dir, f"{i:02d}.tif") 105 | _slice_img = Image.open(_slice_filepath).convert('F') 106 | # print(f"Slice original size: {original_size}") 107 | _slice_img = _slice_img.resize( 108 | self.resized_size, resample=INTERPOLATION_MODES[interpolation]) 109 | # print(f"Slice resized size: {_slice_img.size}") 110 | _slice = torch.from_numpy(np.array(_slice_img)/65535.0) 111 | # print(f"Slice tensor shape: {_slice.shape}") 112 | # print(f"Slice tensor dtype: {_slice.dtype}") 113 | self.fragment[i, :, :] = _slice 114 | 115 | print(f"Fragment tensor shape: {self.fragment.shape}") 116 | print(f"Fragment tensor dtype: {self.fragment.dtype}") 117 | print(f"Fragment tensor min: {self.fragment.min()}") 118 | print(f"Fragment tensor max: {self.fragment.max()}") 119 | # print(f"Fragment tensor mean: {self.fragment.mean()}") 120 | # print(f"Fragment tensor std: {self.fragment.std()}") 121 | 122 | # Get mean/std for fragment only on mask indices 123 | _fragment_mask = _mask.unsqueeze(0).expand(self.slice_depth, -1, -1) 124 | self.mean = self.fragment[_fragment_mask].mean() 125 | self.std = self.fragment[_fragment_mask].std() 126 | # print(f"Fragment tensor mean (no mask): {self.mean}") 127 | # print(f"Fragment tensor std (no mask): {self.std}") 128 | 129 | # Get indices where mask is 1 130 | self.mask_indices = torch.nonzero(_mask).to(torch.int32) 131 | # print(f"Mask indices shape: {self.mask_indices.shape}") 132 | # print(f"Mask indices dtype: {self.mask_indices.dtype}") 133 | 134 | # TODO: Use Predictions to additionally balance the dataset 135 | # if self.train: 136 | # # Get indices where labels are 1 137 | # self.labels_indices = torch.nonzero(self.labels).to(torch.int32) 138 | # # print(f"Labels indices shape: {self.labels_indices.shape}") 139 | # # print(f"Labels indices dtype: {self.labels_indices.dtype}") 140 | 141 | # # Indices where mask is 0 and labels is 1 142 | # self.mask_0_labels_1_indices = torch.nonzero( 143 | # (~_mask) & self.labels 144 | # ).to(torch.int32) 145 | 146 | # Pad the fragment with zeros based on patch size 147 | self.fragment = F.pad( 148 | self.fragment, 149 | ( 150 | # Padding in Y 151 | self.patch_size_y_half, self.patch_size_y_half, 152 | # Padding in X 153 | self.patch_size_x_half, self.patch_size_x_half, 154 | # No padding on z 155 | 0, 0, 156 | ), 157 | mode='constant', 158 | value=0, 159 | ) 160 | 161 | def __len__(self): 162 | return self.mask_indices.shape[0] 163 | 164 | def __getitem__(self, index): 165 | 166 | # Get the x, y from the mask indices 167 | x, y = self.mask_indices[index] 168 | # print(f"Index: {index}, x: {x}, y: {y}") 169 | 170 | # Pre-allocate the patch 171 | patch = self.fragment[ 172 | :, 173 | x: x + self.patch_size_x, 174 | y: y + self.patch_size_y, 175 | ] 176 | # print(f"Patch tensor shape: {patch.shape}") 177 | # print(f"Patch tensor dtype: {patch.dtype}") 178 | # print(f"Patch tensor min: {patch.min()}") 179 | # print(f"Patch tensor max: {patch.max()}") 180 | 181 | # Label is going to be the label of the center voxel 182 | if self.train: 183 | label = self.labels[ 184 | x, 185 | y, 186 | ] 187 | return patch, label.unsqueeze(0).to(torch.float32) 188 | else: 189 | # If we're not training, we don't have labels 190 | return patch 191 | 192 | 193 | def get_device(device: str = None): 194 | if device == None or device == "gpu": 195 | if torch.cuda.is_available(): 196 | print("Using GPU") 197 | return torch.device("cuda") 198 | print("Using CPU") 199 | return torch.device("cpu") 200 | 201 | 202 | def image_to_rle(img): 203 | starts = np.array((img[:-1] == 0) & (img[1:] == 1)) 204 | ends = np.array((img[:-1] == 1) & (img[1:] == 0)) 205 | starts_ix = np.where(starts)[0] + 2 206 | ends_ix = np.where(ends)[0] + 2 207 | lengths = ends_ix - starts_ix 208 | return " ".join(map(str, sum(zip(starts_ix, lengths), ()))) 209 | 210 | def image_to_rle_fast(img): 211 | img[0] = 0 212 | img[-1] = 0 213 | runs = np.where(img[1:] != img[:-1])[0] + 2 214 | runs[1::2] = runs[1::2] - runs[:-1:2] 215 | f = StringIO() 216 | np.savetxt(f, runs.reshape(1, -1), delimiter=" ", fmt="%d") 217 | return f.getvalue().strip() 218 | 219 | 220 | def save_rle_as_image(rle_csv_path, output_dir, subtest_name, image_shape): 221 | with open(rle_csv_path, 'r') as csvfile: 222 | csv_reader = csv.reader(csvfile) 223 | next(csv_reader) # Skip header 224 | for row in csv_reader: 225 | _subtest_name, rle_data = row 226 | if _subtest_name != subtest_name: 227 | continue 228 | rle_pairs = list(map(int, rle_data.split())) 229 | 230 | # Decode RLE data 231 | img = np.zeros(image_shape[0] * image_shape[1], dtype=np.uint8) 232 | for i in range(0, len(rle_pairs), 2): 233 | start = rle_pairs[i] - 1 234 | end = start + rle_pairs[i + 1] 235 | img[start:end] = 1 236 | 237 | # Reshape decoded image data to original shape 238 | img = img.reshape(image_shape) 239 | img = Image.fromarray(img * 255).convert('1') 240 | _image_filepath = os.path.join( 241 | output_dir, f"pred_{subtest_name}_rle.png") 242 | img.save(_image_filepath) 243 | 244 | 245 | def dice_score(preds, label, beta=0.5, epsilon=1e-6): 246 | # Implementation of DICE coefficient 247 | # https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient 248 | preds = torch.sigmoid(preds) 249 | preds = preds.flatten() 250 | # print(f"Predictions tensor shape: {preds.shape}") 251 | # print(f"Predictions tensor dtype: {preds.dtype}") 252 | # print(f"Predictions tensor min: {preds.min()}") 253 | # print(f"Predictions tensor max: {preds.max()}") 254 | label = label.flatten() 255 | # print(f"Label tensor shape: {label.shape}") 256 | # print(f"Label tensor dtype: {label.dtype}") 257 | # print(f"Label tensor min: {label.min()}") 258 | # print(f"Label tensor max: {label.max()}") 259 | tp = preds[label == 1].sum() 260 | fp = preds[label == 0].sum() 261 | fn = label.sum() - tp 262 | p = tp / (tp + fp + epsilon) 263 | r = tp / (tp + fn + epsilon) 264 | _score = (1 + beta * beta) * (p * r) / (beta * beta * p + r + epsilon) 265 | # print(f"DICE score: {_score}") 266 | return _score 267 | 268 | 269 | def get_gpu_memory(): 270 | result = subprocess.run(['nvidia-smi', '--query-gpu=memory.used,memory.free', 271 | '--format=csv,nounits,noheader'], stdout=subprocess.PIPE, text=True) 272 | gpu_memory = [tuple(map(int, line.split(','))) 273 | for line in result.stdout.strip().split('\n')] 274 | for i, (used, free) in enumerate(gpu_memory): 275 | print(f"GPU {i}: Memory Used: {used} MiB | Memory Available: {free} MiB") 276 | 277 | 278 | def clear_gpu_memory(): 279 | if torch.cuda.is_available(): 280 | print('Clearing GPU memory') 281 | torch.cuda.empty_cache() 282 | gc.collect() 283 | 284 | 285 | def print_size_of_model(model: nn.Module): 286 | """ Prints the real size of the model """ 287 | torch.save(model.state_dict(), "temp.p") 288 | print('Model Size (MB):', os.path.getsize("temp.p")/1e6) 289 | os.remove('temp.p') 290 | 291 | 292 | # Types of interpolation used in resizing 293 | INTERPOLATION_MODES = { 294 | 'bilinear': Image.BILINEAR, 295 | 'bicubic': Image.BICUBIC, 296 | 'nearest': Image.NEAREST, 297 | } 298 | 299 | 300 | class ImageModel(nn.Module): 301 | 302 | models = { 303 | 'convnext_tiny': (models.convnext_tiny, models.ConvNeXt_Tiny_Weights.DEFAULT), 304 | 'convnext_small': (models.convnext_small, models.ConvNeXt_Small_Weights.DEFAULT), 305 | 'convnext_base': (models.convnext_base, models.ConvNeXt_Base_Weights.DEFAULT), 306 | 'convnext_large': (models.convnext_large, models.ConvNeXt_Large_Weights.DEFAULT), 307 | 'resnext50_32x4d': (models.resnext50_32x4d, models.ResNeXt50_32X4D_Weights.DEFAULT), 308 | 'resnext101_32x8d': (models.resnext101_32x8d, models.ResNeXt101_32X8D_Weights.DEFAULT), 309 | 'resnext101_64x4d': (models.resnext101_64x4d, models.ResNeXt101_64X4D_Weights.DEFAULT), 310 | 'vit_b_32': (models.vit_b_32, models.ViT_B_32_Weights.DEFAULT), 311 | 'vit_l_32': (models.vit_l_32, models.ViT_L_32_Weights.DEFAULT), 312 | 'vit_h_14': (models.vit_h_14, models.ViT_H_14_Weights.DEFAULT), 313 | } 314 | 315 | def __init__( 316 | self, 317 | slice_depth: int = 65, 318 | model: str = 'convnext_tiny', 319 | load_fresh: bool = False, 320 | freeze: bool = False, 321 | ): 322 | super().__init__() 323 | print(f"Initializing new model: {model}") 324 | # Channel reduction 325 | self.channel_reduce = nn.Sequential( 326 | nn.Conv2d(slice_depth, 3, 1), 327 | nn.Tanh(), 328 | ) 329 | assert model in self.models, f"Model {model} not supported" 330 | _f, weights = self.models[model] 331 | # Optionally load fresh pre-trained model from scratch 332 | self.model = _f(weights=weights if load_fresh else None) 333 | # Put model in training mode 334 | if freeze: 335 | self.model.eval() 336 | else: 337 | self.model.train() 338 | # Binary classification head on top 339 | self.head = nn.Sequential( 340 | nn.BatchNorm1d(1000), 341 | nn.ReLU(), 342 | nn.Dropout(0.2), 343 | nn.Linear(1000, 1), 344 | ) 345 | 346 | def forward(self, x): 347 | x = self.channel_reduce(x) 348 | x = self.model(x) 349 | x = self.head(x) 350 | return x 351 | 352 | 353 | def train_valid( 354 | train_dir: str = "data/train/", 355 | valid_dir: str = "data/valid/", 356 | model: str = "simplenet", 357 | freeze: bool = False, 358 | weights_filepath: str = None, 359 | optimizer: str = "adam", 360 | weight_decay: float = 0., 361 | curriculum: str = "1", 362 | num_samples_train: int = 100, 363 | num_samples_valid: int = 100, 364 | num_workers: int = 1, 365 | output_dir: str = "output/train", 366 | image_augs: bool = False, 367 | slice_depth: int = 3, 368 | patch_size_x: int = 512, 369 | patch_size_y: int = 128, 370 | resize_ratio: float = 1.0, 371 | interpolation: str = "bilinear", 372 | batch_size: int = 16, 373 | lr: float = 0.001, 374 | lr_gamma: float = None, 375 | num_epochs: int = 2, 376 | max_time_hours: float = 8, 377 | writer: SummaryWriter = None, 378 | save_model: bool = True, 379 | device: str = "gpu", 380 | **kwargs, 381 | ): 382 | # Notebook will only run for this amount of time 383 | print(f"Training will run for {max_time_hours} hours") 384 | time_train_max_seconds = max_time_hours * 60 * 60 385 | time_start = time.time() 386 | time_elapsed = 0 387 | 388 | # Get GPU 389 | device = get_device(device) 390 | clear_gpu_memory() 391 | 392 | # Load the model, try to fit on GPU 393 | if isinstance(model, str): 394 | model = ImageModel( 395 | model=model, 396 | slice_depth=slice_depth, 397 | freeze=freeze, 398 | ) 399 | if weights_filepath is not None: 400 | print(f"Loading weights from {weights_filepath}") 401 | model.load_state_dict(torch.load( 402 | weights_filepath, 403 | map_location=device, 404 | )) 405 | model = model.to(device) 406 | model.train() 407 | 408 | # Create optimizers 409 | if optimizer == "adam": 410 | optimizer = torch.optim.Adam( 411 | model.parameters(), lr=lr, weight_decay=weight_decay) 412 | elif optimizer == "sgd": 413 | optimizer = torch.optim.SGD( 414 | model.parameters(), lr=lr, weight_decay=weight_decay) 415 | loss_fn = nn.BCEWithLogitsLoss() 416 | 417 | # Scaler for mixed precision training 418 | scaler = torch.cuda.amp.GradScaler() 419 | 420 | if lr_gamma is not None: 421 | scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=lr_gamma) 422 | 423 | # Writer for Tensorboard 424 | if writer: 425 | writer = SummaryWriter(output_dir) 426 | 427 | # Train the model 428 | step = 0 429 | best_score_dict: Dict[str, float] = {} 430 | for epoch in range(num_epochs): 431 | print(f"Epoch: {epoch}") 432 | 433 | # Curriculum defines the order of the training 434 | for train_dataset_id in curriculum: 435 | 436 | train_dataset_filepath = os.path.join(train_dir, train_dataset_id) 437 | print(f"Training on dataset: {train_dataset_filepath}") 438 | 439 | # Training dataset 440 | train_dataset = PatchDataset( 441 | # Directory containing the dataset 442 | train_dataset_filepath, 443 | # Expected slices per fragment 444 | slice_depth=slice_depth, 445 | # Size of an individual patch 446 | patch_size_x=patch_size_x, 447 | patch_size_y=patch_size_y, 448 | # Image resize ratio 449 | resize_ratio=resize_ratio, 450 | interpolation=interpolation, 451 | # Training vs Testing mode 452 | train=True, 453 | ) 454 | total_dataset_size = len(train_dataset) 455 | print(f"Raw train dataset size: {total_dataset_size}") 456 | 457 | # Image augmentations 458 | img_transform_list = [ 459 | transforms.Normalize(train_dataset.mean, train_dataset.std) 460 | ] 461 | if image_augs: 462 | img_transform_list += [ 463 | transforms.RandomHorizontalFlip(), 464 | transforms.RandomVerticalFlip(), 465 | # transforms.GaussianBlur(), 466 | ] 467 | img_transform = transforms.Compose(img_transform_list) 468 | 469 | # DataLoaders 470 | train_dataloader = DataLoader( 471 | train_dataset, 472 | batch_size=batch_size, 473 | sampler=RandomSampler( 474 | train_dataset, num_samples=num_samples_train), 475 | num_workers=num_workers, 476 | # This will make it go faster if it is loaded into a GPU 477 | pin_memory=True, 478 | ) 479 | 480 | print(f"Training...") 481 | train_loss = 0 482 | _loader = tqdm(train_dataloader) 483 | score = 0 484 | for patch, label in _loader: 485 | # writer.add_histogram('patch_input', patch, step) 486 | # writer.add_histogram('label_input', label, step) 487 | patch = patch.to(device) 488 | label = label.to(device) 489 | with torch.cuda.amp.autocast(): 490 | pred = model(img_transform(patch)) 491 | loss = loss_fn(pred, label) 492 | optimizer.zero_grad() 493 | scaler.scale(loss).backward() 494 | scaler.step(optimizer) 495 | scaler.update() 496 | 497 | _loader.set_postfix_str( 498 | f"Train.{train_dataset_id}.{loss_fn.__class__.__name__}: {loss.item():.4f}") 499 | 500 | step += 1 501 | with torch.no_grad(): 502 | train_loss += loss.item() 503 | score += dice_score(pred, label).item() 504 | 505 | # Check if we have exceeded the time limit 506 | time_elapsed = time.time() - time_start 507 | if time_elapsed > time_train_max_seconds: 508 | print("Time limit exceeded, stopping batches") 509 | break 510 | 511 | if writer: 512 | train_loss /= num_samples_train 513 | writer.add_scalar( 514 | f'{loss_fn.__class__.__name__}/{train_dataset_id}/train', train_loss, step) 515 | 516 | # Score is average dice score for all batches 517 | score /= len(train_dataloader) 518 | if writer: 519 | writer.add_scalar( 520 | f'dice.{train_dataset_id}.train', score, step) 521 | 522 | # Check if we have exceeded the time limit 523 | time_elapsed = time.time() - time_start 524 | if time_elapsed > time_train_max_seconds: 525 | print("Time limit exceeded, stopping curriculum") 526 | break 527 | print(f"Time elapsed: {time_elapsed} seconds") 528 | 529 | if lr_gamma is not None: 530 | before_lr = optimizer.param_groups[0]["lr"] 531 | scheduler.step() 532 | after_lr = optimizer.param_groups[0]["lr"] 533 | print("Epoch %d: SGD lr %.4f -> %.4f" % 534 | (epoch, before_lr, after_lr)) 535 | 536 | # Check if we have exceeded the time limit 537 | time_elapsed = time.time() - time_start 538 | if time_elapsed > time_train_max_seconds: 539 | print("Time limit exceeded, stopping training") 540 | break 541 | 542 | # Validate 543 | for valid_dataset_id in os.listdir(valid_dir): 544 | score_name = f'dice.{valid_dataset_id}.valid' 545 | if score_name not in best_score_dict: 546 | best_score_dict[score_name] = 0 547 | 548 | # Name of sub-directory inside test dir 549 | valid_dataset_filepath = os.path.join(valid_dir, valid_dataset_id) 550 | 551 | # Evaluation dataset 552 | valid_dataset = PatchDataset( 553 | # Directory containing the dataset 554 | valid_dataset_filepath, 555 | # Expected slices per fragment 556 | slice_depth=slice_depth, 557 | # Size of an individual patch 558 | patch_size_x=patch_size_x, 559 | patch_size_y=patch_size_y, 560 | # Image resize ratio 561 | resize_ratio=resize_ratio, 562 | interpolation=interpolation, 563 | # Training vs Testing mode 564 | train=True, 565 | ) 566 | img_transforms = transforms.Compose([ 567 | transforms.Normalize(valid_dataset.mean, valid_dataset.std), 568 | ]) 569 | 570 | # DataLoaders 571 | valid_dataloader = DataLoader( 572 | valid_dataset, 573 | batch_size=batch_size, 574 | sampler=RandomSampler( 575 | valid_dataset, 576 | num_samples=num_samples_valid, 577 | # Generator with constant seed for reproducibility 578 | generator=torch.Generator().manual_seed(42), 579 | ), 580 | num_workers=num_workers, 581 | # This will make it go faster if it is loaded into a GPU 582 | pin_memory=True, 583 | ) 584 | 585 | # Make a blank prediction image 586 | pred_image = np.zeros(valid_dataset.resized_size, dtype=np.float32).T 587 | print(f"Pred image {valid_dataset_id} shape: {pred_image.shape}") 588 | print(f"Pred image min: {pred_image.min()}, max: {pred_image.max()}") 589 | 590 | score = 0 591 | for i, (patch, label) in enumerate(tqdm(valid_dataloader, postfix=f"Valid {valid_dataset_id}")): 592 | patch = patch.to(device) 593 | patch = img_transforms(patch) 594 | with torch.no_grad(): 595 | preds = model(patch) 596 | preds = torch.sigmoid(preds) 597 | score += dice_score(preds, label).item() 598 | 599 | # Iterate through each image and prediction in the batch 600 | for j, pred in enumerate(preds): 601 | pixel_index = valid_dataset.mask_indices[i * batch_size + j] 602 | pred_image[pixel_index[0], pixel_index[1]] = pred 603 | 604 | # Score is average dice score for all batches 605 | score /= len(valid_dataloader) 606 | if writer: 607 | writer.add_scalar(score_name, score, step) 608 | 609 | # Overwrite best score if it is better 610 | if score > best_score_dict[score_name]: 611 | best_score_dict[score_name] = score 612 | if save_model: 613 | print("Saving model...") 614 | torch.save(model.state_dict(), f"{output_dir}/model_best_{score_name}.pth") 615 | 616 | if writer is not None: 617 | print("Writing prediction image to TensorBoard...") 618 | # Add batch dimmension to pred_image for Tensorboard 619 | writer.add_image(f'pred_{valid_dataset_id}', 620 | np.expand_dims(pred_image, axis=0)) 621 | 622 | return best_score_dict, model 623 | 624 | 625 | def eval( 626 | eval_dir: str = "data/test/", 627 | weights_filepath: str = None, 628 | model: Union[str, nn.Module] = "convnext_tiny", 629 | output_dir: str = "output", 630 | slice_depth: int = 3, 631 | patch_size_x: int = 512, 632 | patch_size_y: int = 128, 633 | resize_ratio: float = 1.0, 634 | interpolation: str = "bilinear", 635 | freeze: bool = False, 636 | batch_size: int = 16, 637 | num_workers: int = 1, 638 | save_pred_img: bool = True, 639 | save_submit_csv: bool = False, 640 | threshold: float = 0.5, 641 | postproc_kernel: int = 3, 642 | writer: SummaryWriter = None, 643 | quantize: bool = False, 644 | device: str = "gpu", 645 | save_histograms: bool = False, 646 | **kwargs, 647 | ): 648 | # Get GPU 649 | device = get_device(device) 650 | clear_gpu_memory() 651 | 652 | # Load the model, try to fit on GPU 653 | if isinstance(model, str): 654 | model = ImageModel( 655 | model=model, 656 | slice_depth=slice_depth, 657 | freeze=freeze, 658 | ) 659 | if weights_filepath is not None: 660 | print(f"Loading weights from {weights_filepath}") 661 | model.load_state_dict(torch.load( 662 | weights_filepath, 663 | map_location=device, 664 | )) 665 | model = model.to(device) 666 | model = model.eval() 667 | print_size_of_model(model) 668 | 669 | if quantize: 670 | model.qconfig = quantization.get_default_qconfig('fbgemm') 671 | quantization.prepare(model, inplace=True) 672 | quantization.convert(model, inplace=True) 673 | print_size_of_model(model) 674 | 675 | if save_submit_csv: 676 | submission_filepath = os.path.join(output_dir, 'submission.csv') 677 | with open(submission_filepath, 'w') as f: 678 | # Write header 679 | f.write("Id,Predicted\n") 680 | 681 | # Baseline is to use image mask to create guess submission 682 | for subtest_name in os.listdir(eval_dir): 683 | 684 | # Name of sub-directory inside test dir 685 | subtest_filepath = os.path.join(eval_dir, subtest_name) 686 | 687 | # Evaluation dataset 688 | eval_dataset = PatchDataset( 689 | # Directory containing the dataset 690 | subtest_filepath, 691 | # Expected slices per fragment 692 | slice_depth=slice_depth, 693 | # Size of an individual patch 694 | patch_size_x=patch_size_x, 695 | patch_size_y=patch_size_y, 696 | # Image resize ratio 697 | resize_ratio=resize_ratio, 698 | interpolation=interpolation, 699 | # Training vs Testing mode 700 | train=False, 701 | ) 702 | img_transforms = transforms.Compose([ 703 | transforms.Normalize(eval_dataset.mean, eval_dataset.std), 704 | ]) 705 | 706 | # DataLoaders 707 | eval_dataloader = DataLoader( 708 | eval_dataset, 709 | batch_size=batch_size, 710 | sampler=SequentialSampler(eval_dataset), 711 | num_workers=num_workers, 712 | # This will make it go faster if it is loaded into a GPU 713 | pin_memory=True, 714 | ) 715 | 716 | # Make a blank prediction image 717 | pred_image = np.zeros(eval_dataset.resized_size, dtype=np.float32).T 718 | print(f"Pred image {subtest_name} shape: {pred_image.shape}") 719 | print(f"Pred image min: {pred_image.min()}, max: {pred_image.max()}") 720 | 721 | for i, patch in enumerate(tqdm(eval_dataloader, postfix=f"Eval {subtest_name}")): 722 | patch = patch.to(device) 723 | patch = img_transforms(patch) 724 | with torch.no_grad(): 725 | preds = model(patch) 726 | preds = torch.sigmoid(preds) 727 | 728 | # Iterate through each image and prediction in the batch 729 | for j, pred in enumerate(preds): 730 | pixel_index = eval_dataset.mask_indices[i * batch_size + j] 731 | pred_image[pixel_index[0], pixel_index[1]] = pred 732 | 733 | if writer is not None: 734 | print("Writing prediction image to TensorBoard...") 735 | writer.add_image(f'pred_{subtest_name}', np.expand_dims(pred_image, axis=0)) 736 | 737 | if save_histograms: 738 | # Save histogram of predictions as image 739 | print("Saving prediction histogram...") 740 | _num_bins = 100 741 | np_hist, _ = np.histogram(pred_image.flatten(), bins=_num_bins, range=(0, 1), density=True) 742 | np_hist = np_hist / np_hist.sum() 743 | hist = np.zeros((_num_bins, 100), dtype=np.uint8) 744 | for bin in range(_num_bins): 745 | _height = int(np_hist[bin] * 100) 746 | hist[bin, 0:_height] = 255 747 | hist_img = Image.fromarray(hist) 748 | _histogram_filepath = os.path.join( 749 | output_dir, f"pred_{subtest_name}_histogram.png") 750 | hist_img.save(_histogram_filepath) 751 | 752 | if writer is not None: 753 | print("Writing prediction histogram to TensorBoard...") 754 | writer.add_histogram(f'pred_{subtest_name}', pred_image) 755 | 756 | # Resize pred_image to original size 757 | img = Image.fromarray(pred_image * 255).convert('1') 758 | img = img.resize(( 759 | eval_dataset.original_size[0], 760 | eval_dataset.original_size[1], 761 | ), resample=INTERPOLATION_MODES[interpolation]) 762 | 763 | if save_pred_img: 764 | print("Saving prediction image...") 765 | _image_filepath = os.path.join( 766 | output_dir, f"pred_{subtest_name}.png") 767 | img.save(_image_filepath) 768 | 769 | if postproc_kernel is not None: 770 | print("Postprocessing...") 771 | # Erosion then Dilation 772 | img = img.filter(ImageFilter.MinFilter(postproc_kernel)) 773 | img = img.filter(ImageFilter.MaxFilter(postproc_kernel)) 774 | 775 | if save_pred_img: 776 | print("Saving prediction image...") 777 | _image_filepath = os.path.join( 778 | output_dir, f"pred_{subtest_name}_post.png") 779 | img.save(_image_filepath) 780 | 781 | if save_submit_csv: 782 | print("Saving submission csv...") 783 | img = np.array(img).flatten() 784 | # Convert image to binary using threshold 785 | img = np.where(img > threshold, 1, 0).astype(np.uint8) 786 | # Convert image to RLE 787 | # start_time = time.time() 788 | # inklabels_rle_original = image_to_rle(img) 789 | # print(f"RLE conversion (ORIGINAL) took {time.time() - start_time:.2f} seconds") 790 | start_time = time.time() 791 | inklabels_rle_fast = image_to_rle_fast(img) 792 | print(f"RLE conversion (FAST) took {time.time() - start_time:.2f} seconds") 793 | # assert inklabels_rle_original == inklabels_rle_fast, "RLE conversion is not the same!" 794 | with open(submission_filepath, 'a') as f: 795 | f.write(f"{subtest_name},{inklabels_rle_fast}\n") 796 | 797 | if save_pred_img and save_submit_csv: 798 | save_rle_as_image(submission_filepath, output_dir, 799 | subtest_name, pred_image.shape) 800 | 801 | 802 | def eval_from_episode_dir( 803 | episode_dir: str = None, 804 | output_dir: str = None, 805 | hparams_filename: str = 'hparams.yaml', 806 | weights_filename: str = 'model.pth', 807 | **kwargs, 808 | ): 809 | # Get hyperparams from text file 810 | _hparams_filepath = os.path.join(episode_dir, hparams_filename) 811 | with open(_hparams_filepath, 'r') as f: 812 | hparams = yaml.load(f, Loader=yaml.FullLoader) 813 | hparams['output_dir'] = output_dir 814 | _weights_filepath = os.path.join(episode_dir, weights_filename) 815 | # Merge kwargs with hparams, kwargs takes precedence 816 | hparams = {**hparams, **kwargs} 817 | print(f"Hyperparams:\n{pprint.pformat(hparams)}\n") 818 | eval( 819 | weights_filepath=_weights_filepath, 820 | **hparams, 821 | ) 822 | 823 | 824 | def sweep_episode(hparams) -> float: 825 | 826 | # Print hyperparam dict with logging 827 | print(f"\n\nHyperparams:\n\n{pprint.pformat(hparams)}\n\n") 828 | 829 | # Create directory based on run_name 830 | run_name: str = str(uuid.uuid4())[:8] 831 | hparams['output_dir'] = os.path.join(hparams['output_dir'], run_name) 832 | os.makedirs(hparams['output_dir'], exist_ok=True) 833 | 834 | # HACK: Simpler parameters based 835 | # split input size string into 3 integers 836 | _input_size = [int(x) for x in hparams['input_size'].split('.')] 837 | hparams['patch_size_x'] = _input_size[0] 838 | hparams['patch_size_y'] = _input_size[1] 839 | hparams['slice_depth'] = _input_size[2] 840 | 841 | # Save hyperparams to file with YAML 842 | with open(os.path.join(hparams['output_dir'], 'hparams.yaml'), 'w') as f: 843 | yaml.dump(hparams, f) 844 | 845 | try: 846 | writer = SummaryWriter(log_dir=hparams['output_dir']) 847 | # Train and evaluate a TFLite model 848 | score_dict, model = train_valid( 849 | **hparams, 850 | writer=writer, 851 | ) 852 | writer.add_hparams(hparams, score_dict) 853 | writer.close() 854 | # Score is average of all scores 855 | score = sum(score_dict.values()) / len(score_dict) 856 | except Exception as e: 857 | print(f"\n\n Model Training FAILED with \n{e}\n\n") 858 | score = 0 859 | # Maximize score is minimize negative score 860 | return -score 861 | --------------------------------------------------------------------------------