├── .github └── ISSUE_TEMPLATE │ ├── bug.md │ └── everything-else.md ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── REPRODUCE_RESULTS.md ├── environment.yml ├── main.py ├── neptune.yaml ├── notebooks └── results_exploration.ipynb └── src ├── __init__.py ├── augmentation.py ├── callbacks.py ├── cocoeval.py ├── loaders.py ├── models.py ├── pipeline_config.py ├── pipeline_manager.py ├── pipelines.py ├── postprocessing.py ├── preparation.py ├── steps ├── __init__.py ├── base.py ├── keras │ ├── __init__.py │ ├── architectures.py │ ├── callbacks.py │ ├── contrib.py │ ├── embeddings.py │ ├── loaders.py │ └── models.py ├── misc.py ├── postprocessing.py ├── preprocessing │ ├── __init__.py │ ├── misc.py │ └── text.py ├── pytorch │ ├── __init__.py │ ├── architectures │ │ ├── __init__.py │ │ ├── unet.py │ │ └── utils.py │ ├── callbacks.py │ ├── loaders.py │ ├── models.py │ ├── utils.py │ └── validation.py ├── resources │ └── apostrophes.json ├── sklearn │ ├── __init__.py │ └── models.py └── utils.py ├── unet_models.py └── utils.py /.github/ISSUE_TEMPLATE/bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: bug 3 | about: Create bug report 4 | 5 | --- 6 | 7 | There are two things that will make the processing of your issue faster: 8 | 1. Make sure that you are using the latest version of the code, 9 | 1. In case of bug issue, it would be nice to provide more technical details such like execution command, error message or script that reproduces your bug. 10 | # 11 | 12 | Thanks! 13 | 14 | Kamil & Jakub, 15 | 16 | *core contributors to the Open Solution* 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/everything-else.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: everything else 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | Suggest an idea for this project 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # neptune, pycharm 10 | .cache 11 | .cache/ 12 | .idea/ 13 | .idea_modules/ 14 | neptune_local.yaml 15 | neptune_random_search_local.yaml 16 | out/ 17 | output 18 | output/ 19 | neptune.log 20 | offline_job.log 21 | target/ 22 | data/ 23 | 24 | # Distribution / packaging 25 | .Python 26 | env/ 27 | build/ 28 | develop-eggs/ 29 | dist/ 30 | downloads/ 31 | eggs/ 32 | .eggs/ 33 | lib/ 34 | lib64/ 35 | parts/ 36 | sdist/ 37 | var/ 38 | wheels/ 39 | *.egg-info/ 40 | .installed.cfg 41 | *.egg 42 | 43 | # PyInstaller 44 | # Usually these files are written by a python script from a template 45 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 46 | *.manifest 47 | *.spec 48 | 49 | # Installer logs 50 | pip-log.txt 51 | pip-delete-this-directory.txt 52 | 53 | # Unit test / coverage reports 54 | htmlcov/ 55 | .tox/ 56 | .coverage 57 | .coverage.* 58 | nosetests.xml 59 | coverage.xml 60 | *.cover 61 | .hypothesis/ 62 | 63 | # Translations 64 | *.mo 65 | *.pot 66 | 67 | # Django stuff: 68 | *.log 69 | local_settings.py 70 | 71 | # Flask stuff: 72 | instance/ 73 | .webassets-cache 74 | 75 | # Scrapy stuff: 76 | .scrapy 77 | 78 | # Sphinx documentation 79 | docs/_build/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # celery beat schedule file 88 | celerybeat-schedule 89 | 90 | # SageMath parsed files 91 | *.sage.py 92 | 93 | # dotenv 94 | .env 95 | 96 | # virtualenv 97 | .venv 98 | venv/ 99 | ENV/ 100 | 101 | # Spyder project settings 102 | .spyderproject 103 | .spyproject 104 | 105 | # Rope project settings 106 | .ropeproject 107 | 108 | # mkdocs documentation 109 | /site 110 | 111 | # mypy 112 | .mypy_cache/ 113 | 114 | # data 115 | data/ 116 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at ml-team@neptune.ai. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing to [Mapping Challenge](https://www.crowdai.org/challenges/mapping-challenge) Open Solution 2 | 3 | ### Get involved 4 | You are welcome to contribute to this Open Solution. To get started: 5 | 1. Check [our kanban board](https://github.com/neptune-ai/open-solution-mapping-challenge/projects/1) to see what we are working on right now. 6 | 1. Express your interest in a particular [issue](https://github.com/neptune-ai/open-solution-mapping-challenge/issues) by submitting a comment or, 7 | * submit your own [issue](https://github.com/neptune-ai/open-solution-mapping-challenge/issues). 8 | 1. We will get back to you in order to start working together. 9 | 10 | ### Code contributions 11 | Major - and most appreciated - contribution is [pull request](https://github.com/neptune-ai/open-solution-mapping-challenge/pulls) with feature or bug fix. 12 | 13 | ### Remarks 14 | In case of custom ideas, please contact core contributors directly at ml-team@neptune.ai. 15 | # 16 | 17 | Thanks! 18 | 19 | Kuba & Kamil, 20 | 21 | *core contributors to the Open Solution* 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2020 neptune.ai 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # create env 2 | conda env create -f environment.yml 3 | 4 | # create directories 5 | mkdir data 6 | mkdir data/raw data/meta data/experiments 7 | 8 | # set default env variable for NEPTUNE_API_TOKEN and CONFIG_PATH 9 | export NEPTUNE_API_TOKEN=eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vdWkubmVwdHVuZS5tbCIsImFwaV9rZXkiOiJiNzA2YmM4Zi03NmY5LTRjMmUtOTM5ZC00YmEwMzZmOTMyZTQifQ== 10 | export CONFIG_PATH=neptune.yaml -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Pull Request template to the [Mapping Challenge](https://www.crowdai.org/challenges/mapping-challenge) Open Solution 2 | 3 | Major - and most appreciated - contribution is pull request with feature or bug fix. Each pull request initiates discussion about your code contribution. 4 | 5 | Each pull request should be provided with minimal description about its contents. 6 | # 7 | 8 | Thanks! 9 | 10 | Kuba & Kamil, 11 | 12 | _core contributors to the Open Solutions_ 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Open Solution to the Mapping Challenge Competition 2 | 3 | [![Gitter](https://badges.gitter.im/minerva-ml/open-solution-mapping-challenge.svg)](https://gitter.im/minerva-ml/open-solution-mapping-challenge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) 4 | [![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/LICENSE) 5 | 6 | # Note 7 | **Unfortunately, we can no longer provide support for this repo. Hopefully, it should still work, but if it doesn't, we cannot really help.** 8 | 9 | ## More competitions :sparkler: 10 | Check collection of [public projects :gift:](https://ui.neptune.ai/-/explore), where you can find multiple Kaggle competitions with code, experiments and outputs. 11 | 12 | ## Poster :earth_africa: 13 | Poster that summarizes our project is [available here](https://gist.github.com/kamil-kaczmarek/b3b939797fb39752c45fdadfedba3ed9/raw/7fa365392997e9eae91c911c1837b45bfca45687/EP_poster.pdf). 14 | 15 | ## Intro 16 | Open solution to the [CrowdAI Mapping Challenge](https://www.crowdai.org/challenges/mapping-challenge) competition. 17 | 1. Check **live preview of our work** on public projects page: [Mapping Challenge](https://ui.neptune.ai/neptune-ai/Mapping-Challenge) [:chart_with_upwards_trend:](https://ui.neptune.ai/neptune-ai/Mapping-Challenge). 18 | 1. Source code and [issues](https://github.com/neptune-ai/open-solution-mapping-challenge/issues) are publicly available. 19 | 20 | ## Results 21 | `0.943` **Average Precision** :rocket: 22 | 23 | `0.954` **Average Recall** :rocket: 24 | 25 | 26 | 27 | _No cherry-picking here, I promise :wink:. The results exceded our expectations. The output from the network is so good that not a lot of morphological shenanigans is needed. Happy days:)_ 28 | 29 | Average Precision and Average Recall were calculated on [stage 1 data](https://www.crowdai.org/challenges/mapping-challenge/dataset_files) using [pycocotools](https://github.com/cocodataset/cocoapi/tree/master/PythonAPI/pycocotools). Check this [blog post](https://medium.com/@jonathan_hui/map-mean-average-precision-for-object-detection-45c121a31173) for average precision explanation. 30 | 31 | ## Disclaimer 32 | In this open source solution you will find references to the neptune.ai. It is free platform for community Users, which we use daily to keep track of our experiments. Please note that using neptune.ai is not necessary to proceed with this solution. You may run it as plain Python script :wink:. 33 | 34 | ## Reproduce it! 35 | Check [REPRODUCE_RESULTS](REPRODUCE_RESULTS.md) 36 | 37 | # Solution write-up 38 | ## Pipeline diagram 39 | 40 | 41 | 42 | ## Preprocessing 43 | ### :heavy_check_mark: What Worked 44 | * Overlay binary masks for each image is produced ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/preparation.py) :computer:). 45 | * Distances to the two closest objects are calculated creating the distance map that is used for weighing ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/preparation.py) :computer:). 46 | * Size masks for each image is produced ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/preparation.py) :computer:). 47 | * Dropped small masks on the edges ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/preparation.py#L141-L142) :computer:). 48 | * We load training and validation data in batches: using [torch.utils.data.Dataset](https://pytorch.org/docs/stable/data.html#torch.utils.data.Dataset) and [torch.utils.data.DataLoader](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader) makes it easy and clean ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/loaders.py) :computer:). 49 | * Only some basic augmentations (due to speed constraints) from the [imgaug package](https://github.com/aleju/imgaug) are applied to images ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/augmentation.py) :computer:). 50 | * Image is resized before feeding it to the network. Surprisingly this worked better than cropping ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/loaders.py#L246-L263) :computer: and [config](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/neptune.yaml#L47) :bookmark_tabs:). 51 | 52 | ### :heavy_multiplication_x: What didn't Work 53 | * Ground truth masks are prepared by first eroding them per mask creating non overlapping masks and only after that the distances are calculated ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/preparation.py) :computer:). 54 | * Dilated small objects to increase the signal ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/preparation.py) :computer:). 55 | * Network is fed with random crops ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/loaders.py#L225-L243) :computer: and [config](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/neptune.yaml#L47) :bookmark_tabs:). 56 | 57 | ### :thinking: What could have worked but we haven't tried it 58 | * Ground truth masks for overlapping contours ([DSB-2018 winners](https://www.kaggle.com/c/data-science-bowl-2018/discussion/54741) approach). 59 | 60 | ## Network 61 | ### :heavy_check_mark: What Worked 62 | * Unet with Resnet34, Resnet101 and Resnet152 as an encoder where Resnet101 gave us the best results. This approach is explained in the [TernausNetV2](https://arxiv.org/abs/1806.00844) paper (our [code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/unet_models.py#L315-L403) :computer: and [config](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/neptune.yaml#L63) :bookmark_tabs:). Also take a look at our parametrizable [implementation of the U-Net](https://github.com/neptune-ai/steppy-toolkit/blob/master/toolkit/pytorch_transformers/architectures/unet.py#L9). 63 | 64 | ### :heavy_multiplication_x: What didn't Work 65 | * Network architecture based on dilated convolutions described in [this paper](https://arxiv.org/abs/1709.00179). 66 | 67 | ### :thinking: What could have worked but we haven't tried it 68 | * Unet with contextual blocks explained in [this paper](https://openreview.net/pdf?id=S1F-dpjjM). 69 | 70 | ## Loss function 71 | ### :heavy_check_mark: What Worked 72 | * Distance weighted cross entropy explained in the famous [U-Net paper](https://arxiv.org/pdf/1505.04597.pdf) (our [code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/models.py#L227-L371) :computer: and [config](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/neptune.yaml#L79-L80) :bookmark_tabs:). 73 | * Using linear combination of soft dice and distance weighted cross entropy ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/models.py#L227-L371) :computer: and [config](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/neptune.yaml#L65-L67) :bookmark_tabs:). 74 | * Adding component weighted by building size (smaller buildings has greater weight) to the weighted cross entropy that penalizes misclassification on pixels belonging to the small objects ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/models.py#L227-L371) :computer:). 75 | 76 | ### Weights visualization 77 | For both weights: the darker the color the higher value. 78 | * distance weights: high values corresponds to pixels between buildings. 79 | * size weights: high values denotes small buildings (the smaller the building the darker the color). Note that no-building is fixed to black. 80 | 81 | 82 | 83 | ## Training 84 | ### :heavy_check_mark: What Worked 85 | * Use pretrained models! 86 | * Our multistage training procedure: 87 | 1. train on a 50000 examples subset of the dataset with `lr=0.0001` and `dice_weight=0.5` 88 | 1. train on a full dataset with `lr=0.0001` and `dice_weight=0.5` 89 | 1. train with smaller `lr=0.00001` and `dice_weight=0.5` 90 | 1. increase dice weight to `dice_weight=5.0` to make results smoother 91 | * Multi-GPU training 92 | * Use very simple augmentations 93 | 94 | The entire configuration can be tweaked from the [config file](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/neptune.yaml) :bookmark_tabs:. 95 | 96 | ### :thinking: What could have worked but we haven't tried it 97 | * Set different learning rates to different layers. 98 | * Use cyclic optimizers. 99 | * Use warm start optimizers. 100 | 101 | ## Postprocessing 102 | ### :heavy_check_mark: What Worked 103 | * Test time augmentation (tta). Make predictions on image rotations (90-180-270 degrees) and flips (up-down, left-right) and take geometric mean on the predictions ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/loaders.py#L338-L497) :computer: and [config](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/pipeline_config.py#L119-L125) :bookmark_tabs:). 104 | * Simple morphological operations. At the beginning we used erosion followed by labeling and per label dilation with structure elements chosed by cross-validation. As the models got better, erosion was removed and very small dilation was the only one showing improvements ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/postprocessing.py) :computer:). 105 | * Scoring objects. In the beginning we simply used score `1.0` for every object which was a huge mistake. Changing that to average probability over the object region improved results. What improved scores even more was weighing those probabilities with the object size ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/postprocessing.py#L173-L181) :computer:). 106 | * Second level model. We tried Light-GBM and Random Forest trained on U-Net outputs and features calculated during postprocessing. 107 | 108 | ### :heavy_multiplication_x: What didn't Work 109 | * Test time augmentations by using colors ([config](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/pipeline_config.py#L122) :bookmark_tabs:). 110 | * Inference on reflection-padded images was not a way to go. What worked better (but not for the very best models) was replication padding where border pixel value was replicated for all the padded regions ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/loaders.py#L313) :computer:). 111 | * Conditional Random Fields. It was so slow that we didn't check it for the best models ([code](https://github.com/neptune-ai/open-solution-mapping-challenge/blob/master/src/postprocessing.py#L128-L170) :computer:). 112 | 113 | ### :thinking: What could have worked but we haven't tried it 114 | * Ensembling 115 | * Recurrent neural networks for postprocessing (instead of our current approach) 116 | 117 | # Model Weights 118 | 119 | Model weights for the winning solution are available [here](https://ui.neptune.ai/o/neptune-ai/org/Mapping-Challenge/e/MC-1057/artifacts) 120 | 121 | You can use those weights and run the pipeline as explained in [REPRODUCE_RESULTS](REPRODUCE_RESULTS.md). 122 | 123 | # User support 124 | There are several ways to seek help: 125 | 1. crowdai [discussion](https://www.crowdai.org/challenges/mapping-challenge/topics). 126 | 1. You can submit an [issue](https://github.com/neptune-ai/open-solution-mapping-challenge/issues) directly in this repo. 127 | 1. Join us on [Gitter](https://gitter.im/minerva-ml/open-solution-mapping-challenge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge). 128 | 129 | # Contributing 130 | 1. Check [CONTRIBUTING](CONTRIBUTING.md) for more information. 131 | 1. Check [issues](https://github.com/neptune-ai/open-solution-mapping-challenge/issues) to check if there is something you would like to contribute to. 132 | -------------------------------------------------------------------------------- /REPRODUCE_RESULTS.md: -------------------------------------------------------------------------------- 1 | ## Prepare sources and data 2 | * clone this repository 3 | ```bash 4 | git clone https://github.com/neptune-ai/open-solution-mapping-challenge.git 5 | ``` 6 | * install conda environment mapping 7 | 8 | You can setup the project with default env variables and open `NEPTUNE_API_TOKEN` by running: 9 | 10 | ```bash 11 | source Makefile 12 | ``` 13 | 14 | I suggest at least reading the step-by-step instructions to know what is happening. 15 | 16 | Install conda environment mapping 17 | 18 | ```bash 19 | conda env create -f environment.yml 20 | ``` 21 | 22 | After it is installed you can activate/deactivate it by running: 23 | 24 | ```bash 25 | conda activate mapping 26 | ``` 27 | 28 | ```bash 29 | conda deactivate 30 | ``` 31 | 32 | Register to the [neptune.ai](https://neptune.ai) _(if you wish to use it)_ even if you don't register you can still 33 | see your experiment in Neptune. Just go to [shared/showroom project](https://ui.neptune.ai/o/shared/org/showroom/experiments) and find it. 34 | 35 | Set environment variables `NEPTUNE_API_TOKEN` and `CONFIG_PATH`. 36 | 37 | If you are using the default `neptune.yaml` config then run: 38 | ```bash 39 | export export CONFIG_PATH=neptune.yaml 40 | ``` 41 | 42 | otherwise you can change to your config. 43 | 44 | **Registered in Neptune**: 45 | 46 | Set `NEPTUNE_API_TOKEN` variable with your personal token: 47 | 48 | ```bash 49 | export NEPTUNE_API_TOKEN=your_account_token 50 | ``` 51 | 52 | Create new project in Neptune and go to your config file (`neptune.yaml`) and change `project` name: 53 | 54 | ```yaml 55 | project: USER_NAME/PROJECT_NAME 56 | ``` 57 | 58 | **Not registered in Neptune**: 59 | 60 | open token 61 | ```bash 62 | export NEPTUNE_API_TOKEN=eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vdWkubmVwdHVuZS5tbCIsImFwaV9rZXkiOiJiNzA2YmM4Zi03NmY5LTRjMmUtOTM5ZC00YmEwMzZmOTMyZTQifQ== 63 | ``` 64 | 65 | ## Prepare training data 66 | 67 | * download the data from the [competition site](https://www.aicrowd.com/challenges/mapping-challenge#datasets) 68 | 69 | We suggest setting up a following directory structure: 70 | 71 | ``` 72 | project 73 | |-- README.md 74 | |-- ... 75 | |-- data 76 | |-- raw 77 | |-- train 78 | |-- images 79 | |-- annotation.json 80 | |-- val 81 | |-- images 82 | |-- annotation.json 83 | |-- test_images 84 | |-- img1.jpg 85 | |-- img2.jpg 86 | |-- ... 87 | |-- meta 88 | |-- masks_overlayed_eroded_{}_dilated_{} # it is generated automatically 89 | |-- train 90 | |-- distances 91 | |-- masks 92 | |-- sizes 93 | |-- val 94 | |-- distances 95 | |-- masks 96 | |-- sizes 97 | |-- experiments 98 | |-- mapping_challenge_baseline # this is where your experiment files will be dumped 99 | |-- checkpoints # neural network checkpoints 100 | |-- transformers # serialized transformers after fitting 101 | |-- outputs # outputs of transformers if you specified save_output=True anywhere 102 | |-- prediction.json # prediction on valid 103 | ``` 104 | 105 | * set paths in `neptune.yaml` if you wish to use different project structure. 106 | ```yaml 107 | data_dir: data/raw 108 | meta_dir: data/meta 109 | masks_overlayed_prefix: masks_overlayed 110 | experiment_dir: data/experiments 111 | ``` 112 | 113 | * change erosion/dilation setup in `neptune.yaml` if you want to. Suggested setup 114 | ```yaml 115 | border_width: 0 116 | small_annotations_size: 14 117 | erode_selem_size: 0 118 | dilate_selem_size: 0 119 | ``` 120 | 121 | * prepare target masks and metadata for training 122 | ```bash 123 | python main.py prepare_masks 124 | python main.py prepare_metadata --train_data --valid_data 125 | ``` 126 | 127 | ## Train model :rocket: 128 | 129 | ### Unet Network 130 | This will train your neural network 131 | 132 | ```bash 133 | python main.py train --pipeline_name unet_weighted 134 | ``` 135 | 136 | **NOTE** 137 | 138 | Model weights for the winning solution are available [here](https://ui.neptune.ai/o/neptune-ai/org/Mapping-Challenge/e/MC-1057/artifacts) 139 | 140 | ### Second level model (optional) 141 | This will train a lightgbm to be able to get the best threshold. 142 | Go to `pipeline_config.py` and change the number of thresholds to choose from for the building class. 143 | 19 means that your scoring model will learn which out of 19 threshold options (0.05...0.95) to choose for 144 | a particular image. 145 | 146 | ```python 147 | CATEGORY_LAYERS = [1, 19] 148 | ``` 149 | 150 | ```bash 151 | python main.py train --pipeline_name scoring_model 152 | ``` 153 | 154 | ## Evaluate model and predict on test data: 155 | 156 | Change values in the configuration file `neptune.yaml`. 157 | Suggested setup: 158 | 159 | ```yaml 160 | tta_aggregation_method: gmean 161 | loader_mode: resize 162 | erode_selem_size: 0 163 | dilate_selem_size: 2 164 | ``` 165 | 166 | ### Standard Unet evaluation 167 | 168 | ```bash 169 | python main.py evaluate --pipeline_name unet 170 | ``` 171 | 172 | With Test time augmentation 173 | 174 | ```bash 175 | python main.py evaluate --pipeline_name unet_tta --chunk_size 1000 176 | ``` 177 | 178 | ### Second level model (optional) 179 | 180 | If you trained the second layer model go to the `pipeline_config.py` and change the `CATEGORY_LAYER` to 181 | what you chose during training. 182 | For example, 183 | 184 | ```python 185 | CATEGORY_LAYERS = [1, 19] 186 | ``` 187 | 188 | ```bash 189 | python main.py evaluate --pipeline_name unet_tta_scoring_model --chunk_size 1000 190 | ``` 191 | 192 | 193 | ## Predict on new data 194 | 195 | Put your images in some `inference_directory`. 196 | 197 | Change values in the configuration file `neptune.yaml`. 198 | Suggested setup: 199 | 200 | ```yaml 201 | tta_aggregation_method: gmean 202 | loader_mode: resize 203 | erode_selem_size: 0 204 | dilate_selem_size: 2 205 | ``` 206 | 207 | Run prediction on this directory: 208 | 209 | ```bash 210 | python main.py predict_on_dir \ 211 | --pipeline_name unet_tta_scoring_model \ 212 | --chunk_size 1000 \ 213 | --dir_path path/to/inference_directory \ 214 | --prediction_path path/to/predictions.json 215 | 216 | ``` 217 | 218 | ## Enjoy results :trophy: 219 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: mapping 2 | 3 | dependencies: 4 | - pip=19.1.1 5 | - python=3.6.8 6 | - psutil 7 | - matplotlib 8 | - scikit-image 9 | - lightgbm=2.2.1 10 | 11 | - pip: 12 | - click==6.7 13 | - tqdm==4.23.0 14 | - pydot_ng==1.0.0 15 | - git+https://github.com/lucasb-eyer/pydensecrf.git 16 | - xgboost==0.90 17 | - neptune-client==0.3.0 18 | - neptune-contrib==0.9.2 19 | - imgaug==0.2.5 20 | - opencv_python==3.4.0.12 21 | - torch==0.3.1 22 | - torchvision==0.2.0 23 | - pretrainedmodels==0.7.0 24 | - pandas==0.24.2 25 | - numpy==1.16.4 26 | - cython==0.28.2 27 | - pycocotools==2.0.0 28 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import click 2 | from src.pipeline_manager import PipelineManager 3 | 4 | pipeline_manager = PipelineManager() 5 | 6 | 7 | @click.group() 8 | def main(): 9 | pass 10 | 11 | 12 | @main.command() 13 | @click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False) 14 | def prepare_masks(dev_mode): 15 | pipeline_manager.prepare_masks(dev_mode) 16 | 17 | 18 | @main.command() 19 | @click.option('-tr', '--train_data', help='calculate for train data', is_flag=True, required=False) 20 | @click.option('-val', '--valid_data', help='calculate for validation data', is_flag=True, required=False) 21 | def prepare_metadata(train_data, valid_data): 22 | pipeline_manager.prepare_metadata(train_data, valid_data) 23 | 24 | 25 | @main.command() 26 | @click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True) 27 | @click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False) 28 | def train(pipeline_name, dev_mode): 29 | pipeline_manager.start_experiment() 30 | pipeline_manager.train(pipeline_name, dev_mode) 31 | pipeline_manager.finish_experiment() 32 | 33 | 34 | @main.command() 35 | @click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True) 36 | @click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False) 37 | @click.option('-c', '--chunk_size', help='size of the chunks to run evaluation on', type=int, default=None, 38 | required=False) 39 | def evaluate(pipeline_name, dev_mode, chunk_size): 40 | pipeline_manager.start_experiment() 41 | pipeline_manager.evaluate(pipeline_name, dev_mode, chunk_size) 42 | pipeline_manager.finish_experiment() 43 | 44 | 45 | @main.command() 46 | @click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True) 47 | @click.option('-d', '--dir_path', help='directory with images to score', required=True) 48 | @click.option('-r', '--prediction_path', help='path to the prediction .json file', required=True) 49 | @click.option('-c', '--chunk_size', help='size of the chunks to run prediction on', type=int, default=None, 50 | required=False) 51 | def predict_on_dir(pipeline_name, dir_path, prediction_path, chunk_size): 52 | pipeline_manager.predict_on_dir(pipeline_name, dir_path, prediction_path, chunk_size) 53 | 54 | 55 | @main.command() 56 | @click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True) 57 | @click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False) 58 | @click.option('-c', '--chunk_size', help='size of the chunks to run evaluation and prediction on', type=int, 59 | default=None, required=False) 60 | def train_evaluate(pipeline_name, dev_mode, chunk_size): 61 | pipeline_manager.start_experiment() 62 | pipeline_manager.train(pipeline_name, dev_mode) 63 | pipeline_manager.evaluate(pipeline_name, dev_mode, chunk_size) 64 | pipeline_manager.finish_experiment() 65 | 66 | 67 | if __name__ == "__main__": 68 | main() 69 | -------------------------------------------------------------------------------- /neptune.yaml: -------------------------------------------------------------------------------- 1 | project: shared/showroom 2 | 3 | name: mapping_challenge_baseline 4 | tags: [solution_5] 5 | 6 | 7 | parameters: 8 | # Data Paths 9 | data_dir: data/raw 10 | meta_dir: data/meta 11 | masks_overlayed_prefix: masks_overlayed 12 | experiment_dir: data/experiments/mapping_challenge_baseline 13 | 14 | # Execution 15 | overwrite: 0 16 | num_workers: 4 17 | num_threads: 1000 18 | load_in_memory: 0 19 | pin_memory: 1 20 | evaluation_data_sample: 1000 21 | border_width: 0 22 | small_annotations_size: 14 23 | loader_mode: resize 24 | stream_mode: 0 25 | 26 | # General parameters 27 | image_h: 256 28 | image_w: 256 29 | image_channels: 3 30 | 31 | # U-Net parameters (multi-output) 32 | nr_unet_outputs: 1 33 | channels_per_output: 2 34 | n_filters: 16 35 | conv_kernel: 3 36 | pool_kernel: 3 37 | pool_stride: 2 38 | repeat_blocks: 4 39 | encoder: ResNet101 40 | 41 | # U-Net loss weights (multi-output) 42 | bce_mask: 1.0 43 | dice_mask: 0.2 44 | 45 | # Training schedule 46 | epochs_nr: 100 47 | batch_size_train: 20 48 | batch_size_inference: 20 49 | lr: 0.0005 50 | momentum: 0.9 51 | gamma: 1.0 52 | patience: 30 53 | lr_factor: 0.3 54 | lr_patience: 30 55 | w0: 50 56 | sigma: 10 57 | dice_smooth: 1 58 | dice_activation: 'softmax' 59 | validate_with_map: 1 60 | 61 | # Regularization 62 | use_batch_norm: 1 63 | l2_reg_conv: 0.0001 64 | l2_reg_dense: 0.0 65 | dropout_conv: 0.1 66 | dropout_dense: 0.0 67 | 68 | # Postprocessing 69 | erode_selem_size: 0 70 | dilate_selem_size: 0 71 | tta_aggregation_method: gmean 72 | nms__iou_threshold: 0.5 73 | 74 | # Inference padding 75 | crop_image_h: 300 76 | crop_image_w: 300 77 | h_pad: 10 78 | w_pad: 10 79 | pad_method: 'replicate' 80 | 81 | #Neptune monitor 82 | unet_outputs_to_plot: '["multichannel_map",]' 83 | 84 | #Scoring model 85 | scoring_model: 'lgbm' 86 | scoring_model__num_training_examples: 10000 87 | 88 | #LightGBM 89 | lgbm__learning_rate: 0.01 90 | lgbm__num_leaves: 500 91 | lgbm__min_data: 100 92 | lgbm__max_depth: 20 93 | lgbm__number_of_trees: 50000 94 | lgbm__early_stopping: 10 95 | lgbm__train_size: 0.7 96 | lgbm__target: 'iou' 97 | 98 | #Random Forest 99 | rf__n_estimators: 500 100 | rf__criterion: "mse" 101 | rf__max_depth: 20 102 | rf__min_samples_split: 100 103 | rf__min_samples_leaf: 100 104 | rf__max_features: 'auto' 105 | rf__max_leaf_nodes: 500 106 | rf__n_jobs: 10 107 | rf__verbose: 0 108 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/neptune-ai/open-solution-mapping-challenge/2f1f5f17bb9dfb5ba8dfc3c312533479997bd4c9/src/__init__.py -------------------------------------------------------------------------------- /src/augmentation.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | from imgaug import augmenters as iaa 4 | 5 | fast_seq = iaa.SomeOf((1, 2), 6 | [iaa.Fliplr(0.5), 7 | iaa.Flipud(0.5), 8 | iaa.Affine(rotate=(-10, 10), 9 | translate_percent=(-0.1, 0.1)), 10 | ], random_order=True) 11 | 12 | color_seq = iaa.Sequential([ 13 | # Color 14 | iaa.OneOf([ 15 | iaa.Sequential([ 16 | iaa.ChangeColorspace(from_colorspace="RGB", to_colorspace="HSV"), 17 | iaa.WithChannels(0, iaa.Add((0, 100))), 18 | iaa.ChangeColorspace(from_colorspace="HSV", to_colorspace="RGB")]), 19 | iaa.Sequential([ 20 | iaa.ChangeColorspace(from_colorspace="RGB", to_colorspace="HSV"), 21 | iaa.WithChannels(1, iaa.Add((0, 100))), 22 | iaa.ChangeColorspace(from_colorspace="HSV", to_colorspace="RGB")]), 23 | iaa.Sequential([ 24 | iaa.ChangeColorspace(from_colorspace="RGB", to_colorspace="HSV"), 25 | iaa.WithChannels(2, iaa.Add((0, 100))), 26 | iaa.ChangeColorspace(from_colorspace="HSV", to_colorspace="RGB")]), 27 | iaa.WithChannels(0, iaa.Add((0, 100))), 28 | iaa.WithChannels(1, iaa.Add((0, 100))), 29 | iaa.WithChannels(2, iaa.Add((0, 100))) 30 | ]) 31 | ], random_order=True) 32 | 33 | 34 | def crop_seq(crop_size): 35 | seq = iaa.Sequential([fast_seq, 36 | RandomCropFixedSize(px=crop_size)], random_order=False) 37 | return seq 38 | 39 | 40 | def padding_seq(pad_size, pad_method): 41 | seq = iaa.Sequential([PadFixed(pad=pad_size, pad_method=pad_method), 42 | ]).to_deterministic() 43 | return seq 44 | 45 | 46 | class PadFixed(iaa.Augmenter): 47 | PAD_FUNCTION = {'reflect': cv2.BORDER_REFLECT_101, 48 | 'replicate': cv2.BORDER_REPLICATE, 49 | } 50 | 51 | def __init__(self, pad=None, pad_method=None, name=None, deterministic=False, random_state=None): 52 | super().__init__(name, deterministic, random_state) 53 | self.pad = pad 54 | self.pad_method = pad_method 55 | 56 | def _augment_images(self, images, random_state, parents, hooks): 57 | result = [] 58 | for i, image in enumerate(images): 59 | image_pad = self._pad(image) 60 | result.append(image_pad) 61 | return result 62 | 63 | def _augment_keypoints(self, keypoints_on_images, random_state, parents, hooks): 64 | result = [] 65 | return result 66 | 67 | def _pad(self, img): 68 | img_ = img.copy() 69 | 70 | if self._is_expanded_grey_format(img): 71 | img_ = np.squeeze(img_, axis=-1) 72 | 73 | h_pad, w_pad = self.pad 74 | img_ = cv2.copyMakeBorder(img_.copy(), h_pad, h_pad, w_pad, w_pad, PadFixed.PAD_FUNCTION[self.pad_method]) 75 | 76 | if self._is_expanded_grey_format(img): 77 | img_ = np.expand_dims(img_, axis=-1) 78 | 79 | return img_ 80 | 81 | def get_parameters(self): 82 | return [] 83 | 84 | def _is_expanded_grey_format(self, img): 85 | if len(img.shape) == 3 and img.shape[2] == 1: 86 | return True 87 | else: 88 | return False 89 | 90 | 91 | class RandomCropFixedSize(iaa.Augmenter): 92 | def __init__(self, px=None, name=None, deterministic=False, random_state=None): 93 | super(RandomCropFixedSize, self).__init__(name=name, deterministic=deterministic, random_state=random_state) 94 | self.px = px 95 | if isinstance(self.px, tuple): 96 | self.px_h, self.px_w = self.px 97 | elif isinstance(self.px, int): 98 | self.px_h = self.px 99 | self.px_w = self.px 100 | else: 101 | raise NotImplementedError 102 | 103 | def _augment_images(self, images, random_state, parents, hooks): 104 | 105 | result = [] 106 | seeds = random_state.randint(0, 10 ** 6, (len(images),)) 107 | for i, image in enumerate(images): 108 | seed = seeds[i] 109 | image_cr = self._random_crop(seed, image) 110 | result.append(image_cr) 111 | return result 112 | 113 | def _augment_keypoints(self, keypoints_on_images, random_state, parents, hooks): 114 | result = [] 115 | return result 116 | 117 | def _random_crop(self, seed, image): 118 | height, width = image.shape[:2] 119 | 120 | np.random.seed(seed) 121 | crop_top = np.random.randint(height - self.px_h) 122 | crop_bottom = crop_top + self.px_h 123 | 124 | np.random.seed(seed + 1) 125 | crop_left = np.random.randint(width - self.px_w) 126 | crop_right = crop_left + self.px_w 127 | 128 | if len(image.shape) == 2: 129 | image_cropped = image[crop_top:crop_bottom, crop_left:crop_right] 130 | else: 131 | image_cropped = image[crop_top:crop_bottom, crop_left:crop_right, :] 132 | return image_cropped 133 | 134 | def get_parameters(self): 135 | return [] 136 | -------------------------------------------------------------------------------- /src/callbacks.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import torch 4 | import json 5 | import subprocess 6 | from PIL import Image 7 | import neptune 8 | from torch.autograd import Variable 9 | from tempfile import TemporaryDirectory 10 | 11 | from . import postprocessing as post 12 | from .steps.base import Step, Dummy 13 | from .steps.utils import get_logger 14 | from .steps.pytorch.callbacks import NeptuneMonitor, ValidationMonitor 15 | from .utils import softmax, coco_evaluation, create_annotations, make_apply_transformer 16 | from .pipeline_config import CATEGORY_IDS, Y_COLUMNS_SCORING, CATEGORY_LAYERS 17 | 18 | logger = get_logger() 19 | 20 | 21 | class NeptuneMonitorSegmentation(NeptuneMonitor): 22 | def __init__(self, image_nr, image_resize, model_name, outputs_to_plot): 23 | super().__init__(model_name) 24 | self.image_nr = image_nr 25 | self.image_resize = image_resize 26 | self.outputs_to_plot = outputs_to_plot 27 | 28 | def on_epoch_end(self, *args, **kwargs): 29 | self._send_numeric_channels() 30 | self._send_image_channels() 31 | self.epoch_id += 1 32 | 33 | def _send_image_channels(self): 34 | self.model.eval() 35 | pred_masks = self.get_prediction_masks() 36 | self.model.train() 37 | 38 | for name, pred_mask in pred_masks.items(): 39 | for i, image_duplet in enumerate(pred_mask): 40 | h, w = image_duplet.shape[1:] 41 | image_glued = np.zeros((h, 2 * w + 10)) 42 | 43 | image_glued[:, :w] = image_duplet[0, :, :] 44 | image_glued[:, (w + 10):] = image_duplet[1, :, :] 45 | 46 | pill_image = Image.fromarray((image_glued * 255.).astype(np.uint8)) 47 | h_, w_ = image_glued.shape 48 | pill_image = pill_image.resize((int(self.image_resize * w_), int(self.image_resize * h_)), 49 | Image.ANTIALIAS) 50 | 51 | neptune.send_image('{} {}'.format(self.model_name, name), pill_image) 52 | 53 | if i == self.image_nr: 54 | break 55 | 56 | def get_prediction_masks(self): 57 | prediction_masks = {} 58 | batch_gen, steps = self.validation_datagen 59 | for batch_id, data in enumerate(batch_gen): 60 | if len(data) != len(self.output_names) + 1: 61 | raise ValueError('incorrect targets provided') 62 | X = data[0] 63 | targets_tensors = data[1:] 64 | 65 | if (targets_tensors[0].size()[1] > 1): 66 | targets_tensors = [target_tensor[:, :1] for target_tensor in targets_tensors] 67 | 68 | if torch.cuda.is_available(): 69 | X = Variable(X, volatile=True).cuda() 70 | else: 71 | X = Variable(X, volatile=True) 72 | 73 | outputs_batch = self.model(X) 74 | if len(outputs_batch) == len(self.output_names): 75 | for name, output, target in zip(self.output_names, outputs_batch, targets_tensors): 76 | if name in self.outputs_to_plot: 77 | prediction = [] 78 | for image in softmax(output.data.cpu().numpy()): 79 | prediction.append(post.categorize_image(image)) 80 | prediction = np.stack(prediction) 81 | ground_truth = np.squeeze(target.cpu().numpy(), axis=1) 82 | n_channels = output.data.cpu().numpy().shape[1] 83 | for channel_nr in range(n_channels): 84 | category_id = CATEGORY_IDS[channel_nr] 85 | if category_id != None: 86 | channel_ground_truth = np.where(ground_truth == channel_nr, 1, 0) 87 | mask_key = '{}_{}'.format(name, category_id) 88 | prediction_masks[mask_key] = np.stack([prediction, channel_ground_truth], axis=1) 89 | else: 90 | for name, target in zip(self.output_names, targets_tensors): 91 | if name in self.outputs_to_plot: 92 | prediction = [] 93 | for image in softmax(outputs_batch.data.cpu().numpy()): 94 | prediction.append(post.categorize_image(image)) 95 | prediction = np.stack(prediction) 96 | ground_truth = np.squeeze(target.cpu().numpy(), axis=1) 97 | n_channels = outputs_batch.data.cpu().numpy().shape[1] 98 | for channel_nr in range(n_channels): 99 | category_id = CATEGORY_IDS[channel_nr] 100 | if category_id != None: 101 | channel_ground_truth = np.where(ground_truth == channel_nr, 1, 0) 102 | mask_key = '{}_{}'.format(name, category_id) 103 | prediction_masks[mask_key] = np.stack([prediction, channel_ground_truth], axis=1) 104 | break 105 | return prediction_masks 106 | 107 | 108 | class ValidationMonitorSegmentation(ValidationMonitor): 109 | def __init__(self, data_dir, small_annotations_size, validate_with_map=False, *args, **kwargs): 110 | super().__init__(*args, **kwargs) 111 | self.data_dir = data_dir 112 | self.small_annotations_size = small_annotations_size 113 | self.validate_with_map = validate_with_map 114 | self.validation_pipeline = postprocessing_pipeline_simplified 115 | self.validation_loss = None 116 | self.meta_valid = None 117 | 118 | def set_params(self, transformer, validation_datagen, meta_valid=None, *args, **kwargs): 119 | self.model = transformer.model 120 | self.optimizer = transformer.optimizer 121 | self.loss_function = transformer.loss_function 122 | self.output_names = transformer.output_names 123 | self.validation_datagen = validation_datagen 124 | self.meta_valid = meta_valid 125 | self.validation_loss = transformer.validation_loss 126 | 127 | def get_validation_loss(self): 128 | if self.validate_with_map: 129 | return self._get_validation_loss() 130 | else: 131 | return super().get_validation_loss() 132 | 133 | def _get_validation_loss(self): 134 | with TemporaryDirectory() as temp_dir: 135 | outputs = self._transform() 136 | prediction = self._generate_prediction(temp_dir, outputs) 137 | if len(prediction) == 0: 138 | return self.validation_loss.setdefault(self.epoch_id, {'sum': Variable(torch.Tensor([0]))}) 139 | prediction_filepath = os.path.join(temp_dir, 'prediction.json') 140 | with open(prediction_filepath, "w") as fp: 141 | fp.write(json.dumps(prediction)) 142 | 143 | annotation_file_path = os.path.join(self.data_dir, 'val', "annotation.json") 144 | 145 | logger.info('Calculating mean precision and recall') 146 | average_precision, average_recall = coco_evaluation(gt_filepath=annotation_file_path, 147 | prediction_filepath=prediction_filepath, 148 | image_ids=self.meta_valid[Y_COLUMNS_SCORING].values, 149 | category_ids=CATEGORY_IDS[1:], 150 | small_annotations_size=self.small_annotations_size) 151 | return self.validation_loss.setdefault(self.epoch_id, {'sum': Variable(torch.Tensor([average_precision]))}) 152 | 153 | def _transform(self): 154 | self.model.eval() 155 | batch_gen, steps = self.validation_datagen 156 | outputs = {} 157 | for batch_id, data in enumerate(batch_gen): 158 | if isinstance(data, list): 159 | X = data[0] 160 | else: 161 | X = data 162 | 163 | if torch.cuda.is_available(): 164 | X = Variable(X, volatile=True).cuda() 165 | else: 166 | X = Variable(X, volatile=True) 167 | 168 | outputs_batch = self.model(X) 169 | if len(self.output_names) == 1: 170 | outputs.setdefault(self.output_names[0], []).append(outputs_batch.data.cpu().numpy()) 171 | else: 172 | for name, output in zip(self.output_names, outputs_batch): 173 | output_ = output.data.cpu().numpy() 174 | outputs.setdefault(name, []).append(output_) 175 | if batch_id == steps: 176 | break 177 | self.model.train() 178 | outputs = {'{}_prediction'.format(name): np.vstack(outputs_) for name, outputs_ in outputs.items()} 179 | for name, prediction in outputs.items(): 180 | outputs[name] = softmax(prediction, axis=1) 181 | 182 | return outputs 183 | 184 | def _generate_prediction(self, cache_dirpath, outputs): 185 | data = {'callback_input': {'meta': self.meta_valid, 186 | 'meta_valid': None, 187 | 'target_sizes': [(300, 300)] * len(self.meta_valid), 188 | }, 189 | 'unet_output': {**outputs} 190 | } 191 | 192 | pipeline = self.validation_pipeline(cache_dirpath) 193 | for step_name in pipeline.all_steps: 194 | cmd = 'touch {}'.format(os.path.join(cache_dirpath, 'transformers', step_name)) 195 | subprocess.call(cmd, shell=True) 196 | output = pipeline.transform(data) 197 | y_pred = output['y_pred'] 198 | 199 | prediction = create_annotations(self.meta_valid, y_pred, logger, CATEGORY_IDS, CATEGORY_LAYERS) 200 | return prediction 201 | 202 | 203 | def postprocessing_pipeline_simplified(cache_dirpath): 204 | mask_resize = Step(name='mask_resize', 205 | transformer=make_apply_transformer(post.resize_image, 206 | output_name='resized_images', 207 | apply_on=['images', 'target_sizes']), 208 | input_data=['unet_output', 'callback_input'], 209 | adapter={'images': ([('unet_output', 'multichannel_map_prediction')]), 210 | 'target_sizes': ([('callback_input', 'target_sizes')]), 211 | }, 212 | cache_dirpath=cache_dirpath) 213 | 214 | category_mapper = Step(name='category_mapper', 215 | transformer=make_apply_transformer(post.categorize_image, 216 | output_name='categorized_images'), 217 | input_steps=[mask_resize], 218 | adapter={'images': ([('mask_resize', 'resized_images')]), 219 | }, 220 | cache_dirpath=cache_dirpath) 221 | 222 | labeler = Step(name='labeler', 223 | transformer=make_apply_transformer(post.label_multiclass_image, 224 | output_name='labeled_images'), 225 | input_steps=[category_mapper], 226 | adapter={'images': ([(category_mapper.name, 'categorized_images')]), 227 | }, 228 | cache_dirpath=cache_dirpath) 229 | 230 | score_builder = Step(name='score_builder', 231 | transformer=make_apply_transformer(post.build_score, 232 | output_name='images_with_scores', 233 | apply_on=['images', 'probabilities']), 234 | input_steps=[labeler, mask_resize], 235 | adapter={'images': ([(labeler.name, 'labeled_images')]), 236 | 'probabilities': ([(mask_resize.name, 'resized_images')]), 237 | }, 238 | cache_dirpath=cache_dirpath) 239 | 240 | output = Step(name='output', 241 | transformer=Dummy(), 242 | input_steps=[score_builder], 243 | adapter={'y_pred': ([(score_builder.name, 'images_with_scores')]), 244 | }, 245 | cache_dirpath=cache_dirpath) 246 | 247 | return output 248 | -------------------------------------------------------------------------------- /src/pipeline_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from attrdict import AttrDict 4 | 5 | from .utils import read_config, check_env_vars 6 | 7 | check_env_vars() 8 | 9 | config = read_config(config_path=os.getenv('CONFIG_PATH')) 10 | params = config.parameters 11 | 12 | SIZE_COLUMNS = ['height', 'width'] 13 | X_COLUMNS = ['file_path_image'] 14 | Y_COLUMNS = ['file_path_mask_eroded_0_dilated_0'] 15 | Y_COLUMNS_SCORING = ['ImageId'] 16 | SEED = 1234 17 | CATEGORY_IDS = [None, 100] 18 | CATEGORY_LAYERS = [1, 1] # thresholds, 1 means [0.5], 19 means [0.05, ... 0.95] use only with second layer model 19 | MEAN = [0.485, 0.456, 0.406] 20 | STD = [0.229, 0.224, 0.225] 21 | 22 | GLOBAL_CONFIG = {'exp_root': params.experiment_dir, 23 | 'load_in_memory': params.load_in_memory, 24 | 'num_workers': params.num_workers, 25 | 'num_classes': 2, 26 | 'img_H-W': (params.image_h, params.image_w), 27 | 'batch_size_train': params.batch_size_train, 28 | 'batch_size_inference': params.batch_size_inference, 29 | 'loader_mode': params.loader_mode, 30 | 'stream_mode': params.stream_mode 31 | } 32 | 33 | SOLUTION_CONFIG = AttrDict({ 34 | 'env': {'cache_dirpath': params.experiment_dir}, 35 | 'execution': GLOBAL_CONFIG, 36 | 'xy_splitter': {'x_columns': X_COLUMNS, 37 | 'y_columns': Y_COLUMNS, 38 | }, 39 | 'reader_single': {'x_columns': X_COLUMNS, 40 | 'y_columns': Y_COLUMNS, 41 | }, 42 | 'loader': {'dataset_params': {'h_pad': params.h_pad, 43 | 'w_pad': params.w_pad, 44 | 'h': params.image_h, 45 | 'w': params.image_w, 46 | 'pad_method': params.pad_method 47 | }, 48 | 'loader_params': {'training': {'batch_size': params.batch_size_train, 49 | 'shuffle': True, 50 | 'num_workers': params.num_workers, 51 | 'pin_memory': params.pin_memory 52 | }, 53 | 'inference': {'batch_size': params.batch_size_inference, 54 | 'shuffle': False, 55 | 'num_workers': params.num_workers, 56 | 'pin_memory': params.pin_memory 57 | }, 58 | }, 59 | }, 60 | 61 | 'unet': { 62 | 'architecture_config': {'model_params': {'n_filters': params.n_filters, 63 | 'conv_kernel': params.conv_kernel, 64 | 'pool_kernel': params.pool_kernel, 65 | 'pool_stride': params.pool_stride, 66 | 'repeat_blocks': params.repeat_blocks, 67 | 'batch_norm': params.use_batch_norm, 68 | 'dropout': params.dropout_conv, 69 | 'in_channels': params.image_channels, 70 | 'out_channels': params.channels_per_output, 71 | 'nr_outputs': params.nr_unet_outputs, 72 | 'encoder': params.encoder 73 | }, 74 | 'optimizer_params': {'lr': params.lr, 75 | }, 76 | 'regularizer_params': {'regularize': True, 77 | 'weight_decay_conv2d': params.l2_reg_conv, 78 | }, 79 | 'weights_init': {'function': 'he', 80 | }, 81 | 'loss_weights': {'bce_mask': params.bce_mask, 82 | 'dice_mask': params.dice_mask, 83 | }, 84 | 'weighted_cross_entropy': {'w0': params.w0, 85 | 'sigma': params.sigma, 86 | 'imsize': (params.image_h, params.image_w)}, 87 | 'dice': {'smooth': params.dice_smooth, 88 | 'dice_activation': params.dice_activation}, 89 | }, 90 | 'training_config': {'epochs': params.epochs_nr, 91 | }, 92 | 'callbacks_config': { 93 | 'model_checkpoint': { 94 | 'filepath': os.path.join(GLOBAL_CONFIG['exp_root'], 'checkpoints', 'unet', 'best.torch'), 95 | 'epoch_every': 1, 96 | 'minimize': not params.validate_with_map 97 | }, 98 | 'exp_lr_scheduler': {'gamma': params.gamma, 99 | 'epoch_every': 1}, 100 | 'plateau_lr_scheduler': {'lr_factor': params.lr_factor, 101 | 'lr_patience': params.lr_patience, 102 | 'epoch_every': 1}, 103 | 'training_monitor': {'batch_every': 1, 104 | 'epoch_every': 1}, 105 | 'experiment_timing': {'batch_every': 10, 106 | 'epoch_every': 1}, 107 | 'validation_monitor': { 108 | 'epoch_every': 1, 109 | 'data_dir': params.data_dir, 110 | 'validate_with_map': params.validate_with_map, 111 | 'small_annotations_size': params.small_annotations_size, 112 | }, 113 | 'neptune_monitor': {'model_name': 'unet', 114 | 'image_nr': 16, 115 | 'image_resize': 0.2, 116 | 'outputs_to_plot': params.unet_outputs_to_plot}, 117 | 'early_stopping': {'patience': params.patience, 118 | 'minimize': not params.validate_with_map}, 119 | }, 120 | }, 121 | 'tta_generator': {'flip_ud': True, 122 | 'flip_lr': True, 123 | 'rotation': True, 124 | 'color_shift_runs': False}, 125 | 'tta_aggregator': {'method': params.tta_aggregation_method, 126 | 'num_threads': params.num_threads 127 | }, 128 | 'postprocessor': {'mask_dilation': {'dilate_selem_size': params.dilate_selem_size 129 | }, 130 | 'mask_erosion': {'erode_selem_size': params.erode_selem_size 131 | }, 132 | 'prediction_crop': {'h_crop': params.crop_image_h, 133 | 'w_crop': params.crop_image_w 134 | }, 135 | 'scoring_model': params.scoring_model, 136 | 'lightGBM': {'model_params': {'learning_rate': params.lgbm__learning_rate, 137 | 'boosting_type': 'gbdt', 138 | 'objective': 'regression', 139 | 'metric': 'regression_l2', 140 | 'sub_feature': 1.0, 141 | 'num_leaves': params.lgbm__num_leaves, 142 | 'min_data': params.lgbm__min_data, 143 | 'max_depth': params.lgbm__max_depth, 144 | 'num_threads': params.num_threads}, 145 | 'training_params': {'number_boosting_rounds': params.lgbm__number_of_trees, 146 | 'early_stopping_rounds': params.lgbm__early_stopping}, 147 | 'train_size': params.lgbm__train_size, 148 | 'target': params.lgbm__target 149 | }, 150 | 'random_forest': {'train_size': params.lgbm__train_size, 151 | 'target': params.lgbm__target, 152 | 'model_params': {'n_estimators': params.rf__n_estimators, 153 | 'criterion': params.rf__criterion, 154 | 'max_depth': params.rf__max_depth, 155 | 'min_samples_split': params.rf__min_samples_split, 156 | 'min_samples_leaf': params.rf__min_samples_leaf, 157 | 'max_features': params.rf__max_features, 158 | 'max_leaf_nodes': params.rf__max_leaf_nodes, 159 | 'n_jobs': params.rf__n_jobs, 160 | 'verbose': params.rf__verbose, 161 | } 162 | }, 163 | 'nms': {'iou_threshold': params.nms__iou_threshold, 164 | 'num_threads': params.num_threads}, 165 | } 166 | }) 167 | -------------------------------------------------------------------------------- /src/pipeline_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | import pandas as pd 5 | import neptune 6 | import json 7 | from pycocotools.coco import COCO 8 | 9 | from .pipeline_config import SOLUTION_CONFIG, Y_COLUMNS_SCORING, CATEGORY_IDS, SEED, CATEGORY_LAYERS 10 | from .pipelines import PIPELINES 11 | from .preparation import overlay_masks 12 | from .utils import init_logger, read_config, get_filepaths, generate_metadata, set_seed, coco_evaluation, \ 13 | create_annotations, generate_data_frame_chunks, generate_inference_metadata 14 | 15 | 16 | class PipelineManager: 17 | def __init__(self): 18 | self.logger = init_logger() 19 | self.seed = SEED 20 | set_seed(self.seed) 21 | self.config = read_config(config_path=os.getenv('CONFIG_PATH')) 22 | self.params = self.config.parameters 23 | 24 | def start_experiment(self): 25 | neptune.init(project_qualified_name=self.config.project) 26 | neptune.create_experiment(name=self.config.name, 27 | params=self.params, 28 | upload_source_files=get_filepaths(), 29 | tags=self.config.tags) 30 | 31 | def prepare_masks(self, dev_mode): 32 | prepare_masks(dev_mode, self.logger, self.params) 33 | 34 | def prepare_metadata(self, train_data, valid_data): 35 | prepare_metadata(train_data, valid_data, self.logger, self.params) 36 | 37 | def train(self, pipeline_name, dev_mode): 38 | if 'scoring' in pipeline_name: 39 | assert CATEGORY_LAYERS[1] > 1, """You are running training on a second layer model that chooses 40 | which threshold should be chosen for a particular image. You need to specify a larger number of 41 | possible thresholds in the CATEGORY_LAYERS, suggested is 19""" 42 | train(pipeline_name, dev_mode, self.logger, self.params, self.seed) 43 | 44 | def evaluate(self, pipeline_name, dev_mode, chunk_size): 45 | if 'scoring' in pipeline_name: 46 | assert CATEGORY_LAYERS[1] > 1, """You are running inference with a second layer model that chooses 47 | which threshold should be chosen for a particular image. You need to specify a larger number of 48 | possible thresholds in the CATEGORY_LAYERS, suggested is 19""" 49 | else: 50 | assert CATEGORY_LAYERS[1] == 1, """You are running inference without a second layer model. 51 | Change thresholds setup in CATEGORY_LAYERS to [1,1]""" 52 | evaluate(pipeline_name, dev_mode, chunk_size, self.logger, self.params, self.seed) 53 | 54 | def predict_on_dir(self, pipeline_name, dir_path, prediction_path, chunk_size): 55 | if 'scoring' in pipeline_name: 56 | assert CATEGORY_LAYERS[1] > 1, """You are running inference with a second layer model that chooses 57 | which threshold should be chosen for a particular image. You need to specify a larger number of 58 | possible thresholds in the CATEGORY_LAYERS, suggested is 19""" 59 | else: 60 | assert CATEGORY_LAYERS[1] == 1, """You are running inference without a second layer model. 61 | Change thresholds setup in CATEGORY_LAYERS to [1,1]""" 62 | predict_on_dir(pipeline_name, dir_path, prediction_path, chunk_size, self.logger, self.params) 63 | 64 | def finish_experiment(self): 65 | neptune.stop() 66 | 67 | 68 | def prepare_masks(dev_mode, logger, params): 69 | for dataset in ["train", "val"]: 70 | logger.info('Overlaying masks, dataset: {}'.format(dataset)) 71 | 72 | mask_dirname = "masks_overlayed_eroded_{}_dilated_{}".format(params.erode_selem_size, params.dilate_selem_size) 73 | target_dir = os.path.join(params.meta_dir, mask_dirname) 74 | logger.info('Output directory: {}'.format(target_dir)) 75 | 76 | overlay_masks(data_dir=params.data_dir, 77 | dataset=dataset, 78 | target_dir=target_dir, 79 | category_ids=CATEGORY_IDS, 80 | erode=params.erode_selem_size, 81 | dilate=params.dilate_selem_size, 82 | is_small=dev_mode, 83 | num_threads=params.num_threads, 84 | border_width=params.border_width, 85 | small_annotations_size=params.small_annotations_size) 86 | 87 | 88 | def prepare_metadata(train_data, valid_data, logger, params): 89 | logger.info('creating metadata') 90 | 91 | meta = generate_metadata(data_dir=params.data_dir, 92 | meta_dir=params.meta_dir, 93 | masks_overlayed_prefix=params.masks_overlayed_prefix, 94 | process_train_data=train_data, 95 | process_validation_data=valid_data) 96 | 97 | metadata_filepath = os.path.join(params.meta_dir, 'metadata.csv') 98 | logger.info('saving metadata to {}'.format(metadata_filepath)) 99 | meta.to_csv(metadata_filepath, index=None) 100 | 101 | 102 | def train(pipeline_name, dev_mode, logger, params, seed): 103 | logger.info('training') 104 | if bool(params.overwrite) and os.path.isdir(params.experiment_dir): 105 | shutil.rmtree(params.experiment_dir) 106 | 107 | meta = pd.read_csv(os.path.join(params.meta_dir, 'metadata.csv'), low_memory=False) 108 | meta_train = meta[meta['is_train'] == 1] 109 | meta_valid = meta[meta['is_valid'] == 1] 110 | 111 | train_mode = True 112 | 113 | meta_valid = meta_valid.sample(int(params.evaluation_data_sample), random_state=seed) 114 | 115 | if dev_mode: 116 | meta_train = meta_train.sample(20, random_state=seed) 117 | meta_valid = meta_valid.sample(10, random_state=seed) 118 | 119 | if pipeline_name == 'scoring_model': 120 | train_mode = False 121 | meta_train, annotations = _get_scoring_model_data(params.data_dir, meta_train, 122 | params.scoring_model__num_training_examples, seed) 123 | else: 124 | annotations = None 125 | 126 | data = {'input': {'meta': meta_train, 127 | 'target_sizes': [(300, 300)] * len(meta_train), 128 | 'annotations': annotations}, 129 | 'specs': {'train_mode': train_mode, 130 | 'num_threads': params.num_threads}, 131 | 'callback_input': {'meta_valid': meta_valid} 132 | } 133 | 134 | pipeline = PIPELINES[pipeline_name]['train'](SOLUTION_CONFIG) 135 | pipeline.clean_cache() 136 | pipeline.fit_transform(data) 137 | pipeline.clean_cache() 138 | 139 | 140 | def evaluate(pipeline_name, dev_mode, chunk_size, logger, params, seed): 141 | logger.info('evaluating') 142 | meta = pd.read_csv(os.path.join(params.meta_dir, 'metadata.csv'), low_memory=False) 143 | 144 | meta_valid = meta[meta['is_valid'] == 1] 145 | 146 | meta_valid = meta_valid.sample(int(params.evaluation_data_sample), random_state=seed) 147 | 148 | if dev_mode: 149 | meta_valid = meta_valid.sample(30, random_state=seed) 150 | 151 | pipeline = PIPELINES[pipeline_name]['inference'](SOLUTION_CONFIG) 152 | prediction = generate_prediction(meta_valid, pipeline, logger, CATEGORY_IDS, chunk_size, params.num_threads) 153 | 154 | prediction_filepath = os.path.join(params.experiment_dir, 'prediction.json') 155 | with open(prediction_filepath, "w") as fp: 156 | fp.write(json.dumps(prediction)) 157 | 158 | annotation_file_path = os.path.join(params.data_dir, 'val', "annotation.json") 159 | 160 | logger.info('Calculating mean precision and recall') 161 | average_precision, average_recall = coco_evaluation(gt_filepath=annotation_file_path, 162 | prediction_filepath=prediction_filepath, 163 | image_ids=meta_valid[Y_COLUMNS_SCORING].values, 164 | category_ids=CATEGORY_IDS[1:], 165 | small_annotations_size=params.small_annotations_size) 166 | logger.info('Mean precision on validation is {}'.format(average_precision)) 167 | logger.info('Mean recall on validation is {}'.format(average_recall)) 168 | neptune.send_metric('Precision', average_precision) 169 | neptune.send_metric('Recall', average_recall) 170 | 171 | 172 | def predict_on_dir(pipeline_name, dir_path, prediction_path, chunk_size, logger, params): 173 | logger.info('creating metadata') 174 | meta = generate_inference_metadata(images_dir=dir_path) 175 | 176 | logger.info('predicting') 177 | pipeline = PIPELINES[pipeline_name]['inference'](SOLUTION_CONFIG) 178 | prediction = generate_prediction(meta, pipeline, logger, CATEGORY_IDS, chunk_size, params.num_threads) 179 | 180 | with open(prediction_path, "w") as fp: 181 | fp.write(json.dumps(prediction)) 182 | logger.info('submission saved to {}'.format(prediction_path)) 183 | logger.info('submission head \n\n{}'.format(prediction[0])) 184 | 185 | 186 | def generate_prediction(meta_data, pipeline, logger, category_ids, chunk_size, num_threads=1): 187 | if chunk_size is not None: 188 | return _generate_prediction_in_chunks(meta_data, pipeline, logger, category_ids, chunk_size, num_threads) 189 | else: 190 | return _generate_prediction(meta_data, pipeline, logger, category_ids, num_threads) 191 | 192 | 193 | def _generate_prediction(meta_data, pipeline, logger, category_ids, num_threads=1): 194 | data = {'input': {'meta': meta_data, 195 | 'target_sizes': [(300, 300)] * len(meta_data), 196 | }, 197 | 'specs': {'train_mode': False, 198 | 'num_threads': num_threads}, 199 | 'callback_input': {'meta_valid': None} 200 | } 201 | 202 | pipeline.clean_cache() 203 | output = pipeline.transform(data) 204 | pipeline.clean_cache() 205 | y_pred = output['y_pred'] 206 | 207 | prediction = create_annotations(meta_data, y_pred, logger, category_ids, CATEGORY_LAYERS) 208 | return prediction 209 | 210 | 211 | def _generate_prediction_in_chunks(meta_data, pipeline, logger, category_ids, chunk_size, num_threads=1): 212 | prediction = [] 213 | for meta_chunk in generate_data_frame_chunks(meta_data, chunk_size): 214 | data = {'input': {'meta': meta_chunk, 215 | 'target_sizes': [(300, 300)] * len(meta_chunk) 216 | }, 217 | 'specs': {'train_mode': False, 218 | 'num_threads': num_threads}, 219 | 'callback_input': {'meta_valid': None} 220 | } 221 | pipeline.clean_cache() 222 | output = pipeline.transform(data) 223 | pipeline.clean_cache() 224 | y_pred = output['y_pred'] 225 | 226 | prediction_chunk = create_annotations(meta_chunk, y_pred, logger, category_ids, CATEGORY_LAYERS) 227 | prediction.extend(prediction_chunk) 228 | 229 | return prediction 230 | 231 | 232 | def _get_scoring_model_data(data_dir, meta, num_training_examples, random_seed): 233 | annotation_file_path = os.path.join(data_dir, 'train', "annotation.json") 234 | coco = COCO(annotation_file_path) 235 | meta = meta.sample(num_training_examples, random_state=random_seed) 236 | annotations = [] 237 | for image_id in meta['ImageId'].values: 238 | image_annotations = {} 239 | for category_id in CATEGORY_IDS: 240 | annotation_ids = coco.getAnnIds(imgIds=image_id, catIds=category_id) 241 | category_annotations = coco.loadAnns(annotation_ids) 242 | image_annotations[category_id] = category_annotations 243 | annotations.append(image_annotations) 244 | return meta, annotations 245 | -------------------------------------------------------------------------------- /src/postprocessing.py: -------------------------------------------------------------------------------- 1 | import multiprocessing as mp 2 | 3 | import numpy as np 4 | from skimage.transform import resize 5 | from skimage.morphology import erosion, dilation, rectangle 6 | from tqdm import tqdm 7 | from pydensecrf.densecrf import DenseCRF2D 8 | from pydensecrf.utils import unary_from_softmax 9 | from pycocotools import mask as cocomask 10 | import pandas as pd 11 | import cv2 12 | 13 | from .steps.base import BaseTransformer 14 | from .utils import denormalize_img, add_dropped_objects, label, rle_from_binary 15 | from .pipeline_config import MEAN, STD, CATEGORY_LAYERS, CATEGORY_IDS 16 | 17 | 18 | class FeatureExtractor(BaseTransformer): 19 | def transform(self, images, probabilities, annotations=None): 20 | if annotations is None: 21 | annotations = [{}] * len(images) 22 | all_features = [] 23 | for image, im_probabilities, im_annotations in zip(images, probabilities, annotations): 24 | all_features.append(get_features_for_image(image, im_probabilities, im_annotations)) 25 | return {'features': all_features} 26 | 27 | 28 | class ScoreImageJoiner(BaseTransformer): 29 | def transform(self, images, scores): 30 | images_with_scores = [] 31 | for image, score in tqdm(zip(images, scores)): 32 | images_with_scores.append((image, score)) 33 | return {'images_with_scores': images_with_scores} 34 | 35 | 36 | class NonMaximumSupression(BaseTransformer): 37 | def __init__(self, iou_threshold, num_threads=1): 38 | self.iou_threshold = iou_threshold 39 | self.num_threads = num_threads 40 | 41 | def transform(self, images_with_scores): 42 | with mp.pool.ThreadPool(self.num_threads) as executor: 43 | cleaned_images_with_scores = executor.map( 44 | lambda p: remove_overlapping_masks(*p, iou_threshold=self.iou_threshold), images_with_scores) 45 | return {'images_with_scores': cleaned_images_with_scores} 46 | 47 | 48 | def resize_image(image, target_size): 49 | """Resize image to target size 50 | 51 | Args: 52 | image (numpy.ndarray): Image of shape (C x H x W). 53 | target_size (tuple): Target size (H, W). 54 | 55 | Returns: 56 | numpy.ndarray: Resized image of shape (C x H x W). 57 | 58 | """ 59 | n_channels = image.shape[0] 60 | resized_image = resize(image, (n_channels,) + target_size, mode='constant') 61 | return resized_image 62 | 63 | 64 | def categorize_image(image): 65 | """Maps probability map to categories. Each pixel is assigned with a category with highest probability. 66 | 67 | Args: 68 | image (numpy.ndarray): Probability map of shape (C x H x W). 69 | 70 | Returns: 71 | numpy.ndarray: Categorized image of shape (H x W). 72 | 73 | """ 74 | return np.argmax(image, axis=0) 75 | 76 | 77 | def categorize_multilayer_image(image): 78 | categorized_image = [] 79 | for category_id, category_output in enumerate(image): 80 | threshold_step = 1. / (CATEGORY_LAYERS[category_id] + 1) 81 | thresholds = np.arange(threshold_step, 1, threshold_step) 82 | for threshold in thresholds: 83 | categorized_image.append(category_output > threshold) 84 | return np.stack(categorized_image) 85 | 86 | 87 | def label_multiclass_image(mask): 88 | """Label separate class instances on a mask. 89 | 90 | Input mask is a 2D numpy.ndarray, cell (h, w) contains class number of that cell. 91 | Class number has to be an integer from 0 to C - 1, where C is a number of classes. 92 | This function splits input mask into C masks. Each mask contains separate instances of this class 93 | labeled starting from 1 and 0 as background. 94 | 95 | Example: 96 | Input mask (C = 2): 97 | [[0, 0, 1, 1], 98 | [1, 0, 0, 0], 99 | [1, 1, 1, 0], 100 | [0, 0, 1, 0]] 101 | 102 | Output: 103 | [[[1, 1, 0, 0], 104 | [0, 1, 1, 1], 105 | [0, 0, 0, 1], 106 | [2, 2, 0, 1]], 107 | 108 | [[0, 0, 1, 1], 109 | [2, 0, 0, 0], 110 | [2, 2, 2, 0], 111 | [0, 0, 2, 0]]] 112 | 113 | Args: 114 | mask (numpy.ndarray): Mask of shape (H x W). Each cell contains contains cell's class number. 115 | 116 | Returns: 117 | numpy.ndarray: Labeled mask of shape (C x H x W). 118 | 119 | """ 120 | labeled_channels = [] 121 | for label_nr in range(0, mask.max() + 1): 122 | labeled_channels.append(label(mask == label_nr)) 123 | labeled_image = np.stack(labeled_channels) 124 | return labeled_image 125 | 126 | 127 | def label_multilayer_image(mask): 128 | labeled_channels = [] 129 | for channel in mask: 130 | labeled_channels.append(label(channel)) 131 | labeled_image = np.stack(labeled_channels) 132 | return labeled_image 133 | 134 | 135 | def erode_image(mask, erode_selem_size): 136 | """Erode mask. 137 | 138 | Args: 139 | mask (numpy.ndarray): Mask of shape (H x W) or multiple masks of shape (C x H x W). 140 | erode_selem_size (int): Size of rectangle structuring element used for erosion. 141 | 142 | Returns: 143 | numpy.ndarray: Eroded mask of shape (H x W) or multiple masks of shape (C x H x W). 144 | 145 | """ 146 | if not erode_selem_size > 0: 147 | return mask 148 | selem = rectangle(erode_selem_size, erode_selem_size) 149 | if mask.ndim == 2: 150 | eroded_image = erosion(mask, selem=selem) 151 | else: 152 | eroded_image = [] 153 | for category_mask in mask: 154 | eroded_image.append(erosion(category_mask, selem=selem)) 155 | eroded_image = np.stack(eroded_image) 156 | return add_dropped_objects(mask, eroded_image) 157 | 158 | 159 | def dilate_image(mask, dilate_selem_size): 160 | """Dilate mask. 161 | 162 | Args: 163 | mask (numpy.ndarray): Mask of shape (H x W) or multiple masks of shape (C x H x W). 164 | dilate_selem_size (int): Size of rectangle structuring element used for dilation. 165 | 166 | Returns: 167 | numpy.ndarray: dilated Mask of shape (H x W) or multiple masks of shape (C x H x W). 168 | 169 | """ 170 | if not dilate_selem_size > 0: 171 | return mask 172 | selem = rectangle(dilate_selem_size, dilate_selem_size) 173 | if mask.ndim == 2: 174 | dilated_image = dilation(mask, selem=selem) 175 | else: 176 | dilated_image = [] 177 | for category_mask in mask: 178 | dilated_image.append(dilation(category_mask, selem=selem)) 179 | dilated_image = np.stack(dilated_image) 180 | return dilated_image 181 | 182 | 183 | def dense_crf(img, output_probs, compat_gaussian=3, sxy_gaussian=1, 184 | compat_bilateral=10, sxy_bilateral=1, srgb=50, iterations=5): 185 | """Perform fully connected CRF. 186 | 187 | This function performs CRF method described in the following paper: 188 | 189 | Efficient Inference in Fully Connected CRFs with Gaussian Edge Potentials 190 | Philipp Krähenbühl and Vladlen Koltun 191 | NIPS 2011 192 | https://arxiv.org/abs/1210.5644 193 | 194 | Args: 195 | img (numpy.ndarray): RGB image of shape (3 x H x W). 196 | output_probs (numpy.ndarray): Probability map of shape (C x H x W). 197 | compat_gaussian: Compat value for Gaussian case. 198 | sxy_gaussian: x/y standard-deviation, theta_gamma from the CRF paper. 199 | compat_bilateral: Compat value for RGB case. 200 | sxy_bilateral: x/y standard-deviation, theta_alpha from the CRF paper. 201 | srgb: RGB standard-deviation, theta_beta from the CRF paper. 202 | iterations: Number of CRF iterations. 203 | 204 | Returns: 205 | numpy.ndarray: Probability map of shape (C x H x W) after applying CRF. 206 | 207 | """ 208 | height = output_probs.shape[1] 209 | width = output_probs.shape[2] 210 | 211 | crf = DenseCRF2D(width, height, 2) 212 | unary = unary_from_softmax(output_probs) 213 | org_img = denormalize_img(img, mean=MEAN, std=STD) * 255. 214 | org_img = org_img.transpose(1, 2, 0) 215 | org_img = np.ascontiguousarray(org_img, dtype=np.uint8) 216 | 217 | crf.setUnaryEnergy(unary) 218 | 219 | crf.addPairwiseGaussian(sxy=sxy_gaussian, compat=compat_gaussian) 220 | crf.addPairwiseBilateral(sxy=sxy_bilateral, srgb=srgb, rgbim=org_img, compat=compat_bilateral) 221 | 222 | crf_image = crf.inference(iterations) 223 | crf_image = np.array(crf_image).reshape(output_probs.shape) 224 | 225 | return crf_image 226 | 227 | 228 | def build_score(image, probabilities): 229 | total_score = [] 230 | for category_instances, category_probabilities in zip(image, probabilities): 231 | score = [] 232 | for label_nr in range(1, category_instances.max() + 1): 233 | masked_instance = np.ma.masked_array(category_probabilities, mask=category_instances != label_nr) 234 | score.append(masked_instance.mean() * np.sqrt(np.count_nonzero(category_instances == label_nr))) 235 | total_score.append(score) 236 | return image, total_score 237 | 238 | 239 | def crop_image_center_per_class(image, h_crop, w_crop): 240 | """Crop image center. 241 | 242 | Args: 243 | image (numpy.ndarray): Image of shape (C x H x W). 244 | h_crop: Height of a cropped image. 245 | w_crop: Width of a cropped image. 246 | 247 | Returns: 248 | numpy.ndarray: Cropped image of shape (C x H x W). 249 | 250 | """ 251 | cropped_per_class_prediction = [] 252 | for class_prediction in image: 253 | h, w = class_prediction.shape[:2] 254 | h_start, w_start = int((h - h_crop) / 2.), int((w - w_crop) / 2.) 255 | cropped_prediction = class_prediction[h_start:-h_start, w_start:-w_start] 256 | cropped_per_class_prediction.append(cropped_prediction) 257 | cropped_per_class_prediction = np.stack(cropped_per_class_prediction) 258 | return cropped_per_class_prediction 259 | 260 | 261 | def get_features_for_image(image, probabilities, annotations): 262 | image_features = [] 263 | category_layers_inds = np.cumsum(CATEGORY_LAYERS) 264 | thresholds = get_thresholds() 265 | for category_ind, category_instances in enumerate(image): 266 | layer_features = [] 267 | threshold = round(thresholds[category_ind], 2) 268 | for mask, iou, category_probabilities in get_mask_with_iou(category_ind, category_instances, 269 | category_layers_inds, annotations, probabilities): 270 | layer_features.append(get_features_for_mask(mask, iou, threshold, category_probabilities)) 271 | image_features.append(pd.DataFrame(layer_features)) 272 | return image_features 273 | 274 | 275 | def get_mask_with_iou(category_ind, category_instances, category_layers_inds, annotations, probabilities): 276 | category_nr = np.searchsorted(category_layers_inds, category_ind, side='right') 277 | category_annotations = annotations.get(CATEGORY_IDS[category_nr], []) 278 | iou_matrix = get_iou_matrix(category_instances, category_annotations) 279 | category_probabilities = probabilities[category_nr] 280 | for label_nr in range(1, category_instances.max() + 1): 281 | mask = category_instances == label_nr 282 | iou = get_iou(iou_matrix, label_nr) 283 | yield mask, iou, category_probabilities 284 | 285 | 286 | def get_features_for_mask(mask, iou, threshold, category_probabilities): 287 | mask_probabilities = np.where(mask, category_probabilities, 0) 288 | area = np.count_nonzero(mask) 289 | mean_prob = mask_probabilities.sum() / area 290 | max_prob = mask_probabilities.max() 291 | bbox = get_bbox(mask) 292 | bbox_height = bbox[1] - bbox[0] 293 | bbox_width = bbox[3] - bbox[2] 294 | bbox_aspect_ratio = bbox_height / bbox_width 295 | bbox_area = bbox_width * bbox_height 296 | bbox_fill = area / bbox_area 297 | min_dist_to_border, max_dist_to_border = get_min_max_distance_to_border(bbox, mask.shape) 298 | contour_length = get_contour_length(mask) 299 | mask_features = {'iou': iou, 'threshold': threshold, 'area': area, 'mean_prob': mean_prob, 300 | 'max_prob': max_prob, 'bbox_ar': bbox_aspect_ratio, 301 | 'bbox_area': bbox_area, 'bbox_fill': bbox_fill, 'min_dist_to_border': min_dist_to_border, 302 | 'max_dist_to_border': max_dist_to_border, 'contour_length': contour_length} 303 | return mask_features 304 | 305 | 306 | def get_iou_matrix(labels, annotations): 307 | mask_anns = [] 308 | if annotations is None or annotations == []: 309 | return None 310 | else: 311 | for annotation in annotations: 312 | if not isinstance(annotation['segmentation'], dict): 313 | annotation['segmentation'] = \ 314 | cocomask.frPyObjects(annotation['segmentation'], labels.shape[0], labels.shape[1])[0] 315 | annotations = [annotation['segmentation'] for annotation in annotations] 316 | for label_nr in range(1, labels.max() + 1): 317 | mask = labels == label_nr 318 | mask_ann = rle_from_binary(mask.astype('uint8')) 319 | mask_anns.append(mask_ann) 320 | iou_matrix = cocomask.iou(mask_anns, annotations, [0, ] * len(annotations)) 321 | return iou_matrix 322 | 323 | 324 | def get_iou(iou_matrix, label_nr): 325 | if iou_matrix is not None: 326 | return iou_matrix[label_nr - 1].max() 327 | else: 328 | return None 329 | 330 | 331 | def get_thresholds(): 332 | thresholds = [] 333 | for n_thresholds in CATEGORY_LAYERS: 334 | threshold_step = 1. / (n_thresholds + 1) 335 | category_thresholds = np.arange(threshold_step, 1, threshold_step) 336 | thresholds.extend(category_thresholds) 337 | return thresholds 338 | 339 | 340 | def get_bbox(mask): 341 | '''taken from https://stackoverflow.com/questions/31400769/bounding-box-of-numpy-array and 342 | modified to prevent bbox of zero area''' 343 | rows = np.any(mask, axis=1) 344 | cols = np.any(mask, axis=0) 345 | rmin, rmax = np.where(rows)[0][[0, -1]] 346 | cmin, cmax = np.where(cols)[0][[0, -1]] 347 | return rmin, rmax + 1, cmin, cmax + 1 348 | 349 | 350 | def get_min_max_distance_to_border(bbox, im_size): 351 | min_distance = min(bbox[0], im_size[0] - bbox[1], bbox[2], im_size[1] - bbox[3]) 352 | max_distance = max(bbox[0], im_size[0] - bbox[1], bbox[2], im_size[1] - bbox[3]) 353 | return min_distance, max_distance 354 | 355 | 356 | def get_contour(mask): 357 | mask_contour = np.zeros_like(mask).astype(np.uint8) 358 | _, contours, hierarchy = cv2.findContours(mask.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) 359 | cv2.drawContours(mask_contour, contours, -1, (255, 255, 255), 1) 360 | return mask_contour 361 | 362 | 363 | def get_contour_length(mask): 364 | return np.count_nonzero(get_contour(mask)) 365 | 366 | 367 | def remove_overlapping_masks(image, scores, iou_threshold=0.5): 368 | scores_with_labels = [] 369 | for layer_nr, layer_scores in enumerate(scores): 370 | scores_with_labels.extend([(score, layer_nr, label_nr + 1) for label_nr, score in enumerate(layer_scores)]) 371 | scores_with_labels.sort(key=lambda x: x[0], reverse=True) 372 | for i, (score_i, layer_nr_i, label_nr_i) in enumerate(scores_with_labels): 373 | base_mask = image[layer_nr_i] == label_nr_i 374 | for score_j, layer_nr_j, label_nr_j in scores_with_labels[i + 1:]: 375 | mask_to_check = image[layer_nr_j] == label_nr_j 376 | iou = get_iou_for_mask_pair(base_mask, mask_to_check) 377 | if iou > iou_threshold: 378 | scores_with_labels.remove((score_j, layer_nr_j, label_nr_j)) 379 | scores[layer_nr_j][label_nr_j - 1] = 0 380 | return image, scores 381 | 382 | 383 | def get_iou_for_mask_pair(mask1, mask2): 384 | intersection = np.count_nonzero(mask1 * mask2) 385 | union = np.count_nonzero(mask1 + mask2) 386 | return intersection / union 387 | -------------------------------------------------------------------------------- /src/preparation.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | import multiprocessing as mp 3 | import os 4 | 5 | import numpy as np 6 | from imageio import imwrite 7 | from pycocotools import mask as cocomask 8 | from pycocotools.coco import COCO 9 | from skimage.morphology import binary_erosion, rectangle, binary_dilation 10 | from scipy.ndimage.morphology import distance_transform_edt 11 | from sklearn.externals import joblib 12 | 13 | from .utils import get_logger, add_dropped_objects, label 14 | 15 | logger = get_logger() 16 | 17 | 18 | def overlay_masks(data_dir, dataset, target_dir, category_ids, erode=0, dilate=0, is_small=False, num_threads=1, 19 | border_width=0, small_annotations_size=14): 20 | if is_small: 21 | suffix = "-small" 22 | else: 23 | suffix = "" 24 | annotation_file_name = "annotation{}.json".format(suffix) 25 | annotation_file_path = os.path.join(data_dir, dataset, annotation_file_name) 26 | coco = COCO(annotation_file_path) 27 | image_ids = coco.getImgIds() 28 | 29 | _overlay_mask_one_image = partial(overlay_mask_one_image, 30 | dataset=dataset, 31 | target_dir=target_dir, 32 | coco=coco, 33 | category_ids=category_ids, 34 | erode=erode, 35 | dilate=dilate, 36 | border_width=border_width, 37 | small_annotations_size=small_annotations_size) 38 | 39 | process_nr = min(num_threads, len(image_ids)) 40 | with mp.pool.ThreadPool(process_nr) as executor: 41 | executor.map(_overlay_mask_one_image, image_ids) 42 | 43 | 44 | def overlay_mask_one_image(image_id, dataset, target_dir, coco, category_ids, erode, dilate, border_width, 45 | small_annotations_size): 46 | image = coco.loadImgs(image_id)[0] 47 | image_size = (image["height"], image["width"]) 48 | mask_overlayed = np.zeros(image_size).astype('uint8') 49 | distances = np.zeros(image_size) 50 | for category_nr, category_id in enumerate(category_ids): 51 | if category_id is not None: 52 | annotation_ids = coco.getAnnIds(imgIds=image_id, catIds=[category_id, ]) 53 | annotations = coco.loadAnns(annotation_ids) 54 | 55 | if erode < 0 or dilate < 0: 56 | raise ValueError('erode and dilate cannot be negative') 57 | 58 | if erode == 0: 59 | mask, distances = overlay_masks_from_annotations(annotations=annotations, 60 | image_size=image_size, 61 | distances=distances) 62 | elif dilate == 0: 63 | mask, _ = overlay_masks_from_annotations(annotations=annotations, 64 | image_size=image_size) 65 | mask_eroded, distances = overlay_eroded_masks_from_annotations(annotations=annotations, 66 | image_size=image_size, 67 | erode=erode, 68 | distances=distances, 69 | small_annotations_size=small_annotations_size) 70 | mask = add_dropped_objects(mask, mask_eroded) 71 | else: 72 | mask, distances = overlay_eroded_dilated_masks_from_annotations(annotations=annotations, 73 | image_size=image_size, 74 | erode=erode, 75 | dilate=dilate, 76 | distances=distances, 77 | small_annotations_size=small_annotations_size) 78 | mask_overlayed = np.where(mask, category_nr, mask_overlayed) 79 | 80 | sizes = get_size_matrix(mask_overlayed) 81 | distances, second_nearest_distances = clean_distances(distances) 82 | 83 | if border_width > 0: 84 | borders = (second_nearest_distances < border_width) & (~mask_overlayed) 85 | borders_class_id = mask_overlayed.max() + 1 86 | mask_overlayed = np.where(borders, borders_class_id, mask_overlayed) 87 | 88 | target_filepath = os.path.join(target_dir, dataset, "masks", os.path.splitext(image["file_name"])[0]) + ".png" 89 | target_filepath_dist = os.path.join(target_dir, dataset, "distances", os.path.splitext(image["file_name"])[0]) 90 | target_filepath_sizes = os.path.join(target_dir, dataset, "sizes", os.path.splitext(image["file_name"])[0]) 91 | os.makedirs(os.path.dirname(target_filepath), exist_ok=True) 92 | os.makedirs(os.path.dirname(target_filepath_dist), exist_ok=True) 93 | os.makedirs(os.path.dirname(target_filepath_sizes), exist_ok=True) 94 | try: 95 | imwrite(target_filepath, mask_overlayed) 96 | joblib.dump(distances, target_filepath_dist) 97 | joblib.dump(sizes, target_filepath_sizes) 98 | except: 99 | logger.info("Failed to save image: {}".format(image_id)) 100 | 101 | 102 | 103 | def overlay_masks_from_annotations(annotations, image_size, distances=None): 104 | mask = np.zeros(image_size) 105 | for ann in annotations: 106 | rle = cocomask.frPyObjects(ann['segmentation'], image_size[0], image_size[1]) 107 | m = cocomask.decode(rle) 108 | 109 | for i in range(m.shape[-1]): 110 | mi = m[:, :, i] 111 | mi = mi.reshape(image_size) 112 | if is_on_border(mi, 2): 113 | continue 114 | if distances is not None: 115 | distances = update_distances(distances, mi) 116 | mask += mi 117 | return np.where(mask > 0, 1, 0).astype('uint8'), distances 118 | 119 | 120 | def overlay_eroded_masks_from_annotations(annotations, image_size, erode, distances, small_annotations_size): 121 | mask = np.zeros(image_size) 122 | for ann in annotations: 123 | rle = cocomask.frPyObjects(ann['segmentation'], image_size[0], image_size[1]) 124 | m = cocomask.decode(rle) 125 | m = m.reshape(image_size) 126 | if is_on_border(m, 2): 127 | continue 128 | m_eroded = get_simple_eroded_mask(m, erode, small_annotations_size) 129 | if distances is not None: 130 | distances = update_distances(distances, m_eroded) 131 | mask += m_eroded 132 | return np.where(mask > 0, 1, 0).astype('uint8'), distances 133 | 134 | 135 | def overlay_eroded_dilated_masks_from_annotations(annotations, image_size, erode, dilate, distances, 136 | small_annotations_size): 137 | mask = np.zeros(image_size) 138 | for ann in annotations: 139 | rle = cocomask.frPyObjects(ann['segmentation'], image_size[0], image_size[1]) 140 | m = cocomask.decode(rle) 141 | m = m.reshape(image_size) 142 | if is_on_border(m, 2): 143 | continue 144 | m_ = get_simple_eroded_dilated_mask(m, erode, dilate, small_annotations_size) 145 | if distances is not None: 146 | distances = update_distances(distances, m_) 147 | mask += m_ 148 | return np.where(mask > 0, 1, 0).astype('uint8'), distances 149 | 150 | 151 | def update_distances(dist, mask): 152 | if dist.sum() == 0: 153 | distances = distance_transform_edt(1 - mask) 154 | else: 155 | distances = np.dstack([dist, distance_transform_edt(1 - mask)]) 156 | return distances 157 | 158 | 159 | def clean_distances(distances): 160 | if len(distances.shape) < 3: 161 | distances = np.dstack([distances, distances]) 162 | else: 163 | distances.sort(axis=2) 164 | distances = distances[:, :, :2] 165 | second_nearest_distances = distances[:, :, 1] 166 | distances_clean = np.sum(distances, axis=2) 167 | return distances_clean.astype(np.float16), second_nearest_distances 168 | 169 | 170 | def get_simple_eroded_mask(mask, selem_size, small_annotations_size): 171 | if mask.sum() > small_annotations_size**2: 172 | selem = rectangle(selem_size, selem_size) 173 | mask_eroded = binary_erosion(mask, selem=selem) 174 | else: 175 | mask_eroded = mask 176 | return mask_eroded 177 | 178 | 179 | def get_simple_eroded_dilated_mask(mask, erode_selem_size, dilate_selem_size, small_annotations_size): 180 | if mask.sum() > small_annotations_size**2: 181 | selem = rectangle(erode_selem_size, erode_selem_size) 182 | mask_ = binary_erosion(mask, selem=selem) 183 | else: 184 | selem = rectangle(dilate_selem_size, dilate_selem_size) 185 | mask_ = binary_dilation(mask, selem=selem) 186 | return mask_ 187 | 188 | 189 | def get_size_matrix(mask): 190 | sizes = np.ones_like(mask) 191 | labeled = label(mask) 192 | for label_nr in range(1, labeled.max() + 1): 193 | label_size = (labeled == label_nr).sum() 194 | sizes = np.where(labeled == label_nr, label_size, sizes) 195 | return sizes 196 | 197 | def is_on_border(mask, border_width): 198 | return not np.any(mask[border_width:-border_width, border_width:-border_width]) -------------------------------------------------------------------------------- /src/steps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/neptune-ai/open-solution-mapping-challenge/2f1f5f17bb9dfb5ba8dfc3c312533479997bd4c9/src/steps/__init__.py -------------------------------------------------------------------------------- /src/steps/base.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pprint 3 | import shutil 4 | 5 | import numpy as np 6 | from scipy import sparse 7 | from sklearn.externals import joblib 8 | 9 | from .utils import view_graph, plot_graph, get_logger, initialize_logger 10 | 11 | initialize_logger() 12 | logger = get_logger() 13 | 14 | 15 | class Step: 16 | def __init__(self, name, transformer, input_steps=[], input_data=[], adapter=None, 17 | cache_dirpath=None, is_trainable=False, cache_output=False, save_output=False, load_saved_output=False, 18 | save_graph=False, force_fitting=False): 19 | self.name = name 20 | 21 | self.transformer = transformer 22 | 23 | self.input_steps = input_steps 24 | self.input_data = input_data 25 | self.adapter = adapter 26 | 27 | self.is_trainable = is_trainable 28 | self.force_fitting = force_fitting 29 | self.cache_output = cache_output 30 | self.save_output = save_output 31 | self.load_saved_output = load_saved_output 32 | 33 | self.cache_dirpath = cache_dirpath 34 | self._prep_cache(cache_dirpath) 35 | 36 | if save_graph: 37 | graph_filepath = os.path.join(self.cache_dirpath, '{}_graph.json'.format(self.name)) 38 | logger.info('Saving graph to {}'.format(graph_filepath)) 39 | joblib.dump(self.graph_info, graph_filepath) 40 | 41 | def _copy_transformer(self, step, name, dirpath): 42 | self.transformer = self.transformer.transformer 43 | 44 | original_filepath = os.path.join(step.cache_dirpath, 'transformers', step.name) 45 | copy_filepath = os.path.join(dirpath, 'transformers', name) 46 | logger.info('copying transformer from {} to {}'.format(original_filepath, copy_filepath)) 47 | shutil.copyfile(original_filepath, copy_filepath) 48 | 49 | def _prep_cache(self, cache_dirpath): 50 | for dirname in ['transformers', 'outputs', 'tmp']: 51 | os.makedirs(os.path.join(cache_dirpath, dirname), exist_ok=True) 52 | 53 | self.cache_dirpath_transformers = os.path.join(cache_dirpath, 'transformers') 54 | self.save_dirpath_outputs = os.path.join(cache_dirpath, 'outputs') 55 | self.save_dirpath_tmp = os.path.join(cache_dirpath, 'tmp') 56 | 57 | self.cache_filepath_step_transformer = os.path.join(self.cache_dirpath_transformers, self.name) 58 | self.save_filepath_step_output = os.path.join(self.save_dirpath_outputs, '{}'.format(self.name)) 59 | self.save_filepath_step_tmp = os.path.join(self.save_dirpath_tmp, '{}'.format(self.name)) 60 | self._cached_output = None 61 | 62 | def clean_cache(self): 63 | for name, step in self.all_steps.items(): 64 | step._clean_cache() 65 | 66 | def _clean_cache(self): 67 | if os.path.exists(self.save_filepath_step_tmp): 68 | os.remove(self.save_filepath_step_tmp) 69 | self._cached_output = None 70 | 71 | @property 72 | def named_steps(self): 73 | return {step.name: step for step in self.input_steps} 74 | 75 | def get_step(self, name): 76 | return self.all_steps[name] 77 | 78 | @property 79 | def transformer_is_cached(self): 80 | if isinstance(self.transformer, Step): 81 | self._copy_transformer(self.transformer, self.name, self.cache_dirpath) 82 | return os.path.exists(self.cache_filepath_step_transformer) 83 | 84 | @property 85 | def output_is_cached(self): 86 | return self._cached_output is not None 87 | 88 | @property 89 | def output_is_saved(self): 90 | return os.path.exists(self.save_filepath_step_output) 91 | 92 | def fit_transform(self, data): 93 | if self.output_is_cached and not self.force_fitting: 94 | logger.info('step {} loading output...'.format(self.name)) 95 | return self._cached_output 96 | elif self.output_is_saved and self.load_saved_output and not self.force_fitting: 97 | logger.info('step {} loading output...'.format(self.name)) 98 | return self._load_output(self.save_filepath_step_output) 99 | else: 100 | step_inputs = {} 101 | if self.input_data is not None: 102 | for input_data_part in self.input_data: 103 | step_inputs[input_data_part] = data[input_data_part] 104 | 105 | for input_step in self.input_steps: 106 | step_inputs[input_step.name] = input_step.fit_transform(data) 107 | 108 | if self.adapter: 109 | step_inputs = self.adapt(step_inputs) 110 | else: 111 | step_inputs = self.unpack(step_inputs) 112 | return self._cached_fit_transform(step_inputs) 113 | 114 | def _cached_fit_transform(self, step_inputs): 115 | if self.is_trainable: 116 | if self.transformer_is_cached and not self.force_fitting: 117 | logger.info('step {} loading transformer...'.format(self.name)) 118 | self.transformer.load(self.cache_filepath_step_transformer) 119 | logger.info('step {} transforming...'.format(self.name)) 120 | step_output_data = self.transformer.transform(**step_inputs) 121 | else: 122 | logger.info('step {} fitting and transforming...'.format(self.name)) 123 | step_output_data = self.transformer.fit_transform(**step_inputs) 124 | logger.info('step {} saving transformer...'.format(self.name)) 125 | self.transformer.save(self.cache_filepath_step_transformer) 126 | else: 127 | logger.info('step {} transforming...'.format(self.name)) 128 | step_output_data = self.transformer.transform(**step_inputs) 129 | 130 | if self.cache_output: 131 | logger.info('step {} caching outputs...'.format(self.name)) 132 | self._cached_output = step_output_data 133 | if self.save_output: 134 | logger.info('step {} saving outputs...'.format(self.name)) 135 | self._save_output(step_output_data, self.save_filepath_step_output) 136 | return step_output_data 137 | 138 | def _load_output(self, filepath): 139 | return joblib.load(filepath) 140 | 141 | def _save_output(self, output_data, filepath): 142 | joblib.dump(output_data, filepath) 143 | 144 | def transform(self, data): 145 | if self.output_is_cached: 146 | logger.info('step {} loading output...'.format(self.name)) 147 | return self._cached_output 148 | elif self.output_is_saved and self.load_saved_output: 149 | logger.info('step {} loading output...'.format(self.name)) 150 | return self._load_output(self.save_filepath_step_output) 151 | else: 152 | step_inputs = {} 153 | if self.input_data is not None: 154 | for input_data_part in self.input_data: 155 | step_inputs[input_data_part] = data[input_data_part] 156 | 157 | for input_step in self.input_steps: 158 | step_inputs[input_step.name] = input_step.transform(data) 159 | 160 | if self.adapter: 161 | step_inputs = self.adapt(step_inputs) 162 | else: 163 | step_inputs = self.unpack(step_inputs) 164 | return self._cached_transform(step_inputs) 165 | 166 | def _cached_transform(self, step_inputs): 167 | if self.is_trainable: 168 | if self.transformer_is_cached: 169 | logger.info('step {} loading transformer...'.format(self.name)) 170 | self.transformer.load(self.cache_filepath_step_transformer) 171 | logger.info('step {} transforming...'.format(self.name)) 172 | step_output_data = self.transformer.transform(**step_inputs) 173 | else: 174 | raise ValueError('No transformer cached {}'.format(self.name)) 175 | else: 176 | logger.info('step {} transforming...'.format(self.name)) 177 | step_output_data = self.transformer.transform(**step_inputs) 178 | 179 | if self.cache_output: 180 | logger.info('step {} caching outputs...'.format(self.name)) 181 | self._cached_output = step_output_data 182 | if self.save_output: 183 | logger.info('step {} saving outputs...'.format(self.name)) 184 | self._save_output(step_output_data, self.save_filepath_step_output) 185 | return step_output_data 186 | 187 | def adapt(self, step_inputs): 188 | logger.info('step {} adapting inputs'.format(self.name)) 189 | adapted_steps = {} 190 | for adapted_name, mapping in self.adapter.items(): 191 | if isinstance(mapping, str): 192 | adapted_steps[adapted_name] = step_inputs[mapping] 193 | else: 194 | if len(mapping) == 2: 195 | (step_mapping, func) = mapping 196 | elif len(mapping) == 1: 197 | step_mapping = mapping 198 | func = identity_inputs 199 | else: 200 | raise ValueError('wrong mapping specified') 201 | 202 | raw_inputs = [step_inputs[step_name][step_var] for step_name, step_var in step_mapping] 203 | adapted_steps[adapted_name] = func(raw_inputs) 204 | return adapted_steps 205 | 206 | def unpack(self, step_inputs): 207 | logger.info('step {} unpacking inputs'.format(self.name)) 208 | unpacked_steps = {} 209 | for step_name, step_dict in step_inputs.items(): 210 | unpacked_steps = {**unpacked_steps, **step_dict} 211 | return unpacked_steps 212 | 213 | @property 214 | def all_steps(self): 215 | all_steps = {} 216 | all_steps = self._get_steps(all_steps) 217 | return all_steps 218 | 219 | def _get_steps(self, all_steps): 220 | for input_step in self.input_steps: 221 | all_steps = input_step._get_steps(all_steps) 222 | all_steps[self.name] = self 223 | return all_steps 224 | 225 | @property 226 | def graph_info(self): 227 | graph_info = {'edges': set(), 228 | 'nodes': set()} 229 | 230 | graph_info = self._get_graph_info(graph_info) 231 | 232 | return graph_info 233 | 234 | def _get_graph_info(self, graph_info): 235 | for input_step in self.input_steps: 236 | graph_info = input_step._get_graph_info(graph_info) 237 | graph_info['edges'].add((input_step.name, self.name)) 238 | graph_info['nodes'].add(self.name) 239 | for input_data in self.input_data: 240 | graph_info['nodes'].add(input_data) 241 | graph_info['edges'].add((input_data, self.name)) 242 | return graph_info 243 | 244 | def plot_graph(self, filepath): 245 | plot_graph(self.graph_info, filepath) 246 | 247 | def __str__(self): 248 | return pprint.pformat(self.graph_info) 249 | 250 | def _repr_html_(self): 251 | return view_graph(self.graph_info) 252 | 253 | 254 | class BaseTransformer: 255 | def fit(self, *args, **kwargs): 256 | return self 257 | 258 | def transform(self, *args, **kwargs): 259 | return NotImplementedError 260 | 261 | def fit_transform(self, *args, **kwargs): 262 | self.fit(*args, **kwargs) 263 | return self.transform(*args, **kwargs) 264 | 265 | def load(self, filepath): 266 | return self 267 | 268 | def save(self, filepath): 269 | joblib.dump({}, filepath) 270 | 271 | 272 | class MockTransformer(BaseTransformer): 273 | def fit(self, *args, **kwargs): 274 | return self 275 | 276 | def transform(self, *args, **kwargs): 277 | return 278 | 279 | def fit_transform(self, *args, **kwargs): 280 | self.fit(*args, **kwargs) 281 | return self.transform(*args, **kwargs) 282 | 283 | 284 | class Dummy(BaseTransformer): 285 | def transform(self, **kwargs): 286 | return kwargs 287 | 288 | 289 | def to_tuple_inputs(inputs): 290 | return tuple(inputs) 291 | 292 | 293 | def identity_inputs(inputs): 294 | return inputs[0] 295 | 296 | 297 | def sparse_hstack_inputs(inputs): 298 | return sparse.hstack(inputs) 299 | 300 | 301 | def hstack_inputs(inputs): 302 | return np.hstack(inputs) 303 | 304 | 305 | def vstack_inputs(inputs): 306 | return np.vstack(inputs) 307 | 308 | 309 | def stack_inputs(inputs): 310 | stacked = np.stack(inputs, axis=0) 311 | return stacked 312 | 313 | 314 | def sum_inputs(inputs): 315 | stacked = np.stack(inputs, axis=0) 316 | return np.sum(stacked, axis=0) 317 | 318 | 319 | def average_inputs(inputs): 320 | stacked = np.stack(inputs, axis=0) 321 | return np.mean(stacked, axis=0) 322 | 323 | 324 | def exp_transform(inputs): 325 | return np.exp(inputs[0]) 326 | -------------------------------------------------------------------------------- /src/steps/keras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/neptune-ai/open-solution-mapping-challenge/2f1f5f17bb9dfb5ba8dfc3c312533479997bd4c9/src/steps/keras/__init__.py -------------------------------------------------------------------------------- /src/steps/keras/callbacks.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from deepsense import neptune 4 | from keras import backend as K 5 | from keras.callbacks import Callback 6 | 7 | 8 | class NeptuneMonitor(Callback): 9 | def __init__(self, model_name): 10 | super().__init__() 11 | self.model_name = model_name 12 | self.ctx = neptune.Context() 13 | self.batch_loss_channel_name = get_correct_channel_name(self.ctx, 14 | '{} Batch Log-loss training'.format(self.model_name)) 15 | self.epoch_loss_channel_name = get_correct_channel_name(self.ctx, 16 | '{} Log-loss training'.format(self.model_name)) 17 | self.epoch_val_loss_channel_name = get_correct_channel_name(self.ctx, 18 | '{} Log-loss validation'.format(self.model_name)) 19 | 20 | self.epoch_id = 0 21 | self.batch_id = 0 22 | 23 | def on_batch_end(self, batch, logs={}): 24 | self.batch_id += 1 25 | self.ctx.channel_send(self.batch_loss_channel_name, self.batch_id, logs['loss']) 26 | 27 | def on_epoch_end(self, epoch, logs={}): 28 | self.epoch_id += 1 29 | self.ctx.channel_send(self.epoch_loss_channel_name, self.epoch_id, logs['loss']) 30 | self.ctx.channel_send(self.epoch_val_loss_channel_name, self.epoch_id, logs['val_loss']) 31 | 32 | 33 | class ReduceLR(Callback): 34 | def __init__(self, gamma): 35 | self.gamma = gamma 36 | 37 | def on_epoch_end(self, epoch, logs={}): 38 | if self.gamma is not None: 39 | K.set_value(self.model.optimizer.lr, self.gamma * K.get_value(self.model.optimizer.lr)) 40 | 41 | 42 | class UnfreezeLayers(Callback): 43 | def __init__(self, unfreeze_on_epoch, from_layer=0, to_layer=1): 44 | self.unfreeze_on_epoch = unfreeze_on_epoch 45 | self.from_layer = from_layer 46 | self.to_layer = to_layer 47 | 48 | self.epoch_id = 0 49 | self.batch_id = 0 50 | 51 | def on_epoch_end(self, epoch, logs={}): 52 | if self.epoch_id == self.unfreeze_on_epoch: 53 | for i, layer in enumerate(self.model.layers): 54 | if i >= self.from_layer and i <= self.to_layer: 55 | layer.trainable = True 56 | self.epoch_id += 1 57 | 58 | 59 | def get_correct_channel_name(ctx, name): 60 | channels_with_name = [channel for channel in ctx._experiment._channels if name in channel.name] 61 | if len(channels_with_name) == 0: 62 | return name 63 | else: 64 | channel_ids = [re.split('[^\d]', channel.name)[-1] for channel in channels_with_name] 65 | channel_ids = sorted([int(idx) if idx != '' else 0 for idx in channel_ids]) 66 | last_id = channel_ids[-1] 67 | corrected_name = '{} {}'.format(name, last_id + 1) 68 | return corrected_name 69 | -------------------------------------------------------------------------------- /src/steps/keras/contrib.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import, division 3 | 4 | import sys 5 | from os.path import dirname 6 | 7 | sys.path.append(dirname(dirname(__file__))) 8 | from keras import initializers 9 | from keras.engine import InputSpec, Layer 10 | from keras import backend as K 11 | import tensorflow as tf 12 | 13 | 14 | class AttentionWeightedAverage(Layer): 15 | """ 16 | Computes a weighted average of the different channels across timesteps. 17 | Uses 1 parameter pr. channel to compute the attention value for a single timestep. 18 | """ 19 | 20 | def __init__(self, return_attention=False, **kwargs): 21 | self.init = initializers.get('uniform') 22 | self.supports_masking = True 23 | self.return_attention = return_attention 24 | super(AttentionWeightedAverage, self).__init__(**kwargs) 25 | 26 | def build(self, input_shape): 27 | self.input_spec = [InputSpec(ndim=3)] 28 | assert len(input_shape) == 3 29 | 30 | self.W = self.add_weight(shape=(input_shape[2], 1), 31 | name='{}_W'.format(self.name), 32 | initializer=self.init) 33 | self.trainable_weights = [self.W] 34 | super(AttentionWeightedAverage, self).build(input_shape) 35 | 36 | def call(self, x, mask=None): 37 | # computes a probability distribution over the timesteps 38 | # uses 'max trick' for numerical stability 39 | # reshape is done to avoid issue with Tensorflow 40 | # and 1-dimensional weights 41 | logits = K.dot(x, self.W) 42 | x_shape = K.shape(x) 43 | logits = K.reshape(logits, (x_shape[0], x_shape[1])) 44 | ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True)) 45 | 46 | # masked timesteps have zero weight 47 | if mask is not None: 48 | mask = K.cast(mask, K.floatx()) 49 | ai = ai * mask 50 | att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon()) 51 | weighted_input = x * K.expand_dims(att_weights) 52 | result = K.sum(weighted_input, axis=1) 53 | if self.return_attention: 54 | return [result, att_weights] 55 | return result 56 | 57 | def get_output_shape_for(self, input_shape): 58 | return self.compute_output_shape(input_shape) 59 | 60 | def compute_output_shape(self, input_shape): 61 | output_len = input_shape[2] 62 | if self.return_attention: 63 | return [(input_shape[0], output_len), (input_shape[0], input_shape[1])] 64 | return (input_shape[0], output_len) 65 | 66 | def compute_mask(self, input, input_mask=None): 67 | if isinstance(input_mask, list): 68 | return [None] * len(input_mask) 69 | else: 70 | return None 71 | 72 | 73 | def pair_loss(y_true, y_pred): 74 | y_true = tf.cast(y_true, tf.int32) 75 | parts = tf.dynamic_partition(y_pred, y_true, 2) 76 | y_pos = parts[1] 77 | y_neg = parts[0] 78 | y_pos = tf.expand_dims(y_pos, 0) 79 | y_neg = tf.expand_dims(y_neg, -1) 80 | out = K.sigmoid(y_neg - y_pos) 81 | return K.mean(out) 82 | -------------------------------------------------------------------------------- /src/steps/keras/embeddings.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from gensim.models import KeyedVectors 3 | from sklearn.externals import joblib 4 | 5 | from ..base import BaseTransformer 6 | 7 | 8 | class EmbeddingsMatrix(BaseTransformer): 9 | def __init__(self, pretrained_filepath, max_features, embedding_size): 10 | self.pretrained_filepath = pretrained_filepath 11 | self.max_features = max_features 12 | self.embedding_size = embedding_size 13 | 14 | def fit(self, tokenizer): 15 | self.embedding_matrix = self._get_embedding_matrix(tokenizer) 16 | return self 17 | 18 | def transform(self, tokenizer): 19 | return {'embeddings_matrix': self.embedding_matrix} 20 | 21 | def _get_embedding_matrix(self, tokenizer): 22 | return NotImplementedError 23 | 24 | def save(self, filepath): 25 | joblib.dump(self.embedding_matrix, filepath) 26 | 27 | def load(self, filepath): 28 | self.embedding_matrix = joblib.load(filepath) 29 | return self 30 | 31 | 32 | class GloveEmbeddingsMatrix(EmbeddingsMatrix): 33 | def _get_embedding_matrix(self, tokenizer): 34 | return load_glove_embeddings(self.pretrained_filepath, 35 | tokenizer, 36 | self.max_features, 37 | self.embedding_size) 38 | 39 | 40 | class Word2VecEmbeddingsMatrix(EmbeddingsMatrix): 41 | def _get_embedding_matrix(self, tokenizer): 42 | return load_word2vec_embeddings(self.pretrained_filepath, 43 | tokenizer, 44 | self.max_features, 45 | self.embedding_size) 46 | 47 | 48 | class FastTextEmbeddingsMatrix(EmbeddingsMatrix): 49 | def _get_embedding_matrix(self, tokenizer): 50 | return load_fasttext_embeddings(self.pretrained_filepath, 51 | tokenizer, 52 | self.max_features, 53 | self.embedding_size) 54 | 55 | 56 | def load_glove_embeddings(filepath, tokenizer, max_features, embedding_size): 57 | embeddings_index = dict() 58 | with open(filepath) as f: 59 | for line in f: 60 | # Note: use split(' ') instead of split() if you get an error. 61 | values = line.split(' ') 62 | word = values[0] 63 | coefs = np.asarray(values[1:], dtype='float32') 64 | embeddings_index[word] = coefs 65 | 66 | all_embs = np.stack(embeddings_index.values()) 67 | emb_mean, emb_std = all_embs.mean(), all_embs.std() 68 | 69 | word_index = tokenizer.word_index 70 | nb_words = min(max_features, len(word_index)) 71 | embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embedding_size)) 72 | for word, i in word_index.items(): 73 | if i >= max_features: 74 | continue 75 | embedding_vector = embeddings_index.get(word) 76 | if embedding_vector is not None: 77 | embedding_matrix[i] = embedding_vector 78 | return embedding_matrix 79 | 80 | 81 | def load_word2vec_embeddings(filepath, tokenizer, max_features, embedding_size): 82 | model = KeyedVectors.load_word2vec_format(filepath, binary=True) 83 | 84 | emb_mean, emb_std = model.wv.syn0.mean(), model.wv.syn0.std() 85 | 86 | word_index = tokenizer.word_index 87 | nb_words = min(max_features, len(word_index)) 88 | embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embedding_size)) 89 | for word, i in word_index.items(): 90 | if i >= max_features: 91 | continue 92 | try: 93 | embedding_vector = model[word] 94 | embedding_matrix[i] = embedding_vector 95 | except KeyError: 96 | continue 97 | return embedding_matrix 98 | 99 | 100 | def load_fasttext_embeddings(filepath, tokenizer, max_features, embedding_size): 101 | embeddings_index = dict() 102 | with open(filepath) as f: 103 | for i, line in enumerate(f): 104 | line = line.strip() 105 | if i == 0: 106 | continue 107 | values = line.split(' ') 108 | word = values[0] 109 | coefs = np.asarray(values[1:], dtype='float32') 110 | if coefs.shape[0] != embedding_size: 111 | continue 112 | embeddings_index[word] = coefs 113 | 114 | all_embs = np.stack(embeddings_index.values()) 115 | emb_mean, emb_std = all_embs.mean(), all_embs.std() 116 | 117 | word_index = tokenizer.word_index 118 | nb_words = min(max_features, len(word_index)) 119 | embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embedding_size)) 120 | for word, i in word_index.items(): 121 | if i >= max_features: 122 | continue 123 | embedding_vector = embeddings_index.get(word) 124 | if embedding_vector is not None: 125 | embedding_matrix[i] = embedding_vector 126 | return embedding_matrix 127 | -------------------------------------------------------------------------------- /src/steps/keras/loaders.py: -------------------------------------------------------------------------------- 1 | from keras.preprocessing import text, sequence 2 | from sklearn.externals import joblib 3 | 4 | from ..base import BaseTransformer 5 | 6 | 7 | class Tokenizer(BaseTransformer): 8 | def __init__(self, char_level, maxlen, num_words): 9 | self.char_level = char_level 10 | self.maxlen = maxlen 11 | self.num_words = num_words 12 | 13 | self.tokenizer = text.Tokenizer(char_level=self.char_level, num_words=self.num_words) 14 | 15 | def fit(self, X, X_valid=None, train_mode=True): 16 | self.tokenizer.fit_on_texts(X) 17 | return self 18 | 19 | def transform(self, X, X_valid=None, train_mode=True): 20 | X_tokenized = self._transform(X) 21 | 22 | if X_valid is not None: 23 | X_valid_tokenized = self._transform(X_valid) 24 | else: 25 | X_valid_tokenized = None 26 | return {'X': X_tokenized, 27 | 'X_valid': X_valid_tokenized, 28 | 'tokenizer': self.tokenizer} 29 | 30 | def _transform(self, X): 31 | list_tokenized = self.tokenizer.texts_to_sequences(list(X)) 32 | X_tokenized = sequence.pad_sequences(list_tokenized, maxlen=self.maxlen) 33 | return X_tokenized 34 | 35 | def load(self, filepath): 36 | object_pickle = joblib.load(filepath) 37 | self.char_level = object_pickle['char_level'] 38 | self.maxlen = object_pickle['maxlen'] 39 | self.num_words = object_pickle['num_words'] 40 | self.tokenizer = object_pickle['tokenizer'] 41 | return self 42 | 43 | def save(self, filepath): 44 | object_pickle = {'char_level': self.char_level, 45 | 'maxlen': self.maxlen, 46 | 'num_words': self.num_words, 47 | 'tokenizer': self.tokenizer} 48 | joblib.dump(object_pickle, filepath) 49 | 50 | 51 | class TextAugmenter(BaseTransformer): 52 | pass 53 | """ 54 | Augmentations by Thesaurus synonim substitution or typos 55 | """ 56 | -------------------------------------------------------------------------------- /src/steps/keras/models.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | from keras.models import load_model 4 | 5 | from .architectures import vdcnn, scnn, dpcnn, cudnn_gru, cudnn_lstm 6 | from .contrib import AttentionWeightedAverage 7 | from ..base import BaseTransformer 8 | 9 | 10 | class KerasModelTransformer(BaseTransformer): 11 | """ 12 | Todo: 13 | load the best model at the end of the fit and save it 14 | """ 15 | 16 | def __init__(self, architecture_config, training_config, callbacks_config): 17 | self.architecture_config = architecture_config 18 | self.training_config = training_config 19 | self.callbacks_config = callbacks_config 20 | 21 | def reset(self): 22 | self.model = self._build_model(**self.architecture_config) 23 | 24 | def _compile_model(self, model_params, optimizer_params): 25 | model = self._build_model(**model_params) 26 | optimizer = self._build_optimizer(**optimizer_params) 27 | loss = self._build_loss() 28 | model.compile(optimizer=optimizer, loss=loss) 29 | return model 30 | 31 | def _create_callbacks(self, **kwargs): 32 | return NotImplementedError 33 | 34 | def _build_model(self, **kwargs): 35 | return NotImplementedError 36 | 37 | def _build_optimizer(self, **kwargs): 38 | return NotImplementedError 39 | 40 | def _build_loss(self, **kwargs): 41 | return NotImplementedError 42 | 43 | def save(self, filepath): 44 | checkpoint_callback = self.callbacks_config.get('model_checkpoint') 45 | if checkpoint_callback: 46 | checkpoint_filepath = checkpoint_callback['filepath'] 47 | shutil.copyfile(checkpoint_filepath, filepath) 48 | else: 49 | self.model.save(filepath) 50 | 51 | def load(self, filepath): 52 | self.model = load_model(filepath, 53 | custom_objects={'AttentionWeightedAverage': AttentionWeightedAverage}) 54 | return self 55 | 56 | 57 | class ClassifierXY(KerasModelTransformer): 58 | def fit(self, X, y, validation_data, *args, **kwargs): 59 | self.callbacks = self._create_callbacks(**self.callbacks_config) 60 | self.model = self._compile_model(**self.architecture_config) 61 | 62 | self.model.fit(X, y, 63 | validation_data=validation_data, 64 | callbacks=self.callbacks, 65 | verbose=1, 66 | **self.training_config) 67 | return self 68 | 69 | def transform(self, X, y=None, validation_data=None, *args, **kwargs): 70 | predictions = self.model.predict(X, verbose=1) 71 | return {'prediction_probability': predictions} 72 | 73 | 74 | class ClassifierGenerator(KerasModelTransformer): 75 | def fit(self, datagen, validation_datagen, *args, **kwargs): 76 | self.callbacks = self._create_callbacks(**self.callbacks_config) 77 | self.model = self._compile_model(**self.architecture_config) 78 | 79 | train_flow, train_steps = datagen 80 | valid_flow, valid_steps = validation_datagen 81 | self.model.fit_generator(train_flow, 82 | steps_per_epoch=train_steps, 83 | validation_data=valid_flow, 84 | validation_steps=valid_steps, 85 | callbacks=self.callbacks, 86 | verbose=1, 87 | **self.training_config) 88 | return self 89 | 90 | def transform(self, datagen, validation_datagen=None, *args, **kwargs): 91 | test_flow, test_steps = datagen 92 | predictions = self.model.predict_generator(test_flow, test_steps, verbose=1) 93 | return {'prediction_probability': predictions} 94 | 95 | 96 | class PretrainedEmbeddingModel(ClassifierXY): 97 | def fit(self, X, y, validation_data, embedding_matrix): 98 | X_valid, y_valid = validation_data 99 | self.callbacks = self._create_callbacks(**self.callbacks_config) 100 | self.architecture_config['model_params']['embedding_matrix'] = embedding_matrix 101 | self.model = self._compile_model(**self.architecture_config) 102 | self.model.fit(X, y, 103 | validation_data=[X_valid, y_valid], 104 | callbacks=self.callbacks, 105 | verbose=1, 106 | **self.training_config) 107 | return self 108 | 109 | def transform(self, X, y=None, validation_data=None, embedding_matrix=None): 110 | predictions = self.model.predict(X, verbose=1) 111 | return {'prediction_probability': predictions} 112 | 113 | 114 | class CharVDCNNTransformer(ClassifierXY): 115 | def _build_model(self, embedding_size, maxlen, max_features, 116 | filter_nr, kernel_size, repeat_block, 117 | dense_size, repeat_dense, output_size, output_activation, 118 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 119 | dropout_embedding, conv_dropout, dense_dropout, dropout_mode, 120 | conv_kernel_reg_l2, conv_bias_reg_l2, 121 | dense_kernel_reg_l2, dense_bias_reg_l2, 122 | use_prelu, use_batch_norm, batch_norm_first): 123 | return vdcnn(embedding_size, maxlen, max_features, 124 | filter_nr, kernel_size, repeat_block, 125 | dense_size, repeat_dense, output_size, output_activation, 126 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 127 | dropout_embedding, conv_dropout, dense_dropout, dropout_mode, 128 | conv_kernel_reg_l2, conv_bias_reg_l2, 129 | dense_kernel_reg_l2, dense_bias_reg_l2, 130 | use_prelu, use_batch_norm, batch_norm_first) 131 | 132 | 133 | class WordSCNNTransformer(PretrainedEmbeddingModel): 134 | def _build_model(self, embedding_matrix, embedding_size, trainable_embedding, maxlen, max_features, 135 | filter_nr, kernel_size, repeat_block, 136 | dense_size, repeat_dense, output_size, output_activation, 137 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 138 | dropout_embedding, conv_dropout, dense_dropout, dropout_mode, 139 | conv_kernel_reg_l2, conv_bias_reg_l2, 140 | dense_kernel_reg_l2, dense_bias_reg_l2, 141 | use_prelu, use_batch_norm, batch_norm_first): 142 | return scnn(embedding_matrix, embedding_size, trainable_embedding, maxlen, max_features, 143 | filter_nr, kernel_size, repeat_block, 144 | dense_size, repeat_dense, output_size, output_activation, 145 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 146 | dropout_embedding, conv_dropout, dense_dropout, dropout_mode, 147 | conv_kernel_reg_l2, conv_bias_reg_l2, 148 | dense_kernel_reg_l2, dense_bias_reg_l2, 149 | use_prelu, use_batch_norm, batch_norm_first) 150 | 151 | 152 | class WordDPCNNTransformer(PretrainedEmbeddingModel): 153 | def _build_model(self, embedding_matrix, embedding_size, trainable_embedding, maxlen, max_features, 154 | filter_nr, kernel_size, repeat_block, 155 | dense_size, repeat_dense, output_size, output_activation, 156 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 157 | dropout_embedding, conv_dropout, dense_dropout, dropout_mode, 158 | conv_kernel_reg_l2, conv_bias_reg_l2, 159 | dense_kernel_reg_l2, dense_bias_reg_l2, 160 | use_prelu, use_batch_norm, batch_norm_first): 161 | """ 162 | Implementation of http://ai.tencent.com/ailab/media/publications/ACL3-Brady.pdf 163 | """ 164 | return dpcnn(embedding_matrix, embedding_size, trainable_embedding, maxlen, max_features, 165 | filter_nr, kernel_size, repeat_block, 166 | dense_size, repeat_dense, output_size, output_activation, 167 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 168 | dropout_embedding, conv_dropout, dense_dropout, dropout_mode, 169 | conv_kernel_reg_l2, conv_bias_reg_l2, 170 | dense_kernel_reg_l2, dense_bias_reg_l2, 171 | use_prelu, use_batch_norm, batch_norm_first) 172 | 173 | 174 | class WordCuDNNLSTMTransformer(PretrainedEmbeddingModel): 175 | def _build_model(self, embedding_matrix, embedding_size, trainable_embedding, 176 | maxlen, max_features, 177 | unit_nr, repeat_block, 178 | dense_size, repeat_dense, output_size, output_activation, 179 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 180 | dropout_embedding, rnn_dropout, dense_dropout, dropout_mode, 181 | rnn_kernel_reg_l2, rnn_recurrent_reg_l2, rnn_bias_reg_l2, 182 | dense_kernel_reg_l2, dense_bias_reg_l2, 183 | use_prelu, use_batch_norm, batch_norm_first): 184 | return cudnn_lstm(embedding_matrix, embedding_size, trainable_embedding, 185 | maxlen, max_features, 186 | unit_nr, repeat_block, 187 | dense_size, repeat_dense, output_size, output_activation, 188 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 189 | dropout_embedding, rnn_dropout, dense_dropout, dropout_mode, 190 | rnn_kernel_reg_l2, rnn_recurrent_reg_l2, rnn_bias_reg_l2, 191 | dense_kernel_reg_l2, dense_bias_reg_l2, 192 | use_prelu, use_batch_norm, batch_norm_first) 193 | 194 | 195 | class WordCuDNNGRUTransformer(PretrainedEmbeddingModel): 196 | def _build_model(self, embedding_matrix, embedding_size, trainable_embedding, 197 | maxlen, max_features, 198 | unit_nr, repeat_block, 199 | dense_size, repeat_dense, output_size, output_activation, 200 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 201 | dropout_embedding, rnn_dropout, dense_dropout, dropout_mode, 202 | rnn_kernel_reg_l2, rnn_recurrent_reg_l2, rnn_bias_reg_l2, 203 | dense_kernel_reg_l2, dense_bias_reg_l2, 204 | use_prelu, use_batch_norm, batch_norm_first): 205 | return cudnn_gru(embedding_matrix, embedding_size, trainable_embedding, 206 | maxlen, max_features, 207 | unit_nr, repeat_block, 208 | dense_size, repeat_dense, output_size, output_activation, 209 | max_pooling, mean_pooling, weighted_average_attention, concat_mode, 210 | dropout_embedding, rnn_dropout, dense_dropout, dropout_mode, 211 | rnn_kernel_reg_l2, rnn_recurrent_reg_l2, rnn_bias_reg_l2, 212 | dense_kernel_reg_l2, dense_bias_reg_l2, 213 | use_prelu, use_batch_norm, batch_norm_first) 214 | -------------------------------------------------------------------------------- /src/steps/misc.py: -------------------------------------------------------------------------------- 1 | import lightgbm as lgb 2 | from attrdict import AttrDict 3 | from sklearn.externals import joblib 4 | 5 | from .base import BaseTransformer 6 | from .utils import get_logger 7 | 8 | logger = get_logger() 9 | 10 | 11 | class LightGBM(BaseTransformer): 12 | def __init__(self, model_config, training_config): 13 | self.model_config = AttrDict(model_config) 14 | self.training_config = AttrDict(training_config) 15 | self.evaluation_function = None 16 | 17 | def fit(self, X, y, X_valid, y_valid, feature_names, categorical_features, **kwargs): 18 | train = lgb.Dataset(X, label=y, 19 | feature_name=feature_names, 20 | categorical_feature=categorical_features 21 | ) 22 | valid = lgb.Dataset(X_valid, label=y_valid, 23 | feature_name=feature_names, 24 | categorical_feature=categorical_features 25 | ) 26 | 27 | evaluation_results = {} 28 | self.estimator = lgb.train(self.model_config, 29 | train, valid_sets=[train, valid], valid_names=['train', 'valid'], 30 | evals_result=evaluation_results, 31 | num_boost_round=self.training_config.number_boosting_rounds, 32 | early_stopping_rounds=self.training_config.early_stopping_rounds, 33 | verbose_eval=self.model_config.verbose, 34 | feval=self.evaluation_function) 35 | return self 36 | 37 | def transform(self, X, y=None, **kwargs): 38 | prediction = self.estimator.predict(X) 39 | return {'prediction': prediction} 40 | 41 | def load(self, filepath): 42 | self.estimator = joblib.load(filepath) 43 | return self 44 | 45 | def save(self, filepath): 46 | joblib.dump(self.estimator, filepath) 47 | -------------------------------------------------------------------------------- /src/steps/postprocessing.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from sklearn.externals import joblib 4 | 5 | from .base import BaseTransformer 6 | 7 | 8 | class ClassPredictor(BaseTransformer): 9 | def transform(self, prediction_proba): 10 | predictions_class = np.argmax(prediction_proba, axis=1) 11 | return {'y_pred': predictions_class} 12 | 13 | def load(self, filepath): 14 | return ClassPredictor() 15 | 16 | def save(self, filepath): 17 | joblib.dump({}, filepath) 18 | 19 | 20 | class PredictionAverage(BaseTransformer): 21 | def __init__(self, weights=None): 22 | self.weights = weights 23 | 24 | def transform(self, prediction_proba_list): 25 | if self.weights is not None: 26 | reshaped_weights = self._reshape_weights(prediction_proba_list.shape) 27 | prediction_proba_list *= reshaped_weights 28 | avg_pred = np.sum(prediction_proba_list, axis=0) 29 | else: 30 | avg_pred = np.mean(prediction_proba_list, axis=0) 31 | return {'prediction_probability': avg_pred} 32 | 33 | def load(self, filepath): 34 | params = joblib.load(filepath) 35 | self.weights = params['weights'] 36 | return self 37 | 38 | def save(self, filepath): 39 | joblib.dump({'weights': self.weights}, filepath) 40 | 41 | def _reshape_weights(self, prediction_shape): 42 | dim = len(prediction_shape) 43 | reshape_dim = (-1,) + tuple([1] * (dim - 1)) 44 | reshaped_weights = np.array(self.weights).reshape(reshape_dim) 45 | return reshaped_weights 46 | 47 | 48 | class PredictionAverageUnstack(BaseTransformer): 49 | def transform(self, prediction_probability, id_list): 50 | df = pd.DataFrame(prediction_probability) 51 | df['id'] = id_list 52 | avg_pred = df.groupby('id').mean().reset_index().drop(['id'], axis=1).values 53 | return {'prediction_probability': avg_pred} 54 | 55 | def load(self, filepath): 56 | return self 57 | 58 | def save(self, filepath): 59 | joblib.dump({}, filepath) 60 | 61 | 62 | class ProbabilityCalibration(BaseTransformer): 63 | def __init__(self, power): 64 | self.power = power 65 | 66 | def fit(self, prediction_proba): 67 | return self 68 | 69 | def transform(self, prediction_probability): 70 | prediction_probability = np.array(prediction_probability) ** self.power 71 | return {'prediction_probability': prediction_probability} 72 | 73 | def load(self, filepath): 74 | return self 75 | 76 | def save(self, filepath): 77 | joblib.dump({}, filepath) 78 | -------------------------------------------------------------------------------- /src/steps/preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/neptune-ai/open-solution-mapping-challenge/2f1f5f17bb9dfb5ba8dfc3c312533479997bd4c9/src/steps/preprocessing/__init__.py -------------------------------------------------------------------------------- /src/steps/preprocessing/misc.py: -------------------------------------------------------------------------------- 1 | from sklearn.externals import joblib 2 | 3 | from ..base import BaseTransformer 4 | 5 | 6 | class XYSplit(BaseTransformer): 7 | def __init__(self, x_columns, y_columns): 8 | self.x_columns = x_columns 9 | self.y_columns = y_columns 10 | 11 | def transform(self, meta, train_mode): 12 | X = meta[self.x_columns].values 13 | if train_mode: 14 | y = meta[self.y_columns].values 15 | else: 16 | y = None 17 | 18 | return {'X': X, 19 | 'y': y} 20 | 21 | def load(self, filepath): 22 | params = joblib.load(filepath) 23 | self.columns_to_get = params['x_columns'] 24 | self.target_columns = params['y_columns'] 25 | return self 26 | 27 | def save(self, filepath): 28 | params = {'x_columns': self.x_columns, 29 | 'y_columns': self.y_columns 30 | } 31 | joblib.dump(params, filepath) -------------------------------------------------------------------------------- /src/steps/preprocessing/text.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | import string 4 | 5 | import nltk 6 | import numpy as np 7 | import pandas as pd 8 | from nltk.corpus import stopwords 9 | from nltk.stem.wordnet import WordNetLemmatizer 10 | from nltk.tokenize import TweetTokenizer 11 | from sklearn.externals import joblib 12 | 13 | from ..base import BaseTransformer 14 | 15 | lem = WordNetLemmatizer() 16 | tokenizer = TweetTokenizer() 17 | nltk.download('wordnet') 18 | nltk.download('stopwords') 19 | eng_stopwords = set(stopwords.words("english")) 20 | with open('steps/resources/apostrophes.json', 'r') as f: 21 | APPO = json.load(f) 22 | 23 | 24 | class WordListFilter(BaseTransformer): 25 | def __init__(self, word_list_filepath): 26 | self.word_set = self._read_data(word_list_filepath) 27 | 28 | def transform(self, X): 29 | X = self._transform(X) 30 | return {'X': X} 31 | 32 | def _transform(self, X): 33 | X = pd.DataFrame(X, columns=['text']).astype(str) 34 | X['text'] = X['text'].apply(self._filter_words) 35 | return X['text'].values 36 | 37 | def _filter_words(self, x): 38 | x = x.lower() 39 | x = ' '.join([w for w in x.split() if w in self.word_set]) 40 | return x 41 | 42 | def _read_data(self, filepath): 43 | with open(filepath, 'r+') as f: 44 | data = f.read() 45 | return set(data.split('\n')) 46 | 47 | def load(self, filepath): 48 | return self 49 | 50 | def save(self, filepath): 51 | joblib.dump({}, filepath) 52 | 53 | 54 | class TextCleaner(BaseTransformer): 55 | def __init__(self, 56 | drop_punctuation, 57 | drop_newline, 58 | drop_multispaces, 59 | all_lower_case, 60 | fill_na_with, 61 | deduplication_threshold, 62 | anonymize, 63 | apostrophes, 64 | use_stopwords): 65 | self.drop_punctuation = drop_punctuation 66 | self.drop_newline = drop_newline 67 | self.drop_multispaces = drop_multispaces 68 | self.all_lower_case = all_lower_case 69 | self.fill_na_with = fill_na_with 70 | self.deduplication_threshold = deduplication_threshold 71 | self.anonymize = anonymize 72 | self.apostrophes = apostrophes 73 | self.use_stopwords = use_stopwords 74 | 75 | def transform(self, X): 76 | X = pd.DataFrame(X, columns=['text']).astype(str) 77 | X['text'] = X['text'].apply(self._transform) 78 | if self.fill_na_with: 79 | X['text'] = X['text'].fillna(self.fill_na_with).values 80 | return {'X': X['text'].values} 81 | 82 | def _transform(self, x): 83 | if self.all_lower_case: 84 | x = self._lower(x) 85 | if self.drop_punctuation: 86 | x = self._remove_punctuation(x) 87 | if self.drop_newline: 88 | x = self._remove_newline(x) 89 | if self.drop_multispaces: 90 | x = self._substitute_multiple_spaces(x) 91 | if self.deduplication_threshold is not None: 92 | x = self._deduplicate(x) 93 | if self.anonymize: 94 | x = self._anonymize(x) 95 | if self.apostrophes: 96 | x = self._apostrophes(x) 97 | if self.use_stopwords: 98 | x = self._use_stopwords(x) 99 | return x 100 | 101 | def _use_stopwords(self, x): 102 | words = tokenizer.tokenize(x) 103 | words = [w for w in words if not w in eng_stopwords] 104 | x = " ".join(words) 105 | return x 106 | 107 | def _apostrophes(self, x): 108 | words = tokenizer.tokenize(x) 109 | words = [APPO[word] if word in APPO else word for word in words] 110 | words = [lem.lemmatize(word, "v") for word in words] 111 | words = [w for w in words if not w in eng_stopwords] 112 | x = " ".join(words) 113 | return x 114 | 115 | def _anonymize(self, x): 116 | # remove leaky elements like ip,user 117 | x = re.sub("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", x) 118 | # removing usernames 119 | x = re.sub("\[\[.*\]", " ", x) 120 | return x 121 | 122 | def _lower(self, x): 123 | return x.lower() 124 | 125 | def _remove_punctuation(self, x): 126 | return re.sub(r'[^\w\s]', ' ', x) 127 | 128 | def _remove_newline(self, x): 129 | x = x.replace('\n', ' ') 130 | x = x.replace('\n\n', ' ') 131 | return x 132 | 133 | def _substitute_multiple_spaces(self, x): 134 | return ' '.join(x.split()) 135 | 136 | def _deduplicate(self, x): 137 | word_list = x.split() 138 | num_words = len(word_list) 139 | if num_words == 0: 140 | return x 141 | else: 142 | num_unique_words = len(set(word_list)) 143 | unique_ratio = num_words / num_unique_words 144 | if unique_ratio > self.deduplication_threshold: 145 | x = ' '.join(x.split()[:num_unique_words]) 146 | return x 147 | 148 | def load(self, filepath): 149 | params = joblib.load(filepath) 150 | self.drop_punctuation = params['drop_punctuation'] 151 | self.all_lower_case = params['all_lower_case'] 152 | self.fill_na_with = params['fill_na_with'] 153 | return self 154 | 155 | def save(self, filepath): 156 | params = {'drop_punctuation': self.drop_punctuation, 157 | 'all_lower_case': self.all_lower_case, 158 | 'fill_na_with': self.fill_na_with, 159 | } 160 | joblib.dump(params, filepath) 161 | 162 | 163 | class TextCounter(BaseTransformer): 164 | def transform(self, X): 165 | X = pd.DataFrame(X, columns=['text']).astype(str) 166 | X = X['text'].apply(self._transform) 167 | X['caps_vs_length'] = self._caps_vs_length(X) 168 | X['num_symbols'] = X['text'].apply(lambda comment: sum(comment.count(w) for w in '*&$%')) 169 | X['num_words'] = X['text'].apply(lambda comment: len(comment.split())) 170 | X['num_unique_words'] = X['text'].apply(lambda comment: len(set(w for w in comment.split()))) 171 | X['words_vs_unique'] = self._words_vs_unique(X) 172 | X['mean_word_len'] = X['text'].apply(lambda x: np.mean([len(w) for w in str(x).split()])) 173 | X.drop('text', axis=1, inplace=True) 174 | X.fillna(0.0, inplace=True) 175 | return {'X': X} 176 | 177 | def _transform(self, x): 178 | features = {} 179 | features['text'] = x 180 | features['char_count'] = char_count(x) 181 | features['word_count'] = word_count(x) 182 | features['punctuation_count'] = punctuation_count(x) 183 | features['upper_case_count'] = upper_case_count(x) 184 | features['lower_case_count'] = lower_case_count(x) 185 | features['digit_count'] = digit_count(x) 186 | features['space_count'] = space_count(x) 187 | features['newline_count'] = newline_count(x) 188 | return pd.Series(features) 189 | 190 | def _caps_vs_length(self, X): 191 | try: 192 | return X.apply(lambda row: float(row['upper_case_count']) / float(row['char_count']), axis=1) 193 | except ZeroDivisionError: 194 | return 0 195 | 196 | def _words_vs_unique(self, X): 197 | try: 198 | return X['num_unique_words'] / X['num_words'] 199 | except ZeroDivisionError: 200 | return 0 201 | 202 | def load(self, filepath): 203 | return self 204 | 205 | def save(self, filepath): 206 | joblib.dump({}, filepath) 207 | 208 | 209 | def char_count(x): 210 | return len(x) 211 | 212 | 213 | def word_count(x): 214 | return len(x.split()) 215 | 216 | 217 | def newline_count(x): 218 | return x.count('\n') 219 | 220 | 221 | def upper_case_count(x): 222 | return sum(c.isupper() for c in x) 223 | 224 | 225 | def lower_case_count(x): 226 | return sum(c.islower() for c in x) 227 | 228 | 229 | def digit_count(x): 230 | return sum(c.isdigit() for c in x) 231 | 232 | 233 | def space_count(x): 234 | return sum(c.isspace() for c in x) 235 | 236 | 237 | def punctuation_count(x): 238 | return occurence(x, string.punctuation) 239 | 240 | 241 | def occurence(s1, s2): 242 | return sum([1 for x in s1 if x in s2]) 243 | -------------------------------------------------------------------------------- /src/steps/pytorch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/neptune-ai/open-solution-mapping-challenge/2f1f5f17bb9dfb5ba8dfc3c312533479997bd4c9/src/steps/pytorch/__init__.py -------------------------------------------------------------------------------- /src/steps/pytorch/architectures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/neptune-ai/open-solution-mapping-challenge/2f1f5f17bb9dfb5ba8dfc3c312533479997bd4c9/src/steps/pytorch/architectures/__init__.py -------------------------------------------------------------------------------- /src/steps/pytorch/architectures/unet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from .utils import get_downsample_pad, get_upsample_pad 5 | 6 | 7 | class UNet(nn.Module): 8 | def __init__(self, conv_kernel=3, 9 | pool_kernel=3, pool_stride=2, 10 | repeat_blocks=2, n_filters=8, 11 | batch_norm=True, dropout=0.1, 12 | in_channels=3, out_channels=2, 13 | kernel_scale=3, 14 | **kwargs): 15 | 16 | assert conv_kernel % 2 == 1 17 | assert pool_stride > 1 or pool_kernel % 2 == 1 18 | 19 | super(UNet, self).__init__() 20 | 21 | self.conv_kernel = conv_kernel 22 | self.pool_kernel = pool_kernel 23 | self.pool_stride = pool_stride 24 | self.repeat_blocks = repeat_blocks 25 | self.n_filters = n_filters 26 | self.batch_norm = batch_norm 27 | self.dropout = dropout 28 | self.in_channels = in_channels 29 | self.out_channels = out_channels 30 | self.kernel_scale = kernel_scale 31 | 32 | self.input_block = self._input_block() 33 | self.down_convs = self._down_convs() 34 | self.down_pools = self._down_pools() 35 | self.floor_block = self._floor_block() 36 | self.up_convs = self._up_convs() 37 | self.up_samples = self._up_samples() 38 | self.classification_block = self._classification_block() 39 | self.output_layer = self._output_layer() 40 | 41 | def _down_convs(self): 42 | down_convs = [] 43 | for i in range(self.repeat_blocks): 44 | in_channels = int(self.n_filters * 2 ** i) 45 | down_convs.append(DownConv(in_channels, self.conv_kernel, self.batch_norm, self.dropout)) 46 | return nn.ModuleList(down_convs) 47 | 48 | def _up_convs(self): 49 | up_convs = [] 50 | for i in range(self.repeat_blocks): 51 | in_channels = int(self.n_filters * 2 ** (i + 2)) 52 | up_convs.append(UpConv(in_channels, self.conv_kernel, self.batch_norm, self.dropout)) 53 | return nn.ModuleList(up_convs) 54 | 55 | def _down_pools(self): 56 | down_pools = [] 57 | padding = get_downsample_pad(stride=self.pool_stride, kernel=self.pool_kernel) 58 | for _ in range(self.repeat_blocks): 59 | down_pools.append(nn.MaxPool2d(kernel_size=self.pool_kernel, 60 | stride=self.pool_stride, 61 | padding=padding)) 62 | return nn.ModuleList(down_pools) 63 | 64 | def _up_samples(self): 65 | up_samples = [] 66 | kernel_scale = self.kernel_scale 67 | stride = self.pool_stride 68 | kernel_size = kernel_scale * stride 69 | padding, output_padding = get_upsample_pad(stride=stride, kernel=kernel_size) 70 | for i in range(self.repeat_blocks): 71 | in_channels = int(self.n_filters * 2 ** (i + 2)) 72 | out_channels = int(self.n_filters * 2 ** (i + 1)) 73 | up_samples.append(nn.ConvTranspose2d(in_channels=in_channels, 74 | out_channels=out_channels, 75 | kernel_size=kernel_size, 76 | stride=stride, 77 | padding=padding, 78 | output_padding=output_padding, 79 | bias=False 80 | )) 81 | return nn.ModuleList(up_samples) 82 | 83 | def _input_block(self): 84 | stride = 1 85 | padding = get_downsample_pad(stride=stride, kernel=self.conv_kernel) 86 | if self.batch_norm: 87 | input_block = nn.Sequential(nn.Conv2d(in_channels=self.in_channels, out_channels=self.n_filters, 88 | kernel_size=(self.conv_kernel, self.conv_kernel), 89 | stride=stride, padding=padding), 90 | nn.BatchNorm2d(num_features=self.n_filters), 91 | nn.ReLU(), 92 | 93 | nn.Conv2d(in_channels=self.n_filters, out_channels=self.n_filters, 94 | kernel_size=(self.conv_kernel, self.conv_kernel), 95 | stride=stride, padding=padding), 96 | nn.BatchNorm2d(num_features=self.n_filters), 97 | nn.ReLU(), 98 | 99 | nn.Dropout(self.dropout), 100 | ) 101 | else: 102 | input_block = nn.Sequential(nn.Conv2d(in_channels=self.in_channels, out_channels=self.n_filters, 103 | kernel_size=(self.conv_kernel, self.conv_kernel), 104 | stride=stride, padding=padding), 105 | nn.ReLU(), 106 | 107 | nn.Conv2d(in_channels=self.n_filters, out_channels=self.n_filters, 108 | kernel_size=(self.conv_kernel, self.conv_kernel), 109 | stride=stride, padding=padding), 110 | nn.ReLU(), 111 | 112 | nn.Dropout(self.dropout), 113 | ) 114 | return input_block 115 | 116 | def _floor_block(self): 117 | in_channels = int(self.n_filters * 2 ** self.repeat_blocks) 118 | return nn.Sequential(DownConv(in_channels, self.conv_kernel, self.batch_norm, self.dropout), 119 | ) 120 | 121 | def _classification_block(self): 122 | in_block = int(2 * self.n_filters) 123 | stride = 1 124 | padding = get_downsample_pad(stride=stride, kernel=self.conv_kernel) 125 | 126 | if self.batch_norm: 127 | classification_block = nn.Sequential(nn.Conv2d(in_channels=in_block, out_channels=self.n_filters, 128 | kernel_size=(self.conv_kernel, self.conv_kernel), 129 | stride=stride, padding=padding), 130 | nn.BatchNorm2d(num_features=self.n_filters), 131 | nn.ReLU(), 132 | nn.Dropout(self.dropout), 133 | 134 | nn.Conv2d(in_channels=self.n_filters, out_channels=self.n_filters, 135 | kernel_size=(self.conv_kernel, self.conv_kernel), 136 | stride=stride, padding=padding), 137 | nn.BatchNorm2d(num_features=self.n_filters), 138 | nn.ReLU(), 139 | ) 140 | else: 141 | classification_block = nn.Sequential(nn.Conv2d(in_channels=in_block, out_channels=self.n_filters, 142 | kernel_size=(self.conv_kernel, self.conv_kernel), 143 | stride=stride, padding=padding), 144 | nn.ReLU(), 145 | nn.Dropout(self.dropout), 146 | 147 | nn.Conv2d(in_channels=self.n_filters, out_channels=self.n_filters, 148 | kernel_size=(self.conv_kernel, self.conv_kernel), 149 | stride=stride, padding=padding), 150 | nn.ReLU(), 151 | ) 152 | return classification_block 153 | 154 | def _output_layer(self): 155 | return nn.Conv2d(in_channels=self.n_filters, out_channels=self.out_channels, 156 | kernel_size=(1, 1), stride=1, padding=0) 157 | 158 | def forward(self, x): 159 | x = self.input_block(x) 160 | 161 | down_convs_outputs = [] 162 | for block, down_pool in zip(self.down_convs, self.down_pools): 163 | x = block(x) 164 | down_convs_outputs.append(x) 165 | x = down_pool(x) 166 | x = self.floor_block(x) 167 | 168 | for down_conv_output, block, up_sample in zip(reversed(down_convs_outputs), 169 | reversed(self.up_convs), 170 | reversed(self.up_samples)): 171 | x = up_sample(x) 172 | x = torch.cat((down_conv_output, x), dim=1) 173 | 174 | x = block(x) 175 | 176 | x = self.classification_block(x) 177 | x = self.output_layer(x) 178 | return x 179 | 180 | 181 | class UNetMultitask(UNet): 182 | def __init__(self, 183 | conv_kernel, 184 | pool_kernel, 185 | pool_stride, 186 | repeat_blocks, 187 | n_filters, 188 | batch_norm, 189 | dropout, 190 | in_channels, 191 | out_channels, 192 | nr_outputs): 193 | super(UNetMultitask, self).__init__(conv_kernel, 194 | pool_kernel, 195 | pool_stride, 196 | repeat_blocks, 197 | n_filters, 198 | batch_norm, 199 | dropout, 200 | in_channels, 201 | out_channels) 202 | self.nr_outputs = nr_outputs 203 | output_legs = [] 204 | for i in range(self.nr_outputs): 205 | output_legs.append(self._output_layer()) 206 | self.output_legs = nn.ModuleList(output_legs) 207 | 208 | def forward(self, x): 209 | x = self.input_block(x) 210 | 211 | down_convs_outputs = [] 212 | for block, down_pool in zip(self.down_convs, self.down_pools): 213 | x = block(x) 214 | down_convs_outputs.append(x) 215 | x = down_pool(x) 216 | x = self.floor_block(x) 217 | 218 | for down_conv_output, block, up_sample in zip(reversed(down_convs_outputs), 219 | reversed(self.up_convs), 220 | reversed(self.up_samples)): 221 | x = up_sample(x) 222 | x = torch.cat((down_conv_output, x), dim=1) 223 | 224 | x = block(x) 225 | 226 | x = self.classification_block(x) 227 | 228 | outputs = [output_leg(x) for output_leg in self.output_legs] 229 | return outputs 230 | 231 | 232 | class DownConv(nn.Module): 233 | def __init__(self, in_channels, kernel_size, batch_norm, dropout): 234 | super(DownConv, self).__init__() 235 | self.in_channels = in_channels 236 | self.block_channels = int(in_channels * 2.) 237 | self.kernel_size = kernel_size 238 | self.batch_norm = batch_norm 239 | self.dropout = dropout 240 | 241 | self.down_conv = self._down_conv() 242 | 243 | def _down_conv(self): 244 | stride = 1 245 | padding = get_downsample_pad(stride=stride, kernel=self.kernel_size) 246 | if self.batch_norm: 247 | down_conv = nn.Sequential(nn.Conv2d(in_channels=self.in_channels, out_channels=self.block_channels, 248 | kernel_size=(self.kernel_size, self.kernel_size), 249 | stride=stride, padding=padding), 250 | nn.BatchNorm2d(num_features=self.block_channels), 251 | nn.ReLU(), 252 | 253 | nn.Conv2d(in_channels=self.block_channels, out_channels=self.block_channels, 254 | kernel_size=(self.kernel_size, self.kernel_size), 255 | stride=stride, padding=padding), 256 | nn.BatchNorm2d(num_features=self.block_channels), 257 | nn.ReLU(), 258 | 259 | nn.Dropout(self.dropout), 260 | ) 261 | else: 262 | down_conv = nn.Sequential(nn.Conv2d(in_channels=self.in_channels, out_channels=self.block_channels, 263 | kernel_size=(self.kernel_size, self.kernel_size), 264 | stride=stride, padding=padding), 265 | nn.ReLU(), 266 | 267 | nn.Conv2d(in_channels=self.block_channels, out_channels=self.block_channels, 268 | kernel_size=(self.kernel_size, self.kernel_size), 269 | stride=stride, padding=padding), 270 | nn.ReLU(), 271 | 272 | nn.Dropout(self.dropout), 273 | ) 274 | return down_conv 275 | 276 | def forward(self, x): 277 | return self.down_conv(x) 278 | 279 | 280 | class UpConv(nn.Module): 281 | def __init__(self, in_channels, kernel_size, batch_norm, dropout): 282 | super(UpConv, self).__init__() 283 | self.in_channels = in_channels 284 | self.block_channels = int(in_channels / 2.) 285 | self.kernel_size = kernel_size 286 | self.batch_norm = batch_norm 287 | self.dropout = dropout 288 | 289 | self.up_conv = self._up_conv() 290 | 291 | def _up_conv(self): 292 | stride = 1 293 | padding = get_downsample_pad(stride=stride, kernel=self.kernel_size) 294 | if self.batch_norm: 295 | up_conv = nn.Sequential(nn.Conv2d(in_channels=self.in_channels, out_channels=self.block_channels, 296 | kernel_size=(self.kernel_size, self.kernel_size), 297 | stride=stride, padding=padding), 298 | 299 | nn.BatchNorm2d(num_features=self.block_channels), 300 | nn.ReLU(), 301 | 302 | nn.Conv2d(in_channels=self.block_channels, out_channels=self.block_channels, 303 | kernel_size=(self.kernel_size, self.kernel_size), 304 | stride=stride, padding=padding), 305 | nn.BatchNorm2d(num_features=self.block_channels), 306 | nn.ReLU(), 307 | 308 | nn.Dropout(self.dropout) 309 | ) 310 | else: 311 | up_conv = nn.Sequential(nn.Conv2d(in_channels=self.in_channels, out_channels=self.block_channels, 312 | kernel_size=(self.kernel_size, self.kernel_size), 313 | stride=stride, padding=padding), 314 | nn.ReLU(), 315 | 316 | nn.Conv2d(in_channels=self.block_channels, out_channels=self.block_channels, 317 | kernel_size=(self.kernel_size, self.kernel_size), 318 | stride=stride, padding=padding), 319 | nn.ReLU(), 320 | 321 | nn.Dropout(self.dropout) 322 | ) 323 | return up_conv 324 | 325 | def forward(self, x): 326 | return self.up_conv(x) 327 | -------------------------------------------------------------------------------- /src/steps/pytorch/architectures/utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch.nn as nn 4 | 5 | 6 | class Reshape(nn.Module): 7 | def __init__(self, *shape): 8 | super(Reshape, self).__init__() 9 | self.shape = shape 10 | 11 | def forward(self, x): 12 | return x.view(*self.shape) 13 | 14 | 15 | def get_downsample_pad(stride, kernel, dilation=1): 16 | return int(math.ceil((1 - stride + dilation * kernel - 1) / 2)) 17 | 18 | 19 | def get_upsample_pad(stride, kernel, dilation=1): 20 | if kernel - stride >= 0 and (kernel - stride) % 2 == 0: 21 | return (int((kernel - stride) / 2), 0) 22 | elif kernel - stride < 0: 23 | return (0, stride - kernel) 24 | else: 25 | return (int(math.ceil((kernel - stride) / 2)), 1) 26 | -------------------------------------------------------------------------------- /src/steps/pytorch/callbacks.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime, timedelta 3 | 4 | import neptune 5 | from torch.optim.lr_scheduler import ExponentialLR 6 | 7 | from ..utils import get_logger 8 | from .utils import Averager, save_model 9 | from .validation import score_model 10 | 11 | logger = get_logger() 12 | 13 | 14 | class Callback: 15 | def __init__(self): 16 | self.epoch_id = None 17 | self.batch_id = None 18 | 19 | self.model = None 20 | self.optimizer = None 21 | self.loss_function = None 22 | self.output_names = None 23 | self.validation_datagen = None 24 | self.lr_scheduler = None 25 | 26 | def set_params(self, transformer, validation_datagen, *args, **kwargs): 27 | self.model = transformer.model 28 | self.optimizer = transformer.optimizer 29 | self.loss_function = transformer.loss_function 30 | self.output_names = transformer.output_names 31 | self.validation_datagen = validation_datagen 32 | self.validation_loss = transformer.validation_loss 33 | 34 | def on_train_begin(self, *args, **kwargs): 35 | self.epoch_id = 0 36 | self.batch_id = 0 37 | 38 | def on_train_end(self, *args, **kwargs): 39 | pass 40 | 41 | def on_epoch_begin(self, *args, **kwargs): 42 | pass 43 | 44 | def on_epoch_end(self, *args, **kwargs): 45 | self.epoch_id += 1 46 | 47 | def training_break(self, *args, **kwargs): 48 | return False 49 | 50 | def on_batch_begin(self, *args, **kwargs): 51 | pass 52 | 53 | def on_batch_end(self, *args, **kwargs): 54 | self.batch_id += 1 55 | 56 | def get_validation_loss(self): 57 | if self.epoch_id not in self.validation_loss.keys(): 58 | self.validation_loss[self.epoch_id] = score_model(self.model, self.loss_function, self.validation_datagen) 59 | return self.validation_loss[self.epoch_id] 60 | 61 | 62 | class CallbackList: 63 | def __init__(self, callbacks=None): 64 | if callbacks is None: 65 | self.callbacks = [] 66 | elif isinstance(callbacks, Callback): 67 | self.callbacks = [callbacks] 68 | else: 69 | self.callbacks = callbacks 70 | 71 | def __len__(self): 72 | return len(self.callbacks) 73 | 74 | def set_params(self, *args, **kwargs): 75 | for callback in self.callbacks: 76 | callback.set_params(*args, **kwargs) 77 | 78 | def on_train_begin(self, *args, **kwargs): 79 | for callback in self.callbacks: 80 | callback.on_train_begin(*args, **kwargs) 81 | 82 | def on_train_end(self, *args, **kwargs): 83 | for callback in self.callbacks: 84 | callback.on_train_end(*args, **kwargs) 85 | 86 | def on_epoch_begin(self, *args, **kwargs): 87 | for callback in self.callbacks: 88 | callback.on_epoch_begin(*args, **kwargs) 89 | 90 | def on_epoch_end(self, *args, **kwargs): 91 | for callback in self.callbacks: 92 | callback.on_epoch_end(*args, **kwargs) 93 | 94 | def training_break(self, *args, **kwargs): 95 | callback_out = [callback.training_break(*args, **kwargs) for callback in self.callbacks] 96 | return any(callback_out) 97 | 98 | def on_batch_begin(self, *args, **kwargs): 99 | for callback in self.callbacks: 100 | callback.on_batch_begin(*args, **kwargs) 101 | 102 | def on_batch_end(self, *args, **kwargs): 103 | for callback in self.callbacks: 104 | callback.on_batch_end(*args, **kwargs) 105 | 106 | 107 | class TrainingMonitor(Callback): 108 | def __init__(self, epoch_every=None, batch_every=None): 109 | super().__init__() 110 | self.epoch_loss_averagers = {} 111 | if epoch_every == 0: 112 | self.epoch_every = False 113 | else: 114 | self.epoch_every = epoch_every 115 | if batch_every == 0: 116 | self.batch_every = False 117 | else: 118 | self.batch_every = batch_every 119 | 120 | def on_train_begin(self, *args, **kwargs): 121 | self.epoch_loss_averagers = {} 122 | self.epoch_id = 0 123 | self.batch_id = 0 124 | 125 | def on_epoch_end(self, *args, **kwargs): 126 | for name, averager in self.epoch_loss_averagers.items(): 127 | epoch_avg_loss = averager.value 128 | averager.reset() 129 | if self.epoch_every and ((self.epoch_id % self.epoch_every) == 0): 130 | logger.info('epoch {0} {1}: {2:.5f}'.format(self.epoch_id, name, epoch_avg_loss)) 131 | self.epoch_id += 1 132 | 133 | def on_batch_end(self, metrics, *args, **kwargs): 134 | for name, loss in metrics.items(): 135 | loss = loss.data.cpu().numpy()[0] 136 | if name in self.epoch_loss_averagers.keys(): 137 | self.epoch_loss_averagers[name].send(loss) 138 | else: 139 | self.epoch_loss_averagers[name] = Averager() 140 | self.epoch_loss_averagers[name].send(loss) 141 | 142 | if self.batch_every and ((self.batch_id % self.batch_every) == 0): 143 | logger.info('epoch {0} batch {1} {2}: {3:.5f}'.format(self.epoch_id, self.batch_id, name, loss)) 144 | self.batch_id += 1 145 | 146 | 147 | class ValidationMonitor(Callback): 148 | def __init__(self, epoch_every=None, batch_every=None): 149 | super().__init__() 150 | if epoch_every == 0: 151 | self.epoch_every = False 152 | else: 153 | self.epoch_every = epoch_every 154 | if batch_every == 0: 155 | self.batch_every = False 156 | else: 157 | self.batch_every = batch_every 158 | 159 | def on_epoch_end(self, *args, **kwargs): 160 | if self.epoch_every and ((self.epoch_id % self.epoch_every) == 0): 161 | self.model.eval() 162 | val_loss = self.get_validation_loss() 163 | self.model.train() 164 | for name, loss in val_loss.items(): 165 | loss = loss.data.cpu().numpy()[0] 166 | logger.info('epoch {0} validation {1}: {2:.5f}'.format(self.epoch_id, name, loss)) 167 | self.epoch_id += 1 168 | 169 | 170 | class EarlyStopping(Callback): 171 | def __init__(self, patience, minimize=True): 172 | super().__init__() 173 | self.patience = patience 174 | self.minimize = minimize 175 | self.best_score = None 176 | self.epoch_since_best = 0 177 | self._training_break = False 178 | 179 | def on_epoch_end(self, *args, **kwargs): 180 | self.model.eval() 181 | val_loss = self.get_validation_loss() 182 | loss_sum = val_loss['sum'] 183 | loss_sum = loss_sum.data.cpu().numpy()[0] 184 | 185 | self.model.train() 186 | 187 | if not self.best_score: 188 | self.best_score = loss_sum 189 | 190 | if (self.minimize and loss_sum < self.best_score) or (not self.minimize and loss_sum > self.best_score): 191 | self.best_score = loss_sum 192 | self.epoch_since_best = 0 193 | else: 194 | self.epoch_since_best += 1 195 | 196 | if self.epoch_since_best > self.patience: 197 | self._training_break = True 198 | self.epoch_id += 1 199 | 200 | def training_break(self, *args, **kwargs): 201 | return self._training_break 202 | 203 | 204 | class ExponentialLRScheduler(Callback): 205 | def __init__(self, gamma, epoch_every=1, batch_every=None): 206 | super().__init__() 207 | self.gamma = gamma 208 | if epoch_every == 0: 209 | self.epoch_every = False 210 | else: 211 | self.epoch_every = epoch_every 212 | if batch_every == 0: 213 | self.batch_every = False 214 | else: 215 | self.batch_every = batch_every 216 | 217 | def set_params(self, transformer, validation_datagen, *args, **kwargs): 218 | self.validation_datagen = validation_datagen 219 | self.model = transformer.model 220 | self.optimizer = transformer.optimizer 221 | self.loss_function = transformer.loss_function 222 | self.lr_scheduler = ExponentialLR(self.optimizer, self.gamma, last_epoch=-1) 223 | 224 | def on_train_begin(self, *args, **kwargs): 225 | self.epoch_id = 0 226 | self.batch_id = 0 227 | logger.info('initial lr: {0}'.format(self.optimizer.state_dict()['param_groups'][0]['initial_lr'])) 228 | 229 | def on_epoch_end(self, *args, **kwargs): 230 | if self.epoch_every and (((self.epoch_id + 1) % self.epoch_every) == 0): 231 | self.lr_scheduler.step() 232 | logger.info('epoch {0} current lr: {1}'.format(self.epoch_id + 1, 233 | self.optimizer.state_dict()['param_groups'][0]['lr'])) 234 | self.epoch_id += 1 235 | 236 | def on_batch_end(self, *args, **kwargs): 237 | if self.batch_every and ((self.batch_id % self.batch_every) == 0): 238 | self.lr_scheduler.step() 239 | logger.info('epoch {0} batch {1} current lr: {2}'.format( 240 | self.epoch_id + 1, self.batch_id + 1, self.optimizer.state_dict()['param_groups'][0]['lr'])) 241 | self.batch_id += 1 242 | 243 | 244 | class ModelCheckpoint(Callback): 245 | def __init__(self, filepath, epoch_every=1, minimize=True): 246 | super().__init__() 247 | self.filepath = filepath 248 | self.minimize = minimize 249 | self.best_score = None 250 | 251 | if epoch_every == 0: 252 | self.epoch_every = False 253 | else: 254 | self.epoch_every = epoch_every 255 | 256 | def on_train_begin(self, *args, **kwargs): 257 | self.epoch_id = 0 258 | self.batch_id = 0 259 | os.makedirs(os.path.dirname(self.filepath), exist_ok=True) 260 | 261 | def on_epoch_end(self, *args, **kwargs): 262 | if self.epoch_every and ((self.epoch_id % self.epoch_every) == 0): 263 | self.model.eval() 264 | val_loss = self.get_validation_loss() 265 | loss_sum = val_loss['sum'] 266 | loss_sum = loss_sum.data.cpu().numpy()[0] 267 | 268 | self.model.train() 269 | 270 | if self.best_score is None: 271 | self.best_score = loss_sum 272 | 273 | if (self.minimize and loss_sum < self.best_score) or (not self.minimize and loss_sum > self.best_score) or ( 274 | self.epoch_id == 0): 275 | self.best_score = loss_sum 276 | save_model(self.model, self.filepath) 277 | logger.info('epoch {0} model saved to {1}'.format(self.epoch_id, self.filepath)) 278 | 279 | self.epoch_id += 1 280 | 281 | 282 | class NeptuneMonitor(Callback): 283 | def __init__(self, model_name): 284 | super().__init__() 285 | self.model_name = model_name 286 | self.epoch_loss_averager = Averager() 287 | 288 | def on_train_begin(self, *args, **kwargs): 289 | self.epoch_loss_averagers = {} 290 | self.epoch_id = 0 291 | self.batch_id = 0 292 | 293 | def on_batch_end(self, metrics, *args, **kwargs): 294 | for name, loss in metrics.items(): 295 | loss = loss.data.cpu().numpy()[0] 296 | 297 | if name in self.epoch_loss_averagers.keys(): 298 | self.epoch_loss_averagers[name].send(loss) 299 | else: 300 | self.epoch_loss_averagers[name] = Averager() 301 | self.epoch_loss_averagers[name].send(loss) 302 | 303 | neptune.send_metric('{} batch {} loss'.format(self.model_name, name), x=self.batch_id, y=loss) 304 | 305 | self.batch_id += 1 306 | 307 | def on_epoch_end(self, *args, **kwargs): 308 | self._send_numeric_channels() 309 | self.epoch_id += 1 310 | 311 | def _send_numeric_channels(self, *args, **kwargs): 312 | for name, averager in self.epoch_loss_averagers.items(): 313 | epoch_avg_loss = averager.value 314 | averager.reset() 315 | neptune.send_metric('{} epoch {} loss'.format(self.model_name, name), x=self.epoch_id, y=epoch_avg_loss) 316 | 317 | self.model.eval() 318 | val_loss = self.get_validation_loss() 319 | self.model.train() 320 | for name, loss in val_loss.items(): 321 | loss = loss.data.cpu().numpy()[0] 322 | neptune.send_metric('{} epoch_val {} loss/metric'.format(self.model_name, name), x=self.epoch_id, y=loss) 323 | 324 | 325 | class ExperimentTiming(Callback): 326 | def __init__(self, epoch_every=None, batch_every=None): 327 | super().__init__() 328 | if epoch_every == 0: 329 | self.epoch_every = False 330 | else: 331 | self.epoch_every = epoch_every 332 | if batch_every == 0: 333 | self.batch_every = False 334 | else: 335 | self.batch_every = batch_every 336 | self.batch_start = None 337 | self.epoch_start = None 338 | self.current_sum = None 339 | self.current_mean = None 340 | 341 | def on_train_begin(self, *args, **kwargs): 342 | self.epoch_id = 0 343 | self.batch_id = 0 344 | logger.info('starting training...') 345 | 346 | def on_train_end(self, *args, **kwargs): 347 | logger.info('training finished') 348 | 349 | def on_epoch_begin(self, *args, **kwargs): 350 | if self.epoch_id > 0: 351 | epoch_time = datetime.now() - self.epoch_start 352 | if self.epoch_every: 353 | if (self.epoch_id % self.epoch_every) == 0: 354 | logger.info('epoch {0} time {1}'.format(self.epoch_id - 1, str(epoch_time)[:-7])) 355 | self.epoch_start = datetime.now() 356 | self.current_sum = timedelta() 357 | self.current_mean = timedelta() 358 | logger.info('epoch {0} ...'.format(self.epoch_id)) 359 | 360 | def on_batch_begin(self, *args, **kwargs): 361 | if self.batch_id > 0: 362 | current_delta = datetime.now() - self.batch_start 363 | self.current_sum += current_delta 364 | self.current_mean = self.current_sum / self.batch_id 365 | if self.batch_every: 366 | if self.batch_id > 0 and (((self.batch_id - 1) % self.batch_every) == 0): 367 | logger.info('epoch {0} average batch time: {1}'.format(self.epoch_id, str(self.current_mean)[:-5])) 368 | if self.batch_every: 369 | if self.batch_id == 0 or self.batch_id % self.batch_every == 0: 370 | logger.info('epoch {0} batch {1} ...'.format(self.epoch_id, self.batch_id)) 371 | self.batch_start = datetime.now() 372 | 373 | 374 | class ReduceLROnPlateau(Callback): # thank you keras 375 | def __init__(self): 376 | super().__init__() 377 | pass 378 | -------------------------------------------------------------------------------- /src/steps/pytorch/loaders.py: -------------------------------------------------------------------------------- 1 | from math import ceil 2 | 3 | import numpy as np 4 | import torch 5 | import torchvision.transforms as transforms 6 | from PIL import Image 7 | from sklearn.externals import joblib 8 | from torch.utils.data import Dataset, DataLoader 9 | 10 | from ..base import BaseTransformer 11 | 12 | 13 | class MetadataImageDataset(Dataset): 14 | def __init__(self, X, y, image_transform, target_transform, image_augment): 15 | super().__init__() 16 | self.X = X 17 | if y is not None: 18 | self.y = y 19 | else: 20 | self.y = None 21 | 22 | self.image_transform = image_transform 23 | self.image_augment = image_augment 24 | self.target_transform = target_transform 25 | 26 | def load_image(self, img_filepath): 27 | image = np.asarray(Image.open(img_filepath)) 28 | image = image / 255.0 29 | return image 30 | 31 | def __len__(self): 32 | return self.X.shape[0] 33 | 34 | def __getitem__(self, index): 35 | img_filepath = self.X[index] 36 | 37 | Xi = self.load_image(img_filepath) 38 | 39 | if self.image_augment is not None: 40 | Xi = self.image_augment(Xi) 41 | 42 | if self.image_transform is not None: 43 | Xi = self.image_transform(Xi) 44 | if self.y is not None: 45 | yi = self.y[index] 46 | if self.target_transform is not None: 47 | yi = self.target_transform(yi) 48 | return Xi, yi 49 | else: 50 | return Xi 51 | 52 | 53 | class MetadataImageLoader(BaseTransformer): 54 | def __init__(self, loader_params): 55 | super().__init__() 56 | self.loader_params = loader_params 57 | 58 | self.dataset = MetadataImageDataset 59 | self.image_transform = transforms.ToTensor() 60 | self.target_transform = target_transform 61 | self.image_augment = None 62 | 63 | def transform(self, X, y, validation_data, train_mode): 64 | if train_mode: 65 | flow, steps = self.get_datagen(X, y, train_mode, self.loader_params['training']) 66 | else: 67 | flow, steps = self.get_datagen(X, y, train_mode, self.loader_params['inference']) 68 | 69 | if validation_data is not None: 70 | X_valid, y_valid = validation_data 71 | valid_flow, valid_steps = self.get_datagen(X_valid, y_valid, False, self.loader_params['inference']) 72 | else: 73 | valid_flow = None 74 | valid_steps = None 75 | 76 | return {'datagen': (flow, steps), 77 | 'validation_datagen': (valid_flow, valid_steps)} 78 | 79 | def get_datagen(self, X, y, train_mode, loader_params): 80 | if train_mode: 81 | dataset = self.dataset(X, y, 82 | image_augment=self.image_augment, 83 | image_transform=self.image_transform, 84 | target_transform=self.target_transform) 85 | 86 | else: 87 | dataset = self.dataset(X, y, 88 | image_augment=None, 89 | image_transform=self.image_transform, 90 | target_transform=self.target_transform) 91 | datagen = DataLoader(dataset, **loader_params) 92 | steps = ceil(X.shape[0] / loader_params['batch_size']) 93 | return datagen, steps 94 | 95 | def load(self, filepath): 96 | params = joblib.load(filepath) 97 | self.loader_params = params['loader_params'] 98 | return self 99 | 100 | def save(self, filepath): 101 | params = {'loader_params': self.loader_params} 102 | joblib.dump(params, filepath) 103 | 104 | 105 | def target_transform(y): 106 | return torch.from_numpy(y).type(torch.LongTensor) 107 | -------------------------------------------------------------------------------- /src/steps/pytorch/models.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | from functools import partial 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn as nn 8 | from torch.autograd import Variable 9 | from torch.nn import init 10 | 11 | from ..base import BaseTransformer 12 | from ..utils import get_logger 13 | from .utils import save_model 14 | 15 | logger = get_logger() 16 | 17 | 18 | class Model(BaseTransformer): 19 | def __init__(self, architecture_config, training_config, callbacks_config): 20 | super().__init__() 21 | self.architecture_config = architecture_config 22 | self.training_config = training_config 23 | self.callbacks_config = callbacks_config 24 | 25 | self.model = None 26 | self.optimizer = None 27 | self.loss_function = None 28 | self.callbacks = None 29 | self.validation_loss = {} 30 | 31 | @property 32 | def output_names(self): 33 | return [name for (name, func, weight) in self.loss_function] 34 | 35 | def _initialize_model_weights(self): 36 | logger.info('initializing model weights...') 37 | weights_init_config = self.architecture_config['weights_init'] 38 | 39 | if weights_init_config['function'] == 'normal': 40 | weights_init_func = partial(init_weights_normal, **weights_init_config['params']) 41 | elif weights_init_config['function'] == 'xavier': 42 | weights_init_func = init_weights_xavier 43 | elif weights_init_config['function'] == 'he': 44 | weights_init_func = init_weights_he 45 | else: 46 | raise NotImplementedError 47 | 48 | self.model.apply(weights_init_func) 49 | 50 | def fit(self, datagen, validation_datagen=None): 51 | self._initialize_model_weights() 52 | 53 | self.model = nn.DataParallel(self.model) 54 | 55 | if torch.cuda.is_available(): 56 | self.model = self.model.cuda() 57 | 58 | self.callbacks.set_params(self, validation_datagen=validation_datagen) 59 | self.callbacks.on_train_begin() 60 | 61 | batch_gen, steps = datagen 62 | for epoch_id in range(self.training_config['epochs']): 63 | self.callbacks.on_epoch_begin() 64 | for batch_id, data in enumerate(batch_gen): 65 | self.callbacks.on_batch_begin() 66 | metrics = self._fit_loop(data) 67 | self.callbacks.on_batch_end(metrics=metrics) 68 | if batch_id == steps: 69 | break 70 | self.callbacks.on_epoch_end() 71 | if self.callbacks.training_break(): 72 | break 73 | self.callbacks.on_train_end() 74 | return self 75 | 76 | def _fit_loop(self, data): 77 | X = data[0] 78 | targets_tensors = data[1:] 79 | 80 | if torch.cuda.is_available(): 81 | X = Variable(X).cuda() 82 | targets_var = [] 83 | for target_tensor in targets_tensors: 84 | targets_var.append(Variable(target_tensor).cuda()) 85 | else: 86 | X = Variable(X) 87 | targets_var = [] 88 | for target_tensor in targets_tensors: 89 | targets_var.append(Variable(target_tensor)) 90 | 91 | self.optimizer.zero_grad() 92 | outputs_batch = self.model(X) 93 | partial_batch_losses = {} 94 | 95 | # assert len(targets_tensors) == len(outputs_batch) == len(self.loss_function),\ 96 | # '''Number of targets, model outputs and elements of loss function must equal. 97 | # You have n_targets={0}, n_model_outputs={1}, n_loss_function_elements={2}. 98 | # The order of elements must also be preserved.'''.format(len(targets_tensors), 99 | # len(outputs_batch), 100 | # len(self.loss_function)) 101 | 102 | if len(self.output_names) == 1: 103 | for (name, loss_function, weight), target in zip(self.loss_function, targets_var): 104 | batch_loss = loss_function(outputs_batch, target) * weight 105 | else: 106 | for (name, loss_function, weight), output, target in zip(self.loss_function, outputs_batch, targets_var): 107 | partial_batch_losses[name] = loss_function(output, target) * weight 108 | batch_loss = sum(partial_batch_losses.values()) 109 | partial_batch_losses['sum'] = batch_loss 110 | batch_loss.backward() 111 | self.optimizer.step() 112 | 113 | return partial_batch_losses 114 | 115 | def _transform(self, datagen, validation_datagen=None): 116 | self.model.eval() 117 | 118 | batch_gen, steps = datagen 119 | outputs = {} 120 | for batch_id, data in enumerate(batch_gen): 121 | if isinstance(data, list): 122 | X = data[0] 123 | else: 124 | X = data 125 | 126 | if torch.cuda.is_available(): 127 | X = Variable(X, volatile=True).cuda() 128 | else: 129 | X = Variable(X, volatile=True) 130 | 131 | outputs_batch = self.model(X) 132 | if len(self.output_names) == 1: 133 | outputs.setdefault(self.output_names[0], []).append(outputs_batch.data.cpu().numpy()) 134 | else: 135 | for name, output in zip(self.output_names, outputs_batch): 136 | output_ = output.data.cpu().numpy() 137 | outputs.setdefault(name, []).append(output_) 138 | if batch_id == steps: 139 | break 140 | self.model.train() 141 | outputs = {'{}_prediction'.format(name): np.vstack(outputs_) for name, outputs_ in outputs.items()} 142 | return outputs 143 | 144 | def transform(self, datagen, validation_datagen=None): 145 | predictions = self._transform(datagen, validation_datagen) 146 | return NotImplementedError 147 | 148 | def load(self, filepath): 149 | self.model.eval() 150 | 151 | if not isinstance(self.model, nn.DataParallel): 152 | self.model = nn.DataParallel(self.model) 153 | 154 | if torch.cuda.is_available(): 155 | self.model.cpu() 156 | self.model.load_state_dict(torch.load(filepath)) 157 | self.model = self.model.cuda() 158 | else: 159 | self.model.load_state_dict(torch.load(filepath, map_location=lambda storage, loc: storage)) 160 | return self 161 | 162 | def save(self, filepath): 163 | checkpoint_callback = self.callbacks_config.get('model_checkpoint') 164 | if checkpoint_callback: 165 | checkpoint_filepath = checkpoint_callback['filepath'] 166 | if os.path.exists(checkpoint_filepath): 167 | shutil.copyfile(checkpoint_filepath, filepath) 168 | else: 169 | save_model(self.model, filepath) 170 | else: 171 | save_model(self.model, filepath) 172 | 173 | 174 | class PyTorchBasic(nn.Module): 175 | def _flatten_features(self, in_size, features): 176 | f = features(Variable(torch.ones(1, *in_size))) 177 | return int(np.prod(f.size()[1:])) 178 | 179 | def forward(self, x): 180 | features = self.features(x) 181 | flat_features = features.view(-1, self.flat_features) 182 | out = self.classifier(flat_features) 183 | return out 184 | 185 | def forward_target(self, x): 186 | return self.forward(x) 187 | 188 | 189 | def init_weights_normal(model, mean, std_conv2d, std_linear): 190 | if type(model) == nn.Conv2d: 191 | model.weight.data.normal_(mean=mean, std=std_conv2d) 192 | if type(model) == nn.Linear: 193 | model.weight.data.normal_(mean=mean, std=std_linear) 194 | 195 | 196 | def init_weights_xavier(model): 197 | if isinstance(model, nn.Conv2d): 198 | init.xavier_normal(model.weight) 199 | init.constant(model.bias, 0) 200 | 201 | 202 | def init_weights_he(model): 203 | if isinstance(model, nn.Conv2d): 204 | init.kaiming_normal(model.weight) 205 | init.constant(model.bias, 0) 206 | -------------------------------------------------------------------------------- /src/steps/pytorch/utils.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import torch 4 | from imgaug import augmenters as iaa 5 | 6 | 7 | def denormalize_img(img): 8 | mean = [0.28201905, 0.37246801, 0.42341868] 9 | std = [0.13609867, 0.12380088, 0.13325344] 10 | img_ = (img * std) + mean 11 | return img_ 12 | 13 | 14 | def overlay_box(img, predicted_box, true_box, bin_nr): 15 | img_h, img_w, img_c = img.shape 16 | x1, y1, x2, y2 = predicted_box 17 | 18 | x1 = int(1.0 * x1 * img_w / bin_nr) 19 | y1 = int(1.0 * y1 * img_h / bin_nr) 20 | x2 = int(1.0 * x2 * img_h / bin_nr) 21 | y2 = int(1.0 * y2 * img_h / bin_nr) 22 | 23 | tx1, ty1, tx2, ty2 = true_box 24 | 25 | tx1 = int(1.0 * tx1 * img_w / bin_nr) 26 | ty1 = int(1.0 * ty1 * img_h / bin_nr) 27 | tx2 = int(1.0 * tx2 * img_h / bin_nr) 28 | ty2 = int(1.0 * ty2 * img_h / bin_nr) 29 | 30 | img_overlayed = img.copy() 31 | img_overlayed = (denormalize_img(img_overlayed) * 255.).astype(np.uint8) 32 | cv2.rectangle(img_overlayed, (x1, y1), (x2, y2), (255, 0, 0), 2) 33 | cv2.rectangle(img_overlayed, (tx1, ty1), (tx2, ty2), (0, 255, 0), 2) 34 | img_overlayed = (img_overlayed / 255.).astype(np.float64) 35 | return img_overlayed 36 | 37 | 38 | def overlay_keypoints(img, pred_keypoints, true_keypoints, bin_nr): 39 | img_h, img_w, img_c = img.shape 40 | x1, y1, x2, y2 = pred_keypoints[:4] 41 | 42 | x1 = int(1.0 * x1 * img_w / bin_nr) 43 | y1 = int(1.0 * y1 * img_h / bin_nr) 44 | x2 = int(1.0 * x2 * img_h / bin_nr) 45 | y2 = int(1.0 * y2 * img_h / bin_nr) 46 | 47 | tx1, ty1, tx2, ty2 = true_keypoints[:4] 48 | 49 | tx1 = int(1.0 * tx1 * img_w / bin_nr) 50 | ty1 = int(1.0 * ty1 * img_h / bin_nr) 51 | tx2 = int(1.0 * tx2 * img_h / bin_nr) 52 | ty2 = int(1.0 * ty2 * img_h / bin_nr) 53 | 54 | img_overlayed = img.copy() 55 | img_overlayed = (denormalize_img(img_overlayed) * 255.).astype(np.uint8) 56 | 57 | cv2.circle(img_overlayed, (x1, y1), 5, (252, 124, 0), -1) 58 | cv2.circle(img_overlayed, (x2, y2), 5, (139, 46, 87), -1) 59 | 60 | cv2.circle(img_overlayed, (tx1, ty1), 5, (102, 255, 102), -1) 61 | cv2.circle(img_overlayed, (tx2, ty2), 5, (0, 204, 0), -1) 62 | 63 | img_overlayed = (img_overlayed / 255.).astype(np.float64) 64 | return img_overlayed 65 | 66 | 67 | def save_model(model, path): 68 | model.eval() 69 | if torch.cuda.is_available(): 70 | model.cpu() 71 | torch.save(model.state_dict(), path) 72 | model.cuda() 73 | else: 74 | torch.save(model.state_dict(), path) 75 | model.train() 76 | 77 | 78 | class Averager: 79 | """ 80 | Todo: 81 | Rewrite as a coroutine (yield from) 82 | """ 83 | 84 | def __init__(self): 85 | self.current_total = 0.0 86 | self.iterations = 0.0 87 | 88 | def send(self, value): 89 | self.current_total += value 90 | self.iterations += 1 91 | 92 | @property 93 | def value(self): 94 | if self.iterations == 0: 95 | return 0 96 | else: 97 | return 1.0 * self.current_total / self.iterations 98 | 99 | def reset(self): 100 | self.current_total = 0.0 101 | self.iterations = 0.0 102 | 103 | 104 | def sigmoid(x): 105 | return 1. / (1 + np.exp(-x)) 106 | 107 | 108 | class ImgAug: 109 | def __init__(self, augmenters): 110 | if not isinstance(augmenters, list): 111 | augmenters = [augmenters] 112 | self.augmenters = augmenters 113 | self.seq_det = None 114 | 115 | def _pre_call_hook(self): 116 | seq = iaa.Sequential(self.augmenters) 117 | seq.reseed() 118 | self.seq_det = seq.to_deterministic() 119 | 120 | def transform(self, *images): 121 | images = [self.seq_det.augment_image(image) for image in images] 122 | if len(images) == 1: 123 | return images[0] 124 | else: 125 | return images 126 | 127 | def __call__(self, *args): 128 | self._pre_call_hook() 129 | return self.transform(*args) 130 | -------------------------------------------------------------------------------- /src/steps/pytorch/validation.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from sklearn.metrics import accuracy_score 5 | from torch.autograd import Variable 6 | 7 | 8 | class DiceLoss(nn.Module): 9 | def __init__(self, smooth=0, eps = 1e-7): 10 | super(DiceLoss, self).__init__() 11 | self.smooth = smooth 12 | self.eps = eps 13 | 14 | def forward(self, output, target): 15 | return 1 - (2 * torch.sum(output * target) + self.smooth) / ( 16 | torch.sum(output) + torch.sum(target) + self.smooth + self.eps) 17 | 18 | 19 | def segmentation_loss(output, target, weight_bce=1.0, weight_dice=1.0): 20 | bce = nn.BCEWithLogitsLoss() 21 | dice = DiceLoss() 22 | return weight_bce * bce(output, target) + weight_dice * dice(output, target) 23 | 24 | 25 | def multiclass_segmentation_loss(output, target): 26 | target = target.squeeze(1).long() 27 | cross_entropy = nn.CrossEntropyLoss() 28 | return cross_entropy(output, target) 29 | 30 | 31 | def cross_entropy(output, target, squeeze=False): 32 | if squeeze: 33 | target = target.squeeze(1) 34 | return F.nll_loss(output, target) 35 | 36 | 37 | def mse(output, target, squeeze=False): 38 | if squeeze: 39 | target = target.squeeze(1) 40 | return F.mse_loss(output, target) 41 | 42 | 43 | def multi_output_cross_entropy(outputs, targets): 44 | losses = [] 45 | for output, target in zip(outputs, targets): 46 | loss = cross_entropy(output, target) 47 | losses.append(loss) 48 | return sum(losses) / len(losses) 49 | 50 | 51 | def score_model(model, loss_function, datagen): 52 | batch_gen, steps = datagen 53 | partial_batch_losses = {} 54 | for batch_id, data in enumerate(batch_gen): 55 | X = data[0] 56 | targets_tensors = data[1:] 57 | 58 | if torch.cuda.is_available(): 59 | X = Variable(X, volatile=True).cuda() 60 | targets_var = [] 61 | for target_tensor in targets_tensors: 62 | targets_var.append(Variable(target_tensor, volatile=True).cuda()) 63 | else: 64 | X = Variable(X, volatile=True) 65 | targets_var = [] 66 | for target_tensor in targets_tensors: 67 | targets_var.append(Variable(target_tensor, volatile=True)) 68 | 69 | outputs = model(X) 70 | if len(loss_function) == 1: 71 | for (name, loss_function_one, weight), target in zip(loss_function, targets_var): 72 | loss_sum = loss_function_one(outputs, target) * weight 73 | else: 74 | batch_losses = [] 75 | for (name, loss_function_one, weight), output, target in zip(loss_function, outputs, targets_var): 76 | loss = loss_function_one(output, target) * weight 77 | batch_losses.append(loss) 78 | partial_batch_losses.setdefault(name, []).append(loss) 79 | loss_sum = sum(batch_losses) 80 | partial_batch_losses.setdefault('sum', []).append(loss_sum) 81 | if batch_id == steps: 82 | break 83 | average_losses = {name: sum(losses) / steps for name, losses in partial_batch_losses.items()} 84 | return average_losses 85 | 86 | 87 | def torch_acc_score(output, target): 88 | output = output.data.cpu().numpy() 89 | y_true = target.numpy() 90 | y_pred = output.argmax(axis=1) 91 | return accuracy_score(y_true, y_pred) 92 | 93 | 94 | def torch_acc_score_multi_output(outputs, targets, take_first=None): 95 | accuracies = [] 96 | for i, (output, target) in enumerate(zip(outputs, targets)): 97 | if i == take_first: 98 | break 99 | accuracy = torch_acc_score(output, target) 100 | accuracies.append(accuracy) 101 | avg_accuracy = sum(accuracies) / len(accuracies) 102 | return avg_accuracy 103 | -------------------------------------------------------------------------------- /src/steps/resources/apostrophes.json: -------------------------------------------------------------------------------- 1 | { 2 | "arent": "are not", 3 | "cant": "cannot", 4 | "couldnt": "could not", 5 | "didnt": "did not", 6 | "doesnt": "does not", 7 | "dont": "do not", 8 | "hadnt": "had not", 9 | "hasnt": "has not", 10 | "havent": "have not", 11 | "hed": "he would", 12 | "hell": "he will", 13 | "hes": "he is", 14 | "id": "I had", 15 | "ill": "I will", 16 | "im": "I am", 17 | "isnt": "is not", 18 | "its": "it is", 19 | "itll": "it will", 20 | "ive": "I have", 21 | "lets": "let us", 22 | "mightnt": "might not", 23 | "mustnt": "must not", 24 | "shant": "shall not", 25 | "shed" : "she would", 26 | "shell": "she will", 27 | "shes": "she is", 28 | "shouldnt": "should not", 29 | "thats": "that is", 30 | "theres": "there is", 31 | "theyd": "they would", 32 | "theyll": "they will", 33 | "theyre": "they are", 34 | "theyve": "they have", 35 | "wed": "we would", 36 | "were": "we are", 37 | "werent": "were not", 38 | "weve": "we have", 39 | "whatll": "what will", 40 | "whatre": "what are", 41 | "whats": "what is", 42 | "whatve": "what have", 43 | "wheres": "where is", 44 | "whod": "who would", 45 | "wholl": "who will", 46 | "whore": "who are", 47 | "whos": "who is", 48 | "whove": "who have", 49 | "wont": "will not", 50 | "wouldnt": "would not", 51 | "youd": "you would", 52 | "youll": "you will", 53 | "youre": "you are", 54 | "youve": "you have", 55 | "re": "are", 56 | "wasnt": "was not", 57 | "well": "will" 58 | } 59 | -------------------------------------------------------------------------------- /src/steps/sklearn/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/neptune-ai/open-solution-mapping-challenge/2f1f5f17bb9dfb5ba8dfc3c312533479997bd4c9/src/steps/sklearn/__init__.py -------------------------------------------------------------------------------- /src/steps/sklearn/models.py: -------------------------------------------------------------------------------- 1 | import lightgbm as lgb 2 | import numpy as np 3 | import sklearn.linear_model as lr 4 | from attrdict import AttrDict 5 | #from catboost import CatBoostClassifier 6 | from sklearn import ensemble 7 | from sklearn import svm 8 | from sklearn.externals import joblib 9 | from xgboost import XGBClassifier 10 | 11 | from ..base import BaseTransformer 12 | from ..utils import get_logger 13 | 14 | logger = get_logger() 15 | 16 | 17 | class SklearnClassifier(BaseTransformer): 18 | def __init__(self, estimator): 19 | self.estimator = estimator 20 | 21 | def fit(self, X, y, **kwargs): 22 | self.estimator.fit(X, y) 23 | return self 24 | 25 | def transform(self, X, y=None, **kwargs): 26 | prediction = self.estimator.predict_proba(X) 27 | return {'prediction': prediction} 28 | 29 | 30 | class SklearnRegressor(BaseTransformer): 31 | def __init__(self, estimator): 32 | self.estimator = estimator 33 | 34 | def fit(self, X, y, **kwargs): 35 | self.estimator.fit(X, y) 36 | return self 37 | 38 | def transform(self, X, y=None, **kwargs): 39 | prediction = self.estimator.predict(X) 40 | return {'prediction': prediction} 41 | 42 | 43 | class SklearnTransformer(BaseTransformer): 44 | def __init__(self, estimator): 45 | self.estimator = estimator 46 | 47 | def fit(self, X, y, **kwargs): 48 | self.estimator.fit(X, y) 49 | return self 50 | 51 | def transform(self, X, y=None, **kwargs): 52 | transformed = self.estimator.transform(X) 53 | return {'transformed': transformed} 54 | 55 | 56 | class SklearnPipeline(BaseTransformer): 57 | def __init__(self, estimator): 58 | self.estimator = estimator 59 | 60 | def fit(self, X, y, **kwargs): 61 | self.estimator.fit(X, y) 62 | return self 63 | 64 | def transform(self, X, y=None, **kwargs): 65 | transformed = self.estimator.transform(X) 66 | return {'transformed': transformed} 67 | 68 | 69 | class LightGBM(BaseTransformer): 70 | def __init__(self, model_params, training_params): 71 | self.model_params = model_params 72 | self.training_params = AttrDict(training_params) 73 | self.evaluation_function = None 74 | 75 | def fit(self, X, y, X_valid, y_valid, feature_names, categorical_features, **kwargs): 76 | train = lgb.Dataset(X, label=y, 77 | feature_name=feature_names, 78 | categorical_feature=categorical_features 79 | ) 80 | valid = lgb.Dataset(X_valid, label=y_valid, 81 | feature_name=feature_names, 82 | categorical_feature=categorical_features 83 | ) 84 | 85 | evaluation_results = {} 86 | self.estimator = lgb.train(self.model_params, 87 | train, 88 | valid_sets=[train, valid], 89 | valid_names=['train', 'valid'], 90 | evals_result=evaluation_results, 91 | num_boost_round=self.training_params.number_boosting_rounds, 92 | early_stopping_rounds=self.training_params.early_stopping_rounds, 93 | verbose_eval=10, 94 | feval=self.evaluation_function) 95 | return self 96 | 97 | def transform(self, X, y=None, **kwargs): 98 | prediction = self.estimator.predict(X) 99 | return {'prediction': prediction} 100 | 101 | 102 | class MultilabelEstimator(BaseTransformer): 103 | def __init__(self, label_nr, **kwargs): 104 | self.label_nr = label_nr 105 | self.estimators = self._get_estimators(**kwargs) 106 | 107 | @property 108 | def estimator(self): 109 | return NotImplementedError 110 | 111 | def _get_estimators(self, **kwargs): 112 | estimators = [] 113 | for i in range(self.label_nr): 114 | estimators.append((i, self.estimator(**kwargs))) 115 | return estimators 116 | 117 | def fit(self, X, y, **kwargs): 118 | for i, estimator in self.estimators: 119 | logger.info('fitting estimator {}'.format(i)) 120 | estimator.fit(X, y[:, i]) 121 | return self 122 | 123 | def transform(self, X, y=None, **kwargs): 124 | predictions = [] 125 | for i, estimator in self.estimators: 126 | prediction = estimator.predict_proba(X) 127 | predictions.append(prediction) 128 | predictions = np.stack(predictions, axis=0) 129 | predictions = predictions[:, :, 1].transpose() 130 | return {'prediction_probability': predictions} 131 | 132 | def load(self, filepath): 133 | params = joblib.load(filepath) 134 | self.label_nr = params['label_nr'] 135 | self.estimators = params['estimators'] 136 | return self 137 | 138 | def save(self, filepath): 139 | params = {'label_nr': self.label_nr, 140 | 'estimators': self.estimators} 141 | joblib.dump(params, filepath) 142 | 143 | 144 | class LogisticRegressionMultilabel(MultilabelEstimator): 145 | @property 146 | def estimator(self): 147 | return lr.LogisticRegression 148 | 149 | 150 | class SVCMultilabel(MultilabelEstimator): 151 | @property 152 | def estimator(self): 153 | return svm.SVC 154 | 155 | 156 | class LinearSVC_proba(svm.LinearSVC): 157 | def __platt_func(self, x): 158 | return 1 / (1 + np.exp(-x)) 159 | 160 | def predict_proba(self, X): 161 | f = np.vectorize(self.__platt_func) 162 | raw_predictions = self.decision_function(X) 163 | platt_predictions = f(raw_predictions).reshape(-1, 1) 164 | prob_positive = platt_predictions / platt_predictions.sum(axis=1)[:, None] 165 | prob_negative = 1.0 - prob_positive 166 | probs = np.hstack([prob_negative, prob_positive]) 167 | print(prob_positive) 168 | return probs 169 | 170 | 171 | class LinearSVCMultilabel(MultilabelEstimator): 172 | @property 173 | def estimator(self): 174 | return LinearSVC_proba 175 | 176 | 177 | class RandomForestMultilabel(MultilabelEstimator): 178 | @property 179 | def estimator(self): 180 | return ensemble.RandomForestClassifier 181 | 182 | 183 | class CatboostClassifierMultilabel(MultilabelEstimator): 184 | @property 185 | def estimator(self): 186 | return CatBoostClassifier 187 | 188 | 189 | class XGBoostClassifierMultilabel(MultilabelEstimator): 190 | @property 191 | def estimator(self): 192 | return XGBClassifier 193 | 194 | 195 | def make_transformer(estimator, mode='classifier'): 196 | if mode == 'classifier': 197 | transformer = SklearnClassifier(estimator) 198 | elif mode == 'regressor': 199 | transformer = SklearnRegressor(estimator) 200 | elif mode == 'transformer': 201 | transformer = SklearnTransformer(estimator) 202 | elif mode == 'pipeline': 203 | transformer = SklearnPipeline(estimator) 204 | else: 205 | raise NotImplementedError("""Only classifier, regressor and transformer modes are available""") 206 | 207 | return transformer 208 | -------------------------------------------------------------------------------- /src/steps/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | 5 | import pydot_ng as pydot 6 | from IPython.display import Image, display 7 | 8 | 9 | def view_pydot(pydot_object): 10 | plt = Image(pydot_object.create_png()) 11 | display(plt) 12 | 13 | 14 | def create_graph(graph_info): 15 | dot = pydot.Dot() 16 | for node in graph_info['nodes']: 17 | dot.add_node(pydot.Node(node)) 18 | for node1, node2 in graph_info['edges']: 19 | dot.add_edge(pydot.Edge(node1, node2)) 20 | return dot 21 | 22 | 23 | def view_graph(graph_info): 24 | graph = create_graph(graph_info) 25 | view_pydot(graph) 26 | 27 | 28 | def plot_graph(graph_info, filepath): 29 | graph = create_graph(graph_info) 30 | graph.write(filepath, format='png') 31 | 32 | 33 | def create_filepath(filepath): 34 | dirpath = os.path.dirname(filepath) 35 | os.makedirs(dirpath, exist_ok=True) 36 | 37 | 38 | def initialize_logger(): 39 | logger = logging.getLogger('steps') 40 | logger.setLevel(logging.INFO) 41 | message_format = logging.Formatter(fmt='%(asctime)s %(name)s >>> %(message)s', 42 | datefmt='%Y-%m-%d %H-%M-%S') 43 | 44 | # console handler for validation info 45 | ch_va = logging.StreamHandler(sys.stdout) 46 | ch_va.setLevel(logging.INFO) 47 | 48 | ch_va.setFormatter(fmt=message_format) 49 | 50 | # add the handlers to the logger 51 | logger.addHandler(ch_va) 52 | 53 | return logger 54 | 55 | 56 | def get_logger(): 57 | return logging.getLogger('steps') 58 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import math 4 | import os 5 | import ntpath 6 | import random 7 | import sys 8 | import time 9 | from itertools import product, chain 10 | from collections import defaultdict, Iterable 11 | 12 | import glob 13 | import numpy as np 14 | import pandas as pd 15 | import torch 16 | import yaml 17 | import imgaug as ia 18 | from PIL import Image 19 | from attrdict import AttrDict 20 | from pycocotools import mask as cocomask 21 | from pycocotools.coco import COCO 22 | from tqdm import tqdm 23 | from scipy import ndimage as ndi 24 | from .cocoeval import COCOeval 25 | from .steps.base import BaseTransformer 26 | 27 | 28 | def init_logger(): 29 | logger = logging.getLogger('mapping-challenge') 30 | logger.setLevel(logging.INFO) 31 | message_format = logging.Formatter(fmt='%(asctime)s %(name)s >>> %(message)s', 32 | datefmt='%Y-%m-%d %H-%M-%S') 33 | 34 | # console handler for validation info 35 | ch_va = logging.StreamHandler(sys.stdout) 36 | ch_va.setLevel(logging.INFO) 37 | 38 | ch_va.setFormatter(fmt=message_format) 39 | 40 | # add the handlers to the logger 41 | logger.addHandler(ch_va) 42 | 43 | return logger 44 | 45 | 46 | def get_logger(): 47 | return logging.getLogger('mapping-challenge') 48 | 49 | 50 | def get_filepaths(dirpath='.', extensions=None): 51 | if not extensions: 52 | extensions = ['.py', '.yaml', 'yml'] 53 | files = [] 54 | for r, d, f in os.walk(dirpath): 55 | for file in f: 56 | if any(file.endswith(ext) for ext in extensions): 57 | files.append(os.path.join(r, file)) 58 | return files 59 | 60 | 61 | def decompose(labeled): 62 | nr_true = labeled.max() 63 | masks = [] 64 | for i in range(1, nr_true + 1): 65 | msk = labeled.copy() 66 | msk[msk != i] = 0. 67 | msk[msk == i] = 255. 68 | masks.append(msk) 69 | 70 | if not masks: 71 | return [labeled] 72 | else: 73 | return masks 74 | 75 | 76 | def create_annotations(meta, predictions, logger, category_ids, category_layers, save=False, experiment_dir='./'): 77 | """ 78 | 79 | Args: 80 | meta: pd.DataFrame with metadata 81 | predictions: list of labeled masks or numpy array of size [n_images, im_height, im_width] 82 | logger: logging object 83 | category_ids: list with ids of categories, 84 | e.g. [None, 100] means, that no annotations will be created from category 0 data, and annotations 85 | from category 1 will be created with category_id=100 86 | category_layers: 87 | save: True, if one want to save submission, False if one want to return it 88 | experiment_dir: directory of experiment to save annotations, relevant if save==True 89 | 90 | Returns: submission if save==False else True 91 | 92 | """ 93 | annotations = [] 94 | logger.info('Creating annotations') 95 | category_layers_inds = np.cumsum(category_layers) 96 | for image_id, (prediction, image_scores) in zip(meta["ImageId"].values, predictions): 97 | for category_ind, (category_instances, category_scores) in enumerate(zip(prediction, image_scores)): 98 | category_nr = np.searchsorted(category_layers_inds, category_ind, side='right') 99 | if category_ids[category_nr] != None: 100 | masks = decompose(category_instances) 101 | for mask_nr, (mask, score) in enumerate(zip(masks, category_scores)): 102 | annotation = {} 103 | annotation["image_id"] = int(image_id) 104 | annotation["category_id"] = category_ids[category_nr] 105 | annotation["score"] = score 106 | annotation["segmentation"] = rle_from_binary(mask.astype('uint8')) 107 | annotation['segmentation']['counts'] = annotation['segmentation']['counts'].decode("UTF-8") 108 | annotation["bbox"] = bounding_box_from_rle(rle_from_binary(mask.astype('uint8'))) 109 | annotations.append(annotation) 110 | if save: 111 | submission_filepath = os.path.join(experiment_dir, 'submission.json') 112 | with open(submission_filepath, "w") as fp: 113 | fp.write(str(json.dumps(annotations))) 114 | logger.info("Submission saved to {}".format(submission_filepath)) 115 | logger.info('submission head \n\n{}'.format(annotations[0])) 116 | return True 117 | else: 118 | return annotations 119 | 120 | 121 | def rle_from_binary(prediction): 122 | prediction = np.asfortranarray(prediction) 123 | return cocomask.encode(prediction) 124 | 125 | 126 | def bounding_box_from_rle(rle): 127 | return list(cocomask.toBbox(rle)) 128 | 129 | 130 | def read_config(config_path): 131 | with open(config_path) as f: 132 | config = yaml.load(f) 133 | return AttrDict(config) 134 | 135 | 136 | def generate_metadata(data_dir, 137 | meta_dir, 138 | masks_overlayed_prefix, 139 | process_train_data=True, 140 | process_validation_data=True, 141 | ): 142 | def _generate_metadata(dataset): 143 | assert dataset in ["train", "val"], "Unknown dataset!" 144 | 145 | images_path = os.path.join(data_dir, dataset) 146 | 147 | images_path = os.path.join(images_path, "images") 148 | 149 | masks_overlayed_dirs, mask_overlayed_suffix = [], [] 150 | for file_path in glob.glob('{}/*'.format(meta_dir)): 151 | if ntpath.basename(file_path).startswith(masks_overlayed_prefix): 152 | masks_overlayed_dirs.append(file_path) 153 | mask_overlayed_suffix.append(ntpath.basename(file_path).replace(masks_overlayed_prefix, '')) 154 | df_dict = defaultdict(lambda: []) 155 | 156 | annotation_path = os.path.join(data_dir, dataset, 'annotation.json') 157 | 158 | with open(annotation_path) as f: 159 | annotation = json.load(f) 160 | file_name2img_id = {img['file_name']: img['id'] for img in annotation['images']} 161 | 162 | for image_file_path in tqdm(sorted(glob.glob('{}/*'.format(images_path)))): 163 | image_file_name = ntpath.basename(image_file_path) 164 | if dataset == "test_images": 165 | image_id = image_file_name.split('.')[0] 166 | else: 167 | image_id = file_name2img_id[image_file_name] 168 | 169 | n_buildings = None 170 | if dataset == "train": 171 | is_train, is_valid = 1, 0 172 | elif dataset == "val": 173 | is_train, is_valid = 0, 1 174 | else: 175 | raise NotImplementedError 176 | 177 | df_dict['ImageId'].append(image_id) 178 | df_dict['file_path_image'].append(image_file_path) 179 | df_dict['is_train'].append(is_train) 180 | df_dict['is_valid'].append(is_valid) 181 | df_dict['n_buildings'].append(n_buildings) 182 | 183 | for mask_dir, mask_dir_suffix in zip(masks_overlayed_dirs, mask_overlayed_suffix): 184 | file_path_mask = os.path.join(mask_dir, dataset, "masks", 185 | '{}.png'.format(image_file_name.split('.')[0])) 186 | df_dict['file_path_mask' + mask_dir_suffix].append(file_path_mask) 187 | 188 | return pd.DataFrame.from_dict(df_dict) 189 | 190 | metadata = pd.DataFrame() 191 | if process_train_data: 192 | train_metadata = _generate_metadata(dataset="train") 193 | metadata = metadata.append(train_metadata, ignore_index=True) 194 | if process_validation_data: 195 | validation_metadata = _generate_metadata(dataset="val") 196 | metadata = metadata.append(validation_metadata, ignore_index=True) 197 | 198 | if not (process_train_data or process_validation_data): 199 | raise ValueError('At least one of train_data or validation_data has to be set to True') 200 | 201 | return metadata 202 | 203 | 204 | def generate_inference_metadata(images_dir): 205 | df_dict = defaultdict(lambda: []) 206 | for image_id, image_file_path in tqdm(enumerate(sorted(glob.glob('{}/*'.format(images_dir))))): 207 | n_buildings = None 208 | df_dict['ImageId'].append(image_id) 209 | df_dict['file_path_image'].append(image_file_path) 210 | df_dict['is_train'].append(0) 211 | df_dict['is_valid'].append(0) 212 | df_dict['is_test'].append(1) 213 | df_dict['n_buildings'].append(n_buildings) 214 | 215 | return pd.DataFrame.from_dict(df_dict) 216 | 217 | 218 | def check_env_vars(): 219 | assert os.getenv('NEPTUNE_API_TOKEN'), """You must put your Neptune API token in the \ 220 | NEPTUNE_API_TOKEN env variable. You should run: 221 | $ export NEPTUNE_API_TOKEN=your_neptune_api_token""" 222 | assert os.getenv('CONFIG_PATH'), """You must specify path to the config file in \ 223 | CONFIG_PATH env variable. For example run: 224 | $ export CONFIG_PATH=neptune.yaml""" 225 | 226 | 227 | def squeeze_inputs(inputs): 228 | return np.squeeze(inputs[0], axis=1) 229 | 230 | 231 | def softmax(X, theta=1.0, axis=None): 232 | """ 233 | https://nolanbconaway.github.io/blog/2017/softmax-numpy 234 | Compute the softmax of each element along an axis of X. 235 | 236 | Parameters 237 | ---------- 238 | X: ND-Array. Probably should be floats. 239 | theta (optional): float parameter, used as a multiplier 240 | prior to exponentiation. Default = 1.0 241 | axis (optional): axis to compute values along. Default is the 242 | first non-singleton axis. 243 | 244 | Returns an array the same size as X. The result will sum to 1 245 | along the specified axis. 246 | """ 247 | 248 | # make X at least 2d 249 | y = np.atleast_2d(X) 250 | 251 | # find axis 252 | if axis is None: 253 | axis = next(j[0] for j in enumerate(y.shape) if j[1] > 1) 254 | 255 | # multiply y against the theta parameter, 256 | y = y * float(theta) 257 | 258 | # subtract the max for numerical stability 259 | y = y - np.expand_dims(np.max(y, axis=axis), axis) 260 | 261 | # exponentiate y 262 | y = np.exp(y) 263 | 264 | # take the sum along the specified axis 265 | ax_sum = np.expand_dims(np.sum(y, axis=axis), axis) 266 | 267 | # finally: divide elementwise 268 | p = y / ax_sum 269 | 270 | # flatten if X was 1D 271 | if len(X.shape) == 1: p = p.flatten() 272 | 273 | return p 274 | 275 | 276 | def from_pil(*images): 277 | images = [np.array(image) for image in images] 278 | if len(images) == 1: 279 | return images[0] 280 | else: 281 | return images 282 | 283 | 284 | def to_pil(*images): 285 | images = [Image.fromarray((image).astype(np.uint8)) for image in images] 286 | if len(images) == 1: 287 | return images[0] 288 | else: 289 | return images 290 | 291 | 292 | def set_seed(seed): 293 | random.seed(seed) 294 | np.random.seed(seed) 295 | torch.manual_seed(seed) 296 | if torch.cuda.is_available(): 297 | torch.cuda.manual_seed_all(seed) 298 | 299 | 300 | def generate_data_frame_chunks(meta, chunk_size): 301 | n_rows = meta.shape[0] 302 | chunk_nr = math.ceil(n_rows / chunk_size) 303 | for i in tqdm(range(chunk_nr)): 304 | meta_chunk = meta.iloc[i * chunk_size:(i + 1) * chunk_size] 305 | yield meta_chunk 306 | 307 | 308 | def coco_evaluation(gt_filepath, prediction_filepath, image_ids, category_ids, small_annotations_size): 309 | coco = COCO(gt_filepath) 310 | coco_results = coco.loadRes(prediction_filepath) 311 | cocoEval = COCOeval(coco, coco_results) 312 | cocoEval.params.imgIds = image_ids 313 | cocoEval.params.catIds = category_ids 314 | cocoEval.params.areaRng = [[0 ** 2, 1e5 ** 2], [0 ** 2, small_annotations_size ** 2], 315 | [small_annotations_size ** 2, 1e5 ** 2]] 316 | cocoEval.params.areaRngLbl = ['all', 'small', 'large'] 317 | cocoEval.evaluate() 318 | cocoEval.accumulate() 319 | cocoEval.summarize() 320 | 321 | return cocoEval.stats[0], cocoEval.stats[3] 322 | 323 | 324 | def denormalize_img(image, mean, std): 325 | return image * np.array(std).reshape(3, 1, 1) + np.array(mean).reshape(3, 1, 1) 326 | 327 | 328 | def label(mask): 329 | labeled, nr_true = ndi.label(mask) 330 | return labeled 331 | 332 | 333 | def add_dropped_objects(original, processed): 334 | reconstructed = processed.copy() 335 | labeled = label(original) 336 | for i in range(1, labeled.max() + 1): 337 | if not np.any(np.where((labeled == i) & processed)): 338 | reconstructed += (labeled == i) 339 | return reconstructed.astype('uint8') 340 | 341 | 342 | def make_apply_transformer(func, output_name='output', apply_on=None): 343 | class StaticApplyTransformer(BaseTransformer): 344 | def transform(self, *args, **kwargs): 345 | self.check_input(*args, **kwargs) 346 | 347 | if not apply_on: 348 | iterator = zip(*args, *kwargs.values()) 349 | else: 350 | iterator = zip(*args, *[kwargs[key] for key in apply_on]) 351 | 352 | output = [] 353 | for func_args in tqdm(iterator, total=self.get_arg_length(*args, **kwargs)): 354 | output.append(func(*func_args)) 355 | return {output_name: output} 356 | 357 | @staticmethod 358 | def check_input(*args, **kwargs): 359 | if len(args) and len(kwargs) == 0: 360 | raise Exception('Input must not be empty') 361 | 362 | arg_length = None 363 | for arg in chain(args, kwargs.values()): 364 | if not isinstance(arg, Iterable): 365 | raise Exception('All inputs must be iterable') 366 | arg_length_loc = None 367 | try: 368 | arg_length_loc = len(arg) 369 | except: 370 | pass 371 | if arg_length_loc is not None: 372 | if arg_length is None: 373 | arg_length = arg_length_loc 374 | elif arg_length_loc != arg_length: 375 | raise Exception('All inputs must be the same length') 376 | 377 | @staticmethod 378 | def get_arg_length(*args, **kwargs): 379 | arg_length = None 380 | for arg in chain(args, kwargs.values()): 381 | if arg_length is None: 382 | try: 383 | arg_length = len(arg) 384 | except: 385 | pass 386 | if arg_length is not None: 387 | return arg_length 388 | 389 | return StaticApplyTransformer() 390 | 391 | 392 | def make_apply_transformer_stream(func, output_name='output', apply_on=None): 393 | class StaticApplyTransformerStream(BaseTransformer): 394 | def transform(self, *args, **kwargs): 395 | self.check_input(*args, **kwargs) 396 | return {output_name: self._transform(*args, **kwargs)} 397 | 398 | def _transform(self, *args, **kwargs): 399 | if not apply_on: 400 | iterator = zip(*args, *kwargs.values()) 401 | else: 402 | iterator = zip(*args, *[kwargs[key] for key in apply_on]) 403 | 404 | for func_args in tqdm(iterator): 405 | yield func(*func_args) 406 | 407 | @staticmethod 408 | def check_input(*args, **kwargs): 409 | for arg in chain(args, kwargs.values()): 410 | if not isinstance(arg, Iterable): 411 | raise Exception('All inputs must be iterable') 412 | 413 | return StaticApplyTransformerStream() 414 | 415 | 416 | def get_seed(): 417 | seed = int(time.time()) + int(os.getpid()) 418 | return seed 419 | 420 | 421 | def reseed(augmenter_sequence, deterministic=True): 422 | for aug in augmenter_sequence: 423 | aug.random_state = ia.new_random_state(get_seed()) 424 | if deterministic: 425 | aug.deterministic = True 426 | return augmenter_sequence 427 | --------------------------------------------------------------------------------