├── .github ├── .gitignore ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── issue--data-contribution.md └── workflows │ └── pypi_publish.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── ci └── envs │ ├── 310-coastseg.yaml │ ├── 311-coastseg.yaml │ └── 39-coastseg.yaml ├── gdal_retile.py ├── pyproject.toml ├── requirements.txt ├── requirements_dev.txt ├── seg2map.ipynb ├── setup.py ├── src └── seg2map │ ├── __init__.py │ ├── common.py │ ├── downloads.py │ ├── exception_handler.py │ ├── exceptions.py │ ├── log_maker.py │ ├── map_UI.py │ ├── map_functions.py │ ├── map_interface.py │ ├── model_functions.py │ ├── models_UI.py │ ├── new_downloads.py │ ├── roi.py │ ├── sessions.py │ └── zoo_model.py ├── test_models.py ├── unet.ipynb └── unzipper.py /.github/.gitignore: -------------------------------------------------------------------------------- 1 | data\* 2 | src\*\downloaded_models\* 3 | *segmentation_data_* 4 | # Distribution / packaging 5 | md2rst.md 6 | md2rst.rst 7 | testing/ 8 | .Python 9 | env/ 10 | build/ 11 | develop-eggs/ 12 | dist/ 13 | downloads/ 14 | eggs/ 15 | .eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | wheels/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Thanks for finding a bug! Please read and follow these instructions carefully, then delete this introductory text to keep your issue easy to read. Feel free to keep the titles described here to ensure our team can easily understand your issue. Note that the issue tracker is NOT the place for usage questions and technical assistance. 11 | 12 | ## Describe the bug 13 | A clear and concise description of what the bug is. 14 | 15 | ## Steps To Reproduce 16 | Steps to reproduce the behavior: 17 | 1. Go to '...' 18 | 2. Click on '....' 19 | 3. Scroll down to '....' 20 | 4. See error 21 | 22 | ## Expected behavior 23 | A clear and concise description of what you expected to happen. 24 | 25 | ## Screenshots 26 | If applicable, add screenshots to help explain your problem. 27 | ## ALL software version info 28 | **Desktop (please complete the following information):** 29 | (this library, plus any other relevant software, e.g. bokeh, python, notebook, OS, browser, etc) 30 | - OS: [e.g. iOS] 31 | - Browser [e.g. chrome, safari] 32 | - Version [e.g. 22] 33 | 34 | 35 | ### **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue--data-contribution.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Issue: Data Contribution' 3 | about: Contribute a shoreline, transect, or other data 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Thanks for contributing! Please read and follow these instructions carefully, then delete this introductory text to keep your issue easy to read. Feel free to keep the titles described here to ensure our team can easily integrate your contribution. 11 | 12 | ## Describe the Contribution 13 | A clear and concise description of what you are contributing. 14 | - Where is the data from? 15 | - What is its purpose? 16 | - How large is it (MB)? 17 | 18 | ### Data Type (Choose One) 19 | 1. Shoreline geojson file 20 | 2. Transect geojson file 21 | 3. Other 22 | 23 | #### Shoreline Contributions 24 | You must include a json file called `file_bounds.json` that includes a bounding boxes for each of the shorelines you wish to contribute. These bounding boxes are used to check if the user's ROI's intersect with the shoreline. You can check [file_bounds.json](https://zenodo.org/record/7033367#.YxFQFXbMI2w) for an example. 25 | 26 | The shorelines must be contained within geojson files that are no larger than 15MB. You can check [zenodo release for USA county shorelines](https://zenodo.org/record/7033367#.YxFQFXbMI2w) for an example. 27 | 28 | #### Transect Contributions 29 | The transects must be contained within geojson files that are no larger than 15MB. 30 | 31 | 32 | ## Screenshots 33 | If applicable, add screenshots to show your contribution. For example for contributing a shoreline show where the shoreline is on a map. 34 | 35 | 36 | ### **Additional context** 37 | Add any other context about the contribution here. 38 | 39 | ## Data 40 | Upload your file(s) as a zip or link to a safe location to download the data from. 41 | -------------------------------------------------------------------------------- /.github/workflows/pypi_publish.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: 3 | push: 4 | branches: [main] 5 | tags: 6 | - "v[0-9]+.[0-9]+.[0-9]+" 7 | - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+" 8 | - "v[0-9]+.[0-9]+.[0-9]+b[0-9]+" 9 | - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+" 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | name: Build and deploy to PyPI 15 | runs-on: "ubuntu-latest" 16 | defaults: 17 | run: 18 | shell: bash -l {0} 19 | steps: 20 | - uses: actions/checkout@v3 21 | with: 22 | fetch-depth: "100" 23 | - uses: actions/setup-python@v4 24 | with: 25 | python-version: "3.10" 26 | # - name: Install dependencies 27 | # run: | 28 | # python -m pip install --upgrade pip 29 | # pip install setuptools wheel twine 30 | - name: Install pypa/build 31 | run: >- 32 | python -m 33 | pip install 34 | build 35 | --user 36 | - name: Build a binary wheel and a source tarball 37 | run: >- 38 | python -m 39 | build 40 | --sdist 41 | --wheel 42 | --outdir dist/ 43 | . 44 | - name: Publish to PyPI 45 | uses: pypa/gh-action-pypi-publish@release/v1 46 | with: 47 | verbose: true 48 | password: ${{ secrets.SEG2MAP_PYPI_TOKEN}} 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # (Seg2Map only) 2 | # downloaded files to ignore 3 | *chunk* 4 | *merged* 5 | *.tif* 6 | *multiband* 7 | *USDA* 8 | 9 | data\* 10 | sessions\* 11 | ./sessions/* 12 | /sessions/ 13 | src\*\downloaded_models\* 14 | *segmentation_data_* 15 | *downloaded_models* 16 | 17 | data/* 18 | config_gdf.geojson 19 | config.json 20 | 21 | # Byte-compiled / optimized / DLL files 22 | __pycache__/ 23 | *.py[cod] 24 | *$py.class 25 | # C extensions 26 | *.so 27 | 28 | # Distribution / packaging 29 | .Python 30 | build/ 31 | develop-eggs/ 32 | dist/ 33 | downloads/ 34 | eggs/ 35 | .eggs/ 36 | lib/ 37 | lib64/ 38 | parts/ 39 | sdist/ 40 | var/ 41 | wheels/ 42 | pip-wheel-metadata/ 43 | share/python-wheels/ 44 | *.egg-info/ 45 | .installed.cfg 46 | *.egg 47 | MANIFEST 48 | 49 | # PyInstaller 50 | # Usually these files are written by a python script from a template 51 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 52 | *.manifest 53 | *.spec 54 | 55 | # Installer logs 56 | pip-log.txt 57 | pip-delete-this-directory.txt 58 | 59 | # Unit test / coverage reports 60 | htmlcov/ 61 | .tox/ 62 | .nox/ 63 | .coverage 64 | .coverage.* 65 | .cache 66 | nosetests.xml 67 | coverage.xml 68 | *.cover 69 | *.py,cover 70 | .hypothesis/ 71 | .pytest_cache/ 72 | 73 | # Translations 74 | *.mo 75 | *.pot 76 | 77 | # Django stuff: 78 | *.log 79 | local_settings.py 80 | db.sqlite3 81 | db.sqlite3-journal 82 | 83 | # Flask stuff: 84 | instance/ 85 | .webassets-cache 86 | 87 | # Scrapy stuff: 88 | .scrapy 89 | 90 | # Sphinx documentation 91 | docs/_build/ 92 | 93 | # PyBuilder 94 | target/ 95 | 96 | # Jupyter Notebook 97 | .ipynb_checkpoints 98 | 99 | # IPython 100 | profile_default/ 101 | ipython_config.py 102 | 103 | # pyenv 104 | .python-version 105 | 106 | # pipenv 107 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 108 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 109 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 110 | # install all needed dependencies. 111 | #Pipfile.lock 112 | 113 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 114 | __pypackages__/ 115 | 116 | # Celery stuff 117 | celerybeat-schedule 118 | celerybeat.pid 119 | 120 | # SageMath parsed files 121 | *.sage.py 122 | 123 | # Environments 124 | .env 125 | .venv 126 | env/ 127 | venv/ 128 | ENV/ 129 | env.bak/ 130 | venv.bak/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | .spyproject 135 | 136 | # Rope project settings 137 | .ropeproject 138 | 139 | # mkdocs documentation 140 | /site 141 | 142 | # mypy 143 | .mypy_cache/ 144 | .dmypy.json 145 | dmypy.json 146 | 147 | # Pyre type checker 148 | .pyre/ 149 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Doodleverse 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include MANIFEST.in 2 | include LICENSE 3 | include pyproject.toml 4 | include *.md 5 | 6 | # Include src directories in package 7 | graft src 8 | 9 | # Exclude downloaded models 10 | recursive-exclude src/seg2map/downloaded_models * 11 | 12 | # Remove the pycache directory and any pycache files 13 | prune src/seg2map/__pycache__ 14 | recursive-exclude * *.py[co] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Seg2Map :mag_right: :milky_way: 2 | 3 | _An interactive web map app for applying Doodleverse/Zoo models to geospatial imagery_ 4 | 5 | ![](https://user-images.githubusercontent.com/3596509/194389595-82ade668-daf0-4d24-b1a0-6ecf897f40fe.gif) 6 | 7 | ![separate_seg_controls_demo (1)](https://github.com/Doodleverse/seg2map/assets/61564689/d527fe8c-c3f2-4c62-b448-e581162e8475) 8 | 9 | ## Overview: 10 | 11 | - Seg2Map facilitates application of Deep Learning-based image segmentation models and apply them to high-resolution (~1m or less spatial footprint) geospatial imagery, in order to make high-resolution label maps. Please see our [wiki](https://github.com/Doodleverse/seg2map/wiki) for more information. 12 | 13 | - The principle aim is to generate time-series of label maps from a time-series of imagery, in order to detect and assess land use/cover change. This project also demonstrates how to apply generic models for land-use/cover on publicly available high-resolution imagery at arbitrary locations. 14 | 15 | - Imagery comes from Google Earth Engine via [s2m_engine](https://github.com/Doodleverse/s2m_engine). Initially, we focus on [NAIP](https://www.usgs.gov/centers/eros/science/usgs-eros-archive-aerial-photography-national-agriculture-imagery-program-naip) time-series, available for the conterminious United States since 2003. In the future, [Planetscope](https://developers.planet.com/docs/data/planetscope/) imagery may also be made available (for those with access, such as federal researchers). 16 | 17 | - We offer a set of [Segmentation Zoo](https://github.com/Doodleverse/segmentation_zoo) models, especially created and curated for this project based on a publicly available datasets. These datasets have been selected because they are public, large (several hundred to several thousand labeled images), and provide broad class labels for generic land use/cover mapping needs. 18 | 19 | # Installation Instructions 20 | 21 | In order to use seg2map you need to install Python packages in an environment. We recommend you use [Anaconda](https://www.anaconda.com/products/distribution) to install the python packages in an environment for seg2map. After you install Anaconda on your PC, open the Anaconda prompt or Terminal in Mac and Linux and use the `cd` command (change directory) to go the folder where you have downloaded the seg2map repository. 22 | 23 | 1. Create an Anaconda environment 24 | 25 | - This command creates an anaconda environment named `seg2map` and installs `python 3.10` in it 26 | - You can also use `python 3.10` 27 | - We will install the seg2map package and its dependencies in this environment. 28 | ```bash 29 | conda create --name seg2map python=3.10 -y 30 | 31 | 2. Activate your conda environment 32 | 33 | ```bash 34 | conda activate seg2map 35 | ``` 36 | 37 | - If you have successfully activated seg2map you should see that your terminal's command line prompt should now start with `(seg2map)`. 38 | 39 | 3. Install Conda Dependencies 40 | 41 | - seg2map requires `jupyterlab` and `geopandas` to function properly so they will be installed in the `seg2map` environment. 42 | - [Geopandas](https://geopandas.org/en/stable/) has [GDAL](https://gdal.org/) as a dependency so its best to install it with conda. 43 | - Make sure to install geopandas from the `conda-forge` channel to ensure you get the latest version. 44 | - Make sure to install both jupyterlab and geopandas from the conda forge channel to avoid dependency conflicts 45 | 46 | ```bash 47 | conda install -c conda-forge geopandas jupyterlab -y 48 | ``` 49 | 50 | 4. Install the seg2map from PyPi 51 | ```bash 52 | pip install seg2map 53 | ``` 54 | 5. Uninstall the h5py installed by pip and reinstall with conda-forge 55 | ```bash 56 | pip uninstall h5py -y 57 | conda install -c conda-forge h5py -y 58 | ``` 59 | ## **Having Installation Errors?** 60 | 61 | Use the command `conda clean --all` to clean old packages from your anaconda base environment. Ensure you are not in your seg2map environment or any other environment by running `conda deactivate`, to deactivate any environment you're in before running `conda clean --all`. It is recommended that you have Anaconda prompt (terminal for Mac and Linux) open as an administrator before you attempt to install `seg2map` again. 62 | 63 | #### Conda Clean Steps 64 | 65 | ```bash 66 | conda deactivate 67 | conda clean --all 68 | ``` 69 | 70 | # How to Use Seg2Map 71 | 72 | 1. Sign up to use Google Earth Engine Python API 73 | 74 | First, you need to request access to Google Earth Engine at https://signup.earthengine.google.com/. It takes about 1 day for Google to approve requests. 75 | 76 | 2. Activate your conda environment 77 | 78 | ```bash 79 | conda activate seg2map 80 | ``` 81 | 82 | - If you have successfully activated seg2map you should see that your terminal's command line prompt should now start with `(seg2map)`. 83 | 84 | 3. Install the seg2map from PyPi 85 | ```bash 86 | cd 87 | ex: cd C:\1_repos\seg2map 88 | ``` 89 | 4. Launch Jupyter Lab 90 | 91 | - make you run this command in the seg2map directory so you can choose a notebook to use. 92 | ```bash 93 | jupyter lab 94 | ``` 95 | 96 | ## Features 97 | ### 1. Download Imagery from Google Earth Engine 98 | Use google earth engine to download multiple years worth of imagery. 99 | ![download_imagery_demo](https://github.com/Doodleverse/seg2map/assets/61564689/a36421de-e6d2-4a3f-8c08-2e47be99e3e0) 100 | 101 | ### You can download multiple ROIs and years of data at lighting speeds 🌩️ 102 | 103 | ![download_imagery_demo_multi_roi](https://github.com/Doodleverse/seg2map/assets/61564689/46219ca8-beed-46e0-a28f-0d5ceab6d474) 104 | 105 | 106 | ### 2. Apply Models to Imagery 107 | 108 | ![apply_model_demo](https://github.com/Doodleverse/seg2map/assets/61564689/75c55659-56f4-46f3-892d-6bebdcd6a653) 109 | 110 | 111 | ### 3. Load Segmented Imagery onto the Map 112 | 113 | ![load_segmentation_demo](https://github.com/Doodleverse/seg2map/assets/61564689/d6bbf3ba-a8a9-4e90-b47a-61c9c2fe4799) 114 | 115 | ## Generic workflow: 116 | * Provide a web map for navigation to a location, and draw a bounding box 117 | * Provide an interface for controls (set time period, etc) 118 | * Download geospatial imagery (for now, just NAIP) 119 | * Provide tools to select and apply a Zoo model to create a label image 120 | * Provide tools to interact with those label images (download, mosaic, merge classes, etc) 121 | 122 | ## Authors 123 | * [@dbuscombe-usgs](https://github.com/dbuscombe-usgs) 124 | * [@2320sharon](https://github.com/2320sharon) 125 | 126 | Contributions: 127 | * [@venuswku](https://github.com/venuswku) 128 | 129 | We welcome collaboration! Please use our [Discussions](https://github.com/Doodleverse/seg2map/discussions) tab if you're interested in this project. We welcome user-contributed models! They must be trained using [Segmentation Gym](https://github.com/Doodleverse/segmentation_gym), and then served and documented through [Segmentation Zoo](https://github.com/Doodleverse/segmentation_zoo) - get in touch and we'll walk you through the process! 130 | 131 | ## Roadmap / progress 132 | 133 | ### V1 134 | - [X] Develop codes to create a web map for navigation to a location, and draw a bounding box 135 | - [X] Develop codes interface for controls (time period, etc) 136 | - [X] Develop codes for downloading NAIP imagery using GEE 137 | - [X] Put together a prototype jupyter notebook for web map, bounding box, and image downloads 138 | - [ ] Create Seg2Map models 139 | - [X] [Coast Train](https://coasttrain.github.io/CoastTrain/) / aerial / high-res. sat 140 | - [X] 2 class [dataset](https://coasttrain.github.io/CoastTrain/docs/Version%201:%20March%202022/data) (water, other) 141 | - [X] zenodo release for 768x768 imagery [zenodo page](https://doi.org/10.5281/zenodo.7574784) 142 | - [X] [Coast Train](https://coasttrain.github.io/CoastTrain/) / NAIP 143 | - [X] 5 class [dataset](https://coasttrain.github.io/CoastTrain/docs/Version%201:%20March%202022/data) (water, whitewater, sediment, bare terrain, other terrain) 144 | - [X] 8 class [dataset](https://coasttrain.github.io/CoastTrain/docs/Version%201:%20March%202022/data) (water, whitewater, sediment, bare terrain, marsh veg, terrestrial veg, ag., dev.) 145 | - [X] zenodo release of 5-class ResUNet models for 768x768 imagery [zenodo page](https://doi.org/10.5281/zenodo.7566992) 146 | - [X] zenodo release of 8-class ResUNet models for 768x768 imagery [zenodo page](https://doi.org/10.5281/zenodo.7570583) 147 | - [X] zenodo release of 5-class Segformer models for 768x768 imagery [zenodo page](https://doi.org/10.5281/zenodo.7641708) 148 | - [X] zenodo release of 8-class Segformer models for 768x768 imagery [zenodo page](https://doi.org/10.5281/zenodo.7641724) 149 | - [X] [Chesapeake Landcover](https://lila.science/datasets/chesapeakelandcover) (CCLC) / NAIP 150 | - [X] 7 class [dataset](https://lila.science/datasets/chesapeakelandcover) (water, tree canopy / forest, low vegetation / field, barren land, impervious (other), impervious (road), no data) 151 | - [X] zenodo release of 7-class ResUNet models for 512x512 imagery [page](https://doi.org/10.5281/zenodo.7576904) 152 | - [X] zenodo release of 7-class SegFormer models for 512x512 imagery [page](https://doi.org/10.5281/zenodo.7677506) 153 | - [X] [EnviroAtlas](https://zenodo.org/record/6268150#.Y9H3vxzMLRZ) / NAIP 154 | - [X] 6 class dataset (water, impervious, barren, trees, herbaceous, shrubland) 155 | - [X] zenodo release of 6-class ResUNet models for 1024 x 1024 models [zenodo page](https://doi.org/10.5281/zenodo.7576909) 156 | - [X] [OpenEarthMap](https://open-earth-map.org/) / aerial / high-res. sat 157 | - [X] 9 class [dataset](https://zenodo.org/record/7223446#.Y9IN2BzMLRY) (bareland, rangeland, dev., road, tree, water, ag., building, nodata) 158 | - [X] zenodo release of 9-class ResUNet models for 512x512 models [zenodo page](https://doi.org/10.5281/zenodo.7576894) 159 | - [X] [DeepGlobe](https://arxiv.org/abs/1805.06561) / aerial / high-res. sat 160 | - [X] 7 class [dataset](https://www.kaggle.com/datasets/balraj98/deepglobe-land-cover-classification-dataset) (urban, ag., rangeland, forest, water, bare, unknown) 161 | - [X] zenodo release for of 7-class ResUNet models 512x512 imagery [zenodo page](https://doi.org/10.5281/zenodo.7576898) 162 | - [ ] [Barrier Islands](https://www.sciencebase.gov/catalog/item/5d5ece47e4b01d82ce961e36) / orthomosaic / coastlines 163 | - [ ] Substrate data 6 class (dev, sand, mixed, coarse, unknown, water) 164 | - [ ] zenodo release of substrate models for 768x768 imagery 165 | - [ ] Vegetation type data 7 class (shrub/forest, shrub, none/herb., none, herb., herb./shrub, dev) 166 | - [ ] zenodo release of Vegetation type models for 768x768 imagery 167 | - [ ] Vegetation density data 7 class (dense, dev., moderate, moderate/dense, none, none/sparse, sparse) 168 | - [ ] zenodo release of Vegetation density models for 768x768 imagery 169 | - [X] Geomorphic setting data 7 class (beach, backshore, dune, washover, barrier interior, marsh, ridge/swale) 170 | - [ ] zenodo release of Geomorphic setting models for 768x768 imagery 171 | - [X] Supervised classification data 9 class (water, sand, herbaceous veg./low shrub, sparse/moderate, herbaceous veg/low shrub, moderate/dense, high shrub/forest, marsh/sediment, marsh/veg, marsh, high shrub/forest, development) 172 | - [ ] zenodo release of Supervised classification models for 768x768 imagery 173 | - [X] [AAAI](https://github.com/FrontierDevelopmentLab/multi3net) / aerial / high-res. sat 174 | - [X] 2 class dataset (other, building) 175 | - [X] zenodo release for 1024x1024 imagery [zenodo page](https://doi.org/10.5281/zenodo.7607895) 176 | - [X] 2 class dataset (other, flooded building) 177 | - [X] zenodo release for 1024x1024 imagery [zenodo page](https://doi.org/10.5281/zenodo.7613106) 178 | - [X] xBD-hurricanes / aerial / high-res. sat, a subset of the [XView2](https://xview2.org/) dataset 179 | - [X] 4 class building dataset (other, no damage, minor damage, major damage) 180 | - [X] zenodo release for 768x768 imagery [zenodo page](https://doi.org/10.5281/zenodo.7613175) 181 | - [X] 2 class building dataset (other, building) 182 | - [X] zenodo release for 768x768 imagery [zenodo page](https://doi.org/10.5281/zenodo.7613212) 183 | - [ ] Superclass models 184 | - [X] 8 merged datasets for 8 separate superclass models (water, sediment, veg, herb. veg., woody veg., impervious, building, agriculture) 185 | - [ ] zenodo release for 768x768 imagery / water 186 | - [ ] zenodo release for 768x768 imagery / sediment 187 | - [ ] zenodo release for 768x768 imagery / veg 188 | - [ ] zenodo release for 768x768 imagery / herb. veg. 189 | - [ ] zenodo release for 768x768 imagery / woody veg. 190 | - [ ] zenodo release for 768x768 imagery / impervious 191 | - [ ] zenodo release for 768x768 imagery / building 192 | - [ ] zenodo release for 768x768 imagery / agriculture 193 | - [ ] Develop codes/docs for selecting model 194 | - [ ] Develop codes/docs for applying model to make label imagery 195 | - [ ] Tool for mosaicing labels 196 | - [ ] Tool for downloading labels in geotiff format 197 | 198 | ### V2 199 | - [ ] Tool for post-processing/editing labels 200 | - [ ] Tool for detecting change 201 | - [ ] Make [Planetscope](https://developers.planet.com/docs/data/planetscope/) 3m imagery available via Planet API (federal researchers only) 202 | - [ ] Include additional models/datasets (TBD) 203 | 204 | ## Datasets 205 | 206 | ### General Landcover 207 | 208 | #### DeepGlobe 209 | * [paper](https://arxiv.org/abs/1805.06561) 210 | * [challenge](http://deepglobe.org/challenge.html) 211 | * [data](https://www.kaggle.com/datasets/balraj98/deepglobe-land-cover-classification-dataset) 212 | * Zenodo model release (512x512): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Res-UNet models for DeepGlobe/7-class segmentation of RGB 512x512 high-res. images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7576898 213 | 214 | #### EnviroAtlas 215 | * [EnviroAtlas dataset](https://zenodo.org/record/6268150#.Y9H3vxzMLRZ) 216 | * [EnviroAtlas paper](https://www.mdpi.com/2072-4292/12/12/1909) 217 | * [paper using EnviroAtlasdata](https://arxiv.org/pdf/2202.14000.pdf) 218 | * This dataset was organized to accompany the 2022 paper, [Resolving label uncertainty with implicit generative models](https://openreview.net/forum?id=AEa_UepnMDX). More details can be found [here](https://github.com/estherrolf/qr_for_landcover) 219 | * Zenodo model release (512x512): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Res-UNet models for EnviroAtlas/6-class segmentation of RGB 512x512 high-res. images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7576909 220 | 221 | #### OpenEarthMap 222 | * [website](https://open-earth-map.org/) 223 | * [data](https://zenodo.org/record/7223446#.Y7zQLxXMK3A) 224 | * [paper](https://arxiv.org/abs/2210.10732) 225 | * Zenodo model release (512x512): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Res-UNet models for OpenEarthMap/9-class segmentation of RGB 512x512 high-res. images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7576894 226 | 227 | 228 | ### Coastal Landcover 229 | 230 | #### Chesapeake Landcover 231 | * [webpage](https://lila.science/datasets/chesapeakelandcover) 232 | * Zenodo model release (512x512): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Res-UNet models for Chesapeake/7-class segmentation of RGB 512x512 high-res. images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7576904 233 | * Zenodo SegFormer model release (512x512): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Segformer models for Chesapeake/7-class segmentation of RGB 512x512 high-res. images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7677506 234 | 235 | #### Coast Train 236 | * [paper](https://www.nature.com/articles/s41597-023-01929-2) 237 | * [website](https://coasttrain.github.io/CoastTrain/) 238 | * [data](https://cmgds.marine.usgs.gov/data-releases/datarelease/10.5066-P91NP87I/) 239 | * [preprint](https://eartharxiv.org/repository/view/3560/) 240 | * Zenodo model release, 2-class (768x768): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Res-UNet models for CoastTrain water/other segmentation of RGB 768x768 orthomosaic images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7574784 241 | * Zenodo model release, 5-class (768x768): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Res-UNet models for CoastTrain/5-class segmentation of RGB 768x768 NAIP images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7566992 242 | * Zenodo model release, 8-class (768x768): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Res-UNet models for CoastTrain/8-class segmentation of RGB 768x768 NAIP images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7570583 243 | * Zenodo SegFormer model release, 5-class (768x768): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map SegFormer models for CoastTrain/5-class segmentation of RGB 768x768 NAIP images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7641708 244 | * Zenodo SegFormer model release, 8-class (768x768): Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map SegFormer models for CoastTrain/8-class segmentation of RGB 768x768 NAIP images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7641724 245 | 246 | #### AAAI / Buildings / Flooded Buildings 247 | * [data](https://github.com/FrontierDevelopmentLab/multi3net) 248 | * [data](https://github.com/orion29/Satellite-Image-Segmentation-for-Flood-Damage-Analysis) 249 | * [paper](https://arxiv.org/pdf/1812.01756.pdf) 250 | * Zenodo model release (1024x1024) building / no building: Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Res-UNet models for segmentation of buildings of RGB 1024x1024 high-res. images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7607895 251 | * Zenodo model release (1024x1024) flooded building / no flooded building: Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map Res-UNet models for segmentation of AAAI/flooded buildings in RGB 1024x1024 high-res. images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7622733 252 | 253 | #### XBD-hurricanes 254 | * [Xview2 challenge](https://xview2.org/) 255 | * [XBD-hurricanes code](https://github.com/MARDAScience/XBD-hurricanes) 256 | * Zenodo SegFormer model release (768x768) building damage: Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map SegFormer models for segmentation of xBD/damaged buildings in RGB 768x768 high-res. images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7613175 257 | * Zenodo SegFormer model release (768x768) building presence/absence: Buscombe, Daniel. (2023). Doodleverse/Segmentation Zoo/Seg2Map SegFormer models for segmentation of xBD/buildings in RGB 768x768 high-res. images (v1.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7613212 258 | 259 | #### Barrier Islands 260 | * [webpage](https://www.sciencebase.gov/catalog/item/5d5ece47e4b01d82ce961e36) 261 | * [paper](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0209986) 262 | * Zenodo Substrate model release (768x768): 263 | * Zenodo Vegetation type model release (768x768): 264 | * Zenodo Vegetation density model release (768x768): 265 | * Zenodo Geomorphic model release (768x768): 266 | * Zenodo Supervised classification model release (768x768): 267 | 268 | 269 | ## Superclasses 270 | 271 | A. Water: 272 | * Coast Train 273 | * Chesapeake 274 | * EnviroAtlas 275 | * OpenEarthMap 276 | * DeepGlobe 277 | * Barrier Substrate 278 | * NOAA 279 | * [v2: Barrier Substrate] 280 | * [v2: Elwha] 281 | 282 | B. Sediment: 283 | * Coast Train 284 | * NOAA 285 | * [v2: Barrier Substrate (sand, mixed, coarse)] 286 | * [v2: Elwha] 287 | 288 | C. Bare: 289 | * Chesapeake (barren land) 290 | * EnviroAtlas (barren) 291 | * OpenEarthMap (bareland) 292 | 293 | D. Vegetated: 294 | * Coast Train (marsh veg, terrestrial veg, ag) 295 | * FloodNet (tree, grass) 296 | * Chesapeake (tree canopy / forest, low vegetation / field) 297 | * EnviroAtlas (trees, herbaceous, shrubland) 298 | * OpenEarthMap (rangeland, tree, ag) 299 | * DeepGlobe (ag., rangeland, forest) 300 | * NOAA (veg) 301 | * [v2: Elwha] 302 | * [v2: Barrier Substrate] 303 | 304 | E. Impervious: 305 | * FloodNet (Building-flooded, Building-non-flooded, Road-flooded, Road-non-flooded, vehicle) 306 | * Chesapeake (impervious (other), impervious (road)) 307 | * EnviroAtlas (impervious) 308 | * OpenEarthMap (dev, road, building) 309 | * DeepGlobe (urban) 310 | * NOAA (dev) 311 | * [v2: Elwha] 312 | 313 | F. Building: 314 | * OpenEarthMap (building) 315 | * AAAI (building) 316 | 317 | G. Agriculture: 318 | * OpenEarthMap (ag) 319 | * DeepGlobe (ag) 320 | 321 | H. Woody Veg: 322 | * FloodNet (tree) 323 | * Chesapeake (tree canopy / forest) 324 | * EnviroAtlas (trees) 325 | * OpenEarthMap (tree) 326 | * DeepGlobe (forest) 327 | * [v2: Elwha] 328 | * [v2: Barrier Substrate] 329 | 330 | ## References 331 | 332 | ### Notes 333 | * [NLCD classes](https://www.mrlc.gov/data/legends/national-land-cover-database-class-legend-and-description) 334 | * [NAIP imagery](https://doi.org/10.5066/F7QN651G) 335 | 336 | 337 | ## Classes: 338 | 339 | | | Coast Train 1 | Coast Train 2 | Coast Train 3| FloodNet | Chesapeake| EnviroAtlas| OpenEarthMap| DeepGlobe| AAAI | NOAA | Barrier Substrate | 340 | |---|---|---|---|---|---|---|---|---|---|---|---| 341 | |A. Water | X| X|X |X |X |X |X |X | | X|X| 342 | |a. whitewater | |X |X | | | | | | | | | 343 | |a. pool | | | |X | | | | | | | | 344 | |---|---|---|---|---|---|---|---|---|---|---|---| 345 | |B. Sediment | | X|X | | | | | | | X| | 346 | |b. sand | | | | | | | | | | |X| 347 | |b. mixed | | | | | | | | | | | X| 348 | |b. coarse | | | | | | | | | | | X| 349 | |---|---|---|---|---|---|---|---|---|---|---|---| 350 | |C. Bare/barren| |X |X | |X |X | X| X| | | | 351 | |---|---|---|---|---|---|---|---|---|---|---| 352 | |d. marsh | | |X | | | | | | | | | 353 | |d. terrestrial veg| | |X | | | | | | | X| | 354 | |d. agriculture| | | X| | | |X | X| | | | 355 | |d. grass | | | |X | | | | | | | | 356 | |d. herbaceous / low vegetation / field | | | | | X|X | | | | | | 357 | |d. tree/forest | | | |X |X |X | X|X | | | | 358 | |d. shrubland | | | | | |X | | | | | | 359 | |d. rangeland | | | | | |X | X| X| | | | 360 | |---|---|---|---|---|---|---|---|---|---|---|---| 361 | |E. Impervious/urban/developed | | |X | | |X | X| X| | X| X| 362 | |e. impervious (other) | | | | |X | | | | | | | 363 | |e. impervious (road) | | | | |X | | X| | | | | 364 | |e. Building-flooded | | | | X| | | | | | | | 365 | |e. Building-non-flooded | | | |X | | |X | | X| | | 366 | |e. Road-flooded | | | |X | | | | | | | | 367 | |e. Road-non-flooded | | | |X | | | | | | | | 368 | |e. Vehicle | | | |X | | | | | | | | 369 | |---|---|---|---|---|---|---|---|---|---|---|---| 370 | |X. Other | X| X| | | | | | | X| | | 371 | 372 | 373 | -------------------------------------------------------------------------------- /ci/envs/310-coastseg.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.10 6 | # required 7 | - jupyterlab 8 | - geopandas 9 | - pip 10 | - pip: 11 | - tensorflow 12 | -------------------------------------------------------------------------------- /ci/envs/311-coastseg.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.11 6 | # required 7 | - jupyterlab 8 | - geopandas 9 | - pip 10 | - pip: 11 | - tensorflow 12 | -------------------------------------------------------------------------------- /ci/envs/39-coastseg.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.9 6 | # required 7 | - jupyterlab 8 | - geopandas 9 | - pip 10 | - pip: 11 | - tensorflow 12 | -------------------------------------------------------------------------------- /gdal_retile.py: -------------------------------------------------------------------------------- 1 | #!C:/Users/Sharon/anaconda3/envs/seg2map\python.exe 2 | 3 | import sys 4 | 5 | from osgeo.gdal import deprecation_warn 6 | 7 | # import osgeo_utils.gdal_retile as a convenience to use as a script 8 | from osgeo_utils.gdal_retile import * # noqa 9 | from osgeo_utils.gdal_retile import main 10 | 11 | deprecation_warn("gdal_retile") 12 | sys.exit(main(sys.argv)) 13 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "seg2map" 7 | dynamic = ["readme"] 8 | version = "0.0.14" 9 | authors = [ 10 | { name=" Sharon Fitzpatrick", email="sharon.fitzpatrick23@gmail.com" }, 11 | ] 12 | # find` directive with `include` or `exclude` 13 | description = "An interactive jupyter notebook for downloading satellite imagery" 14 | dependencies = [ 15 | "scikit-image", 16 | "imageio", 17 | "transformers", 18 | "rasterio", 19 | "area", 20 | "doodleverse-utils>=0.0.33", 21 | "ipyfilechooser>=0.6.0", 22 | "tqdm", 23 | "leafmap>=0.14.0", 24 | "geemap", 25 | "geojson", 26 | "aiohttp", 27 | "nest-asyncio", 28 | "tensorflow"] 29 | license = { file="LICENSE" } 30 | requires-python = ">=3.9" 31 | classifiers = [ 32 | "Programming Language :: Python :: 3", 33 | "License :: OSI Approved :: MIT License", 34 | "Operating System :: OS Independent", 35 | ] 36 | 37 | # tells setup tools to include the code in the coastseg directory within the src directory 38 | [tool.setuptools.packages.find] 39 | where = ["src"] 40 | 41 | 42 | [project.urls] 43 | "Homepage" = "https://github.com/Doodleverse/seg2map" 44 | "Bug Tracker" = "https://github.com/Doodleverse/seg2map/issues" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | earthengine-api 2 | # google-api-python-client 3 | geopandas 4 | geemap 5 | leafmap 6 | numpy 7 | tqdm 8 | area 9 | jupyterlab 10 | aiohttp 11 | nest-asyncio 12 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | build 3 | twine 4 | black -------------------------------------------------------------------------------- /seg2map.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "7f327b92-3814-44ff-92b6-bd8488e6528f", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import os\n", 11 | "# Local Imports\n", 12 | "from src.seg2map import map_interface\n", 13 | "from src.seg2map import log_maker #must be the first module loaded to create logs folder\n", 14 | "\n", 15 | "# External Imports\n", 16 | "import ee\n", 17 | "from google.auth import exceptions as google_auth_exceptions\n", 18 | "\n", 19 | "# suppress tensorflow warnings\n", 20 | "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "id": "00a32ffb-a5ac-46c6-b728-4aa6fce87acf", 26 | "metadata": {}, 27 | "source": [ 28 | "## Authenticate and Initialize with Google Earth Engine (GEE)\n", 29 | "\n", 30 | "- Run this cell to initialize with GEE which will allow you to download remote sensing data from GEE.\n", 31 | "\n", 32 | "### First Time Users\n", 33 | "\n", 34 | "- In order to use Google Earth Engine (GEE) you will need to sign up to request access to use Google Earth Engine.https://signup.earthengine.google.com. You will only need to do this once and it takes only a day to get your account verified.\n", 35 | "\n", 36 | "### How `ee.Authenticate()` works\n", 37 | "\n", 38 | "- In order to initialize with GEE you will need an authorization token with is obtained by running `ee.Authenticate()`.This token lasts 7 days and during those 7 days you will not need to authenticate with google earth engine with an access code. Once the 7 days are up you will need to reauthenticate to use GEE again.\n" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "id": "eefa9963-0fc1-43f2-bbfa-3b8665481712", 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "import ee\n", 49 | "from google.auth import exceptions as google_auth_exceptions\n", 50 | "\n", 51 | "try:\n", 52 | " ee.Initialize()\n", 53 | "except google_auth_exceptions.RefreshError:\n", 54 | " print(\"Please refresh your Google authentication token.\\n\")\n", 55 | " ee.Authenticate()\n", 56 | " ee.Initialize()\n", 57 | "except ee.EEException:\n", 58 | " print(\"Please authenticate with Google Earth Engine:\\n\")\n", 59 | " ee.Authenticate()\n", 60 | " ee.Initialize()\n", 61 | "except FileNotFoundError:\n", 62 | " print(\"Credentials file not found. Please authenticate with Google Earth Engine:\\n\")\n", 63 | " ee.Authenticate()\n", 64 | " ee.Initialize()" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "id": "61da2172-9ac3-49ce-8d01-052f34ea711d", 70 | "metadata": {}, 71 | "source": [ 72 | "# How to Use The Map\n", 73 | "\n", 74 | "---\n", 75 | "1. Click `Save Settings` Button\n", 76 | "2. Use the rectangle tool to draw a ROI along the coastline.\n", 77 | "3. Load transects into your bounding box with the `Load Transects` button. If any exist for the bounding box you selected they should appear.\n", 78 | "4. Click the ROIs you want to download.\n", 79 | "5. Once you've selected all the ROIs you want to download click `Downlod Imagery`\n", 80 | " - If any of the ROIs succesfully download they will have their own folder with all their data in the `data` directory in the `seg2map` directory" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "id": "61c1cc88-c52a-4006-87dd-091774569cdb", 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "# from coastseg.map_UI import UI\n", 91 | "from src.seg2map.map_UI import UI\n", 92 | "from src.seg2map.map_interface import Seg2Map\n", 93 | "\n", 94 | "seg2map=Seg2Map()\n", 95 | "seg2map_ui = UI(seg2map)\n", 96 | "seg2map_ui.create_dashboard()" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "id": "6b3ea1b4-081c-41c7-8f2b-fab74c24be33", 103 | "metadata": {}, 104 | "outputs": [], 105 | "source": [ 106 | "from src.seg2map import log_maker\n", 107 | "from src.seg2map.models_UI import UI_Models\n", 108 | "models_ui = UI_Models()\n", 109 | "models_ui.create_dashboard()" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "id": "080c0b88-db97-4fcf-9e39-0e12a447c0b9", 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [] 119 | } 120 | ], 121 | "metadata": { 122 | "kernelspec": { 123 | "display_name": "Python 3 (ipykernel)", 124 | "language": "python", 125 | "name": "python3" 126 | }, 127 | "language_info": { 128 | "codemirror_mode": { 129 | "name": "ipython", 130 | "version": 3 131 | }, 132 | "file_extension": ".py", 133 | "mimetype": "text/x-python", 134 | "name": "python", 135 | "nbconvert_exporter": "python", 136 | "pygments_lexer": "ipython3", 137 | "version": "3.10.13" 138 | } 139 | }, 140 | "nbformat": 4, 141 | "nbformat_minor": 5 142 | } 143 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # A setup.py to read markdown on readme and render it correctly on the pypi page 2 | from setuptools import setup 3 | 4 | with open("README.md") as readme_file: 5 | readme = readme_file.read() 6 | 7 | setup( 8 | long_description=readme, 9 | long_description_content_type="text/markdown", 10 | ) -------------------------------------------------------------------------------- /src/seg2map/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Doodleverse/seg2map/9d17656a62764d86746e8d818cd6ea4ea138f8dc/src/seg2map/__init__.py -------------------------------------------------------------------------------- /src/seg2map/exception_handler.py: -------------------------------------------------------------------------------- 1 | # standard python imports 2 | import os 3 | import logging 4 | import traceback 5 | from typing import Union 6 | 7 | # internal python imports 8 | from seg2map import exceptions 9 | from seg2map import common 10 | from seg2map.roi import ROI 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | NO_CONFIG_ROIS = ( 15 | "No ROIs were selected. Cannot save ROIs to config until ROIs are selected." 16 | ) 17 | NO_CONFIG_SETTINGS = "Settings must be loaded before configuration files can be made.\nClick save settings" 18 | ROIS_NOT_DOWNLOADED = ( 19 | "Not all ROI directories exist on your computer. Try downloading the ROIs again." 20 | ) 21 | EMPTY_SELECTED_ROIS = "Must select at least one ROI on the map" 22 | SETTINGS_NOT_FOUND = "No settings found. Click save settings." 23 | NO_ROI_SETTINGS = ( 24 | "No roi settings found. Click download imagery first or upload configs" 25 | ) 26 | 27 | # Separate different exception checking and handling 28 | # Checking decides the message 29 | # Handling it sends message to user 30 | 31 | 32 | def config_check_if_none(feature, feature_type: str = ""): 33 | if feature is None: 34 | message = ( 35 | f"{feature_type} must be loaded before configuration files can be saved." 36 | ) 37 | raise exceptions.Object_Not_Found(feature_type, message) 38 | 39 | 40 | def check_file_not_found(path: str, filename: str, search_path: str): 41 | # if path is None raises FileNotFoundError 42 | if path is None: 43 | logger.error(f"{filename} file was not found at {search_path}") 44 | raise FileNotFoundError(f"{filename} file was not found at {search_path}") 45 | 46 | 47 | def check_if_subset(subset: set, superset: set, superset_name: str, message: str = ""): 48 | if not subset.issubset(superset): 49 | logger.error(f"Missing keys {subset-superset} from {superset_name}\n{message}") 50 | raise ValueError( 51 | f"Missing keys {subset-superset} from {superset_name}\n{message}
Try clicking save settings" 52 | ) 53 | 54 | 55 | def check_if_rois_downloaded(roi_settings: dict, roi_ids: list): 56 | if common.were_rois_downloaded(roi_settings, roi_ids) == False: 57 | logger.error(f"Not all rois were downloaded{roi_settings}") 58 | raise FileNotFoundError(ROIS_NOT_DOWNLOADED) 59 | 60 | 61 | def can_feature_save_to_file(feature, feature_type: str = ""): 62 | if feature is None: 63 | logger.error(f"Feature {feature_type} did not exist. Cannot Save to File") 64 | raise ValueError(f"Feature {feature_type} did not exist. Cannot Save to File") 65 | 66 | 67 | def check_empty_dict(feature, feature_type: str = ""): 68 | if feature == {}: 69 | if feature_type == "roi_settings": 70 | raise Exception(NO_ROI_SETTINGS) 71 | 72 | 73 | def check_empty_layer(layer, feature_type: str = ""): 74 | if layer is None: 75 | if feature_type == ROI.LAYER_NAME: 76 | logger.error(f"No ROI layer found on map") 77 | raise Exception("No ROI layer found on map") 78 | if feature_type == ROI.SELECTED_LAYER_NAME: 79 | logger.error(f"No selected ROI layer found on map") 80 | raise Exception("No selected ROI layer found on map") 81 | logger.error(f"Cannot add an empty {feature_type} layer to the map.") 82 | raise Exception(f"Cannot add an empty {feature_type} layer to the map.") 83 | 84 | 85 | def check_if_None(feature, feature_type: str = "", message: str = ""): 86 | if feature is None: 87 | if feature_type == "settings": 88 | message = SETTINGS_NOT_FOUND 89 | logger.error(f"{feature_type} is None") 90 | raise exceptions.Object_Not_Found(feature_type, message) 91 | 92 | 93 | def check_path_already_exists(full_path: str, dir_name=""): 94 | if os.path.exists(full_path): 95 | msg = ( 96 | f"Directory already exists at {full_path}" 97 | if dir_name == "" 98 | else f"{dir_name} directory already exists at {full_path}" 99 | ) 100 | raise Exception(msg) 101 | 102 | 103 | def check_empty_roi_layer(layer): 104 | if layer is None: 105 | raise Exception(EMPTY_SELECTED_ROIS) 106 | 107 | 108 | def check_selected_set(selected_set): 109 | if selected_set is None: 110 | raise Exception(EMPTY_SELECTED_ROIS) 111 | if len(selected_set) == 0: 112 | raise Exception(EMPTY_SELECTED_ROIS) 113 | 114 | 115 | def check_if_gdf_empty(feature, feature_type: str, message: str = ""): 116 | if feature.empty == True: 117 | logger.error(f"{feature_type} {feature} is empty") 118 | raise exceptions.Object_Not_Found(feature_type, message) 119 | 120 | 121 | def handle_exception(error, row: "ipywidgets.HBox", title: str = None, msg: str = None): 122 | error_message = f"{error}
Additional Information
" + traceback.format_exc() 123 | logger.error(f"{traceback.format_exc()}") 124 | if isinstance(error, exceptions.Object_Not_Found): 125 | error_message = str(error) 126 | logger.error(f"{error_message}") 127 | launch_error_box(row, title="Error", msg=error_message) 128 | 129 | 130 | def handle_bbox_error( 131 | error_msg: Union[exceptions.TooLargeError, exceptions.TooSmallError], 132 | row: "ipywidgets.HBox", 133 | ): 134 | logger.error(f"Bounding Box Error{error_msg}") 135 | launch_error_box(row, title="Error", msg=error_msg) 136 | 137 | 138 | def launch_error_box(row: "ipywidgets.HBox", title: str = None, msg: str = None): 139 | # Show user error message 140 | warning_box = common.create_warning_box(title=title, msg=msg) 141 | # clear row and close all widgets in self.file_row before adding new warning_box 142 | common.clear_row(row) 143 | # add instance of warning_box to row 144 | row.children = [warning_box] 145 | -------------------------------------------------------------------------------- /src/seg2map/exceptions.py: -------------------------------------------------------------------------------- 1 | class Object_Not_Found(Exception): 2 | """Object_Not_Found: raised when the feature does not exist 3 | Args: 4 | Exception: Inherits from the base exception class 5 | """ 6 | 7 | def __init__(self, feature: str, message=""): 8 | self.msg = f"No {feature.lower()} found on the map.\n{message}" 9 | self.feature = feature 10 | super().__init__(self.msg) 11 | 12 | def __str__(self): 13 | return f"{self.msg}" 14 | 15 | 16 | class No_Images_Available(Exception): 17 | """No_Images_Available: raised when nothing can be downloaded 18 | Args: 19 | Exception: Inherits from the base exception class 20 | """ 21 | 22 | def __init__(self, feature: str, message=""): 23 | self.msg = f"No {feature.lower()} found on the map.\n{message}" 24 | self.feature = feature 25 | super().__init__(self.msg) 26 | 27 | def __str__(self): 28 | return f"{self.msg}" 29 | 30 | 31 | class Id_Not_Found(Exception): 32 | """Id_Not_Found: raised when ROI id does not exist 33 | Args: 34 | Exception: Inherits from the base exception class 35 | """ 36 | 37 | def __init__(self, id: int = None, msg="The ROI id does not exist."): 38 | self.msg = msg 39 | if id is not None: 40 | self.msg = f"The ROI id {id} does not exist." 41 | super().__init__(self.msg) 42 | 43 | def __str__(self): 44 | return f"{self.msg}" 45 | 46 | 47 | class TooLargeError(Exception): 48 | """TooLargeError: raised when ROI is larger than MAX_SIZE 49 | Args: 50 | Exception: Inherits from the base exception class 51 | """ 52 | 53 | def __init__(self, msg="The ROI was too large."): 54 | self.msg = msg 55 | super().__init__(self.msg) 56 | 57 | def __str__(self): 58 | return f"{self.msg}" 59 | 60 | 61 | class TooSmallError(Exception): 62 | """TooLargeError: raised when ROI is smaller than MIN_SIZE 63 | Args: 64 | Exception: Inherits from the base exception class 65 | """ 66 | 67 | def __init__(self, msg="The ROI was too small."): 68 | self.msg = msg 69 | super().__init__(self.msg) 70 | 71 | def __str__(self): 72 | return f"{self.msg}" 73 | 74 | 75 | class DownloadError(Exception): 76 | """DownloadError: raised when a download error occurs. 77 | Args: 78 | Exception: Inherits from the base exception class 79 | """ 80 | 81 | def __init__(self, file): 82 | msg = f"\n ERROR\nShoreline file:'{file}' is not online.\nPlease raise an issue on GitHub with the shoreline name.\n https://github.com/SatelliteShorelines/CoastSeg/issues" 83 | self.msg = msg 84 | super().__init__(self.msg) 85 | 86 | def __str__(self): 87 | return f"{self.msg}" 88 | -------------------------------------------------------------------------------- /src/seg2map/log_maker.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | import logging 4 | 5 | 6 | def prepare_logging(): 7 | if not os.path.exists(os.path.abspath(os.path.join(os.getcwd(), "logs"))): 8 | os.mkdir(os.path.abspath(os.path.join(os.getcwd(), "logs"))) 9 | 10 | 11 | def create_root_logger(): 12 | """Creates the root logger. The root logger will write to a log file with the format 13 | log_----. This log file will be written to by all the other loggers 14 | """ 15 | log_filename = "log_" + datetime.now().strftime("%m-%d-%y-%I_%M_%S") + ".log" 16 | log_file = os.path.abspath(os.path.join(os.getcwd(), "logs", log_filename)) 17 | # configure the logger 18 | log_format = "%(asctime)s - %(filename)s at line %(lineno)s in %(funcName)s() - %(levelname)s : %(message)s" 19 | os.path.abspath(os.path.join(os.getcwd(), "logs")) 20 | # Use FileHandler() to log to a file 21 | file_handler = logging.FileHandler(log_file, mode="a") 22 | formatter = logging.Formatter(log_format) 23 | file_handler.setFormatter(formatter) 24 | # Have all loggers write to the same log file 25 | logging.basicConfig( 26 | handlers=[file_handler], 27 | format=log_format, 28 | level=logging.INFO, 29 | datefmt="-%m-%d-%y-%I:%M:%S", 30 | ) 31 | 32 | 33 | # Prepare and create the logger 34 | prepare_logging() 35 | create_root_logger() 36 | -------------------------------------------------------------------------------- /src/seg2map/map_UI.py: -------------------------------------------------------------------------------- 1 | # standard python imports 2 | import os 3 | import datetime 4 | import logging 5 | from collections import defaultdict 6 | 7 | # internal python imports 8 | from seg2map import exception_handler 9 | from seg2map import common 10 | 11 | # external python imports 12 | from IPython.display import display 13 | from ipyfilechooser import FileChooser 14 | 15 | from google.auth import exceptions as google_auth_exceptions 16 | from ipywidgets import Box 17 | from ipywidgets import Button 18 | from ipywidgets import ToggleButton 19 | from ipywidgets import HBox 20 | from ipywidgets import VBox 21 | from ipywidgets import Layout 22 | from ipywidgets import DatePicker 23 | from ipywidgets import HTML 24 | from ipywidgets import RadioButtons 25 | from ipywidgets import Text 26 | from ipywidgets import Output 27 | from ipywidgets import FloatSlider 28 | from ipywidgets import SelectionSlider 29 | from ipywidgets import Dropdown 30 | 31 | 32 | logger = logging.getLogger(__name__) 33 | 34 | 35 | def create_file_chooser(callback, title: str = None): 36 | padding = "0px 0px 0px 5px" # upper, right, bottom, left 37 | # creates a unique instance of filechooser and button to close filechooser 38 | geojson_chooser = FileChooser(os.getcwd()) 39 | geojson_chooser.dir_icon = os.sep 40 | geojson_chooser.filter_pattern = ["*.geojson"] 41 | geojson_chooser.title = "Select a geojson file" 42 | if title is not None: 43 | geojson_chooser.title = f"{title}" 44 | geojson_chooser.register_callback(callback) 45 | 46 | close_button = ToggleButton( 47 | value=False, 48 | tooltip="Close File Chooser", 49 | icon="times", 50 | button_style="primary", 51 | layout=Layout(height="28px", width="28px", padding=padding), 52 | ) 53 | 54 | def close_click(change): 55 | if change["new"]: 56 | geojson_chooser.close() 57 | close_button.close() 58 | 59 | close_button.observe(close_click, "value") 60 | chooser = HBox([geojson_chooser, close_button], layout=Layout(width="100%")) 61 | return chooser 62 | 63 | 64 | class UI: 65 | # all instances of UI will share the same debug_view 66 | # this means that UI and seg2map must have a 1:1 relationship 67 | # Output widget used to print messages and exceptions created by Seg2Map 68 | debug_view = Output(layout={"border": "1px solid black"}) 69 | # Output widget used to print messages and exceptions created by download progress 70 | download_view = Output(layout={"border": "1px solid black"}) 71 | settings_messages = Output(layout={"border": "1px solid black"}) 72 | 73 | def __init__(self, seg2map): 74 | # save an instance of Seg2Map 75 | self.seg2map = seg2map 76 | # button styles 77 | self.get_button_styles() 78 | 79 | # buttons to load configuration files 80 | self.load_configs_button = Button( 81 | description="Load Config", style=self.load_style 82 | ) 83 | self.load_configs_button.on_click(self.on_load_configs_clicked) 84 | 85 | # buttons to load configuration files 86 | 87 | self.save_config_button = Button( 88 | description="Save Config", style=self.save_style 89 | ) 90 | self.save_config_button.on_click(self.on_save_config_clicked) 91 | 92 | self.download_button = Button( 93 | description="Download Imagery", style=self.action_style 94 | ) 95 | self.download_button.on_click(self.download_button_clicked) 96 | 97 | # Remove buttons 98 | self.clear_debug_button = Button( 99 | description="Clear Messages", style=self.clear_stlye 100 | ) 101 | self.clear_debug_button.on_click(self.clear_debug_view) 102 | 103 | # Remove buttons 104 | self.clear_downloads_button = Button( 105 | description="Clear Downloads", style=self.clear_stlye 106 | ) 107 | self.clear_downloads_button.on_click(self.clear_download_view) 108 | 109 | # create the HTML widgets containing the instructions 110 | self._create_HTML_widgets() 111 | 112 | def get_button_styles(self): 113 | self.remove_style = dict(button_color="red") 114 | self.load_style = dict(button_color="#69add1") 115 | self.action_style = dict(button_color="#ae3cf0") 116 | self.save_style = dict(button_color="#50bf8f") 117 | self.clear_stlye = dict(button_color="#a3adac") 118 | 119 | def get_view_settings(self) -> VBox: 120 | # update settings button 121 | update_settings_btn = Button( 122 | description="Update Settings", style=self.action_style 123 | ) 124 | update_settings_btn.on_click(self.update_settings_btn_clicked) 125 | self.settings_html = HTML() 126 | self.settings_html.value = self.get_settings_html(self.seg2map.settings) 127 | view_settings_vbox = VBox([self.settings_html, update_settings_btn]) 128 | return view_settings_vbox 129 | 130 | def get_settings_vbox(self) -> VBox: 131 | # declare settings widgets 132 | dates_vbox = self.get_dates_picker() 133 | self.sitename_field = Text( 134 | value="sitename", 135 | placeholder="Type sitename", 136 | description="Sitename:", 137 | disabled=False, 138 | ) 139 | settings_button = Button(description="Save Settings", style=self.action_style) 140 | settings_button.on_click(self.save_settings_clicked) 141 | box_layout = Layout(width='430px', 142 | height='65px', 143 | flex_flow='row', 144 | overflow='auto', 145 | display='flex') 146 | 147 | # scrollable output to store message in 148 | scrollable_output = Box(children=[UI.settings_messages], layout=box_layout) 149 | 150 | # create settings vbox 151 | settings_vbox = VBox( 152 | [ 153 | dates_vbox, 154 | self.sitename_field, 155 | settings_button, 156 | scrollable_output, 157 | ] 158 | ) 159 | 160 | return settings_vbox 161 | 162 | def get_dates_picker(self): 163 | # Date Widgets 164 | self.start_date = DatePicker( 165 | description="Start Date", 166 | value=datetime.date(2010, 1, 1), 167 | disabled=False, 168 | ) 169 | self.end_date = DatePicker( 170 | description="End Date", 171 | value=datetime.date(2010, 12, 31), # 2019, 1, 1 172 | disabled=False, 173 | ) 174 | date_instr = HTML(value="Pick a date:", layout=Layout(padding="10px")) 175 | dates_box = HBox([self.start_date, self.end_date]) 176 | dates_vbox = VBox([date_instr, dates_box]) 177 | return dates_vbox 178 | 179 | def remove_buttons(self): 180 | # define remove feature radio box button 181 | remove_instr = HTML( 182 | value="

Remove Feature from Map

", 183 | layout=Layout(padding="0px"), 184 | ) 185 | self.remove_button = Button(description=f"Remove ROIs", style=self.remove_style) 186 | self.remove_button.on_click(self.remove_feature_from_map) 187 | 188 | self.remove_seg_button = Button( 189 | description=f"Remove Imagery", style=self.remove_style 190 | ) 191 | self.remove_seg_button.on_click(self.remove_seg_clicked) 192 | # define remove all button 193 | self.remove_all_button = Button( 194 | description="Remove all", style=self.remove_style 195 | ) 196 | self.remove_all_button.on_click(self.remove_all_from_map) 197 | remove_buttons = VBox( 198 | [ 199 | remove_instr, 200 | self.remove_button, 201 | self.remove_seg_button, 202 | self.remove_all_button, 203 | ] 204 | ) 205 | return remove_buttons 206 | 207 | def segmentation_controls(self): 208 | # define remove feature radio box button 209 | instr = HTML( 210 | value="

Load Segmentations on the Map

", 211 | layout=Layout(padding="0px"), 212 | ) 213 | inital_options = ["all"] 214 | classes = list(self.seg2map.get_classes()) 215 | classes = ["all"] + classes 216 | if len(self.seg2map.get_classes()) == 0: 217 | classes = inital_options 218 | self.class_dropdown = Dropdown( 219 | options=classes, 220 | description="Select Class:", 221 | value=classes[0], 222 | style={"description_width": "initial"}, 223 | ) 224 | 225 | self.load_segmentations_button = Button( 226 | description="Load Segmentations", style=self.load_style 227 | ) 228 | self.load_segmentations_button.on_click(self.on_load_session_clicked) 229 | 230 | self.opacity_slider = FloatSlider( 231 | value=1.0, 232 | min=0, 233 | max=1.0, 234 | step=0.01, 235 | description="Opacity:", 236 | disabled=False, 237 | continuous_update=False, 238 | orientation="horizontal", 239 | readout=True, 240 | readout_format=".2f", 241 | ) 242 | 243 | default_years = ["2021"] 244 | years = self.seg2map.get_years() 245 | if len(years) == 0: 246 | years = default_years 247 | self.year_slider = SelectionSlider( 248 | options=years, 249 | value=years[0], 250 | description="Select a year:", 251 | disabled=False, 252 | continuous_update=False, 253 | orientation="horizontal", 254 | readout=True, 255 | ) 256 | 257 | def year_slider_changed(change): 258 | year = self.year_slider.value 259 | seg_layers = self.seg2map.get_seg_layers() 260 | original_layers = self.seg2map.get_original_layers() 261 | # order matters: original layers must be before seg layer otherwise so segmentations will appear on to of image 262 | layers = original_layers + seg_layers 263 | self.seg2map.load_layers_by_year(layers, year) 264 | 265 | self.year_slider.observe(year_slider_changed, "value") 266 | 267 | def opacity_slider_changed(change): 268 | # apply opacity to all layers 269 | year = self.year_slider.value 270 | opacity = self.opacity_slider.value 271 | logger.info(f"self.class_dropdown.value: {self.class_dropdown.value}") 272 | print(f"self.class_dropdown.value: {self.class_dropdown.value}") 273 | logger.info(f"year: {year}") 274 | print(f"year: {year}") 275 | if self.class_dropdown.value == "all": 276 | seg_layers = self.seg2map.get_seg_layers() 277 | if self.class_dropdown.value != "all": 278 | # apply opacity to selected layer name 279 | seg_layers = self.seg2map.get_seg_layers(self.class_dropdown.value) 280 | 281 | if seg_layers == []: 282 | return 283 | self.seg2map.modify_layers_opacity_by_year(seg_layers, year, opacity) 284 | 285 | self.opacity_slider.observe(opacity_slider_changed, "value") 286 | 287 | def handle_class_dropdown(change): 288 | # apply opacity to all layers 289 | logger.info(f"handle_class_dropdown: {change['new']}") 290 | print(f"handle_class_dropdown: {change['new']}") 291 | if change["new"] == "all": 292 | year = self.year_slider.value 293 | seg_layers = self.seg2map.get_seg_layers() 294 | logger.info(f"seg_layers: {seg_layers}") 295 | if seg_layers == []: 296 | return 297 | opacity = self.opacity_slider.value 298 | self.seg2map.modify_layers_opacity_by_year(seg_layers, year, opacity) 299 | if change["new"] != "all": 300 | # apply opacity to selected layer name 301 | year = self.year_slider.value 302 | seg_layers = self.seg2map.get_seg_layers(change["new"]) 303 | logger.info(f"seg_layers: {seg_layers}") 304 | if seg_layers == []: 305 | return 306 | opacity = self.opacity_slider.value 307 | self.seg2map.modify_layers_opacity_by_year(seg_layers, year, opacity) 308 | 309 | self.class_dropdown.observe(handle_class_dropdown, "value") 310 | 311 | opacity_controls = HBox([self.opacity_slider, self.class_dropdown]) 312 | segmentation_box = VBox( 313 | [ 314 | instr, 315 | self.load_segmentations_button, 316 | self.year_slider, 317 | opacity_controls, 318 | ] 319 | ) 320 | return segmentation_box 321 | 322 | def get_settings_html( 323 | self, 324 | settings: dict, 325 | ): 326 | # Modifies setttings html 327 | values = defaultdict(lambda: "unknown", settings) 328 | return """ 329 |

Settings

330 |

dates: {}

331 |

sitename: {}

332 | """.format( 333 | values["dates"], 334 | values["sitename"], 335 | ) 336 | 337 | def _create_HTML_widgets(self): 338 | """create HTML widgets that display the instructions. 339 | widgets created: instr_create_ro, instr_save_roi, instr_load_btns 340 | instr_download_roi 341 | """ 342 | self.instr_download_roi = HTML( 343 | value="

Download Imagery

\ 344 |
  • You must click an ROI on the map before you can download ROIs \ 345 |
  • The downloaded imagery will be saved to the 'data' directory
    • \ 346 | The folder name for each downloaded ROI will consist of the ROI's ID and the time of download.\ 347 |
    Example: 'ID_1_datetime11-03-22__02_33_22'\ 348 | ", 349 | layout=Layout(margin="0px 0px 0px 5px"), 350 | ) 351 | 352 | self.instr_config_btns = HTML( 353 | value="

    Load and Save Config Files

    \ 354 | Load Config: Load ROIs from file: 'config_gdf.geojson'\ 355 |
  • 'config.json' must be in the same directory as 'config_gdf.geojson'.
    • \ 356 | Save Config: Saves the state of the map to file: 'config_gdf.geojson'\ 357 | ", 358 | layout=Layout(margin="0px 5px 0px 5px"), # top right bottom left 359 | ) 360 | 361 | def get_file_controls(self): 362 | # define remove feature radio box button 363 | instr = HTML( 364 | value="

    Load & Save GeoJSON Files

    ", 365 | layout=Layout(padding="0px"), 366 | ) 367 | self.load_file_button = Button( 368 | description=f"Load GeoJSON file", 369 | icon="fa-file-o", 370 | style=self.load_style, 371 | ) 372 | self.load_file_button.on_click(self.load_feature_from_file) 373 | 374 | self.save_button = Button(description=f"Save to GeoJSON", style=self.save_style) 375 | self.save_button.on_click(self.save_to_file_btn_clicked) 376 | control_box = VBox( 377 | [ 378 | instr, 379 | self.load_file_button, 380 | self.save_button, 381 | ] 382 | ) 383 | return control_box 384 | 385 | def create_dashboard(self): 386 | """creates a dashboard containing all the buttons, instructions and widgets organized together.""" 387 | # create settings controls 388 | files_controls = self.get_file_controls() 389 | settings_controls = self.get_settings_vbox() 390 | remove_buttons = self.remove_buttons() 391 | segmentation_controls = self.segmentation_controls() 392 | 393 | self.save_button = Button( 394 | description=f"Save ROIs to file", style=self.save_style 395 | ) 396 | self.save_button.on_click(self.save_to_file_btn_clicked) 397 | 398 | save_vbox = VBox([files_controls, remove_buttons, segmentation_controls]) 399 | config_vbox = VBox( 400 | [self.instr_config_btns, self.load_configs_button, self.save_config_button] 401 | ) 402 | download_vbox = VBox( 403 | [ 404 | self.instr_download_roi, 405 | self.download_button, 406 | config_vbox, 407 | ] 408 | ) 409 | 410 | # Static settings HTML used to show currently loaded settings 411 | static_settings = self.get_view_settings() 412 | 413 | row_0 = HBox([settings_controls, static_settings]) 414 | row_1 = HBox([save_vbox, download_vbox]) 415 | # in this row prints are rendered with UI.debug_view 416 | row_3 = HBox([self.clear_debug_button, UI.debug_view]) 417 | self.error_row = HBox([]) 418 | self.file_chooser_row = HBox([]) 419 | row_5 = HBox([self.seg2map.map]) 420 | row_6 = HBox([self.clear_downloads_button, UI.download_view]) 421 | 422 | return display( 423 | row_0, 424 | row_1, 425 | row_3, 426 | self.error_row, 427 | self.file_chooser_row, 428 | row_5, 429 | row_6, 430 | ) 431 | 432 | @debug_view.capture(clear_output=True) 433 | def update_settings_btn_clicked(self, btn): 434 | UI.debug_view.clear_output(wait=True) 435 | # Display the settings currently loaded into Seg2Map 436 | try: 437 | self.settings_html.value = self.get_settings_html(self.seg2map.settings) 438 | except Exception as error: 439 | exception_handler.handle_exception(error, self.seg2map.warning_box) 440 | 441 | @settings_messages.capture(clear_output=True) 442 | def save_settings_clicked(self, btn): 443 | # Save dates selected by user 444 | dates = [str(self.start_date.value), str(self.end_date.value)] 445 | sitename = self.sitename_field.value.replace(" ", "") 446 | settings = {"dates": dates, "sitename": sitename} 447 | dates = [datetime.datetime.strptime(_, "%Y-%m-%d") for _ in dates] 448 | if dates[1] <= dates[0]: 449 | print("Dates are not correct chronological order") 450 | print("Settings not saved") 451 | return 452 | # check if sitename path exists and if it does tell user they need a new name 453 | parent_path = os.path.join(os.getcwd(), "data") 454 | sitename_path = os.path.join(parent_path, sitename) 455 | if os.path.exists(sitename_path): 456 | print( 457 | f"Sorry this sitename already exists at {sitename_path}\nTry another sitename." 458 | ) 459 | print("Settings not saved") 460 | return 461 | elif not os.path.exists(sitename_path): 462 | print(f"{sitename} will be created at {sitename_path}") 463 | try: 464 | self.seg2map.save_settings(**settings) 465 | self.settings_html.value = self.get_settings_html(self.seg2map.settings) 466 | except Exception as error: 467 | # renders error message as a box on map 468 | exception_handler.handle_exception(error, self.seg2map.warning_box) 469 | 470 | @download_view.capture(clear_output=True) 471 | def download_button_clicked(self, btn): 472 | UI.download_view.clear_output() 473 | UI.debug_view.clear_output() 474 | self.seg2map.map.default_style = {"cursor": "wait"} 475 | print("Scroll down past map to see download progress.") 476 | try: 477 | self.download_button.disabled = True 478 | try: 479 | self.seg2map.download_imagery() 480 | except Exception as error: 481 | # renders error message as a box on map 482 | exception_handler.handle_exception(error, self.seg2map.warning_box) 483 | except google_auth_exceptions.RefreshError as exception: 484 | print(exception) 485 | exception_handler.handle_exception( 486 | error, 487 | self.seg2map.warning_box, 488 | title="Authentication Error", 489 | msg="Please authenticate with Google using the cell above: \n Authenticate and Initialize with Google Earth Engine (GEE)", 490 | ) 491 | self.download_button.disabled = False 492 | self.seg2map.map.default_style = {"cursor": "default"} 493 | 494 | def clear_row(self, row: HBox): 495 | """close widgets in row/column and clear all children 496 | Args: 497 | row (HBox)(VBox): row or column 498 | """ 499 | for index in range(len(row.children)): 500 | row.children[index].close() 501 | row.children = [] 502 | 503 | @debug_view.capture(clear_output=True) 504 | def on_load_session_clicked(self, button): 505 | # Prompt user to select a config geojson file 506 | def load_callback(filechooser: FileChooser) -> None: 507 | try: 508 | if filechooser.selected: 509 | self.seg2map.load_session(filechooser.selected) 510 | years = self.seg2map.years 511 | classes = self.seg2map.get_classes() 512 | classes = list(classes) 513 | if classes: 514 | self.class_dropdown.options = ["all"] + classes 515 | self.class_dropdown.value = classes[0] 516 | if years: 517 | self.year_slider.options = years 518 | self.year_slider.value = years[0] 519 | 520 | except Exception as error: 521 | # renders error message as a box on map 522 | exception_handler.handle_exception(error, self.seg2map.warning_box) 523 | 524 | # create instance of chooser that calls load_callback 525 | dir_chooser = common.create_dir_chooser( 526 | load_callback, 527 | title="Select Session Directory", 528 | starting_directory="sessions", 529 | ) 530 | # clear row and close all widgets in row_4 before adding new file_chooser 531 | self.clear_row(self.file_chooser_row) 532 | # add instance of file_chooser to row 4 533 | self.file_chooser_row.children = [dir_chooser] 534 | 535 | @debug_view.capture(clear_output=True) 536 | def load_segmentations(self, button): 537 | # Prompt user to select a config geojson file 538 | def load_callback(filechooser: FileChooser) -> None: 539 | try: 540 | if filechooser.selected: 541 | self.seg2map.load_configs(filechooser.selected) 542 | self.settings_html.value = self.get_settings_html( 543 | self.seg2map.settings 544 | ) 545 | except Exception as error: 546 | # renders error message as a box on map 547 | exception_handler.handle_exception(error, self.seg2map.warning_box) 548 | 549 | # create instance of chooser that calls load_callback 550 | file_chooser = create_file_chooser(load_callback) 551 | # clear row and close all widgets in file_chooser_row before adding new file_chooser 552 | self.clear_row(self.file_chooser_row) 553 | # add instance of file_chooser to file_chooser_row 554 | self.file_chooser_row.children = [file_chooser] 555 | 556 | @debug_view.capture(clear_output=True) 557 | def on_load_configs_clicked(self, button): 558 | # Prompt user to select a config geojson file 559 | def load_callback(filechooser: FileChooser) -> None: 560 | try: 561 | if filechooser.selected: 562 | self.seg2map.load_configs(filechooser.selected) 563 | self.settings_html.value = self.get_settings_html( 564 | self.seg2map.settings 565 | ) 566 | except Exception as error: 567 | # renders error message as a box on map 568 | exception_handler.handle_exception(error, self.seg2map.warning_box) 569 | 570 | # create instance of chooser that calls load_callback 571 | file_chooser = create_file_chooser(load_callback) 572 | # clear row and close all widgets in file_chooser_row before adding new file_chooser 573 | self.clear_row(self.file_chooser_row) 574 | # add instance of file_chooser to file_chooser_row 575 | self.file_chooser_row.children = [file_chooser] 576 | 577 | @debug_view.capture(clear_output=True) 578 | def on_save_config_clicked(self, button): 579 | try: 580 | self.seg2map.save_config() 581 | except Exception as error: 582 | # renders error message as a box on map 583 | exception_handler.handle_exception(error, self.seg2map.warning_box) 584 | 585 | @debug_view.capture(clear_output=True) 586 | def remove_feature_from_map(self, btn): 587 | UI.debug_view.clear_output(wait=True) 588 | try: 589 | if "rois" in btn.description.lower(): 590 | self.seg2map.launch_delete_box(self.seg2map.remove_box) 591 | except Exception as error: 592 | # renders error message as a box on map 593 | exception_handler.handle_exception(error, self.seg2map.warning_box) 594 | 595 | @debug_view.capture(clear_output=True) 596 | def remove_seg_clicked(self, btn): 597 | UI.debug_view.clear_output(wait=True) 598 | try: 599 | self.seg2map.remove_segmentation_layers() 600 | 601 | except Exception as error: 602 | # renders error message as a box on map 603 | exception_handler.handle_exception(error, self.seg2map.warning_box) 604 | 605 | @debug_view.capture(clear_output=True) 606 | def load_feature_from_file(self, btn): 607 | # Prompt user to select a geojson file 608 | def file_chooser_callback(filechooser: FileChooser) -> None: 609 | try: 610 | if filechooser.selected: 611 | print( 612 | f"Loading ROIs from file: {os.path.abspath(filechooser.selected)}" 613 | ) 614 | self.seg2map.load_feature_on_map( 615 | file=os.path.abspath(filechooser.selected) 616 | ) 617 | except Exception as error: 618 | # renders error message as a box on map 619 | exception_handler.handle_exception(error, self.seg2map.warning_box) 620 | 621 | # create instance of chooser that calls callsfile_chooser_callback 622 | file_chooser = create_file_chooser( 623 | file_chooser_callback, title="Select a geojson file" 624 | ) 625 | # clear row and close all widgets in self.file_chooser_row before adding new file_chooser 626 | self.clear_row(self.file_chooser_row) 627 | # add instance of file_chooser to row 4 628 | self.file_chooser_row.children = [file_chooser] 629 | 630 | @debug_view.capture(clear_output=True) 631 | def save_to_file_btn_clicked(self, btn): 632 | UI.debug_view.clear_output(wait=True) 633 | try: 634 | self.seg2map.save_feature_to_file(self.seg2map.rois) 635 | except Exception as error: 636 | # renders error message as a box on map 637 | exception_handler.handle_exception(error, self.seg2map.warning_box) 638 | 639 | @debug_view.capture(clear_output=True) 640 | def remove_all_from_map(self, btn): 641 | try: 642 | self.seg2map.remove_all() 643 | except Exception as error: 644 | # renders error message as a box on map 645 | exception_handler.handle_exception(error, self.seg2map.warning_box) 646 | 647 | def clear_debug_view(self, btn): 648 | UI.debug_view.clear_output() 649 | 650 | def clear_download_view(self, btn): 651 | UI.download_view.clear_output() 652 | -------------------------------------------------------------------------------- /src/seg2map/map_functions.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | from typing import Tuple, List 4 | from PIL import Image 5 | from io import BytesIO 6 | import numpy as np 7 | from base64 import encodebytes 8 | from ipyleaflet import ImageOverlay 9 | from PIL import Image 10 | from time import perf_counter 11 | 12 | from seg2map import common 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | def time_func(func): 18 | def wrapper(*args, **kwargs): 19 | start = perf_counter() 20 | result = func(*args, **kwargs) 21 | end = perf_counter() 22 | print(f"{func.__name__} took {end - start:.6f} seconds to run.") 23 | return result 24 | 25 | return wrapper 26 | 27 | 28 | def get_existing_class_files(dir_path: str, class_names: list[str]) -> list[str]: 29 | """ 30 | Given a directory path and a list of class names, returns a list of paths to PNG files in that directory 31 | whose filenames match the class names. 32 | 33 | Args: 34 | dir_path (str): The path to the directory to search for PNG files. 35 | class_names (list[str]): A list of class names to match against PNG filenames. 36 | 37 | Returns: 38 | list[str]: A list of paths to PNG files in the directory whose filenames match the class names. 39 | """ 40 | existing_files = [] 41 | for class_name in class_names: 42 | filename = f"{class_name}.png" 43 | file_path = os.path.join(dir_path, f"{class_name}.png") 44 | if os.path.isfile(file_path): 45 | existing_files.append(filename) 46 | return existing_files 47 | 48 | 49 | def get_class_masks_overlay( 50 | tif_file: str, mask_output_dir: str, classes: List[str], year: str, roi_id: str 51 | ) -> List: 52 | """ 53 | Given a path to a TIFF file, create binary masks for each class in the file and 54 | return a list of image overlay layers that can be used to display the masks over 55 | the original image. 56 | 57 | Args: 58 | tif_file (str): The path to the input TIFF file. 59 | mask_output_dir (str): The path to the directory where the output mask images 60 | will be saved. 61 | classes (List[str]): A list of class names to include in the masks. 62 | year(str): year that tif was created 63 | 64 | Returns: 65 | A list of image overlay layers, one for each class mask. 66 | """ 67 | logger.info(f"tif_file: {tif_file}") 68 | # get bounds of tif 69 | bounds = common.get_bounds(tif_file) 70 | 71 | # get class names to create class mapping 72 | class_mapping = get_class_mapping(classes) 73 | 74 | # see if any class masks already exist 75 | class_masks_filenames = get_existing_class_files(mask_output_dir, classes) 76 | 77 | # generate binary masks for each class in tif as a separate PNG in mask_output_dir 78 | if not class_masks_filenames: 79 | class_masks_filenames = generate_class_masks( 80 | tif_file, class_mapping, mask_output_dir 81 | ) 82 | 83 | # for each class mask PNG, create an image overlay 84 | layers = [] 85 | for file_path in class_masks_filenames: 86 | file_path = os.path.join(mask_output_dir, file_path) 87 | layer_name = ( 88 | roi_id + "_" + os.path.basename(file_path).split(".")[0] + "_" + year 89 | ) 90 | # combine mask name with save path 91 | image_overlay = get_overlay_for_image( 92 | file_path, bounds, layer_name, file_format="png" 93 | ) 94 | layers.append(image_overlay) 95 | return layers 96 | 97 | 98 | def get_class_layers(tif_directory, classes, year, roi_id) -> List: 99 | # locate greyscale segmented tif in session directory 100 | greyscale_tif_path = common.find_file( 101 | tif_directory, "Mosaic_greyscale.tif", case_insensitive=True 102 | ) 103 | if greyscale_tif_path is None: 104 | logger.warning( 105 | f"Does not exist {os.path.join(tif_directory, '*merged_multispectral.jp*g*')}" 106 | ) 107 | return [] 108 | # create layers for each class present in greyscale tiff 109 | class_layers = get_class_masks_overlay( 110 | greyscale_tif_path, tif_directory, classes, year, roi_id 111 | ) 112 | return class_layers 113 | 114 | 115 | def get_class_mapping(names: List[str]) -> dict: 116 | """Create a mapping of class names to integer labels. 117 | 118 | Given a list of class names, this function creates a dictionary that maps each 119 | class name to a unique integer label starting from 1. 120 | 121 | Parameters 122 | ---------- 123 | names : list of str 124 | A list of class names to map to integer labels. 125 | 126 | Returns 127 | ------- 128 | dict 129 | A dictionary mapping class names to integer labels. 130 | 131 | Ex: 132 | get_class_mapping(['water','sand']) 133 | { 134 | 1:'water', 135 | 2:'sand' 136 | } 137 | """ 138 | class_mapping = {} 139 | for i, name in enumerate(names, start=1): 140 | class_mapping[i] = name 141 | return class_mapping 142 | 143 | 144 | def generate_color_map(num_colors: int) -> dict: 145 | """ 146 | Generate a color map for the specified number of colors. 147 | 148 | Args: 149 | num_colors (int): The number of colors needed. 150 | 151 | Returns: 152 | A dictionary containing a color map for the specified number of colors. 153 | """ 154 | import colorsys 155 | 156 | # Generate a list of equally spaced hues 157 | hues = [i / num_colors for i in range(num_colors)] 158 | 159 | # Convert each hue to an RGB color tuple 160 | rgb_colors = [colorsys.hsv_to_rgb(h, 1.0, 1.0) for h in hues] 161 | 162 | # Scale each RGB value to the range [0, 255] and add to the color map 163 | color_map = {} 164 | for i, color in enumerate(rgb_colors): 165 | color_map[i] = tuple(int(255 * c) for c in color) 166 | 167 | return color_map 168 | 169 | 170 | # @time_func 171 | def generate_class_masks(file: str, class_mapping: dict, save_path: str) -> List[str]: 172 | """ 173 | Generate binary masks for each class in the given grayscale image, based on a color-to-class mapping. 174 | 175 | Args: 176 | file (str): The path to the grayscale input image file. 177 | class_mapping (dict): A dictionary that maps pixel colors to class names. 178 | save_path (str): The path to the directory where the generated mask images will be saved. 179 | 180 | Returns: 181 | List[str]: A list of filenames of the saved mask images. 182 | 183 | Raises: 184 | None 185 | 186 | Example: 187 | If file='path/to/image.tif', class_mapping={0: 'background', 1: 'water', 2: 'land'}, and save_path='path/to/masks', 188 | generate_class_masks(file, class_mapping, save_path) returns ['background.png', 'water.png', 'land.png']. 189 | 190 | """ 191 | img_gray = Image.open(file) 192 | unique_colors = img_gray.getcolors() 193 | color_map = generate_color_map(len(unique_colors)) 194 | 195 | # Convert the image to a NumPy array 196 | img_gray_np = np.array(img_gray) 197 | 198 | files_saved = [] 199 | for i, (count, color) in enumerate(unique_colors): 200 | filename = class_mapping[color] 201 | image_name = f"{filename}.png" 202 | 203 | # Create a binary mask with 1 where the pixel color matches and 0 elsewhere 204 | mask = (img_gray_np == color).astype(np.uint8) 205 | 206 | # Create a new RGBA image with the same dimensions as the input image 207 | mask_img = np.zeros((img_gray.height, img_gray.width, 4), dtype=np.uint8) 208 | 209 | # Set the RGB values of the mask image to the corresponding color in the color map 210 | mask_img[..., :3] = np.array(color_map[i]) * mask[..., None] 211 | 212 | # Set the alpha channel to 255 where the mask is 1, and 0 elsewhere 213 | mask_img[..., 3] = mask * 255 214 | 215 | # Convert the NumPy array back to a PIL Image object 216 | mask_img_pil = Image.fromarray(mask_img) 217 | 218 | # Save the mask image to disk with a unique filename 219 | img_path = os.path.join(save_path, image_name) 220 | mask_img_pil.save(img_path) 221 | files_saved.append(image_name) 222 | 223 | return files_saved 224 | 225 | 226 | def get_uri(data: bytes, scheme: str = "image/png") -> str: 227 | """Generates a URI (Uniform Resource Identifier) for a given data object and scheme. 228 | 229 | The data is first encoded as base64, and then added to the URI along with the specified scheme. 230 | 231 | Works for both RGB and RGBA imagery 232 | 233 | Scheme : string of character that specifies the purpose of the uri 234 | Available schemes for imagery: 235 | "image/jpeg" 236 | "image/png" 237 | 238 | Parameters 239 | ---------- 240 | data : bytes 241 | The data object to be encoded and added to the URI. 242 | scheme : str, optional (default="image/png") 243 | The URI scheme to use for the generated URI. Defaults to "image/png". 244 | 245 | Returns 246 | ------- 247 | str 248 | The generated URI, as a string. 249 | """ 250 | return f"data:{scheme};base64,{encodebytes(data).decode('ascii')}" 251 | 252 | 253 | def get_overlay_for_image( 254 | image_path: str, bounds: Tuple, name: str, file_format: str 255 | ) -> ImageOverlay: 256 | """Create an ImageOverlay object for an image file. 257 | 258 | Args: 259 | image_path (str): The path to the image file. 260 | bounds (Tuple): The bounding box for the image overlay. 261 | name (str): The name of the image overlay. 262 | file_format (str): The format of the image file, either 'png', 'jpg', or 'jpeg'. 263 | 264 | Returns: 265 | An ImageOverlay object. 266 | """ 267 | if file_format.lower() not in ["png", "jpg", "jpeg"]: 268 | raise ValueError( 269 | f"{file_format} is not recognized. Allowed file formats are: png, jpg, and jpeg." 270 | ) 271 | 272 | if file_format.lower() == "png": 273 | file_format = "png" 274 | scheme = "image/png" 275 | elif file_format.lower() == "jpg" or file_format.lower() == "jpeg": 276 | file_format = "jpeg" 277 | scheme = "image/jpeg" 278 | 279 | logger.info(f"image_path: {image_path}") 280 | logger.info(f"file_format: {file_format}") 281 | 282 | # use pillow to open the image 283 | img_data = Image.open(image_path) 284 | # convert image to bytes 285 | img_bytes = convert_image_to_bytes(img_data, file_format) 286 | # create a uri from bytes 287 | uri = get_uri(img_bytes, scheme) 288 | # create image overlay from uri 289 | return ImageOverlay(url=uri, bounds=bounds, name=name) 290 | 291 | 292 | def convert_image_to_bytes(image, file_format: str = "png"): 293 | if file_format.lower() not in ["png", "jpg", "jpeg"]: 294 | raise ValueError( 295 | f"{file_format} is not recognized. Allowed file formats are: png, jpg, and jpeg." 296 | ) 297 | file_format = "PNG" if file_format.lower() == "png" else "JPEG" 298 | f = BytesIO() 299 | image.save(f, file_format) 300 | # get the bytes from the bytesIO object 301 | return f.getvalue() 302 | -------------------------------------------------------------------------------- /src/seg2map/model_functions.py: -------------------------------------------------------------------------------- 1 | # standard imports 2 | import os, json 3 | import asyncio 4 | import platform 5 | 6 | # external imports 7 | from glob import glob 8 | 9 | # import tqdm 10 | from tqdm.auto import tqdm as auto_tqdm 11 | import tqdm.asyncio 12 | import zipfile 13 | import requests 14 | import aiohttp 15 | 16 | os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" 17 | os.environ["CUDA_VISIBLE_DEVICES"] = "-1" 18 | 19 | import tensorflow as tf 20 | from doodleverse_utils.prediction_imports import do_seg 21 | from doodleverse_utils.model_imports import dice_coef_loss 22 | 23 | # Import the architectures for following models from doodleverse_utils 24 | from doodleverse_utils.model_imports import ( 25 | simple_resunet, 26 | custom_resunet, 27 | custom_unet, 28 | simple_unet, 29 | simple_resunet, 30 | simple_satunet, 31 | segformer, 32 | ) 33 | from joblib import Parallel, delayed 34 | 35 | 36 | def get_model_dir(parent_directory: str, dir_name: str) -> str: 37 | """returns full path to directory named dir_name and if it doesn't exist 38 | creates new directory dir_name within parent directory 39 | 40 | Args: 41 | parent_directory (str): directory to create new directory dir_name within 42 | dir_name (str): name of directory to get full path to 43 | 44 | Returns: 45 | str: full path to dir_name directory 46 | """ 47 | new_dir = os.path.join(parent_directory, dir_name) 48 | if not os.path.isdir(new_dir): 49 | print(f"Creating {new_dir}") 50 | os.mkdir(new_dir) 51 | return new_dir 52 | 53 | 54 | def request_available_files(zenodo_id: str) -> list: 55 | """returns list of available downloadable files for zenodo_id 56 | 57 | Args: 58 | zenodo_id (str): id of zenodo release 59 | 60 | Returns: 61 | list: list of available files downloadable for zenodo_id 62 | """ 63 | # Send request to zenodo for selected model by zenodo_id 64 | root_url = "https://zenodo.org/api/records/" + zenodo_id 65 | r = requests.get(root_url) 66 | # get list of all files associated with zenodo id 67 | js = json.loads(r.text) 68 | files = js["files"] 69 | return files 70 | 71 | 72 | def is_zipped_release(files: list) -> bool: 73 | """returns True if zenodo id contains 'rgb.zip' file otherwise returns false 74 | 75 | Args: 76 | files (list): list of available files for download 77 | 78 | Returns: 79 | bool: returns True if zenodo id contains 'rgb.zip' file otherwise returns false 80 | """ 81 | zipped_model_list = [f for f in files if f["key"].endswith("rgb.zip")] 82 | if zipped_model_list == []: 83 | return False 84 | return True 85 | 86 | 87 | def download_url(url, save_path, chunk_size=128): 88 | r = requests.get(url, stream=True) 89 | with open(save_path, "wb") as fd: 90 | for chunk in r.iter_content(chunk_size=chunk_size): 91 | fd.write(chunk) 92 | 93 | 94 | def get_url_dict_to_download(models_json_dict: dict) -> dict: 95 | """Returns dictionary which contains 96 | paths to save downloaded files to matched with urls to download files 97 | 98 | each key in returned dictionary contains a full path to a file 99 | each value in returned dictionary contains url to download file 100 | ex. 101 | {'C:\Home\Project\file.json':"https://website/file.json"} 102 | 103 | Args: 104 | models_json_dict (dict): full path to files and links 105 | 106 | Returns: 107 | dict: full path to files and links 108 | """ 109 | url_dict = {} 110 | for save_path, link in models_json_dict.items(): 111 | if not os.path.isfile(save_path): 112 | url_dict[save_path] = link 113 | json_filepath = save_path.replace("_fullmodel.h5", ".json") 114 | if not os.path.isfile(json_filepath): 115 | json_link = link.replace("_fullmodel.h5", ".json") 116 | url_dict[json_filepath] = json_link 117 | 118 | return url_dict 119 | 120 | 121 | async def fetch( 122 | session: aiohttp.client.ClientSession, url: str, save_path: str 123 | ) -> None: 124 | """downloads the file at url to be saved at save_path. Generates tqdm progress bar 125 | to track download progress of file 126 | 127 | Args: 128 | session (aiohttp.client.ClientSession): session with server that files are downloaded from 129 | url (str): url to file to download 130 | save_path (str): full path where file will be saved 131 | """ 132 | model_name = url.split("/")[-1] 133 | chunk_size: int = 2048 134 | async with session.get(url, timeout=600) as r: 135 | content_length = r.headers.get("Content-Length") 136 | if content_length is not None: 137 | content_length = int(content_length) 138 | with open(save_path, "wb") as fd: 139 | with auto_tqdm( 140 | total=content_length, 141 | unit="B", 142 | unit_scale=True, 143 | unit_divisor=1024, 144 | desc=f"Downloading {model_name}", 145 | initial=0, 146 | ascii=False, 147 | ) as pbar: 148 | async for chunk in r.content.iter_chunked(chunk_size): 149 | fd.write(chunk) 150 | pbar.update(len(chunk)) 151 | else: 152 | with open(save_path, "wb") as fd: 153 | async for chunk in r.content.iter_chunked(chunk_size): 154 | fd.write(chunk) 155 | 156 | 157 | async def fetch_all(session: aiohttp.client.ClientSession, url_dict: dict) -> None: 158 | """concurrently downloads all urls in url_dict within a single provided session 159 | 160 | Args: 161 | session (aiohttp.client.ClientSession): session with server that files are downloaded from 162 | url_dict (dict): dictionary with keys as full path to file to download and value being ulr to 163 | file to download 164 | """ 165 | tasks = [] 166 | for save_path, url in url_dict.items(): 167 | task = asyncio.create_task(fetch(session, url, save_path)) 168 | tasks.append(task) 169 | await tqdm.asyncio.asyncio.gather(*tasks) 170 | 171 | 172 | async def async_download_urls(url_dict: dict) -> None: 173 | # error raised if downloads dont's complete in 600 seconds (10 mins) 174 | async with aiohttp.ClientSession(raise_for_status=True, timeout=600) as session: 175 | await fetch_all(session, url_dict) 176 | 177 | 178 | def run_async_download(url_dict: dict) -> None: 179 | """concurrently downloads all thr urls in url_dict 180 | 181 | Args: 182 | url_dict (dict): dictionary with keys as full path to file to download and value being ulr to 183 | file to download 184 | """ 185 | if platform.system() == "Windows": 186 | asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) 187 | # wait for async downloads to complete 188 | asyncio.run(async_download_urls(url_dict)) 189 | 190 | 191 | def download_zip(url: str, save_path: str, chunk_size: int = 128) -> None: 192 | """Downloads the zipped model from the given url to the save_path location. 193 | Args: 194 | url (str): url to zip directory to download 195 | save_path (str): directory to save model 196 | chunk_size (int, optional): Defaults to 128. 197 | """ 198 | # make an HTTP request within a context manager 199 | with requests.get(url, stream=True) as r: 200 | # check header to get content length, in bytes 201 | total_length = int(r.headers.get("Content-Length")) 202 | with open(save_path, "wb") as fd: 203 | with auto_tqdm( 204 | total=total_length, 205 | unit="B", 206 | unit_scale=True, 207 | unit_divisor=1024, 208 | desc="Downloading Model", 209 | initial=0, 210 | ascii=True, 211 | ) as pbar: 212 | for chunk in r.iter_content(chunk_size=chunk_size): 213 | fd.write(chunk) 214 | pbar.update(len(chunk)) 215 | 216 | 217 | def download_zipped_model(model_direc: str, url: str) -> str: 218 | """download a zipped model from zenodo located at url 219 | 220 | 'rgb.zip' is name of directory containing model online 221 | this function is used to download older styles of zenodo releases 222 | 223 | Args: 224 | model_direc (str): full path to directory to save model to 225 | url (str): url to download model from 226 | 227 | Returns: 228 | str: full path to unzipped model directory 229 | """ 230 | # 'rgb.zip' is name of directory containing model online 231 | filename = "rgb" 232 | # outfile: full path to directory containing model files 233 | # example: 'c:/model_name/rgb' 234 | outfile = model_direc + os.sep + filename 235 | if os.path.exists(outfile): 236 | print(f"\n Found model weights directory: {os.path.abspath(outfile)}") 237 | # if model directory does not exist download zipped model from Zenodo 238 | if not os.path.exists(outfile): 239 | print(f"\n Downloading to model weights directory: {os.path.abspath(outfile)}") 240 | zip_file = filename + ".zip" 241 | zip_folder = model_direc + os.sep + zip_file 242 | print(f"Retrieving model {url} ...") 243 | download_zip(url, zip_folder) 244 | print(f"Unzipping model to {model_direc} ...") 245 | with zipfile.ZipFile(zip_folder, "r") as zip_ref: 246 | zip_ref.extractall(model_direc) 247 | print(f"Removing {zip_folder}") 248 | os.remove(zip_folder) 249 | 250 | # set weights dir to sub directory (rgb) containing model files 251 | model_direc = os.path.join(model_direc, "rgb") 252 | 253 | # Ensure all files are unzipped 254 | with os.scandir(model_direc) as it: 255 | for entry in it: 256 | if entry.name.endswith(".zip"): 257 | with zipfile.ZipFile(entry, "r") as zip_ref: 258 | zip_ref.extractall(model_direc) 259 | os.remove(entry) 260 | return model_direc 261 | 262 | 263 | def download_BEST_model(files: list, model_direc: str) -> None: 264 | """downloads best model from zenodo. 265 | 266 | Args: 267 | files (list): list of available files for zenodo release 268 | model_direc (str): directory of model to download 269 | 270 | Raises: 271 | FileNotFoundError: if model filename in 'BEST_MODEL.txt' does not 272 | exist online 273 | """ 274 | # dictionary to hold urls and full path to save models at 275 | models_json_dict = {} 276 | # retrieve best model text file 277 | best_model_json = [f for f in files if f["key"].strip() == "BEST_MODEL.txt"][0] 278 | best_model_txt_path = os.path.join(model_direc, "BEST_MODEL.txt") 279 | # if BEST_MODEL.txt file not exist download it 280 | if not os.path.isfile(best_model_txt_path): 281 | download_url( 282 | best_model_json["links"]["self"], 283 | best_model_txt_path, 284 | ) 285 | 286 | # read in BEST_MODEL.txt file 287 | with open(best_model_txt_path) as f: 288 | best_model_filename = f.read().strip() 289 | 290 | print(f"Best Model filename: {best_model_filename}") 291 | # check if json and h5 file in BEST_MODEL.txt exist 292 | model_json = [f for f in files if f["key"].strip() == best_model_filename] 293 | if model_json == []: 294 | FILE_NOT_ONLINE_ERROR = ( 295 | f"File {best_model_filename} not found online. Raise an issue on Github" 296 | ) 297 | raise FileNotFoundError(FILE_NOT_ONLINE_ERROR) 298 | # path to save model 299 | outfile = os.path.join(model_direc, best_model_filename) 300 | # path to save file and json data associated with file saved to dict 301 | models_json_dict[outfile] = model_json[0]["links"]["self"] 302 | url_dict = get_url_dict_to_download(models_json_dict) 303 | # if any files are not found locally download them asynchronous 304 | if url_dict != {}: 305 | run_async_download(url_dict) 306 | 307 | 308 | def download_ENSEMBLE_model(files: list, model_direc: str) -> None: 309 | """downloads all models from zenodo. 310 | 311 | Args: 312 | files (list): list of available files for zenodo release 313 | model_direc (str): directory of model to download 314 | """ 315 | # dictionary to hold urls and full path to save models at 316 | models_json_dict = {} 317 | # list of all models 318 | all_models = [f for f in files if f["key"].endswith(".h5")] 319 | # check if all h5 files in files are in model_direc 320 | for model_json in all_models: 321 | outfile = model_direc + os.sep + model_json["links"]["self"].split("/")[-1] 322 | # path to save file and json data associated with file saved to dict 323 | models_json_dict[outfile] = model_json["links"]["self"] 324 | url_dict = get_url_dict_to_download(models_json_dict) 325 | # if any files are not found locally download them asynchronous 326 | if url_dict != {}: 327 | run_async_download(url_dict) 328 | 329 | 330 | def get_weights_list(model_choice: str, weights_direc: str) -> list: 331 | """Returns of list of full paths to weights files(.h5) within weights_direc 332 | 333 | Args: 334 | model_choice (str): 'ENSEMBLE' or 'BEST' 335 | weights_direc (str): full path to directory containing model weights 336 | 337 | Returns: 338 | list: list of full paths to weights files(.h5) within weights_direc 339 | """ 340 | if model_choice == "ENSEMBLE": 341 | return glob(weights_direc + os.sep + "*.h5") 342 | elif model_choice == "BEST": 343 | with open(weights_direc + os.sep + "BEST_MODEL.txt") as f: 344 | w = f.readlines() 345 | return [weights_direc + os.sep + w[0]] 346 | 347 | 348 | def get_metadatadict(weights_list: list, config_files: list, model_names: list) -> dict: 349 | """returns dictionary of model weights,config_files, and model_names 350 | 351 | Args: 352 | weights_list (list): list of full paths to weights files(.h5) 353 | config_files (list): list of full paths to config files(.json) 354 | model_names (list): list of model names 355 | 356 | Returns: 357 | dict: dictionary of model weights,config_files, and model_names 358 | """ 359 | metadatadict = {} 360 | metadatadict["model_weights"] = weights_list 361 | metadatadict["config_files"] = config_files 362 | metadatadict["model_names"] = model_names 363 | return metadatadict 364 | 365 | 366 | def get_config(weights_list: list) -> dict: 367 | """loads contents of config json files 368 | that have same name of h5 files in weights_list 369 | 370 | Args: 371 | weights_list (list): weight files(.h5) in weights_list 372 | 373 | Returns: 374 | dict: contents of config json files that have same name of h5 files in weights_list 375 | """ 376 | weights_file = weights_list[0] 377 | configfile = ( 378 | weights_file.replace(".h5", ".json").replace("weights", "config").strip() 379 | ) 380 | if "fullmodel" in configfile: 381 | configfile = configfile.replace("_fullmodel", "").strip() 382 | with open(configfile.strip()) as f: 383 | config = json.load(f) 384 | return config 385 | 386 | 387 | def get_model(weights_list: list): 388 | """Loads models in from weights list and loads in corresponding config file 389 | for each model weights file(.h5) in weights_list 390 | 391 | Args: 392 | weights_list (list): full path to model weights files(.h5) 393 | 394 | Raises: 395 | Exception: raised if weights_list is empty 396 | Exception: An unknown model type was loaded from any of weights files in 397 | weights_list 398 | 399 | Returns: 400 | model, model_list, config_files, model_names 401 | """ 402 | model_list = [] 403 | config_files = [] 404 | model_names = [] 405 | if weights_list == []: 406 | raise Exception("No Model Info Passed") 407 | for weights in weights_list: 408 | # "fullmodel" is for serving on zoo they are smaller and more portable between systems than traditional h5 files 409 | # gym makes a h5 file, then you use gym to make a "fullmodel" version then zoo can read "fullmodel" version 410 | configfile = ( 411 | weights.replace(".h5", ".json").replace("weights", "config").strip() 412 | ) 413 | if "fullmodel" in configfile: 414 | configfile = configfile.replace("_fullmodel", "").strip() 415 | with open(configfile) as f: 416 | config = json.load(f) 417 | TARGET_SIZE = config.get("TARGET_SIZE") 418 | MODEL = config.get("MODEL") 419 | NCLASSES = config.get("NCLASSES") 420 | KERNEL = config.get("KERNEL") 421 | STRIDE = config.get("STRIDE") 422 | FILTERS = config.get("FILTERS") 423 | N_DATA_BANDS = config.get("N_DATA_BANDS") 424 | DROPOUT = config.get("DROPOUT") 425 | DROPOUT_CHANGE_PER_LAYER = config.get("DROPOUT_CHANGE_PER_LAYER") 426 | DROPOUT_TYPE = config.get("DROPOUT_TYPE") 427 | USE_DROPOUT_ON_UPSAMPLING = config.get("USE_DROPOUT_ON_UPSAMPLING") 428 | DO_TRAIN = config.get("DO_TRAIN") 429 | LOSS = config.get("LOSS") 430 | PATIENCE = config.get("PATIENCE") 431 | MAX_EPOCHS = config.get("MAX_EPOCHS") 432 | VALIDATION_SPLIT = config.get("VALIDATION_SPLIT") 433 | RAMPUP_EPOCHS = config.get("RAMPUP_EPOCHS") 434 | SUSTAIN_EPOCHS = config.get("SUSTAIN_EPOCHS") 435 | EXP_DECAY = config.get("EXP_DECAY") 436 | START_LR = config.get("START_LR") 437 | MIN_LR = config.get("MIN_LR") 438 | MAX_LR = config.get("MAX_LR") 439 | FILTER_VALUE = config.get("FILTER_VALUE") 440 | DOPLOT = config.get("DOPLOT") 441 | ROOT_STRING = config.get("ROOT_STRING") 442 | USEMASK = config.get("USEMASK") 443 | AUG_ROT = config.get("AUG_ROT") 444 | AUG_ZOOM = config.get("AUG_ZOOM") 445 | AUG_WIDTHSHIFT = config.get("AUG_WIDTHSHIFT") 446 | AUG_HEIGHTSHIFT = config.get("AUG_HEIGHTSHIFT") 447 | AUG_HFLIP = config.get("AUG_HFLIP") 448 | AUG_VFLIP = config.get("AUG_VFLIP") 449 | AUG_LOOPS = config.get("AUG_LOOPS") 450 | AUG_COPIES = config.get("AUG_COPIES") 451 | REMAP_CLASSES = config.get("REMAP_CLASSES") 452 | try: 453 | # Get the selected model based on the weights file's MODEL key provided 454 | # create the model with the data loaded in from the weights file 455 | # Load in the model from the weights which is the location of the weights file 456 | model = tf.keras.models.load_model(weights) 457 | except BaseException: 458 | if MODEL == "resunet": 459 | model = custom_resunet( 460 | (TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS), 461 | FILTERS, 462 | nclasses=NCLASSES, 463 | kernel_size=(KERNEL, KERNEL), 464 | strides=STRIDE, 465 | dropout=DROPOUT, # 0.1, 466 | dropout_change_per_layer=DROPOUT_CHANGE_PER_LAYER, # 0.0, 467 | dropout_type=DROPOUT_TYPE, # "standard", 468 | use_dropout_on_upsampling=USE_DROPOUT_ON_UPSAMPLING, # False, 469 | ) 470 | elif MODEL == "unet": 471 | model = custom_unet( 472 | (TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS), 473 | FILTERS, 474 | nclasses=NCLASSES, 475 | kernel_size=(KERNEL, KERNEL), 476 | strides=STRIDE, 477 | dropout=DROPOUT, # 0.1, 478 | dropout_change_per_layer=DROPOUT_CHANGE_PER_LAYER, # 0.0, 479 | dropout_type=DROPOUT_TYPE, # "standard", 480 | use_dropout_on_upsampling=USE_DROPOUT_ON_UPSAMPLING, # False, 481 | ) 482 | elif MODEL == "simple_resunet": 483 | # num_filters = 8 # initial filters 484 | model = simple_resunet( 485 | (TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS), 486 | kernel=(2, 2), 487 | nclasses=NCLASSES, 488 | activation="relu", 489 | use_batch_norm=True, 490 | dropout=DROPOUT, # 0.1, 491 | dropout_change_per_layer=DROPOUT_CHANGE_PER_LAYER, # 0.0, 492 | dropout_type=DROPOUT_TYPE, # "standard", 493 | use_dropout_on_upsampling=USE_DROPOUT_ON_UPSAMPLING, # False, 494 | filters=FILTERS, # 8, 495 | num_layers=4, 496 | strides=(1, 1), 497 | ) 498 | # 346,564 499 | elif MODEL == "simple_unet": 500 | model = simple_unet( 501 | (TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS), 502 | kernel=(2, 2), 503 | nclasses=NCLASSES, 504 | activation="relu", 505 | use_batch_norm=True, 506 | dropout=DROPOUT, # 0.1, 507 | dropout_change_per_layer=DROPOUT_CHANGE_PER_LAYER, # 0.0, 508 | dropout_type=DROPOUT_TYPE, # "standard", 509 | use_dropout_on_upsampling=USE_DROPOUT_ON_UPSAMPLING, # False, 510 | filters=FILTERS, # 8, 511 | num_layers=4, 512 | strides=(1, 1), 513 | ) 514 | elif MODEL == "satunet": 515 | model = simple_satunet( 516 | (TARGET_SIZE[0], TARGET_SIZE[1], N_DATA_BANDS), 517 | kernel=(2, 2), 518 | num_classes=NCLASSES, # [NCLASSES+1 if NCLASSES==1 else NCLASSES][0], 519 | activation="relu", 520 | use_batch_norm=True, 521 | dropout=DROPOUT, 522 | dropout_change_per_layer=DROPOUT_CHANGE_PER_LAYER, 523 | dropout_type=DROPOUT_TYPE, 524 | use_dropout_on_upsampling=USE_DROPOUT_ON_UPSAMPLING, 525 | filters=FILTERS, 526 | num_layers=4, 527 | strides=(1, 1), 528 | ) 529 | elif MODEL == "segformer": 530 | id2label = {} 531 | for k in range(NCLASSES): 532 | id2label[k] = str(k) 533 | model = segformer(id2label, num_classes=NCLASSES) 534 | # model.compile(optimizer='adam') 535 | else: 536 | raise Exception( 537 | f"An unknown model type {MODEL} was received. Please select a valid model." 538 | ) 539 | # Load in custom loss function from doodleverse_utils 540 | # Load metrics mean_iou, dice_coef from doodleverse_utils 541 | # if MODEL!='segformer': 542 | # model.compile( 543 | # optimizer="adam", loss=dice_coef_loss(NCLASSES) 544 | # ) # , metrics = [iou_multi(NCLASSES), dice_multi(NCLASSES)]) 545 | weights = weights.strip() 546 | model.load_weights(weights) 547 | 548 | model_names.append(MODEL) 549 | model_list.append(model) 550 | config_files.append(configfile) 551 | return model, model_list, config_files, model_names 552 | 553 | 554 | def sort_files(sample_direc: str) -> list: 555 | """returns list of sorted filenames in sample_direc 556 | 557 | Args: 558 | sample_direc (str): full path to directory of imagery/npz 559 | to run models on 560 | 561 | Returns: 562 | list: list of sorted filenames in sample_direc 563 | """ 564 | # prepares data to be predicted 565 | sample_filenames = sorted(glob(sample_direc + os.sep + "*.*")) 566 | 567 | if sample_filenames[0].split(".")[-1] == "npz": 568 | sample_filenames = sorted(tf.io.gfile.glob(sample_direc + os.sep + "*.npz")) 569 | else: 570 | sample_filenames = sorted(tf.io.gfile.glob(sample_direc + os.sep + "*.jpg")) 571 | if len(sample_filenames) == 0: 572 | sample_filenames = sorted(glob(sample_direc + os.sep + "*.png")) 573 | return sample_filenames 574 | 575 | 576 | # ========================================================= 577 | 578 | 579 | def compute_segmentation( 580 | TARGET_SIZE: tuple, 581 | N_DATA_BANDS: int, 582 | NCLASSES: int, 583 | MODEL, 584 | sample_direc: str, 585 | model_list: list, 586 | metadatadict: dict, 587 | do_parallel: bool, 588 | profile: str, 589 | ) -> None: 590 | """applies models in model_list to directory of imagery in sample_direc. 591 | imagery will be resized to TARGET_SIZE and should contain number of bands specified by 592 | N_DATA_BANDS. The outputted segmentation will contain number of classes corresponding to NCLASSES. 593 | Outputted segmented images will be located in a new subdirectory named 'out' created within sample_direc. 594 | Args: 595 | TARGET_SIZE (tuple):imagery will be resized to this size 596 | N_DATA_BANDS (int): number of bands in imagery 597 | NCLASSES (int): number of classes used in segmentation model 598 | sample_direc (str): full path to directory containing imagery to segment 599 | model_list (list): list of loaded models 600 | metadatadict (dict): config files, model weight files, and names of each model in model_list 601 | """ 602 | # look for TTA config 603 | if "TESTTIMEAUG" not in locals(): 604 | TESTTIMEAUG = False 605 | WRITE_MODELMETADATA = False 606 | OTSU_THRESHOLD = False 607 | 608 | # Read in the image filenames as either .npz,.jpg, or .png 609 | files_to_segment = sort_files(sample_direc) 610 | sample_direc = os.path.abspath(sample_direc) 611 | # Compute the segmentation for each of the files 612 | 613 | if do_parallel: 614 | 615 | from joblib import Parallel, delayed 616 | 617 | w = Parallel(n_jobs=-1, verbose=1)( 618 | delayed( 619 | do_seg( 620 | file_to_seg, 621 | model_list, 622 | metadatadict, 623 | MODEL, 624 | sample_direc, 625 | NCLASSES, 626 | N_DATA_BANDS, 627 | TARGET_SIZE, 628 | TESTTIMEAUG, 629 | WRITE_MODELMETADATA, 630 | OTSU_THRESHOLD, 631 | profile, 632 | ) 633 | )() 634 | for file_to_seg in files_to_segment 635 | ) 636 | 637 | else: 638 | 639 | for file_to_seg in tqdm.auto.tqdm(files_to_segment): 640 | do_seg( 641 | file_to_seg, 642 | model_list, 643 | metadatadict, 644 | MODEL, 645 | sample_direc=sample_direc, 646 | NCLASSES=NCLASSES, 647 | N_DATA_BANDS=N_DATA_BANDS, 648 | TARGET_SIZE=TARGET_SIZE, 649 | TESTTIMEAUG=TESTTIMEAUG, 650 | WRITE_MODELMETADATA=WRITE_MODELMETADATA, 651 | OTSU_THRESHOLD=OTSU_THRESHOLD, 652 | profile=profile, 653 | ) 654 | -------------------------------------------------------------------------------- /src/seg2map/models_UI.py: -------------------------------------------------------------------------------- 1 | # standard python imports 2 | import os 3 | import logging 4 | 5 | # internal python imports 6 | from seg2map import common 7 | from seg2map import zoo_model 8 | 9 | # external python imports 10 | import ipywidgets 11 | from IPython.display import display 12 | from ipywidgets import Button 13 | from ipywidgets import ToggleButton 14 | from ipywidgets import HBox 15 | from ipywidgets import VBox 16 | from ipywidgets import Layout 17 | from ipywidgets import HTML 18 | from ipywidgets import RadioButtons 19 | from ipywidgets import Output 20 | from ipyfilechooser import FileChooser 21 | 22 | # icons sourced from https://fontawesome.com/v4/icons/ 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | 27 | def create_dir_chooser(callback, title: str = None): 28 | padding = "0px 0px 0px 5px" # upper, right, bottom, left 29 | data_path = os.path.join(os.getcwd(), "data") 30 | if os.path.exists(data_path): 31 | data_path = os.path.join(os.getcwd(), "data") 32 | else: 33 | data_path = os.getcwd() 34 | # creates a unique instance of filechooser and button to close filechooser 35 | dir_chooser = FileChooser(data_path) 36 | dir_chooser.dir_icon = os.sep 37 | # Switch to folder-only mode 38 | dir_chooser.show_only_dirs = True 39 | if title is not None: 40 | dir_chooser.title = f"{title}" 41 | dir_chooser.register_callback(callback) 42 | 43 | close_button = ToggleButton( 44 | value=False, 45 | tooltip="Close Directory Chooser", 46 | icon="times", 47 | button_style="primary", 48 | layout=Layout(height="28px", width="28px", padding=padding), 49 | ) 50 | 51 | def close_click(change): 52 | if change["new"]: 53 | dir_chooser.close() 54 | close_button.close() 55 | 56 | close_button.observe(close_click, "value") 57 | chooser = HBox([dir_chooser, close_button]) 58 | return chooser 59 | 60 | 61 | class UI_Models: 62 | # all instances of UI will share the same debug_view 63 | model_view = Output() 64 | run_model_view = Output() 65 | 66 | def __init__(self): 67 | # Controls size of ROIs generated on map 68 | self.model_dict = { 69 | "sample_direc": None, 70 | "use_GPU": "0", 71 | "implementation": "BEST", 72 | "model_type": "OpenEarthNet_RGB_9class_7576894", 73 | "otsu": False, 74 | "tta": False, 75 | } 76 | # list of RGB models available 77 | self.generic_landcover_models = [ 78 | "OpenEarthNet_RGB_9class_7576894", 79 | "DeepGlobe_RGB_7class_7576898", 80 | "EnviroAtlas_RGB_6class_7576909", 81 | "AAAI-Buildings_RGB_2class_7607895", 82 | "aaai_floodedbuildings_RGB_2class_7622733", 83 | "xbd_building_RGB_2class_7613212", 84 | "xbd_damagedbuilding_RGB_4class_7613175", 85 | ] 86 | 87 | # list of RGB models available 88 | self.coastal_landcover_models = [ 89 | "chesapeake_RGB_7class_7576904", 90 | "orthoCT_RGB_2class_7574784", 91 | "orthoCT_RGB_5class_7566992", 92 | "orthoCT_RGB_5class_segformer_7641708", 93 | "orthoCT_RGB_8class_7570583", 94 | "orthoCT_RGB_8class_segformer_7641724", 95 | "chesapeake_7class_segformer_7677506", 96 | ] # @todo add barrier islands model when its up 97 | 98 | self.session_name = "" 99 | self.inputs_directory = "" 100 | # Declare widgets and on click callbacks 101 | self._create_HTML_widgets() 102 | self._create_widgets() 103 | self._create_buttons() 104 | 105 | def set_inputs_directory(self, full_path: str): 106 | self.inputs_directory = os.path.abspath(full_path) 107 | 108 | def get_inputs_directory( 109 | self, 110 | ): 111 | return self.inputs_directory 112 | 113 | def set_session_name(self, name: str): 114 | self.session_name = str(name).strip() 115 | 116 | def get_session_name( 117 | self, 118 | ): 119 | return self.session_name 120 | 121 | def get_session_selection(self): 122 | output = Output() 123 | self.session_name_text = ipywidgets.Text( 124 | value="", 125 | placeholder="Enter a session name", 126 | description="Session Name:", 127 | disabled=False, 128 | style={"description_width": "initial"}, 129 | ) 130 | 131 | enter_button = ipywidgets.Button(description="Enter") 132 | 133 | @output.capture(clear_output=True) 134 | def enter_clicked(btn): 135 | session_name = str(self.session_name_text.value).strip() 136 | session_path = common.create_directory(os.getcwd(), "sessions") 137 | new_session_path = os.path.join(session_path, session_name) 138 | if os.path.exists(new_session_path): 139 | print(f"Session {session_name} already exists at {new_session_path}") 140 | elif not os.path.exists(new_session_path): 141 | print(f"Session {session_name} will be created at {new_session_path}") 142 | self.set_session_name(session_name) 143 | 144 | enter_button.on_click(enter_clicked) 145 | session_name_controls = HBox([self.session_name_text, enter_button]) 146 | return VBox([session_name_controls, output]) 147 | 148 | def create_dashboard(self): 149 | model_choices_box = HBox( 150 | [self.model_type_dropdown, self.model_dropdown, self.model_implementation] 151 | ) 152 | checkboxes = HBox([self.otsu_radio, self.tta_radio]) 153 | instr_vbox = VBox( 154 | [ 155 | self.instr_select_images, 156 | self.instr_run_model, 157 | ] 158 | ) 159 | self.file_row = HBox([]) 160 | self.warning_row = HBox([]) 161 | display( 162 | checkboxes, 163 | model_choices_box, 164 | self.get_session_selection(), 165 | instr_vbox, 166 | self.use_select_images_button, 167 | self.warning_row, 168 | self.file_row, 169 | UI_Models.model_view, 170 | self.run_model_button, 171 | UI_Models.run_model_view, 172 | ) 173 | 174 | def _create_widgets(self): 175 | self.model_implementation = RadioButtons( 176 | options=["BEST", "ENSEMBLE"], 177 | value="BEST", 178 | description="Select:", 179 | disabled=False, 180 | ) 181 | self.model_implementation.observe(self.handle_model_implementation, "value") 182 | 183 | self.otsu_radio = RadioButtons( 184 | options=["Enabled", "Disabled"], 185 | value="Disabled", 186 | description="Otsu Threshold:", 187 | disabled=False, 188 | style={"description_width": "initial"}, 189 | ) 190 | self.otsu_radio.observe(self.handle_otsu, "value") 191 | 192 | self.tta_radio = RadioButtons( 193 | options=["Enabled", "Disabled"], 194 | value="Disabled", 195 | description="Test Time Augmentation:", 196 | disabled=False, 197 | style={"description_width": "initial"}, 198 | ) 199 | self.tta_radio.observe(self.handle_tta, "value") 200 | 201 | self.model_type_dropdown = ipywidgets.RadioButtons( 202 | options=["Generic Landcover", "Coastal Landcover"], 203 | value="Generic Landcover", 204 | description="Model Type:", 205 | disabled=False, 206 | style={"description_width": "initial"}, 207 | ) 208 | self.model_type_dropdown.observe(self.handle_model_type_change, names="value") 209 | 210 | self.model_dropdown = ipywidgets.Dropdown( 211 | options=self.generic_landcover_models, 212 | value=self.generic_landcover_models[0], 213 | description="Select Model:", 214 | disabled=False, 215 | style={"description_width": "initial"}, 216 | ) 217 | self.model_dropdown.observe(self.handle_model_dropdown, "value") 218 | 219 | # Allow user to enable GPU 220 | self.GPU_checkbox = ipywidgets.widgets.Checkbox( 221 | value=False, description="Use GPU", disabled=False, indent=False 222 | ) 223 | self.GPU_checkbox.observe(self.handle_GPU_checkbox, "value") 224 | 225 | def _create_buttons(self): 226 | # button styles 227 | load_style = dict(button_color="#69add1") 228 | action_style = dict(button_color="#ae3cf0") 229 | 230 | self.run_model_button = Button( 231 | description="Run Model", 232 | style=action_style, 233 | icon="fa-bolt", 234 | ) 235 | self.run_model_button.on_click(self.run_model_button_clicked) 236 | 237 | self.use_select_images_button = Button( 238 | description="Select Images", 239 | style=load_style, 240 | icon="fa-file-image-o", 241 | ) 242 | self.use_select_images_button.on_click(self.use_select_images_button_clicked) 243 | self.open_results_button = Button( 244 | description="Open Results", 245 | style=load_style, 246 | icon="folder-open-o", 247 | ) 248 | self.open_results_button.on_click(self.open_results_button_clicked) 249 | 250 | def _create_HTML_widgets(self): 251 | """create HTML widgets that display the instructions. 252 | widgets created: instr_create_ro, instr_save_roi, instr_load_btns 253 | instr_download_roi 254 | """ 255 | self.line_widget = HTML( 256 | value="____________________________________________________" 257 | ) 258 | 259 | self.instr_select_images = HTML( 260 | value="1. Select Images \ 261 |
    - Select an ROI directory or a directory containing at least one ROI subdirectory.\nExample: ./data/dataset1/ID_e7CxBi_dates_2010-01-01_to_2014-12-31", 262 | layout=Layout(margin="0px 0px 0px 20px"), 263 | ) 264 | 265 | self.instr_run_model = HTML( 266 | value="2. Run Model \ 267 |
    - Click Select Images first, then click run model", 268 | layout=Layout(margin="0px 0px 0px 20px"), 269 | ) 270 | 271 | def handle_model_implementation(self, change): 272 | self.model_dict["implementation"] = change["new"] 273 | 274 | def handle_model_dropdown(self, change): 275 | # 2 class model has not been selected disable otsu threhold 276 | if "2class" not in change["new"]: 277 | if self.otsu_radio.value == "Enabled": 278 | self.model_dict["otsu"] = False 279 | self.otsu_radio.value = "Disabled" 280 | self.otsu_radio.disabled = True 281 | # 2 class model was selected enable otsu threhold radio button 282 | if "2class" in change["new"]: 283 | self.otsu_radio.disabled = False 284 | 285 | logger.info(f"change: {change}") 286 | self.model_dict["model_type"] = change["new"] 287 | 288 | def handle_GPU_checkbox(self, change): 289 | if change["new"] == True: 290 | self.model_dict["use_GPU"] = "1" 291 | elif change["new"] == False: 292 | self.model_dict["use_GPU"] = "0" 293 | 294 | def handle_otsu(self, change): 295 | if change["new"] == "Enabled": 296 | self.model_dict["otsu"] = True 297 | if change["new"] == "Disabled": 298 | self.model_dict["otsu"] = False 299 | 300 | def handle_tta(self, change): 301 | if change["new"] == "Enabled": 302 | self.model_dict["tta"] = True 303 | if change["new"] == "Disabled": 304 | self.model_dict["tta"] = False 305 | 306 | def handle_model_type_change(self, change): 307 | if change["new"] == "Generic Landcover": 308 | self.model_dropdown.options = self.generic_landcover_models 309 | if change["new"] == "Coastal Landcover": 310 | self.model_dropdown.options = self.coastal_landcover_models 311 | 312 | @run_model_view.capture(clear_output=True) 313 | def run_model_button_clicked(self, button): 314 | session_name = self.get_session_name() 315 | inputs_directory = self.get_inputs_directory() 316 | if session_name == "": 317 | self.launch_error_box( 318 | "Cannot Run Model", 319 | "Must enter a session name first", 320 | ) 321 | return 322 | if inputs_directory == "": 323 | self.launch_error_box( 324 | "Cannot Run Model", 325 | "Must click 'Select Images' first", 326 | ) 327 | return 328 | if not common.check_id_subdirectories_exist(inputs_directory): 329 | self.launch_error_box( 330 | "Cannot Run Model", 331 | "You must select a directory that contains ROI subdirectories. Example ROI name: 'ID_e7CxBi_dates_2010-01-01_to_2014-12-31'", 332 | ) 333 | return 334 | # Disable run and open results buttons while the model is running 335 | self.run_model_button.disabled = True 336 | 337 | # gets GPU or CPU depending on whether use_GPU is True 338 | use_GPU = self.model_dict["use_GPU"] 339 | model_implementation = self.model_dict["implementation"] 340 | model_id = self.model_dict["model_type"] 341 | use_otsu = self.model_dict["otsu"] 342 | use_tta = self.model_dict["tta"] 343 | 344 | zoo_model_instance = zoo_model.ZooModel() 345 | try: 346 | zoo_model_instance.run_model( 347 | model_implementation, 348 | session_name=session_name, 349 | src_directory=inputs_directory, 350 | model_id=model_id, 351 | use_GPU=use_GPU, 352 | use_otsu=use_otsu, 353 | use_tta=use_tta, 354 | ) 355 | finally: 356 | # Enable run and open results buttons when model has executed 357 | self.run_model_button.disabled = False 358 | 359 | @run_model_view.capture(clear_output=True) 360 | def open_results_button_clicked(self, button): 361 | """open_results_button_clicked on click handler for 'open results' button. 362 | 363 | prints location of model outputs 364 | 365 | Args: 366 | button (Button): button that was clicked 367 | 368 | Raises: 369 | FileNotFoundError: raised when the directory where the model outputs are saved does not exist 370 | """ 371 | if self.model_dict["sample_direc"] is None: 372 | self.launch_error_box( 373 | "Cannot Open Results", "You must click 'Run Model' first" 374 | ) 375 | else: 376 | # path to directory containing model outputs 377 | model_results_path = os.path.abspath(self.model_dict["sample_direc"]) 378 | if not os.path.exists(model_results_path): 379 | self.launch_error_box( 380 | "File Not Found", 381 | "The directory for the model outputs could not be found", 382 | ) 383 | raise FileNotFoundError 384 | else: 385 | print(f"Model outputs located at:\n{model_results_path}") 386 | 387 | @model_view.capture(clear_output=True) 388 | def load_callback(self, filechooser: FileChooser) -> None: 389 | if filechooser.selected: 390 | inputs_directory = os.path.abspath(filechooser.selected) 391 | self.set_inputs_directory(inputs_directory) 392 | # for root, dirs, files in os.walk(inputs_directory): 393 | # # if any directory contains jpgs then set inputs directory to selected directory 394 | # jpgs = glob.glob(os.path.join(root, "*jpg")) 395 | # if len(jpgs) > 0: 396 | # self.set_inputs_directory(inputs_directory) 397 | # return 398 | # self.launch_error_box( 399 | # "File Not Found", 400 | # "The directory contains no jpgs! Please select a directory with jpgs.", 401 | # ) 402 | 403 | @model_view.capture(clear_output=True) 404 | def use_select_images_button_clicked(self, button): 405 | # Prompt the user to select a directory of images 406 | file_chooser = create_dir_chooser( 407 | self.load_callback, title="Select directory of images" 408 | ) 409 | # clear row and close all widgets in self.file_row before adding new file_chooser 410 | common.clear_row(self.file_row) 411 | # add instance of file_chooser to self.file_row 412 | self.file_row.children = [file_chooser] 413 | 414 | def launch_error_box(self, title: str = None, msg: str = None): 415 | # Show user error message 416 | warning_box = common.create_warning_box(title=title, msg=msg) 417 | # clear row and close all widgets in self.file_row before adding new warning_box 418 | common.clear_row(self.warning_row) 419 | # add instance of warning_box to self.warning_row 420 | self.warning_row.children = [warning_box] 421 | -------------------------------------------------------------------------------- /src/seg2map/new_downloads.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import json 4 | import math 5 | from typing import List 6 | import platform 7 | import logging 8 | import os, json, shutil 9 | from glob import glob 10 | import concurrent.futures 11 | from datetime import datetime 12 | 13 | from seg2map import exceptions 14 | from seg2map import common 15 | 16 | import asyncio 17 | import nest_asyncio 18 | import aiohttp 19 | import tqdm 20 | import tqdm.auto 21 | import tqdm.asyncio 22 | import ee 23 | import shapely 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | import asyncio 28 | import aiohttp 29 | import logging 30 | from aiohttp import ClientResponseError, ClientConnectionError 31 | 32 | 33 | async def async_download_url_dict(url_dict: dict = {}): 34 | async with aiohttp.ClientSession() as session: 35 | tasks = [] 36 | for save_path, url in url_dict.items(): 37 | task = asyncio.create_task( 38 | download_data_with_retries(session, url, save_path, total_attempts=2) 39 | ) 40 | tasks.append(task) 41 | results = await tqdm.asyncio.tqdm.gather(*tasks) 42 | 43 | 44 | async def download_data_with_retries( 45 | session, 46 | url, 47 | save_location, 48 | retries=3, 49 | initial_delay=1, 50 | max_delay=32, 51 | total_attempts=2, 52 | ): 53 | attempts = 0 54 | while attempts < total_attempts: 55 | success = await download_with_retries( 56 | session, url, save_location, retries, initial_delay, max_delay 57 | ) 58 | if success: 59 | return True # Download successful 60 | attempts += 1 61 | 62 | logger.error(f"Download failed after {total_attempts} attempts: {url}") 63 | print(f"Download failed after {total_attempts} attempts: {url}") 64 | return False 65 | 66 | 67 | async def async_download_url(session, url: str, save_path: str): 68 | model_name = url.split("/")[-1] 69 | chunk_size: int = 2048 70 | async with session.get(url, raise_for_status=True) as r: 71 | content_length = r.headers.get("Content-Length") 72 | if content_length is not None: 73 | content_length = int(content_length) 74 | with open(save_path, "wb") as fd: 75 | with tqdm.auto.tqdm( 76 | total=content_length, 77 | unit="B", 78 | unit_scale=True, 79 | unit_divisor=1024, 80 | desc=f"Downloading {model_name}", 81 | initial=0, 82 | ascii=False, 83 | position=0, 84 | ) as pbar: 85 | async for chunk in r.content.iter_chunked(chunk_size): 86 | fd.write(chunk) 87 | pbar.update(len(chunk)) 88 | else: 89 | with open(save_path, "wb") as fd: 90 | async for chunk in r.content.iter_chunked(chunk_size): 91 | fd.write(chunk) 92 | 93 | 94 | async def download_with_retries( 95 | session, url, save_location, retries, initial_delay, max_delay 96 | ): 97 | delay = initial_delay 98 | for i in range(retries): 99 | try: 100 | async with session.get(url) as response: 101 | response.raise_for_status() 102 | content_length = response.headers.get("Content-Length") 103 | if content_length is not None: 104 | content_length = int(content_length) 105 | with open(save_location, "wb") as fd: 106 | with tqdm.auto.tqdm( 107 | total=content_length, 108 | unit="B", 109 | unit_scale=True, 110 | unit_divisor=1024, 111 | desc=f"Downloading {os.path.dirname(save_location)}", 112 | initial=0, 113 | ascii=False, 114 | position=0, 115 | ) as pbar: 116 | async for chunk in response.content.iter_chunked(1024): 117 | if not chunk: 118 | break 119 | fd.write(chunk) 120 | pbar.update(len(chunk)) 121 | else: 122 | with open(save_location, "wb") as fd: 123 | async for chunk in response.content.iter_chunked(1024): 124 | fd.write(chunk) 125 | return True # Download successful 126 | except (ClientResponseError, ClientConnectionError) as e: 127 | logger.error(f"Error downloading {url}: {e}") 128 | except asyncio.exceptions.TimeoutError as e: 129 | logger.error(f"Timeout error for {url}: {e}") 130 | except Exception as e: 131 | logger.error(f"Unexpected error downloading {url}: {e}") 132 | if i < retries - 1: 133 | logger.warning( 134 | f"Retrying download in {delay} seconds... ({i + 1}/{retries})" 135 | ) 136 | await asyncio.sleep(delay) 137 | # Update delay for the next iteration, doubling it while ensuring it doesn't exceed max_delay 138 | delay = min(delay * 2, max_delay) 139 | else: 140 | return False 141 | 142 | 143 | async def download_file(session, url, save_location): 144 | retries = 3 # number of times to retry download 145 | for i in range(retries): 146 | try: 147 | async with session.get(url) as response: 148 | if response.status != 200: 149 | print(f"An error occurred while downloading.{response}") 150 | logger.error(f"An error occurred while downloading.{response}") 151 | print(response.status) 152 | return 153 | with open(save_location, "wb") as f: 154 | async for chunk in response.content.iter_chunked(1024): 155 | if not chunk: 156 | break 157 | f.write(chunk) 158 | break # break out of retry loop if download is successful 159 | except asyncio.exceptions.TimeoutError as e: 160 | logger.error(e) 161 | logger.error(f"An error occurred while downloading {save_location}.{e}") 162 | print( 163 | f"Timeout error occurred for {url}. Retrying with new session in 1 second... ({i + 1}/{retries})" 164 | ) 165 | await asyncio.sleep(1) 166 | async with aiohttp.ClientSession() as new_session: 167 | return await download_file(new_session, url, save_location) 168 | except Exception as e: 169 | logger.error(e) 170 | logger.error( 171 | f"Download failed for {save_location} {url}. Retrying in 1 second... ({i + 1}/{retries})" 172 | ) 173 | print( 174 | f"Download failed for {url}. Retrying in 1 second... ({i + 1}/{retries})" 175 | ) 176 | await asyncio.sleep(1) 177 | else: 178 | logger.error(f"Download failed for {save_location} {url}.") 179 | print(f"Download failed for {url}.") 180 | return 181 | 182 | 183 | async def download_group(session, group, semaphore): 184 | coroutines = [] 185 | logger.info(f"group: {group}") 186 | for tile_number, tile in enumerate(group): 187 | polygon = tile["polygon"] 188 | filepath = os.path.abspath(tile["filepath"]) 189 | filenames = { 190 | "multiband": "multiband" + str(tile_number), 191 | "singleband": os.path.basename(filepath), 192 | } 193 | for tile_id in tile["ids"]: 194 | logger.info(f"tile_id: {tile_id}") 195 | file_id = tile_id.replace("/", "_") 196 | filename = filenames["multiband"] + "_" + file_id 197 | save_location = os.path.join(filepath, filename.replace("/", "_") + ".zip") 198 | logger.info(f"save_location: {save_location}") 199 | coroutines.append( 200 | async_download_tile( 201 | session, 202 | polygon, 203 | tile_id, 204 | save_location, 205 | filename, 206 | filePerBand=False, 207 | semaphore=semaphore, 208 | ) 209 | ) 210 | 211 | year_name = os.path.basename(group[0]["filepath"]) 212 | await tqdm.asyncio.tqdm.gather( 213 | *coroutines, leave=False, desc=f"Downloading {year_name}" 214 | ) 215 | # await asyncio.gather(*coroutines) 216 | 217 | logger.info(f"Files downloaded to {group[0]['filepath']}") 218 | common.unzip_dir(group[0]["filepath"]) 219 | common.delete_empty_dirs(group[0]["filepath"]) 220 | # delete duplicate tifs. keep tif with most non-black pixels 221 | # common.delete_tifs_at_same_location(group[0]["filepath"]) 222 | 223 | 224 | # Download the information for each year 225 | async def download_groups( 226 | groups, 227 | semaphore: asyncio.Semaphore, 228 | group_id: str = "", 229 | show_progress_bar: bool = False, 230 | ): 231 | coroutines = [] 232 | async with aiohttp.ClientSession() as session: 233 | logger.info(f"group: {groups}") 234 | for key, group in groups.items(): 235 | logger.info(f"key: {key} group: {group}") 236 | if len(group) > 0: 237 | coroutines.append(download_group(session, group, semaphore)) 238 | else: 239 | print(f"No tiles available to download for year: {key}") 240 | logger.warning(f"No tiles available to download for year: {key}") 241 | 242 | if show_progress_bar == False: 243 | await asyncio.gather(*coroutines) 244 | elif show_progress_bar == True: 245 | await tqdm.asyncio.tqdm.gather( 246 | *coroutines, 247 | position=1, 248 | leave=False, 249 | desc=f"Downloading years for ROI {group_id}", 250 | ) 251 | 252 | 253 | async def download_ROIs(ROI_tiles: dict = {}): 254 | tasks = [] 255 | semaphore = asyncio.Semaphore(15) 256 | show_progress_bar = True 257 | for ROI_id, ROI_info in ROI_tiles.items(): 258 | tasks.append( 259 | download_groups( 260 | ROI_info, 261 | semaphore, 262 | group_id=ROI_id, 263 | show_progress_bar=show_progress_bar, 264 | ) 265 | ) 266 | await tqdm.asyncio.tqdm.gather(*tasks, position=0, desc=f"Downloading ROIs") 267 | 268 | 269 | async def async_download_tile( 270 | session: aiohttp.ClientSession, 271 | polygon: List[set], 272 | tile_id: str, 273 | filepath: str, 274 | filename: str, 275 | filePerBand: bool, 276 | semaphore: asyncio.Semaphore, 277 | ) -> None: 278 | """ 279 | Download a single tile of an Earth Engine image and save it to a zip directory. 280 | 281 | This function uses the Earth Engine API to crop the image to a specified polygon and download it to a zip directory with the specified filename. The number of concurrent downloads is limited to 10. 282 | 283 | Parameters: 284 | 285 | session (aiohttp.ClientSession): An instance of aiohttp session to make the download request. 286 | polygon (List[set]): A list of latitude and longitude coordinates that define the region to crop the image to. 287 | tile_id (str): The ID of the Earth Engine image to download. 288 | filepath (str): The path of the directory to save the downloaded zip file to. 289 | filename (str): The name of the zip file to be saved. 290 | filePerBand (bool): Whether to save each band of the image in a separate file or as a single file. 291 | semaphore:asyncio.Semaphore : Limits number of concurrent requests 292 | Returns: 293 | None 294 | """ 295 | # Semaphore limits number of concurrent requests 296 | async with semaphore: 297 | OUT_RES_M = 0.5 # output raster spatial footprint in metres 298 | image_ee = ee.Image(tile_id) 299 | # crop and download 300 | download_id = ee.data.getDownloadId( 301 | { 302 | "image": image_ee, 303 | "region": polygon, 304 | "scale": OUT_RES_M, 305 | "crs": "EPSG:4326", 306 | "filePerBand": filePerBand, 307 | "name": filename, 308 | } 309 | ) 310 | try: 311 | # create download url using id 312 | url = ee.data.makeDownloadUrl(download_id) 313 | await download_file(session, url, filepath) 314 | except Exception as e: 315 | logger.error(e) 316 | raise e 317 | 318 | 319 | def run_async_function(async_callback, **kwargs) -> None: 320 | if platform.system() == "Windows": 321 | asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) 322 | # apply a nested loop to jupyter's event loop for async downloading 323 | nest_asyncio.apply() 324 | # get nested running loop and wait for async downloads to complete 325 | loop = asyncio.get_running_loop() 326 | result = loop.run_until_complete(async_callback(**kwargs)) 327 | logger.info(f"result: {result}") 328 | return result 329 | 330 | 331 | import json 332 | import math 333 | import logging 334 | import os, json, shutil 335 | from glob import glob 336 | import concurrent.futures 337 | from datetime import datetime 338 | import platform 339 | 340 | from seg2map import exceptions 341 | from seg2map import common 342 | 343 | from typing import List, Tuple 344 | 345 | import tqdm 346 | import tqdm.auto 347 | import zipfile 348 | from area import area 349 | import numpy as np 350 | import geopandas as gpd 351 | import asyncio 352 | import aiohttp 353 | import tqdm.asyncio 354 | import nest_asyncio 355 | from shapely.geometry import LineString, MultiPolygon, Polygon 356 | from shapely.ops import split 357 | import ee 358 | from osgeo import gdal 359 | 360 | logger = logging.getLogger(__name__) 361 | 362 | 363 | # GEE allows for 20 concurrent requests at once 364 | limit = asyncio.Semaphore(20) 365 | 366 | 367 | def get_num_splitters(gdf: gpd.GeoDataFrame) -> int: 368 | """ 369 | Calculates the minimum number of splitters required to divide a geographic region represented by a GeoDataFrame into smaller, equal-sized tiles whose 370 | area <= 1 km^2. 371 | 372 | max area per tile is 1 km^2 373 | 374 | Parameters: 375 | gdf (gpd.GeoDataFrame): A GeoDataFrame representing the geographic region to be split. Must contain only a single entry. 376 | 377 | Returns: 378 | int: An integer representing the minimum number of splitters required to divide the region represented by `gdf` into smaller, equal-sized tiles whose 379 | area <= 1 km^2. 380 | 381 | """ 382 | # convert to geojson dictionary 383 | logger.info(f"gdf: {gdf}") 384 | roi_json = gdf.to_json() 385 | logger.info(f"roi_json: {roi_json}") 386 | roi_json = json.loads(roi_json) 387 | # only one feature is present select 1st feature's geometry 388 | roi_geometry = roi_json["features"][0]["geometry"] 389 | # get area of entire shape as squared kilometers 390 | area_km2 = area(roi_geometry) / 1e6 391 | logger.info(f"Area: {area_km2}") 392 | if area_km2 <= 1: 393 | return 0 394 | # get minimum number of horizontal and vertical splitters to split area equally 395 | # max area per tile is 1 km^2 396 | num_splitters = math.ceil(math.sqrt(area_km2)) 397 | return num_splitters 398 | 399 | 400 | def splitPolygon(polygon: gpd.GeoDataFrame, num_splitters: int) -> MultiPolygon: 401 | """ 402 | Split a polygon into a given number of smaller polygons by adding horizontal and vertical lines. 403 | 404 | Parameters: 405 | polygon (gpd.GeoDataFrame): A GeoDataFrame object containing a single polygon. 406 | num_splitters (int): The number of horizontal and vertical lines to add. 407 | 408 | Returns: 409 | MultiPolygon: A MultiPolygon object containing the smaller polygons. 410 | 411 | Example: 412 | >>> import geopandas as gpd 413 | >>> from shapely.geometry import Polygon, MultiPolygon 414 | >>> poly = Polygon([(0, 0), (0, 10), (10, 10), (10, 0)]) 415 | >>> df = gpd.GeoDataFrame(geometry=[poly]) 416 | >>> result = splitPolygon(df, 2) 417 | >>> result # polygon split into 4 equally sized tiles 418 | """ 419 | minx, miny, maxx, maxy = polygon.bounds.iloc[0] 420 | dx = (maxx - minx) / num_splitters # width of a small part 421 | dy = (maxy - miny) / num_splitters # height of a small part 422 | horizontal_splitters = [ 423 | LineString([(minx, miny + i * dy), (maxx, miny + i * dy)]) 424 | for i in range(num_splitters) 425 | ] 426 | vertical_splitters = [ 427 | LineString([(minx + i * dx, miny), (minx + i * dx, maxy)]) 428 | for i in range(num_splitters) 429 | ] 430 | splitters = horizontal_splitters + vertical_splitters 431 | result = polygon["geometry"].iloc[0] 432 | for splitter in splitters: 433 | result = MultiPolygon(split(result, splitter)) 434 | 435 | # convert the Polygon to GeoJSON and write it to a file 436 | with open("splitpolygon.geojson", "w") as outfile: 437 | # json.dump(shapely.geometry.mapping(result), outfile, indent=4) 438 | json.dump(shapely.to_geojson(result), outfile) 439 | return result 440 | 441 | 442 | def remove_zip(path) -> None: 443 | # Get a list of all the zipped files in the directory 444 | zipped_files = [ 445 | os.path.join(path, f) for f in os.listdir(path) if f.endswith(".zip") 446 | ] 447 | # Remove each zip file 448 | for zipped_file in zipped_files: 449 | os.remove(zipped_file) 450 | 451 | 452 | def unzip(path) -> None: 453 | # Get a list of all the zipped files in the directory 454 | zipped_files = [ 455 | os.path.join(path, f) for f in os.listdir(path) if f.endswith(".zip") 456 | ] 457 | # Unzip each file 458 | for zipped_file in zipped_files: 459 | with zipfile.ZipFile(zipped_file, "r") as zip_ref: 460 | zip_ref.extractall(path) 461 | 462 | 463 | def unzip_files(paths): 464 | # Create a thread pool with a fixed number of threads 465 | with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: 466 | # Submit a unzip task for each directory 467 | futures = [executor.submit(unzip, path) for path in paths] 468 | 469 | # Wait for all tasks to complete 470 | concurrent.futures.wait(futures) 471 | 472 | 473 | def remove_zip_files(paths): 474 | # Create a thread pool with a fixed number of threads 475 | with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: 476 | # Submit a remove_zip task for each directory 477 | futures = [executor.submit(remove_zip, path) for path in paths] 478 | 479 | # Wait for all tasks to complete 480 | concurrent.futures.wait(futures) 481 | 482 | 483 | def get_subdirs(parent_dir: str): 484 | # Get a list of all the subdirectories in the parent directory 485 | subdirectories = [] 486 | for root, dirs, files in os.walk(parent_dir): 487 | for d in dirs: 488 | subdirectories.append(os.path.join(root, d)) 489 | return subdirectories 490 | 491 | 492 | def create_dir(dir_path: str, raise_error=True) -> str: 493 | dir_path = os.path.abspath(dir_path) 494 | if os.path.exists(dir_path): 495 | if raise_error: 496 | raise FileExistsError(dir_path) 497 | else: 498 | os.makedirs(dir_path) 499 | return dir_path 500 | 501 | 502 | def copy_multiband_tifs(roi_path: str, multiband_path: str): 503 | for folder in glob( 504 | roi_path + os.sep + "tile*", 505 | recursive=True, 506 | ): 507 | files = glob(folder + os.sep + "*multiband.tif") 508 | [ 509 | shutil.copyfile(file, multiband_path + os.sep + file.split(os.sep)[-1]) 510 | for file in files 511 | ] 512 | 513 | 514 | def mk_filepaths(tiles_info: List[dict]): 515 | """ 516 | Copy multiband TIF files from a source folder to a destination folder. 517 | 518 | This function uses the glob module to search for multiband TIF files in the subfolders of a source folder, and then uses the shutil module to copy each file to a destination folder. The name of the file in the destination folder is set to the name of the file in the source folder. 519 | 520 | Parameters: 521 | 522 | roi_path (str): The path of the source folder to search for multiband TIF files. 523 | multiband_path (str): The path of the destination folder to copy the multiband TIF files to. 524 | Returns: 525 | None""" 526 | filepaths = [tile_info["filepath"] for tile_info in tiles_info] 527 | for filepath in filepaths: 528 | create_dir(filepath, raise_error=False) 529 | 530 | 531 | def create_tasks( 532 | session: aiohttp.ClientSession, 533 | polygon: List[tuple], 534 | tile_id: str, 535 | filepath: str, 536 | multiband_filepath: str, 537 | filenames: dict, 538 | file_id: str, 539 | download_bands: str, 540 | ) -> list: 541 | """ 542 | 543 | creates a list of tasks that are used to download the data. 544 | 545 | Parameters 546 | ---------- 547 | session : aiohttp.ClientSession 548 | The aiohttp.ClientSession object that handles the connection with the 549 | api. 550 | polygon : List[tuple] coordinates of the polygon in lat/lon 551 | ex: [(1,2),(3,4)(4,3),(3,3),(1,2)] 552 | tile_id : str 553 | GEE id of the tile 554 | ex: 'USDA/NAIP/DOQQ/m_4012407_se_10_1_20100612' 555 | filepath : str 556 | full path to tile directory that data will be saved to 557 | multiband_filepath : str 558 | The path to a directory will save multiband files 559 | filenames : dict 560 | A dictionary of filenames: 561 | 'singleband': name of singleband files 562 | 'multiband': name of multiband files 563 | file_id : str 564 | name that file will be saved as based on tile_id 565 | download_bands : str 566 | type of imagery to download 567 | must be one of the following strings "multiband","singleband", or "both" 568 | 569 | Returns 570 | ------- 571 | tasks : list 572 | A list of tasks that are used to download the data. 573 | 574 | """ 575 | tasks = [] 576 | if download_bands == "multiband" or download_bands == "both": 577 | 578 | task = asyncio.create_task( 579 | async_download_tile( 580 | session, 581 | polygon, 582 | tile_id, 583 | multiband_filepath, 584 | filename=filenames["multiband"] + "_" + file_id, 585 | filePerBand=False, 586 | ) 587 | ) 588 | tasks.append(task) 589 | if download_bands == "singleband" or download_bands == "both": 590 | task = asyncio.create_task( 591 | async_download_tile( 592 | session, 593 | polygon, 594 | tile_id, 595 | filepath, 596 | filename=filenames["singleband"] + "_" + file_id, 597 | filePerBand=False, 598 | ) 599 | ) 600 | tasks.append(task) 601 | return tasks 602 | 603 | 604 | async def async_download_all_tiles(tiles_info: List[dict], download_bands: str) -> None: 605 | # creates task for each tile to be downloaded and waits for tasks to complete 606 | tasks = [] 607 | for counter, tile_dict in enumerate(tiles_info): 608 | polygon = tile_dict["polygon"] 609 | filepath = os.path.abspath(tile_dict["filepath"]) 610 | parent_dir = os.path.dirname(filepath) 611 | multiband_filepath = os.path.join(parent_dir, "multiband") 612 | filenames = { 613 | "multiband": "multiband" + str(counter), 614 | "singleband": os.path.basename(filepath), 615 | } 616 | 617 | timeout = aiohttp.ClientTimeout(total=3000) 618 | async with aiohttp.ClientSession(timeout=timeout) as session: 619 | for tile_id in tile_dict["ids"]: 620 | logger.info(f"tile_id: {tile_id}") 621 | file_id = tile_id.replace("/", "_") 622 | # handles edge case where tile has 2 years in the tile ID by extracting the ealier year 623 | year_str = file_id.split("_")[-1][:4] 624 | if len(file_id.split("_")[-2]) == 8: 625 | year_str = file_id.split("_")[-2][:4] 626 | # full path to year directory within multiband dir eg. ./multiband/2012 627 | year_filepath = os.path.join(multiband_filepath, year_str) 628 | logger.info(f"year_filepath: {year_filepath}") 629 | tasks.extend( 630 | create_tasks( 631 | session, 632 | polygon, 633 | tile_id, 634 | filepath, 635 | year_filepath, 636 | filenames, 637 | file_id, 638 | download_bands, 639 | ) 640 | ) 641 | # show a progress bar of all the requests in progress 642 | await tqdm.asyncio.tqdm.gather(*tasks, position=0, desc=f"All Downloads") 643 | 644 | 645 | async def get_ids_for_tile( 646 | year_path, gee_collection, tile, dates, semaphore: asyncio.Semaphore 647 | ) -> dict: 648 | async with semaphore: 649 | collection = ee.ImageCollection(gee_collection) 650 | polygon = ee.Geometry.Polygon(tile) 651 | # logger.info(f"polygon:{polygon}") 652 | # logger.info(f"tile:{tile}") 653 | # Filter the collection to get only the images within the tile and date range 654 | filtered_collection = ( 655 | collection.filterBounds(polygon) 656 | .filterDate(*dates) 657 | .sort("system:time_start", True) 658 | ) 659 | # Get a list of all the image names in the filtered collection 660 | image_list = filtered_collection.getInfo().get("features") 661 | ids = [obj["id"] for obj in image_list] 662 | # logger.info(f"google ids:{ids}") 663 | # Create a dictionary for each tile with the information about the images to be downloaded 664 | image_dict = { 665 | "polygon": tile, 666 | "ids": ids, 667 | "filepath": year_path, 668 | } 669 | return image_dict 670 | 671 | 672 | async def get_tiles_info( 673 | tile_coords: list, 674 | dates: Tuple[str], 675 | roi_path: str, 676 | gee_collection: str, 677 | semaphore: asyncio.Semaphore, 678 | ) -> List[dict]: 679 | """ 680 | Get information about images within the specified tile coordinates, date range, and image collection. 681 | The information includes the image IDs, file path, and polygon geometry of each tile. 682 | 683 | Parameters: 684 | tile_coords (List[List[Tuple[float]]]): A list of tile coordinates, where each tile coordinate is a list of 685 | (latitude, longitude) tuples that define the polygon of the tile. 686 | dates (Tuple[str]): A tuple of two strings representing the start and end dates for the image collection. 687 | roi_path (str): The path to the directory where the images will be saved. 688 | gee_collection (str): The name of the image collection on Google Earth Engine. 689 | 690 | Returns: 691 | List[dict]: A list of dictionaries, where each dictionary contains information about a single tile, including the 692 | polygon geometry, the IDs of the images within the tile, and the file path to the directory where the 693 | images will be saved. 694 | """ 695 | logger.info(f"dates: {dates}") 696 | logger.info(f"roi_path: {roi_path}") 697 | logger.info(f"tile_coords: {tile_coords}") 698 | start_year = dates[0].strftime("%Y") 699 | year_path = os.path.join(roi_path, "multiband", start_year) 700 | tasks = [] 701 | for tile in tile_coords: 702 | tasks.append( 703 | get_ids_for_tile(year_path, gee_collection, tile, dates, semaphore) 704 | ) 705 | 706 | # create a progress bar for the number of tasks 707 | pbar = tqdm.asyncio.tqdm( 708 | total=len(tasks), position=0, desc=f"Getting tiles for year {start_year}" 709 | ) 710 | 711 | # iterate over completed tasks using asyncio.as_completed() 712 | results = [] 713 | for coro in asyncio.as_completed(tasks): 714 | # update progress bar 715 | pbar.update(1) 716 | result = await coro 717 | logger.info(f"coro result {result}") 718 | results.append(result) 719 | 720 | # close progress bar and return results 721 | pbar.close() 722 | return {start_year: results} 723 | 724 | # # results = await asyncio.gather(*tasks) 725 | # start_year = dates[0].strftime("%Y") 726 | # results = await tqdm.asyncio.tqdm.gather( 727 | # *tasks, position=0,desc=f"Getting tiles for year {start_year}" 728 | # ) 729 | # return {start_year: results} 730 | 731 | 732 | def get_tile_coords(num_splitters: int, roi_gdf: gpd.GeoDataFrame) -> list[list[list]]: 733 | """ 734 | Given the number of splitters and a GeoDataFrame,Splits an ROI geodataframe into tiles of 1km^2 area (or less), 735 | and returns a list of lists of tile coordinates. 736 | 737 | Args: 738 | num_splitters (int): The number of splitters to divide the ROI into. 739 | gpd_data (gpd.GeoDataFrame): The GeoDataFrame containing the ROI geometry. 740 | 741 | Returns: 742 | list[list[list[float]]]: A list of lists of tile coordinates, where each inner list represents the coordinates of one tile. 743 | The tile coordinates are in [lat,lon] format. 744 | """ 745 | if num_splitters == 0: 746 | split_polygon = Polygon(roi_gdf["geometry"].iloc[0]) 747 | tile_coords = [list(split_polygon.exterior.coords)] 748 | elif num_splitters > 0: 749 | # split ROI into rectangles of 1km^2 area (or less) 750 | split_polygon = splitPolygon(roi_gdf, num_splitters) 751 | tile_coords = [list(part.exterior.coords) for part in split_polygon.geoms] 752 | return tile_coords 753 | 754 | 755 | def create_ROI_directories(download_path: str, roi_id: str, dates): 756 | """ 757 | Creates directories to store downloaded data for a single region of interest (ROI). 758 | 759 | The function creates a main directory for the ROI, a subdirectory for multiband files, and subdirectories for each year in the date range. 760 | 761 | Parameters: 762 | - download_path (str): The path to the directory where downloaded data should be stored. 763 | - roi_id (str): The ID of the ROI. 764 | - dates (List[str]): A list containing the start and end dates of the date range in the format ['YYYY-MM-DD', 'YYYY-MM-DD']. 765 | 766 | Returns: 767 | - None 768 | """ 769 | # name of ROI folder to contain all downloaded data 770 | roi_name = f"ID_{roi_id}_dates_{dates[0]}_to_{dates[1]}" 771 | roi_path = os.path.join(download_path, roi_name) 772 | # create directory to hold all multiband files 773 | multiband_path = common.create_directory(roi_path, "multiband") 774 | # create subdirectories for each year 775 | start_date = dates[0].split("-")[0] 776 | end_date = dates[1].split("-")[0] 777 | logger.info(f"start_date : {start_date } end_date : {end_date }") 778 | common.create_year_directories(int(start_date), int(end_date), multiband_path) 779 | return roi_path 780 | 781 | 782 | def prepare_ROI_for_download( 783 | download_path: str, 784 | roi_gdf: gpd.GeoDataFrame, 785 | ids: List[str], 786 | dates: Tuple[str], 787 | ) -> None: 788 | for roi_id in ids: 789 | create_ROI_directories(download_path, roi_id, dates) 790 | roi_gdf = roi_gdf.loc[id] 791 | 792 | 793 | async def get_tiles_info_per_year( 794 | roi_path: str, 795 | rois_gdf: gpd.GeoDataFrame, 796 | roi_id: str, 797 | dates: Tuple[str], 798 | semaphore: asyncio.Semaphore, 799 | ): 800 | gee_collection = "USDA/NAIP/DOQQ" 801 | logger.info(f"rois_gdf : {rois_gdf }") 802 | gdf = rois_gdf.loc[[roi_id]] 803 | logger.info(f"gdf : {gdf }") 804 | roi_gdf = gpd.GeoDataFrame(gdf, geometry=gdf.geometry.name) 805 | logger.info(f"roi_gdf : {roi_gdf }") 806 | # get number of splitters need to split ROI into rectangles of 1km^2 area (or less) 807 | num_splitters = get_num_splitters(roi_gdf) 808 | logger.info(f"Splitting ROI into {num_splitters}x{num_splitters} tiles") 809 | 810 | # split ROI into rectangles of 1km^2 area (or less) 811 | tile_coords = get_tile_coords(num_splitters, roi_gdf) 812 | logger.info(f"tile_coords: {tile_coords}") 813 | yearly_ranges = common.get_yearly_ranges(dates) 814 | 815 | tiles_per_year = {} 816 | tasks = [] 817 | # for each year get the tiles available to download 818 | for year_date in yearly_ranges: 819 | tasks.append( 820 | get_tiles_info(tile_coords, year_date, roi_path, gee_collection, semaphore) 821 | ) 822 | # list_of_tiles = await asyncio.gather(*tasks) 823 | # create a progress bar for the number of tasks 824 | # pbar = tqdm.asyncio.tqdm(total=len(tasks), desc=f"Downloading tiles for ROI: {roi_id}") 825 | 826 | # iterate over completed tasks using asyncio.as_completed() 827 | list_of_tiles = [] 828 | for task in asyncio.as_completed(tasks): 829 | result = await task 830 | tile_key = list(result.keys())[0] 831 | tiles_per_year[tile_key] = result[tile_key] 832 | list_of_tiles.append(result) 833 | 834 | logger.info(f"tiles_per_year: {tiles_per_year}") 835 | return {roi_id: tiles_per_year} 836 | 837 | 838 | # call asyncio to run download_ROIs 839 | async def get_tiles_for_ids( 840 | roi_paths: str, 841 | rois_gdf: gpd.GeoDataFrame, 842 | selected_ids: List[str], 843 | dates: Tuple[str], 844 | ) -> None: 845 | """creates a nested loop that's used to asynchronously download imagery and waits for all the imagery to download 846 | Args: 847 | download_path (str): full path to directory to download imagery to 848 | roi_gdf (gpd.GeoDataFrame): geodataframe of ROIs on the map 849 | ids (List[str]): ids of ROIs to download imagery for 850 | dates (Tuple[str]): start and end dates 851 | download_bands (str): type of imagery to download 852 | must be one of the following strings "multiband","singleband", or "both" 853 | """ 854 | ROI_tiles = {} 855 | tasks = [] 856 | semaphore = asyncio.Semaphore(50) 857 | 858 | for roi_path, roi_id in zip(roi_paths, selected_ids): 859 | tasks.append( 860 | get_tiles_info_per_year(roi_path, rois_gdf, roi_id, dates, semaphore) 861 | ) 862 | 863 | list_of_ROIs = await tqdm.asyncio.tqdm.gather( 864 | *tasks, position=0, desc=f"Getting tiles for each ROI" 865 | ) 866 | 867 | for roi in list_of_ROIs: 868 | roi_id = list(roi.keys())[0] 869 | ROI_tiles[roi_id] = roi[roi_id] 870 | 871 | logger.info(f"ROI_tiles: {ROI_tiles}") 872 | for roi_id in ROI_tiles.keys(): 873 | for year in ROI_tiles[roi_id].keys(): 874 | mk_filepaths(ROI_tiles[roi_id][year]) 875 | 876 | return ROI_tiles 877 | 878 | 879 | async def async_download_ROIs(ROI_tiles: List[dict], download_bands: str) -> None: 880 | """ 881 | Downloads the specified bands for each ROI tile asynchronously using aiohttp and asyncio. 882 | 883 | Parameters: 884 | ROI_tiles (List[dict]): A list of dictionaries representing the ROI tiles to download. 885 | download_bands (str): A comma-separated string of band numbers to download. 886 | 887 | Returns: 888 | None: This function does not return anything. 889 | """ 890 | async with aiohttp.ClientSession() as session: 891 | # creates task for each tile to be downloaded and waits for tasks to complete 892 | tasks = [] 893 | for ROI_tile in ROI_tiles: 894 | for year in ROI_tile.keys(): 895 | print(f"YEAR for ROI tile: {year}") 896 | task = asyncio.create_task( 897 | async_download_year(ROI_tile[year], download_bands, session) 898 | ) 899 | tasks.append(task) 900 | # show a progress bar of all the requests in progress 901 | await tqdm.asyncio.tqdm.gather(*tasks, position=0, desc=f"All Downloads") 902 | 903 | 904 | async def async_download_year( 905 | tiles_info: List[dict], download_bands: str, session 906 | ) -> None: 907 | """ 908 | Downloads the specified bands for each tile in the specified year asynchronously using aiohttp and asyncio. 909 | 910 | Parameters: 911 | tiles_info (List[dict]): A list of dictionaries representing the tiles to download. 912 | download_bands (str): A comma-separated string of band numbers to download. 913 | session: The aiohttp session to use for downloading. 914 | 915 | Returns: 916 | None: This function does not return anything. 917 | """ 918 | # creates task for each tile to be downloaded and waits for tasks to complete 919 | tasks = [] 920 | for counter, tile_dict in enumerate(tiles_info): 921 | 922 | polygon = tile_dict["polygon"] 923 | filepath = os.path.abspath(tile_dict["filepath"]) 924 | filenames = { 925 | "multiband": "multiband" + str(counter), 926 | "singleband": os.path.basename(filepath), 927 | } 928 | for tile_id in tile_dict["ids"]: 929 | logger.info(f"tile_id: {tile_id}") 930 | file_id = tile_id.replace("/", "_") 931 | logger.info(f"year_filepath: {filepath}") 932 | tasks.extend( 933 | create_tasks( 934 | session, 935 | polygon, 936 | tile_id, 937 | filepath, 938 | filepath, 939 | filenames, 940 | file_id, 941 | download_bands, 942 | ) 943 | ) 944 | # show a progress bar of all the requests in progress 945 | await tqdm.asyncio.tqdm.gather(*tasks, position=0, desc=f"All Downloads") 946 | common.unzip_data(os.path.dirname(filepath)) 947 | # delete any directories that were empty 948 | common.delete_empty_dirs(os.path.dirname(filepath)) 949 | 950 | 951 | # call asyncio to run download_ROIs 952 | def run_magic_function_to_download(ROI_tiles: List[dict], download_bands: str) -> None: 953 | if platform.system() == "Windows": 954 | asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) 955 | # apply a nested loop to jupyter's event loop for async downloading 956 | nest_asyncio.apply() 957 | # get nested running loop and wait for async downloads to complete 958 | loop = asyncio.get_running_loop() 959 | loop.run_until_complete(async_download_ROIs(ROI_tiles, download_bands)) 960 | -------------------------------------------------------------------------------- /src/seg2map/roi.py: -------------------------------------------------------------------------------- 1 | # Standard library imports 2 | import logging 3 | from typing import Union 4 | from functools import lru_cache 5 | 6 | # Internal dependencies imports 7 | from .exceptions import TooLargeError, TooSmallError 8 | from seg2map import common 9 | 10 | # External dependencies imports 11 | import geopandas as gpd 12 | import pandas as pd 13 | from shapely.geometry import shape 14 | from ipyleaflet import GeoJSON 15 | 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | class ROI: 21 | """ROI 22 | 23 | A Bounding Box drawn by user. 24 | """ 25 | 26 | MAX_AREA = 100000000 # UNITS = Sq. Meters 27 | MIN_AREA = 10 # UNITS = Sq. Meters 28 | LAYER_NAME = "ROI" 29 | 30 | def __init__( 31 | self, 32 | ): 33 | self.gdf = gpd.GeoDataFrame() 34 | self.settings = {} 35 | # self.settings={"sitename":"","filepath":"","ids":[],"dates":""} 36 | self.filename = "roi.geojson" 37 | 38 | @lru_cache() 39 | def get_ids(self) -> list: 40 | return self.gdf.index.to_list() 41 | 42 | def set_settings(self, roi_settings: dict): 43 | logger.info(f"Old settings: {self.get_settings()}") 44 | logger.info(f"New settings to replace old settings: {roi_settings}") 45 | self.settings = roi_settings 46 | logger.info(f"New Settings: {self.settings}") 47 | 48 | def get_settings(self) -> dict: 49 | return self.settings 50 | 51 | def create_geodataframe_from_geometry( 52 | self, rectangle: dict, new_id: str = "", crs: str = "EPSG:4326" 53 | ) -> gpd.GeoDataFrame: 54 | """Creates a geodataframe with the crs specified by crs 55 | Args: 56 | rectangle (dict): geojson dictionary 57 | crs (str, optional): coordinate reference system string. Defaults to 'EPSG:4326'. 58 | 59 | Returns: 60 | gpd.GeoDataFrame: geodataframe with geometry column = rectangle and given crs""" 61 | geom = [shape(rectangle)] 62 | gdf = gpd.GeoDataFrame({"geometry": geom}) 63 | gdf.crs = crs 64 | gdf["id"] = new_id 65 | gdf.index = gdf["id"] 66 | gdf.index = gdf.index.rename("ROI_ID") 67 | logger.info(f"new geodataframe created: {gdf}") 68 | return gdf 69 | 70 | def add_geometry(self, geometry: dict, crs: str = "EPSG:4326"): 71 | """ 72 | Add a new geometry to the main geodataframe. 73 | 74 | Parameters: 75 | - geometry (dict): The new geometry to be added, represented as a dictionary. 76 | - crs (str): The Coordinate Reference System (CRS) of the geometry, with a default value of "EPSG:4326". 77 | 78 | Raises: 79 | - TypeError: If the `geometry` argument is not of type dict. 80 | 81 | Returns: 82 | - None 83 | 84 | """ 85 | logger.info(f"geometry: {geometry}") 86 | if not isinstance(geometry, dict): 87 | logger.error( 88 | f"TypeError: Expected argument of type int, got {type(geometry)}" 89 | ) 90 | raise TypeError( 91 | "Expected argument of type int, got {}".format(type(geometry)) 92 | ) 93 | bbox_area = common.get_area(geometry) 94 | ROI.check_size(bbox_area) 95 | # create id for new geometry 96 | new_id = common.generate_random_string(self.get_ids()) 97 | # create geodataframe from geometry 98 | new_gdf = self.create_geodataframe_from_geometry(geometry, new_id, crs) 99 | # add geodataframe to main geodataframe 100 | self.gdf = self.add_new(new_gdf) 101 | logger.info(f"Add geometry: {geometry}\n self.gdf {self.gdf}") 102 | 103 | def add_geodataframe( 104 | self, new_gdf: gpd.GeoDataFrame, crs: str = "EPSG:4326" 105 | ) -> None: 106 | # check if geodataframe column has 'id' column and add one if one doesn't exist 107 | if "id" not in new_gdf.columns: 108 | logger.info("Id not in columns.") 109 | # none of the new ids can already exist in self.gdf 110 | avoid_list = self.gdf.index.to_list() 111 | ids = [] 112 | # generate a new id for each ROI in the geodataframe 113 | for _ in range(len(new_gdf)): 114 | new_id = common.generate_random_string(avoid_list) 115 | ids.append(new_id) 116 | avoid_list.append(new_id) 117 | logger.info(f"Adding IDs{ids}") 118 | new_gdf["id"] = ids 119 | new_gdf.index = new_gdf["id"] 120 | new_gdf.index = new_gdf.index.rename("ROI_ID") 121 | 122 | logger.info(f"New gdf after adding IDs: {new_gdf}") 123 | 124 | # get row ids of ROIs whose area exceeds MAX AREA 125 | drop_ids = common.get_ids_with_invalid_area(new_gdf, max_area=ROI.MAX_AREA) 126 | if len(drop_ids) > 0: 127 | print("Dropping ROIs that are an invalid size ") 128 | logger.info(f"Dropping ROIs that are an invalid size {drop_ids}") 129 | new_gdf.drop(index=drop_ids, axis=0, inplace=True) 130 | # convert crs of ROIs to the map crs 131 | new_gdf.to_crs(crs) 132 | new_gdf.index = new_gdf["id"] 133 | new_gdf.index = new_gdf.index.rename("ROI_ID") 134 | # add new_gdf to self.gdf 135 | self.gdf = self.add_new(new_gdf) 136 | logger.info(f"self.gdf: {self.gdf}") 137 | 138 | def add_new( 139 | self, 140 | new_gdf: gpd.GeoDataFrame, 141 | ) -> gpd.GeoDataFrame: 142 | """Adds a new roi entry to self.gdf. 143 | Args: 144 | new_gdf (geodataframe): new ROI to add 145 | Returns: 146 | gpd.GeoDataFrame: geodataframe with new roi added to it""" 147 | # concatenate new geodataframe to existing gdf of rois 148 | logger.info(f"self.gdf: {self.gdf}") 149 | logger.info(f"Adding gdf: {new_gdf}") 150 | new_gdf = gpd.GeoDataFrame(pd.concat([self.gdf, new_gdf], ignore_index=False)) 151 | return new_gdf 152 | 153 | def get_geodataframe(self) -> gpd.GeoDataFrame: 154 | return self.gdf 155 | 156 | # def add_new( 157 | # self, new_gdf: gpd.GeoDataFrame, 158 | # ) -> gpd.GeoDataFrame: 159 | # """Adds a new roi entry to self.gdf. 160 | # Args: 161 | # new_gdf (geodataframe): new ROI to add 162 | # Returns: 163 | # gpd.GeoDataFrame: geodataframe with new roi added to it""" 164 | # # concatenate new geodataframe to existing gdf of rois 165 | # new_gdf = gpd.GeoDataFrame(pd.concat([self.gdf, new_gdf], ignore_index=True)) 166 | # return self 167 | 168 | # def add_new( 169 | # self, geometry: dict, id: str = "", crs: str = "EPSG:4326" 170 | # ) -> gpd.GeoDataFrame: 171 | # """Adds a new roi entry to self.gdf. New roi has given geometry and a column called 172 | # "id" with id. 173 | # Args: 174 | # geometry (dict): geojson dictionary of roi shape 175 | # id(str): unique id of roi 176 | # crs (str, optional): coordinate reference system string. Defaults to 'EPSG:4326'. 177 | 178 | # Returns: 179 | # gpd.GeoDataFrame: geodataframe with new roi added to it""" 180 | # # create new geodataframe with geomtry and id 181 | # geom = [shape(geometry)] 182 | # new_roi = gpd.GeoDataFrame({"geometry": geom}) 183 | # new_roi["id"] = id 184 | # new_roi.crs = crs 185 | # # concatenate new geodataframe to existing gdf of rois 186 | # new_gdf = gpd.GeoDataFrame(pd.concat([self.gdf, new_roi], ignore_index=True)) 187 | # # update self gdf to have new roi 188 | # self.gdf = new_gdf 189 | # return self 190 | 191 | # def remove_by_id( 192 | # self, roi_id: str = "", crs: str = "EPSG:4326" 193 | # ) -> gpd.GeoDataFrame: 194 | # """Removes roi with id matching roi_id from self.gdf 195 | # Args: 196 | # roi_id(str): unique id of roi to remove 197 | # crs (str, optional): coordinate reference system string. Defaults to 'EPSG:4326'. 198 | 199 | # Returns: 200 | # gpd.GeoDataFrame: geodataframe without roi roi_id in it""" 201 | # # create new geodataframe with geomtry and roi_id 202 | # new_gdf = self.gdf[self.gdf["id"] != roi_id] 203 | # # update self gdf to have new roi 204 | # self.gdf = new_gdf 205 | # return new_gdf 206 | 207 | def remove_ids(self, roi_id: Union[str, set] = "") -> None: 208 | """Removes roi with id matching roi_id from self.gdf 209 | Args: 210 | roi_id(str): unique id of roi to remove""" 211 | logger.info(f"Dropping IDs: {roi_id}") 212 | if isinstance(roi_id, set): 213 | self.gdf.drop(roi_id, inplace=True) 214 | else: 215 | if roi_id in self.gdf.index: 216 | self.gdf.drop(roi_id, inplace=True) 217 | logger.info(f"ROI.index after drop: {self.gdf.index}") 218 | 219 | def style_layer(self, geojson: dict, layer_name: str) -> "ipyleaflet.GeoJSON": 220 | """Return styled GeoJson object with layer name 221 | 222 | Args: 223 | geojson (dict): geojson dictionary to be styled 224 | layer_name(str): name of the GeoJSON layer 225 | Returns: 226 | "ipyleaflet.GeoJSON": ROIs as GeoJson layer styled as black box that turn 227 | yellow on hover 228 | """ 229 | assert geojson != {}, "ERROR.\n Empty geojson cannot be drawn onto map" 230 | return GeoJSON( 231 | data=geojson, 232 | name=layer_name, 233 | style={ 234 | "color": "#555555", 235 | "fill_color": "#555555", 236 | "fillOpacity": 0.1, 237 | "weight": 1, 238 | }, 239 | hover_style={"color": "yellow", "fillOpacity": 0.1, "color": "yellow"}, 240 | ) 241 | 242 | def check_size(box_area: float): 243 | """ "Raises an exception if the size of the bounding box is too large or small.""" 244 | # Check if the size is greater than MAX_SIZE 245 | if box_area > ROI.MAX_AREA: 246 | raise TooLargeError() 247 | # Check if size smaller than MIN_SIZE 248 | elif box_area < ROI.MIN_AREA: 249 | raise TooSmallError() 250 | -------------------------------------------------------------------------------- /src/seg2map/sessions.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from typing import List 4 | 5 | import os 6 | import re 7 | import shutil 8 | import asyncio 9 | import platform 10 | import json 11 | import logging 12 | from typing import List, Set 13 | 14 | from seg2map import common 15 | 16 | import requests 17 | import skimage 18 | import aiohttp 19 | import tqdm 20 | import numpy as np 21 | from glob import glob 22 | from osgeo import gdal 23 | import tqdm.asyncio 24 | import nest_asyncio 25 | from skimage.io import imread 26 | from tensorflow.keras import mixed_precision 27 | from doodleverse_utils.prediction_imports import do_seg 28 | from doodleverse_utils.model_imports import ( 29 | simple_resunet, 30 | custom_resunet, 31 | custom_unet, 32 | simple_unet, 33 | simple_resunet, 34 | simple_satunet, 35 | segformer, 36 | ) 37 | from doodleverse_utils.model_imports import dice_coef_loss, iou_multi, dice_multi 38 | import tensorflow as tf 39 | 40 | logger = logging.getLogger(__name__) 41 | 42 | 43 | class Session: 44 | """ 45 | A class representing a session, which contains sets of classes, years, and ROI IDs. 46 | """ 47 | 48 | def __init__(self, name: str = None, path: str = None): 49 | """ 50 | Initializes a new Session object. 51 | 52 | Args: 53 | name (str): The name of the session. Default is None. 54 | path (str): The path to the directory where the session will be saved. Default is None. 55 | """ 56 | self.name = name 57 | self.path = path 58 | self.classes = set() 59 | self.years = set() 60 | self.roi_ids = set() 61 | self.roi_info = {} 62 | 63 | def get_session_data(self) -> dict: 64 | session_data = { 65 | "name": self.name, 66 | "path": self.path, 67 | "classes": list(self.classes), 68 | "years": list(self.years), 69 | "roi_ids": list(self.roi_ids), 70 | } 71 | return session_data 72 | 73 | def get_roi_info(self, roi_id: str = None): 74 | if roi_id: 75 | return self.roi_info.get(roi_id, "") 76 | return self.roi_info 77 | 78 | def set_roi_info(self, new_roi_info: dict): 79 | return self.roi_info.update(new_roi_info) 80 | 81 | # def create_roi_directories() 82 | 83 | def add_classes(self, class_names: List[str]): 84 | """ 85 | Adds one or more class names to the session. 86 | 87 | Args: 88 | class_names (str or iterable): The name(s) of the class(es) to add. 89 | """ 90 | if isinstance(class_names, str): 91 | self.classes.add(class_names) 92 | else: 93 | self.classes.update(class_names) 94 | 95 | def add_years(self, years: List[str]): 96 | """ 97 | Adds one or more years to the session. 98 | 99 | Args: 100 | years (int or str or iterable): The year(s) to add. 101 | """ 102 | if isinstance(years, int): 103 | self.years.add(str(years)) 104 | elif isinstance(years, str): 105 | self.years.add(years) 106 | else: 107 | self.years.update(years) 108 | 109 | def add_roi_ids(self, roi_ids: List[str]): 110 | """ 111 | Adds one or more ROI IDs to the session. 112 | 113 | Args: 114 | roi_ids (str or iterable): The ROI ID(s) to add. 115 | """ 116 | if isinstance(roi_ids, str): 117 | self.roi_ids.add(roi_ids) 118 | else: 119 | self.roi_ids.update(roi_ids) 120 | 121 | def find_session_file(self, path: str, filename: str = "session.json"): 122 | # if session.json is found in main directory then session path was identified 123 | session_path = os.path.join(path, filename) 124 | if os.path.isfile(session_path): 125 | return session_path 126 | else: 127 | parent_directory = os.path.dirname(path) 128 | json_path = os.path.join(parent_directory, filename) 129 | if os.path.isfile(json_path): 130 | return json_path 131 | else: 132 | raise ValueError( 133 | f"File '{filename}' not found in the parent directory: {parent_directory} or path" 134 | ) 135 | 136 | def load(self, path: str): 137 | """ 138 | Loads a session from a directory. 139 | 140 | Args: 141 | path (str): The path to the session directory. 142 | """ 143 | json_path = self.find_session_file(path, "session.json") 144 | with open(json_path, "r") as f: 145 | session_data = json.load(f) 146 | self.name = session_data.get("name") 147 | self.path = session_data.get("path") 148 | self.classes = set(session_data.get("classes", [])) 149 | self.years = set(session_data.get("years", [])) 150 | self.roi_ids = set(session_data.get("roi_ids", [])) 151 | 152 | def save(self, path): 153 | """ 154 | Saves the session to a directory. 155 | 156 | Args: 157 | path (str): The path to the directory where the session will be saved. 158 | """ 159 | if not os.path.exists(path): 160 | os.makedirs(path) 161 | 162 | session_data = { 163 | "name": self.name, 164 | "path": path, 165 | "classes": list(self.classes), 166 | "years": list(self.years), 167 | "roi_ids": list(self.roi_ids), 168 | } 169 | 170 | with open(os.path.join(path, "session.json"), "w") as f: 171 | json.dump(session_data, f, indent=4) 172 | 173 | def __str__(self): 174 | """ 175 | Returns a string representation of the session. 176 | 177 | Returns: 178 | str: A string representation of the session. 179 | """ 180 | return f"Session: {self.name}\nPath: {self.path}\nClasses: {self.classes}\nYears: {self.years}\nROI IDs: {self.roi_ids}\n" 181 | 182 | 183 | # Example usage: 184 | 185 | # Create a new session 186 | # session = Session() 187 | # session.classes = {'Math', 'English', 'Science'} 188 | # session.years = {2020, 2021, 2022} 189 | # session.roi_ids = {1, 2, 3} 190 | # session.name = 'session1' 191 | # session.path = '/path/to/sessions/session1' 192 | 193 | # # Save the session 194 | # session.save(session.path) 195 | 196 | # # Load the session from disk 197 | # session2 = Session() 198 | # session2.load(session.path) 199 | # print(os.path.abspath(session.path)) 200 | 201 | # # Check that the loaded session has the same values as the original session 202 | # print(session2) 203 | -------------------------------------------------------------------------------- /test_models.py: -------------------------------------------------------------------------------- 1 | # Testing & Debugging Script for Seg2Map 2 | # This script is designed for testing and debugging the Seg2Map application. It runs multiple models on a provided input directory containing RGB images, 3 | # using the specified implementation and model types.The results are logged and saved for analysis and evaluation. 4 | # The script allows for command-line arguments to specify the input directory path. 5 | # 6 | # # Author: Sharon Fitzpatrick 7 | # Date: 7/18/2023 8 | # 9 | # To run this script, follow these steps: 10 | # Make sure you have Python installed and the necessary dependencies for the script. 11 | # 1. Open a command prompt or terminal. 12 | # 2. Replace in the command below with the path to the ROI's RGB directory: 13 | # python test_models.py -P """ -I "BEST" 14 | # 2. Execute the command. For example, if the RGB directory is located at C:\development\doodleverse\seg2map\seg2map\data\new_data, the command would be: 15 | # python test_models.py -P "C:\development\doodleverse\seg2map\seg2map\data\new_data" -I "BEST" 16 | 17 | 18 | import argparse 19 | from seg2map import log_maker 20 | from seg2map.zoo_model import ZooModel 21 | 22 | # from transformers import TFSegformerForSemanticSegmentation 23 | # import tensorflow as tf 24 | 25 | # alternatively you can hard code your own variables 26 | # INPUT_DIRECTORY = r"C:\development\doodleverse\seg2map\seg2map\data\new_data" 27 | INPUT_DIRECTORY = r"C:\development\doodleverse\seg2map\seg2map\data\download_group1" 28 | 29 | IMPLEMENTATION = "BEST" # "ENSEMBLE" or "BEST" 30 | 31 | 32 | def parse_arguments(): 33 | """Parse command-line arguments.""" 34 | parser = argparse.ArgumentParser( 35 | description="Run models on provided input directory." 36 | ) 37 | parser.add_argument( 38 | "-P", 39 | "--path", 40 | type=str, 41 | help="Path to an ROI's RGB directory from the data directory", 42 | ) 43 | parser.add_argument( 44 | "-I", 45 | "--implementation", 46 | type=str, 47 | help="BEST or ENSEMBLE", 48 | ) 49 | return parser.parse_args() 50 | 51 | 52 | def print_model_info(model_selected, session_name, input_directory): 53 | """Print information about the selected model.""" 54 | print(f"Running model {model_selected}") 55 | print(f"session_name: {session_name}") 56 | print(f"model_selected: {model_selected}") 57 | print(f"sample_directory: {input_directory}") 58 | 59 | 60 | def run_model(model_dict): 61 | """Run the Seg2Map model with given parameters.""" 62 | zoo_model_instance = ZooModel() 63 | zoo_model_instance.run_model( 64 | model_dict["implementation"], 65 | model_dict["session_name"], 66 | model_dict["sample_direc"], 67 | model_id=model_dict["model_type"], 68 | use_GPU="0", 69 | use_otsu=model_dict["otsu"], 70 | use_tta=model_dict["tta"], 71 | ) 72 | 73 | 74 | def main(): 75 | args = parse_arguments() 76 | 77 | # Get input directory and implementation from command-line arguments or use default values 78 | input_directory = args.path or INPUT_DIRECTORY 79 | implementation = args.implementation or IMPLEMENTATION 80 | 81 | print(f"Using input_directory: {input_directory}") 82 | print(f"Using implementation: {implementation}") 83 | 84 | # List of models that will be tested 85 | available_models = [ 86 | "OpenEarthNet_RGB_9class_7576894", 87 | "DeepGlobe_RGB_7class_7576898", 88 | "EnviroAtlas_RGB_6class_7576909", 89 | "AAAI-Buildings_RGB_2class_7607895", 90 | "aaai_floodedbuildings_RGB_2class_7622733", 91 | "xbd_building_RGB_2class_7613212", 92 | "xbd_damagedbuilding_RGB_4class_7613175", 93 | "chesapeake_RGB_7class_7576904", 94 | "orthoCT_RGB_2class_7574784", 95 | "orthoCT_RGB_5class_7566992", 96 | "orthoCT_RGB_5class_segformer_7641708", 97 | "orthoCT_RGB_8class_7570583", 98 | "orthoCT_RGB_8class_segformer_7641724", 99 | "chesapeake_7class_segformer_7677506", 100 | ] 101 | 102 | for model_selected in available_models: 103 | session_name = model_selected + "_" + implementation + "_" + "session" 104 | print_model_info(model_selected, session_name, input_directory) 105 | 106 | # Load the basic zoo_model settings 107 | model_dict = { 108 | "sample_direc": input_directory, 109 | "session_name": session_name, 110 | "use_GPU": "0", 111 | "implementation": implementation, 112 | "model_type": model_selected, 113 | "otsu": False, 114 | "tta": False, 115 | } 116 | run_model(model_dict) 117 | 118 | 119 | if __name__ == "__main__": 120 | main() 121 | -------------------------------------------------------------------------------- /unzipper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import threading 3 | import zipfile 4 | import concurrent.futures 5 | 6 | 7 | def remove_zip(path): 8 | # Get a list of all the zipped files in the directory 9 | zipped_files = [ 10 | os.path.join(path, f) for f in os.listdir(path) if f.endswith(".zip") 11 | ] 12 | # Remove each zip file 13 | for zipped_file in zipped_files: 14 | os.remove(zipped_file) 15 | 16 | 17 | def unzip(path): 18 | # Get a list of all the zipped files in the directory 19 | zipped_files = [ 20 | os.path.join(path, f) for f in os.listdir(path) if f.endswith(".zip") 21 | ] 22 | # Unzip each file 23 | for zipped_file in zipped_files: 24 | with zipfile.ZipFile(zipped_file, "r") as zip_ref: 25 | zip_ref.extractall(path) 26 | 27 | 28 | def unzip_files(paths): 29 | # Create a thread pool with a fixed number of threads 30 | with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: 31 | # Submit a unzip task for each directory 32 | futures = [executor.submit(unzip, path) for path in paths] 33 | 34 | # Wait for all tasks to complete 35 | concurrent.futures.wait(futures) 36 | 37 | 38 | def remove_zip_files(paths): 39 | # Create a thread pool with a fixed number of threads 40 | with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: 41 | # Submit a remove_zip task for each directory 42 | futures = [executor.submit(remove_zip, path) for path in paths] 43 | 44 | # Wait for all tasks to complete 45 | concurrent.futures.wait(futures) 46 | 47 | 48 | def get_subdirs(parent_dir): 49 | # Get a list of all the subdirectories in the parent directory 50 | subdirs = [ 51 | os.path.join(parent_dir, d) 52 | for d in os.listdir(parent_dir) 53 | if os.path.isdir(os.path.join(parent_dir, d)) 54 | ] 55 | return subdirs 56 | 57 | def unzip_data(parent_dir:str): 58 | subdirs = get_subdirs(parent_dir) 59 | unzip_files(subdirs) 60 | # remove_zip_files(subdirs) 61 | 62 | 63 | parent_dir = r"C:\1_USGS\5_Doodleverse\1_Seg2Map_fork\seg2map\ROIs\ROI9" 64 | unzip_data(parent_dir) 65 | # roi_8 = r"C:\1_USGS\5_Doodleverse\1_Seg2Map_fork\seg2map\ROIs\ROI8\multiband" 66 | # unzip(roi_8) --------------------------------------------------------------------------------