├── .coveragerc ├── .github └── workflows │ ├── build-and-publish_docker.yaml │ └── create-and-publish-wheel.yaml ├── .gitignore ├── .readthedocs.yaml ├── AUTHORS.md ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── bulldozer ├── __init__.py ├── _version.py ├── eoscale │ ├── __init__.py │ ├── eo_executors.py │ ├── manager.py │ ├── shared.py │ └── utils.py ├── extraction │ ├── __init__.py │ ├── cython │ │ ├── c_springforce.cpp │ │ ├── c_springforce.h │ │ ├── idw.c │ │ └── springforce.pyx │ └── drape_cloth.py ├── pipeline │ ├── __init__.py │ ├── bulldozer_parameters.py │ └── bulldozer_pipeline.py ├── postprocessing │ ├── __init__.py │ └── fill_pits.py ├── preprocessing │ ├── __init__.py │ ├── border_detection │ │ ├── __init__.py │ │ ├── border_detector.py │ │ └── cython │ │ │ ├── border.pyx │ │ │ ├── c_border.cpp │ │ │ └── c_border.h │ ├── dsm_filling │ │ ├── __init__.py │ │ ├── cython │ │ │ ├── c_fill.cpp │ │ │ ├── c_fill.h │ │ │ └── fill.pyx │ │ └── dsm_filler.py │ ├── ground_detection │ │ ├── __init__.py │ │ └── ground_anchors_detector.py │ └── regular_detection │ │ ├── __init__.py │ │ ├── cython │ │ ├── c_regular.cpp │ │ ├── c_regular.h │ │ └── regular.pyx │ │ └── regular_detector.py ├── scale │ └── __init__.py └── utils │ ├── __init__.py │ ├── bulldozer_argparse.py │ ├── bulldozer_logger.py │ ├── config_parser.py │ └── helper.py ├── conf ├── basic_conf_template.yaml └── configuration_template.yaml ├── docs ├── css │ └── extra.css ├── index.md └── source │ └── images │ ├── logo.png │ ├── logo_icon.ico │ ├── logo_with_text.png │ └── result_overview.gif ├── mkdocs.yaml ├── pyproject.toml ├── setup.cfg ├── setup.py └── tests ├── __init__.py └── utils ├── __init__.py ├── data └── config_parser │ ├── parser_test.yaml │ └── wrong_syntax.yaml └── test_config_parser.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = bulldozer 3 | omit = 4 | # Omit tests 5 | /tests/* -------------------------------------------------------------------------------- /.github/workflows/build-and-publish_docker.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 2 | # 3 | # This file is part of Bulldozer 4 | # (see https://github.com/CNES/bulldozer). 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # This file is mainly based on the github documentation (https://docs.github.com/en/actions/use-cases-and-examples/publishing-packages/publishing-docker-images) 19 | 20 | 21 | name: Build and publish Docker image 22 | 23 | on: 24 | push: 25 | tags: 26 | - "v*" # Push to every tag containing version number 27 | 28 | jobs: 29 | push_to_registry: 30 | name: Push Docker image to Docker Hub 31 | runs-on: ubuntu-latest 32 | environment: publish 33 | permissions: 34 | packages: write 35 | contents: read 36 | attestations: write 37 | id-token: write 38 | steps: 39 | - name: Check out the repo 40 | uses: actions/checkout@v4 41 | 42 | - name: Log in to Docker Hub 43 | uses: docker/login-action@v3 44 | with: 45 | username: ${{ secrets.DOCKER_USERNAME }} 46 | password: ${{ secrets.DOCKER_TOKEN }} 47 | 48 | - name: Extract metadata (tags, labels) for Docker 49 | id: meta 50 | uses: docker/metadata-action@v5 51 | with: 52 | images: cnes/bulldozer 53 | 54 | - name: Build and push Docker image 55 | id: push 56 | uses: docker/build-push-action@v6 57 | with: 58 | context: . 59 | push: true 60 | tags: ${{ steps.meta.outputs.tags }} 61 | labels: ${{ steps.meta.outputs.labels }} 62 | 63 | - name: Generate artifact attestation 64 | uses: actions/attest-build-provenance@v2 65 | with: 66 | subject-name: index.docker.io/my-docker-hub-namespace/my-docker-hub-repository 67 | subject-digest: ${{ steps.push.outputs.digest }} 68 | push-to-registry: true 69 | -------------------------------------------------------------------------------- /.github/workflows/create-and-publish-wheel.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 2 | # 3 | # This file is part of Bulldozer 4 | # (see https://github.com/CNES/bulldozer). 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # This file is mainly based on the cibuildwheel documentation (https://github.com/pypa/cibuildwheel) 19 | 20 | name: Create and Publish wheel 21 | 22 | on: 23 | push: 24 | tags: 25 | - "v*" # Push to every tag containing version number 26 | 27 | jobs: 28 | # Build Ubuntu, Windows and macOS wheels 29 | build_wheels: 30 | name: Build wheels on ${{ matrix.os }} 31 | runs-on: ${{ matrix.os }} 32 | strategy: 33 | matrix: 34 | os: [ubuntu-latest, windows-latest, macos-13, macos-latest] 35 | 36 | steps: 37 | - uses: actions/checkout@v4 38 | 39 | - uses: actions/setup-python@v5 40 | 41 | - name: Install cibuildwheel 42 | run: python -m pip install cibuildwheel==2.22.0 43 | 44 | - name: Build wheels 45 | run: python -m cibuildwheel --output-dir wheelhouse 46 | 47 | - uses: actions/upload-artifact@v4 48 | with: 49 | name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} 50 | path: ./wheelhouse/*.whl 51 | 52 | build_sdist: 53 | name: Build source distribution 54 | runs-on: ubuntu-latest 55 | steps: 56 | - uses: actions/checkout@v4 57 | 58 | - name: Build sdist 59 | run: pipx run build --sdist 60 | 61 | - uses: actions/upload-artifact@v4 62 | with: 63 | name: cibw-sdist 64 | path: dist/*.tar.gz 65 | 66 | # Upload the wheels generated with cibuildwheel on PyPI 67 | upload_pypi: 68 | name: Publish package on PyPI 69 | needs: [build_wheels, build_sdist] 70 | runs-on: ubuntu-latest 71 | environment: publish 72 | permissions: 73 | id-token: write 74 | steps: 75 | - uses: actions/download-artifact@v4 76 | with: 77 | pattern: cibw-* 78 | path: dist 79 | merge-multiple: true 80 | 81 | - uses: pypa/gh-action-pypi-publish@release/v1 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 2 | # 3 | # This file is part of Bulldozer 4 | # (see https://github.com/CNES/bulldozer). 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # Inspired by https://www.gitignore.io/api/python 20 | 21 | ### C/C++ ### 22 | # C extensions 23 | *.so 24 | # Ignore C++ files that don't start with "c_" (e.g. files automaticly generated by Cython) 25 | *.cpp 26 | !c_*.cpp 27 | 28 | ### Python ### 29 | # Virtual env 30 | *venv*/ 31 | 32 | # Byte-compiled / optimized / DLL files 33 | __pycache__/ 34 | *.py[cod] 35 | *$py.class 36 | 37 | # Distribution / packaging 38 | .Python 39 | build/ 40 | develop-eggs/ 41 | dist/ 42 | downloads/ 43 | eggs/ 44 | .eggs/ 45 | lib/ 46 | lib64/ 47 | parts/ 48 | sdist/ 49 | var/ 50 | wheels/ 51 | pip-wheel-metadata/ 52 | share/python-wheels/ 53 | *.egg-info/ 54 | .installed.cfg 55 | *.egg 56 | MANIFEST 57 | 58 | # Unit test / coverage reports 59 | htmlcov/ 60 | .tox/ 61 | .nox/ 62 | .coverage 63 | .coverage.* 64 | .cache 65 | nosetests.xml 66 | coverage.xml 67 | *.cover 68 | .hypothesis/ 69 | .pytest_cache/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # IPython 78 | profile_default/ 79 | ipython_config.py 80 | 81 | # pyenv 82 | .python-version 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # mkdocs documentation 94 | /site 95 | 96 | # mypy 97 | .mypy_cache/ 98 | .dmypy.json 99 | dmypy.json 100 | 101 | # Pyre type checker 102 | .pyre/ 103 | 104 | # IDE 105 | .vscode 106 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 2 | # 3 | # This file is part of Bulldozer 4 | # (see https://github.com/CNES/bulldozer). 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # Read the Docs configuration file. See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 19 | version: 2 20 | 21 | build: 22 | os: ubuntu-24.04 23 | tools: 24 | python: "3.8" 25 | 26 | jobs: 27 | pre_install: 28 | #TODO update to install from requirements 29 | - pip install mkdocs 30 | mkdocs: 31 | configuration: mkdocs.yml 32 | fail_on_warning: false -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Credits 2 | 3 | Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 4 | 5 | BULLDOZER is licensed under permissive Apache 2 license (See [LICENSE](LICENSE) file). 6 | The copyright is kept CNES only for long term maintenance ease. 7 | 8 | If you want to contribute, please refers to [CONTRIBUTING.md](CONTRIBUTING.md). 9 | ] 10 | This file keeps track of authors contributions. 11 | 12 | ## Development Lead 13 | 14 | * [Dimitri Lallement](mailto:dimitri.lallement@cnes.fr) - current maintainer 15 | 16 | ## Contributors 17 | 18 | * [Pierre Lassalle](mailto:pierre.lassalle@cnes.fr) - previous co-maintainer 19 | * [Yannick Ott](mailto:yannick.ott@thalesgroup.com) - designer & developer 20 | * [Hugo Fournier](mailto:hugo.fournier@cnes.fr) - developer (CI and Docker) 21 | * [Florian Bueno](mailto:florian.bueno@thalesgroup.com) - developer 22 | * [Aurélie Emilien](mailto:aurelie.emilien@thalesgroup.com) - developer 23 | * [Celine Raille](mailto:celine.raille@thalesgroup.com) - developer 24 | * [Andrea Piacentinu](mailto:andrea.piacentini@gmail.com) - designer & tester 25 | * [Alexia Mondot](mailto:alexia.mondot@thalesgroup.com) - QGIS plugin developer 26 | * [Tarek Benhnini](tarek.benhnini@thalesgroup.com) - developer (documentation) 27 | * [Rémi Demortier](mailto:remi.demortier@thalesgroup.com) - previous developer 28 | * [Sophie Lefier](mailto:slefier.anim@gmail.com) - logo designer 29 | 30 | We also want to thank the developers of the amazing open-source libraries on which **Bulldozer** relies: 31 | * [rasterio](https://github.com/rasterio/rasterio) 32 | * [numpy](https://github.com/numpy/numpy) 33 | * [scipy](https://github.com/scipy/scipy) 34 | * [Cython](https://github.com/cython/cython) 35 | * And many others. -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.1.2 QGIS plugin release (April 2025) 4 | 5 | ### Added 6 | - [QGIS plugin](https://github.com/CNES/bulldozer-qgis-plugin) released 7 | - New filling DSM method 8 | - Github action for publishing Docker image on DockerHub 9 | - Adding documentation skeleton (work in progress) 10 | 11 | ### Changed 12 | - Allows the user to set the `reg_filtering_iter` to 0 in order to desactivate the regular mask fitlering 13 | - Remove `cloth_tension_force` parameter 14 | - Eoscale can now handle specific tile size 15 | - Increase the number of filtering iteration of the regular mask 16 | - Rename `dev_mode` alias to `dev` and `unhook_iter` alias to `unhook_it` 17 | 18 | ### Fixed 19 | - Fix the remaining nodata issue after DSM filling step 20 | - Fix the issue of `get_max_pyramid_level` function returning negative result 21 | - Change logfile name in order to comply with the Windows filename rule 22 | - Fix the nodata value in intermediate profile for `dev_mode` during the DSM filling step 23 | 24 | --- 25 | 26 | ## 1.1.1 Docker release (March 2025) 27 | 28 | ### Added 29 | - Adding Dockerfile and publishing it on [dockerhub](https://hub.docker.com/r/cnes/bulldozer) 30 | - Adding expert mode CLI 31 | - Adding deploy pipeline in gitlab CI 32 | - Adding `reg_filtering_iter` parameter in expert mode 33 | 34 | ### Changed 35 | - C++ files refactoring 36 | - The user can override the configuration file with parameters in the CLI 37 | - Adding a new step in developer mode to write regular mask before filtering 38 | - New border noadata mask computation 39 | - Including border nodata mask option in filling method 40 | 41 | ### Fixed 42 | - Github actions workflow file for publishing wheels on PyPI fixed 43 | - Configuration file templates refactoring 44 | - Fixing gitPython issue (dependency removed as it's rarely used) 45 | - Adding default value (2) for dezoom factor if the provided value is less than 2 46 | - Adding default value (1) for nb_iterations during the regular mask filtering if the provided value is less than 1 47 | 48 | --- 49 | 50 | ## 1.1.0 Cross-platforms compatibility (February 2025) 51 | 52 | ### Added 53 | - Building and providing wheels for Windows, macOS and Ubuntu on PyPI 54 | - Filter small element in the regular mask to avoid clipping on noisy elevation points 55 | - Adding `--long-usage` option 56 | - Adding raster profile to logs 57 | 58 | ### Changed 59 | - New version number management 60 | - Update CLI helper 61 | 62 | ### Fixed 63 | - Reduce DSM filling runtime by using a new filling method 64 | - Regular areas detection on the DSM edges issue => fixed 65 | 66 | --- 67 | 68 | ## 1.0.2 New pipeline (November 2024) 69 | 70 | ### Added 71 | - Update of the version release policy: increased release frequency to be expected 72 | - Major method improvement (pseudo-local) 73 | - Adding ground pre-detection: ground_clipping (optional) 74 | - Adding ground mask entry (optional) 75 | - Parallel computation method updated (EOScale) 76 | - Adding CLI with options (in addition to CLI with config file) 77 | - Adding Makefile 78 | - New nodata filling method 79 | - Warning on unused parameters in the configuration 80 | - New logo 81 | 82 | ### Changed 83 | - Parameters name change 84 | - Quality mask split into different masks available in the output directory 85 | - Global refactoring 86 | - Documentation update (README, CONTRIBUTING, etc.) 87 | - Update the version location 88 | - Update the pits filling method 89 | - New log file name 90 | - Remove `nodata` and `min_valid_height` parameters and replace them with an explanation in documentation on how to change the 'nodata' value in raster metadata 91 | - `max_object_width` parameter is renamed `max_object_size` 92 | - `keep_inter_dtm` parameter is removed due to the new pipeline 93 | - `four_connexity` parameter is removed due to the new pipeline 94 | - `mp_tile_size` parameter is removed due to the new pipeline 95 | 96 | ### Fixed 97 | - Remove problematic `uniform_filter_size` parameter 98 | - Remove `output_resolution` parameter since it was a degraded downsampling method 99 | - Git exception is handled if the user doesn't have git installer 100 | - Border nodata detection => fixed 101 | - Add missing logging level 102 | - DSM with nodata set to NaN/None value => fixed 103 | - DHM nodata issue (missing value) => fixed 104 | - Optimize memory release in the pipeline 105 | - DSM with CRS for which to_authority() returns None => fixed 106 | - Bulldozer version added in logfile 107 | - Remove unused documentation until the new version of the official documentation is published 108 | - Remove unnecessary memory allocation in anchor prediction 109 | 110 | --- 111 | 112 | ## 1.0.1 New interface (July 2023) 113 | 114 | ### Added 115 | - New API: all parameters are no longer required 116 | - New API: It is possible to launch Bulldozer without a configuration file, simply by supplying the values of the parameters 117 | - Adding default values for input parameters (except DSM path and output directory) 118 | - New option: store intermediate DTM 119 | 120 | ### Changed 121 | - New API: parameters name update 122 | - Multi-processing improvement (metadata taking in consideration) 123 | - New quality mask format 124 | - Unhook method update 125 | - Global runtime improvement 126 | 127 | ### Fixed 128 | - DHM was not generated when the output resolution was the same as the input DSM => Fixed 129 | - Logger file did not display the plateform information => Fixed 130 | - Nodata retrieval improved 131 | - Disturbed areas missing line => Fixed 132 | - Border nodata multi-processing issue => Fixed 133 | 134 | --- 135 | 136 | ## 1.0.0 Open Source Release (January 2023) 137 | 138 | ### Added 139 | - Publication of the code in open-source -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Bulldozer Contributing guide 2 | 3 | 1. [Report issues](#report-issues) 4 | 2. [Contributing workflow](#contributing-workflow) 5 | 3. [Coding guide](#coding-guide) 6 | 4. [Merge request acceptation process](#merge-request-acceptation-process) 7 | 8 | **Contributions are welcome and greatly appreciated!** 9 | 10 | # Report issues 11 | 12 | Any proven or suspected malfunction should be traced in a bug report, the latter being an issue in the **Bulldozer** [github repository](https://github.com/CNES/bulldozer). 13 | 14 | **Don't hesitate to do so: It is best to open a bug report and quickly resolve it than to let a problem remains in the project.** 15 | **Notifying the potential bugs is the first way for contributing to a software.** 16 | 17 | 18 | 19 | In the problem description, be as accurate as possible. Include: 20 | * The procedure used to initialize the environment 21 | * The incriminated command line or python function 22 | * The content of the output log file 23 | 24 | # Contributing workflow 25 | 26 | Any code modification requires a Merge Request. It is forbidden to push patches directly into master (this branch is protected). 27 | 28 | It is recommended to open your Merge Request as soon as possible in order to inform the developers of your ongoing work. 29 | Please add `WIP:` before your Merge Request title if your work is in progress: this prevents an accidental merge and informs the other developers of the unfinished state of your work. 30 | 31 | The Merge Request shall have a short description of the proposed changes. If it is relative to an issue, you can signal it by adding `Closes xx` where xx is the reference number of the issue. 32 | 33 | Likewise, if you work on a branch (which is recommended), prefix the branch's name by `xx-` in order to link it to the xx issue. 34 | 35 | **Bulldozer** classical workflow is : 36 | * Create an issue (or begin from an existing one) 37 | * Create a Merge Request from the issue: a MR is created accordingly with "WIP:", "Closes xx" and associated "xx-name-issue" branch 38 | * Hack code from a local working directory 39 | * If you use Cython, you must name your C++ files with the following format: `c_.[cpp/h]` 40 | * Git add, commit and push from local working clone directory 41 | * Follow [Conventional commits](https://www.conventionalcommits.org/) specifications for commit messages 42 | * Launch the tests on your modifications 43 | * When finished, change your Merge Request name (erase "WIP:" in title) and ask to review the code (see below [Merge request acceptation process] section(#merge-request-acceptation-process)) 44 | 45 | # Coding guide 46 | 47 | Here are some rules to apply when developing a new functionality: 48 | * Include a comments ratio high enough and use explicit variables names. A comment by code block of several lines is necessary to explain a new functionality. 49 | * The usage of the `print()` function is forbidden: use the **Bulldozer** internal logger instead. 50 | * Each new functionality shall have a corresponding test in its module's test file. This test shall, if possible, check the function's outputs and the corresponding degraded cases. 51 | * All functions shall be documented (object, parameters, return values). 52 | * If major modifications of the user interface or of the tool's behaviour are done, update the user documentation (and the notebooks if necessary). 53 | * Do not add new dependencies unless it is absolutely necessary, and only if it has a permissive license. 54 | * Use the type hints provided by the `typing` python module. 55 | 56 | # Merge request acceptation process 57 | 58 | The Merge Request will be merged into master after being reviewed by **Bullodzer** steering committee. Only the members of this committee can merge into master. 59 | 60 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim 2 | 3 | LABEL maintainer="CNES" 4 | 5 | RUN apt-get update \ 6 | && apt-get install -y \ 7 | build-essential \ 8 | binutils \ 9 | libproj-dev \ 10 | gdal-bin \ 11 | && pip install --no-cache-dir --upgrade pip \ 12 | && pip install --no-cache-dir bulldozer-dtm \ 13 | && rm -rf /var/lib/apt/lists/*; 14 | 15 | # Adding a default user to prevent root access 16 | RUN groupadd bulldozer && useradd bulldozer -g bulldozer; 17 | 18 | USER bulldozer 19 | 20 | ENTRYPOINT ["bulldozer"] 21 | CMD ["-lu"] 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2022 Centre National d'Etudes Spatiales (CNES) 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Include Cython files in the package 2 | global-include *.pyx 3 | global-include *.pxd 4 | global-include c_*.cpp 5 | global-include c_*.h -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 2 | # 3 | # This file is part of Bulldozer 4 | # (see https://github.com/CNES/bulldozer). 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # Initially based on Autodocumented Makefile 19 | # see: https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html 20 | # Dependencies : python3 venv 21 | # Some Makefile global variables can be set in make command line 22 | # Recall: .PHONY defines special targets not associated with files 23 | 24 | ############### GLOBAL VARIABLES ###################### 25 | .DEFAULT_GOAL := help 26 | # Set shell to BASH 27 | SHELL := /bin/bash 28 | 29 | # Set Virtualenv directory name 30 | # Example: VENV="other-venv/" make install 31 | ifndef VENV 32 | VENV = "bulldozer_venv" 33 | endif 34 | 35 | # Python global variables definition 36 | PYTHON_VERSION_MIN = 3.8 37 | 38 | # Set PYTHON if not defined in command line 39 | # Example: PYTHON="python3.10" make venv to use python 3.10 for the venv 40 | # By default the default python3 of the system. 41 | ifndef PYTHON 42 | PYTHON = "python3" 43 | endif 44 | PYTHON_CMD=$(shell command -v $(PYTHON)) 45 | 46 | PYTHON_VERSION_CUR=$(shell $(PYTHON_CMD) -c 'import sys; print("%d.%d"% sys.version_info[0:2])') 47 | PYTHON_VERSION_OK=$(shell $(PYTHON_CMD) -c 'import sys; cur_ver = sys.version_info[0:2]; min_ver = tuple(map(int, "$(PYTHON_VERSION_MIN)".split("."))); print(int(cur_ver >= min_ver))') 48 | 49 | ############### Check python version supported ############ 50 | 51 | ifeq (, $(PYTHON_CMD)) 52 | $(error "PYTHON_CMD=$(PYTHON_CMD) not found in $(PATH)") 53 | endif 54 | 55 | ifeq ($(PYTHON_VERSION_OK), 0) 56 | $(error "Requires python version >= $(PYTHON_VERSION_MIN). Current version is $(PYTHON_VERSION_CUR)") 57 | endif 58 | 59 | ################ MAKE targets by sections ###################### 60 | .PHONY: help 61 | help: ## help on Bulldozer command line usage 62 | @echo " BULLDOZER MAKE HELP" 63 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 64 | 65 | .PHONY: venv 66 | venv: ## create virtualenv in "bulldozer_venv" directory if it doesn't exist 67 | @test -d ${VENV} || $(PYTHON_CMD) -m venv ${VENV} 68 | @touch ${VENV}/bin/activate 69 | @${VENV}/bin/python -m pip install --upgrade wheel setuptools pip # no check to upgrade each time 70 | 71 | 72 | .PHONY: install 73 | install: venv ## install environment for development target (depends bulldozer_venv) 74 | @test -f ${VENV}/bin/bulldozer || echo "Install bulldozer package from local directory" 75 | @test -f ${VENV}/bin/bulldozer || ${VENV}/bin/pip install -e .[dev,docs,notebook] 76 | @echo "Bulldozer installed in dev mode in virtualenv ${VENV}" 77 | @echo "Bulldozer virtual environment usage: source ${VENV}/bin/activate; bulldozer -h" 78 | 79 | ## Test section 80 | 81 | .PHONY: test 82 | test: ## run tests and coverage quickly with the default Python 83 | @${VENV}/bin/pytest -o log_cli=true --cov-config=.coveragerc --cov 84 | 85 | .PHONY: coverage 86 | coverage: ## check code coverage quickly with the default Python 87 | @${VENV}/bin/coverage run --source bulldozer -m pytest 88 | @${VENV}/bin/coverage report -m 89 | @${VENV}/bin/coverage html 90 | 91 | ## Documentation section 92 | 93 | .PHONY: docs 94 | docs: install ## generate Mkdocs HTML documentation 95 | @${VENV}/bin/mkdocs build --clean --strict 96 | 97 | ## Notebook section 98 | 99 | .PHONY: notebook 100 | notebook: ## install Jupyter notebook kernel with venv and bulldozer install 101 | @echo "Install Jupyter Kernel and run Jupyter notebooks environment" 102 | @${VENV}/bin/python -m ipykernel install --sys-prefix --name=jupyter-${VENV} --display-name=jupyter-${VENV} 103 | @echo "Use jupyter kernelspec list to know where is the kernel" 104 | @echo " --> After Bulldozer virtualenv activation, please use following command to run local jupyter notebook to open Notebooks:" 105 | @echo "jupyter notebook" 106 | 107 | .PHONY: notebook-clean-output ## Clean Jupyter notebooks outputs 108 | notebook-clean-output: 109 | @echo "Clean Jupyter notebooks" 110 | @${VENV}/bin/jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace docs/notebooks/*.ipynb 111 | 112 | ## Release section 113 | 114 | .PHONY: dist 115 | dist: clean install ## clean, install, builds source and wheel package 116 | @${VENV}/bin/python -m pip install --upgrade build 117 | @${VENV}/bin/python -m build --sdist 118 | ls -l dist 119 | 120 | .PHONY: release 121 | release: dist ## package and upload a release 122 | @${VENV}/bin/twine check dist/* 123 | @${VENV}/bin/twine upload --verbose --config-file ~/.pypirc -r pypi dist/* ## update your .pypirc accordingly 124 | 125 | ## Clean section 126 | 127 | .PHONY: clean 128 | clean: clean-venv clean-build clean-precommit clean-pyc clean-test clean-lint clean-docs clean-notebook clean-libs ## clean all 129 | 130 | .PHONY: clean-venv 131 | clean-venv: ## clean venv 132 | @echo "+ $@" 133 | @rm -rf ${VENV} 134 | 135 | .PHONY: clean-build 136 | clean-build: ## clean build artifacts 137 | @echo "+ $@" 138 | @rm -rf build/ 139 | @rm -rf dist/ 140 | @rm -rf .eggs/ 141 | @find . -name '*.egg-info' -exec rm -rf {} + 142 | @find . -name '*.egg' -exec rm -rf {} + 143 | 144 | .PHONY: clean-precommit 145 | clean-precommit: ## clean precommit hooks in .git/hooks 146 | @rm -f .git/hooks/pre-commit 147 | @rm -f .git/hooks/pre-push 148 | 149 | .PHONY: clean-pyc 150 | clean-pyc: ## clean Python file artifacts 151 | @echo "+ $@" 152 | @find . -type f -name "*.py[co]" -exec rm -rf {} + 153 | @find . -type d -name "__pycache__" -exec rm -rf {} + 154 | @find . -name '*~' -exec rm -rf {} + 155 | 156 | .PHONY: clean-test 157 | clean-test: ## clean test and coverage artifacts 158 | @echo "+ $@" 159 | @rm -rf .tox/ 160 | @rm -f .coverage 161 | @rm -rf .coverage.* 162 | @rm -rf coverage.xml 163 | @rm -rf htmlcov/ 164 | @rm -rf .pytest_cache 165 | @rm -f pytest-report.xml 166 | @find . -type f -name "debug.log" -exec rm -rf {} + 167 | 168 | .PHONY: clean-lint 169 | clean-lint: ## clean linting artifacts 170 | @echo "+ $@" 171 | @rm -f pylint-report.txt 172 | @rm -f pylint-report.xml 173 | @rm -rf .mypy_cache/ 174 | 175 | .PHONY: clean-docs 176 | clean-docs: ## clean builded documentations 177 | @echo "+ $@" 178 | @rm -rf site/ 179 | 180 | .PHONY: clean-notebook 181 | clean-notebook: ## clean notebooks cache 182 | @echo "+ $@" 183 | @find . -type d -name ".ipynb_checkpoints" -exec rm -rf {} + 184 | 185 | .PHONY: clean-libs 186 | clean-libs: ## clean .so compiled files and cpp files (except ones starting with "c_.cpp") 187 | @echo "+ $@" 188 | @find . -type f -name "*.so" -exec rm -rf {} + 189 | @find . -type f -name "*.cpp" -not -name "c_*.cpp" -exec rm -rf {} + -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | **Bulldozer, a DTM extraction tool that requires only a DSM as input.** 5 | 6 | [![pypi](https://img.shields.io/pypi/v/bulldozer-dtm?color=%2334D058&label=pypi)](https://pypi.org/project/bulldozer-dtm/) 7 | [![docker](https://badgen.net/docker/size/cnes/bulldozer?icon=docker&label=image%20size)](https://hub.docker.com/r/cnes/bulldozer) 8 | [![contributions welcome](https://img.shields.io/badge/contributions-welcome-orange.svg)](CONTRIBUTING.md) 9 | [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 10 |
11 | 12 | 13 | # Overview 14 |
15 | demo 16 |
17 | 18 | **Bulldozer** is a pipeline designed to extract a *Digital Terrain Model* (DTM) from a *Digital Surface Model* (DSM). It supports both noisy satellite DSMs and high-quality LiDAR DSMs. 19 | 20 | # Quick Start 21 | 22 | ## Installation 23 | You can install **Bulldozer** by running the following command: 24 | ```sh 25 | pip install bulldozer-dtm 26 | ``` 27 | Or you can clone the github repository and use the `Makefile`: 28 | ```sh 29 | # Clone the project 30 | git clone https://github.com/CNES/bulldozer.git 31 | cd bulldozer/ 32 | 33 | # Create the virtual environment and install required depencies 34 | make install 35 | 36 | # Activate the virtual env 37 | source bulldozer_venv/bin/activate 38 | ``` 39 | ## Run **Bulldozer** 40 | 41 | There are different ways to launch **Bulldozer**: 42 | 43 | 1. Using the CLI *(Command Line Interface)* - Run the folowing command line after updating the parameters `input_dsm.tif` and `output_dir`: 44 | ```console 45 | bulldozer -dsm input_dsm.tif -out output_dir 46 | ``` 47 | *You can also add optional parameters such as `-dhm`, please refer to the helper (`bulldozer -h`) command to see all the options.* 48 | 49 | ✅ Done! Your DTM is available in the `output_dir`. 50 | 51 | 2. Using the Python API - You can directly provide the input parameters to the `dsm_to_dtm` function: 52 | ```python 53 | from bulldozer.pipeline.bulldozer_pipeline import dsm_to_dtm 54 | 55 | dsm_to_dtm(dsm_path="input_dsm.tif", output_dir="output_dir") 56 | ``` 57 | ✅ Done! Your DTM is available in the `output_dir`. 58 | 59 | 3. Using a configuration file (CLI) - Based on provided [configuration file](conf) templates, you can run the following command line: 60 | ```console 61 | bulldozer conf/configuration_template.yaml 62 | ``` 63 | ✅ Done! Your DTM is available in the directory defined in the configuration file. 64 | 65 | ## **Bulldozer** docker image 66 | 67 | [![Docker Status](http://dockeri.co/image/cnes/bulldozer)](https://hub.docker.com/r/cnes/bulldozer) 68 | 69 | **Bulldozer** is available on Docker Hub and can be downloaded using: 70 | ``` bash 71 | docker pull cnes/bulldozer 72 | ``` 73 | And you can run **Bulldozer** with the following command: 74 | ``` bash 75 | docker run --user $(id -u):$(id -g) --shm-size=10gb -v :/data cnes/bulldozer:latest /data/.yaml 76 | ``` 77 | ⚠️ You have to change the `` to provide a valide absolute path to a directory where the input data are stored and where **Bulldozer** will write the ouput DTM. You also have to provide a configuration file (and rename `.yaml` in the command line) in this directory with an `ouput_dir` value using the `data` folder given to docker, e.g.: `output_dir : "/data/outputdir"`. If you want to run **Bulldozer** on a huge DSM, please improve the shared memory value of the command line (`--shm-size`) argument. 78 | 79 | 80 | 81 | # License 82 | 83 | **Bulldozer** is licensed under Apache License v2.0. Please refer to the [LICENSE](LICENSE) file for more details. 84 | 85 | # Citation 86 | If you use **Bulldozer** in your research, please cite the following paper: 87 | ```text 88 | @article{bulldozer2023, 89 | title={Bulldozer, a free open source scalable software for DTM extraction}, 90 | author={Dimitri, Lallement and Pierre, Lassalle and Yannick, Ott}, 91 | journal = {The International Archives of the Photogrammetry, Remote Sensing and Spatial Information Sciences}, 92 | volume = {XLVIII-4/W7-2023}, 93 | year = {2023}, 94 | pages = {89--94}, 95 | url = {https://isprs-archives.copernicus.org/articles/XLVIII-4-W7-2023/89/2023/}, 96 | doi = {10.5194/isprs-archives-XLVIII-4-W7-2023-89-2023} 97 | } 98 | ``` 99 | -------------------------------------------------------------------------------- /bulldozer/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | Bulldozer module init file 23 | """ 24 | from ._version import __version__ 25 | -------------------------------------------------------------------------------- /bulldozer/_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module is used to centralize the current version of Bulldozer. 23 | """ 24 | __version__ = "1.1.2" 25 | -------------------------------------------------------------------------------- /bulldozer/eoscale/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | -------------------------------------------------------------------------------- /bulldozer/eoscale/eo_executors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | from typing import Callable 22 | import concurrent.futures 23 | import tqdm 24 | import numpy 25 | import math 26 | import copy 27 | 28 | import bulldozer.eoscale.shared as eosh 29 | import bulldozer.eoscale.utils as eotools 30 | import bulldozer.eoscale.manager as eom 31 | 32 | 33 | def isPowerOfTwo(x: int): 34 | # First x in the below expression 35 | # is for the case when x is 0 36 | return x and (not(x & (x - 1))) 37 | 38 | 39 | def compute_mp_strips(image_height: int, 40 | image_width: int, 41 | nb_workers: int, 42 | stable_margin: int, 43 | along_line: bool = False) -> list: 44 | """ 45 | Return a list of strips 46 | """ 47 | if along_line: 48 | strip_width = image_width // nb_workers 49 | else: 50 | strip_height = image_height // nb_workers 51 | 52 | strips = [] 53 | if along_line: 54 | end_x: int = None 55 | end_y: int = image_height - 1 56 | start_x: int = None 57 | start_y: int = 0 58 | else: 59 | end_x: int = image_width - 1 60 | end_y: int = None 61 | start_x: int = 0 62 | start_y: int = None 63 | 64 | top_margin: int = None 65 | right_margin: int = 0 66 | bottom_margin: int = None 67 | left_margin: int = 0 68 | 69 | for worker in range(nb_workers): 70 | if along_line: 71 | start_x = worker * strip_width 72 | else: 73 | start_y = worker * strip_height 74 | 75 | if worker == nb_workers - 1: 76 | if along_line: 77 | end_x = image_width - 1 78 | else: 79 | end_y = image_height - 1 80 | else: 81 | if along_line: 82 | end_x = (worker + 1) * strip_width - 1 83 | else: 84 | end_y = (worker + 1) * strip_height - 1 85 | 86 | top_margin = stable_margin if start_y - stable_margin >= 0 else start_y 87 | bottom_margin = stable_margin if end_y + stable_margin <= image_height - 1 else image_height - 1 - end_y 88 | strips.append(eotools.MpTile(start_x=start_x, 89 | start_y=start_y, 90 | end_x=end_x, 91 | end_y=end_y, 92 | top_margin=top_margin, 93 | right_margin=right_margin, 94 | bottom_margin=bottom_margin, 95 | left_margin=left_margin)) 96 | 97 | return strips 98 | 99 | 100 | def compute_mp_tiles(inputs: list, 101 | stable_margin: int, 102 | nb_workers: int, 103 | tile_mode: bool, 104 | context_manager: eom.EOContextManager, 105 | specific_tile_size: int = None, 106 | strip_along_lines: bool = False): 107 | """ 108 | Given an input eoscale virtual path and nb_workers, 109 | this method computes the list of strips that will 110 | be processed in parallel within a stream strip or tile 111 | """ 112 | 113 | img_idx: int = 0 114 | image_width: int = None 115 | image_height: int = None 116 | for img_key in inputs: 117 | # Warning, the key can be either a memory view or a shared resource key 118 | arr = context_manager.get_array(key=img_key) 119 | if img_idx < 1: 120 | image_height = arr.shape[1] 121 | image_width = arr.shape[2] 122 | else: 123 | if image_width != arr.shape[2] or image_height != arr.shape[1]: 124 | raise ValueError("ERROR: all input images must have the same width and the same height !") 125 | img_idx += 1 126 | 127 | if tile_mode: 128 | 129 | nb_tiles_x: int = 0 130 | nb_tiles_y: int = 0 131 | end_x: int = 0 132 | start_y: int = 0 133 | end_y: int = 0 134 | top_margin: int = 0 135 | right_margin: int = 0 136 | bottom_margin: int = 0 137 | left_margin: int = 0 138 | 139 | # Force to make square tiles (except the last one unfortunately) 140 | nb_pixels_per_worker: int = (image_width * image_height) // nb_workers 141 | if specific_tile_size: 142 | tile_size = specific_tile_size 143 | else: 144 | tile_size = int(math.sqrt(nb_pixels_per_worker)) 145 | nb_tiles_x = image_width // tile_size 146 | nb_tiles_y = image_height // tile_size 147 | if image_width % tile_size > 0: 148 | nb_tiles_x += 1 149 | if image_height % tile_size > 0: 150 | nb_tiles_y += 1 151 | 152 | strips: list = [] 153 | 154 | for ty in range(nb_tiles_y): 155 | 156 | for tx in range(nb_tiles_x): 157 | 158 | # Determine the stable and unstable boundaries of the tile 159 | start_x = tx * tile_size 160 | start_y = ty * tile_size 161 | end_x = min((tx+1) * tile_size - 1, image_width - 1) 162 | end_y = min((ty+1) * tile_size - 1, image_height - 1) 163 | top_margin = stable_margin if start_y - stable_margin >= 0 else start_y 164 | left_margin = stable_margin if start_x - stable_margin >= 0 else start_x 165 | bottom_margin = stable_margin if end_y + stable_margin <= image_height - 1 else image_height - 1 - end_y 166 | right_margin = stable_margin if end_x + stable_margin <= image_width - 1 else image_width - 1 - end_x 167 | 168 | strips.append(eotools.MpTile(start_x=start_x, 169 | start_y=start_y, 170 | end_x=end_x, 171 | end_y=end_y, 172 | top_margin=top_margin, 173 | right_margin=right_margin, 174 | bottom_margin=bottom_margin, 175 | left_margin=left_margin)) 176 | 177 | return strips 178 | 179 | else: 180 | return compute_mp_strips(image_height=image_height, 181 | image_width=image_width, 182 | nb_workers=nb_workers, 183 | stable_margin=stable_margin, 184 | along_line=strip_along_lines) 185 | 186 | 187 | def default_generate_output_profiles(input_profiles: list) -> list: 188 | """ 189 | This method makes a deep copy of the input profiles 190 | """ 191 | return [copy.deepcopy(input_profile) for input_profile in input_profiles] 192 | 193 | 194 | def allocate_outputs(profiles: list, 195 | context_manager: eom.EOContextManager) -> list: 196 | """ 197 | Given a list of profiles, this method creates 198 | shared memory instances of the outputs 199 | """ 200 | 201 | output_eoshared_instances: list = [eosh.EOShared() for i in range(len(profiles))] 202 | 203 | for i in range(len(profiles)): 204 | output_eoshared_instances[i].create_array(profile=profiles[i]) 205 | # Be careful to not close theses shared instances, because they are referenced in 206 | # the context manager. 207 | context_manager.shared_resources[output_eoshared_instances[i].virtual_path] = output_eoshared_instances[i] 208 | 209 | return output_eoshared_instances 210 | 211 | 212 | def execute_filter_n_images_to_n_images(image_filter: Callable, 213 | filter_parameters: dict, 214 | inputs: list, 215 | tile: eotools.MpTile, 216 | context_manager: eom.EOContextManager) -> tuple: 217 | 218 | """ 219 | This method execute the filter on the inputs and then extract the stable 220 | area from the resulting outputs before returning them. 221 | """ 222 | 223 | # Get references to input numpy array buffers 224 | input_buffers = [] 225 | input_profiles = [] 226 | for i in range(len(inputs)): 227 | input_buffers.append(context_manager.get_array(key=inputs[i], tile=tile)) 228 | input_profiles.append(context_manager.get_profile(key=inputs[i])) 229 | 230 | output_buffers = image_filter(input_buffers, input_profiles, filter_parameters) 231 | 232 | if not isinstance(output_buffers, list): 233 | if not isinstance(output_buffers, numpy.ndarray): 234 | raise ValueError("Output of the image filter must be either a Python list or a numpy array") 235 | else: 236 | output_buffers = [output_buffers] 237 | 238 | # Reshape some output buffers if necessary since even for one channel image eoscale 239 | # needs a shape like this (channel, height, width) and it is really shitty to ask 240 | # the developer to take care of this... 241 | for o in range(len(output_buffers)): 242 | if len(output_buffers[o].shape) == 2: 243 | output_buffers[o] = output_buffers[o].reshape((1, output_buffers[o].shape[0], output_buffers[o].shape[1])) 244 | # We need to check now that input image dimensions are the same of outputs 245 | if output_buffers[o].shape[1] != input_buffers[0].shape[1] or output_buffers[o].shape[2] != input_buffers[0].shape[2]: 246 | raise ValueError("ERROR: Output images must have the same height and width of input images for this filter !") 247 | 248 | stable_start_x: int = None 249 | stable_start_y: int = None 250 | stable_end_x: int = None 251 | stable_end_y: int = None 252 | 253 | for i in range(len(output_buffers)): 254 | stable_start_x = tile.left_margin 255 | stable_start_y = tile.top_margin 256 | stable_end_x = stable_start_x + tile.end_x - tile.start_x + 1 257 | stable_end_y = stable_start_y + tile.end_y - tile.start_y + 1 258 | output_buffers[i] = output_buffers[i][:, stable_start_y:stable_end_y, stable_start_x:stable_end_x] 259 | 260 | return output_buffers, tile 261 | 262 | 263 | def default_reduce(outputs: list, 264 | chunk_output_buffers: list, 265 | tile: eotools.MpTile) -> None: 266 | """ Fill the outputs buffer with the results provided by the map filter from a strip """ 267 | for c in range(len(chunk_output_buffers)): 268 | outputs[c][:, tile.start_y: tile.end_y + 1, tile.start_x : tile.end_x + 1] = chunk_output_buffers[c][:, :, :] 269 | 270 | 271 | def n_images_to_m_images_filter(inputs: list = None, 272 | image_filter: Callable = None, 273 | filter_parameters: dict = None, 274 | generate_output_profiles: Callable = None, 275 | output_map = None, 276 | concatenate_filter: Callable = None, 277 | stable_margin: int = 0, 278 | context_manager: eom.EOContextManager = None, 279 | filter_desc: str = "N Images to M images MultiProcessing...", 280 | tile_mode: bool = None, 281 | specific_tile_size : int = None, 282 | strip_along_lines: bool = None) -> list: 283 | """ 284 | Generic paradigm to process n images providing m resulting images using a paradigm 285 | similar to the good old map/reduce 286 | 287 | image_filter is processed in parallel 288 | 289 | generate_output_profiles is a callable taking as input a list of rasterio profiles and the dictionnary 290 | filter_parameters and returning a list of output profiles. 291 | This callable is used by EOScale to allocate new shared images given their profile. It determines the value m 292 | of the n_image_to_m_image executor. 293 | 294 | concatenate_filter is processed by the master node to aggregate results 295 | 296 | If tile_mode is set to False, the image will be cropped as strips. 297 | specific_tile_size: hotfix to handle dezoom in filling dsm method. 298 | If strip_along_line is set to True, those strips will be vertical. 299 | 300 | Strong hypothesis: all input image are in the same geometry and have the same size 301 | """ 302 | 303 | if len(inputs) < 1: 304 | raise ValueError("At least one input image must be given.") 305 | 306 | if image_filter is None: 307 | raise ValueError("A filter must be set !") 308 | 309 | if context_manager is None: 310 | raise ValueError("The EOScale Context Manager must be given !") 311 | 312 | # Sometimes filter does not need parameters 313 | if filter_parameters is None: 314 | filter_parameters = dict() 315 | 316 | # compute the strips 317 | tiles = compute_mp_tiles(inputs=inputs, 318 | stable_margin=stable_margin, 319 | nb_workers=context_manager.nb_workers, 320 | tile_mode=tile_mode if tile_mode is not None else context_manager.tile_mode, 321 | context_manager=context_manager, 322 | specific_tile_size = specific_tile_size, 323 | strip_along_lines=strip_along_lines) 324 | 325 | # Call the generate output profile callable. Use the default one 326 | # if the developer did not assign one 327 | output_profiles: list = [] 328 | if generate_output_profiles is None: 329 | for key in inputs: 330 | output_profiles.append(context_manager.get_profile(key=key)) 331 | else: 332 | copied_input_mtds: list = [] 333 | for key in inputs: 334 | copied_input_mtds.append(context_manager.get_profile(key=key)) 335 | output_profiles = generate_output_profiles( copied_input_mtds, filter_parameters) 336 | if not isinstance(output_profiles, list): 337 | output_profiles = [output_profiles] 338 | 339 | # Allocate and share the outputs 340 | output_eoshareds = allocate_outputs(profiles=output_profiles, 341 | context_manager=context_manager) 342 | 343 | outputs = [ eoshared_inst.get_array() for eoshared_inst in output_eoshareds] 344 | 345 | # For debug, comment this section below in production 346 | # for tile in tiles: 347 | # print("process tile ", tile) 348 | # chunk_output_buffers, tile = execute_filter_n_images_to_n_images(image_filter, 349 | # filter_parameters, 350 | # inputs, 351 | # tile) 352 | # default_reduce(outputs, chunk_output_buffers, tile ) 353 | 354 | # # Multi processing execution 355 | with concurrent.futures.ThreadPoolExecutor(max_workers=min(context_manager.nb_workers, len(tiles))) as executor: 356 | 357 | futures = { executor.submit(execute_filter_n_images_to_n_images, 358 | image_filter, 359 | filter_parameters, 360 | inputs, 361 | tile, 362 | context_manager) for tile in tiles} 363 | 364 | for future in tqdm.tqdm(concurrent.futures.as_completed(futures), total=len(futures), desc=filter_desc): 365 | 366 | chunk_output_buffers, tile = future.result() 367 | if concatenate_filter is None: 368 | default_reduce(outputs, chunk_output_buffers, tile) 369 | else : 370 | concatenate_filter(outputs, chunk_output_buffers, tile) 371 | 372 | output_virtual_paths = [eoshared_inst.virtual_path for eoshared_inst in output_eoshareds] 373 | 374 | return output_virtual_paths 375 | 376 | 377 | def execute_filter_n_images_to_m_scalars(image_filter: Callable, 378 | filter_parameters: dict, 379 | inputs: list, 380 | tile: eotools.MpTile, 381 | context_manager: eom.EOContextManager) -> tuple: 382 | 383 | """ 384 | This method execute the filter on the inputs and then extract the stable 385 | area from the resulting outputs before returning them. 386 | """ 387 | # Get references to input numpy array buffers 388 | input_buffers = [] 389 | input_profiles = [] 390 | for i in range(len(inputs)): 391 | input_buffers.append(context_manager.get_array(key=inputs[i], tile = tile)) 392 | input_profiles.append(context_manager.get_profile(key=inputs[i])) 393 | 394 | output_scalars = image_filter(input_buffers, input_profiles, filter_parameters) 395 | 396 | if not isinstance(output_scalars, list): 397 | output_scalars = [output_scalars] 398 | 399 | return output_scalars, tile 400 | 401 | 402 | def n_images_to_m_scalars(inputs: list = None, 403 | image_filter: Callable = None, 404 | filter_parameters: dict = None, 405 | nb_output_scalars: int = None, 406 | output_scalars: list = None, 407 | concatenate_filter: Callable = None, 408 | context_manager: eom.EOContextManager = None, 409 | filter_desc: str = "N Images to M Scalars MultiProcessing...") -> list: 410 | """ 411 | Generic paradigm to process n images providing m resulting scalars using a paradigm 412 | similar to the good old map/reduce 413 | 414 | image_filter is processed in parallel 415 | 416 | concatenate_filter is processed by the master node to aggregate results 417 | 418 | Strong hypothesis: all input image are in the same geometry and have the same size 419 | """ 420 | 421 | if len(inputs) < 1: 422 | raise ValueError("At least one input image must be given.") 423 | 424 | if image_filter is None: 425 | raise ValueError("A filter must be set !") 426 | 427 | if concatenate_filter is None: 428 | raise ValueError("A concatenate filter must be set !") 429 | 430 | if nb_output_scalars is None: 431 | raise ValueError("The number of output scalars must be set (integer value expected) !") 432 | 433 | if context_manager is None: 434 | raise ValueError("The EOScale Context Manager must be given !") 435 | 436 | # Sometimes filter does not need parameters 437 | if filter_parameters is None: 438 | filter_parameters = dict() 439 | 440 | # compute the strips 441 | tiles = compute_mp_tiles(inputs=inputs, 442 | stable_margin=0, 443 | nb_workers=context_manager.nb_workers, 444 | tile_mode=context_manager.tile_mode, 445 | context_manager=context_manager) 446 | 447 | # Initialize the output scalars if the user doesn't provide it 448 | if output_scalars is None: 449 | output_scalars: list = [0.0 for i in range(nb_output_scalars) ] 450 | 451 | with concurrent.futures.ProcessPoolExecutor(max_workers= min(context_manager.nb_workers, len(tiles))) as executor: 452 | 453 | futures = {executor.submit(execute_filter_n_images_to_m_scalars, 454 | image_filter, 455 | filter_parameters, 456 | inputs, 457 | tile, 458 | context_manager, 459 | ) for tile in tiles} 460 | 461 | for future in tqdm.tqdm(concurrent.futures.as_completed(futures), total=len(futures), desc=filter_desc): 462 | 463 | chunk_output_scalars, tile = future.result() 464 | concatenate_filter(output_scalars, chunk_output_scalars, tile) 465 | 466 | return output_scalars 467 | -------------------------------------------------------------------------------- /bulldozer/eoscale/manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | import rasterio 22 | import uuid 23 | import numpy 24 | import copy 25 | 26 | import bulldozer.eoscale.utils as eoutils 27 | import bulldozer.eoscale.shared as eosh 28 | 29 | class EOContextManager: 30 | 31 | def __init__(self, 32 | nb_workers:int, 33 | tile_mode: bool = False): 34 | 35 | self.nb_workers = nb_workers 36 | self.tile_mode = tile_mode 37 | self.shared_resources: dict = dict() 38 | # Key is a unique memview key and value is a tuple (shared_resource_key, array subset, profile_subset) 39 | self.shared_mem_views: dict = dict() 40 | 41 | def __enter__(self): 42 | self.start() 43 | return self 44 | 45 | def __exit__(self, exc_type, exc_value, traceback): 46 | self.end() 47 | 48 | # Private methods 49 | 50 | def _release_all(self): 51 | 52 | self.shared_mem_views = dict() 53 | 54 | for key in self.shared_resources: 55 | self.shared_resources[key].release() 56 | 57 | self.shared_resources = dict() 58 | 59 | # Public methods 60 | 61 | def open_raster(self, 62 | raster_path: str) -> str: 63 | """ 64 | Create a new shared instance from file and return its virtual path 65 | """ 66 | 67 | new_shared_resource = eosh.EOShared() 68 | new_shared_resource.create_from_raster_path(raster_path=raster_path) 69 | self.shared_resources[new_shared_resource.virtual_path] = new_shared_resource 70 | return new_shared_resource.virtual_path 71 | 72 | def create_image(self, profile: dict) -> str: 73 | """ 74 | Given a profile with at least the following keys: 75 | count 76 | height 77 | width 78 | dtype 79 | this method allocates a shared image and its metadata 80 | """ 81 | eoshared_instance = eosh.EOShared() 82 | eoshared_instance.create_array(profile = profile) 83 | self.shared_resources[eoshared_instance.virtual_path] = eoshared_instance 84 | return eoshared_instance.virtual_path 85 | 86 | def create_memview(self, key: str, arr_subset: numpy.ndarray, arr_subset_profile: dict) -> str: 87 | """ 88 | This method allows the developper to indicate a subset memory view of a shared resource he wants to use as input 89 | of an executor. 90 | """ 91 | mem_view_key: str = str(uuid.uuid4()) 92 | self.shared_mem_views[mem_view_key] = (key, arr_subset, arr_subset_profile) 93 | return mem_view_key 94 | 95 | def get_array(self, key: str, tile: eoutils.MpTile = None) -> numpy.ndarray: 96 | """ 97 | This method returns a memory view from the key given by the user. 98 | This key can be a shared resource key or a memory view key 99 | """ 100 | if key in self.shared_mem_views: 101 | if tile is None: 102 | return self.shared_mem_views[key][1] 103 | else: 104 | start_y = tile.start_y - tile.top_margin 105 | end_y = tile.end_y + tile.bottom_margin + 1 106 | start_x = tile.start_x - tile.left_margin 107 | end_x = tile.end_x + tile.right_margin + 1 108 | return self.shared_mem_views[key][1][:, start_y:end_y, start_x:end_x] 109 | else: 110 | return self.shared_resources[key].get_array(tile = tile) 111 | 112 | def get_profile(self, key: str) -> dict: 113 | """ 114 | This method returns a profile from the key given by the user. 115 | This key can be a shared resource key or a memory view key 116 | """ 117 | if key in self.shared_mem_views: 118 | return copy.deepcopy(self.shared_mem_views[key][2]) 119 | else: 120 | return self.shared_resources[key].get_profile() 121 | 122 | def release(self, key: str): 123 | """ 124 | Release definitely the corresponding shared resource 125 | """ 126 | 127 | mem_view_keys_to_remove: list = [] 128 | # Remove from the mem view dictionnary all the key related to the share resource key 129 | for k in self.shared_mem_views: 130 | if self.shared_mem_views[k][0] == key: 131 | mem_view_keys_to_remove.append(k) 132 | for k in mem_view_keys_to_remove: 133 | del self.shared_mem_views[k] 134 | 135 | if key in self.shared_resources: 136 | self.shared_resources[key].release() 137 | del self.shared_resources[key] 138 | 139 | def write(self, key: str, img_path: str, binary: bool = False, profile: dict = None): 140 | """ 141 | Write the corresponding shared resource to disk 142 | """ 143 | if key in self.shared_resources: 144 | if profile: 145 | target_profile = profile 146 | else: 147 | target_profile = self.shared_resources[key].get_profile() 148 | target_profile['driver'] = 'GTiff' 149 | target_profile['interleave'] = 'band' 150 | img_buffer = self.shared_resources[key].get_array() 151 | if binary: 152 | with rasterio.open(img_path, "w", nbits=1,**target_profile) as out_dataset: 153 | out_dataset.write(img_buffer) 154 | else: 155 | with rasterio.open(img_path, "w", **target_profile) as out_dataset: 156 | out_dataset.write(img_buffer) 157 | else: 158 | print(f"WARNING: the key {key} to write is not known by the context manager") 159 | 160 | def update_profile(self, key: str, profile: dict) -> str: 161 | """ 162 | This method update the profile of a given key and returns the new key 163 | """ 164 | tmp_value = self.shared_resources[key] 165 | del self.shared_resources[key] 166 | tmp_value._release_profile() 167 | new_key: str = tmp_value._update_profile(profile) 168 | self.shared_resources[new_key] = tmp_value 169 | return new_key 170 | 171 | def start(self): 172 | if len(self.shared_resources) > 0: 173 | self._release_all() 174 | 175 | def end(self): 176 | self._release_all() -------------------------------------------------------------------------------- /bulldozer/eoscale/shared.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | from multiprocessing import shared_memory 22 | import rasterio 23 | import numpy 24 | import uuid 25 | import os 26 | import json 27 | import copy 28 | import bulldozer.eoscale.utils as eoutils 29 | 30 | EOSHARED_PREFIX: str = "eoshared" 31 | EOSHARED_MTD: str = "metadata" 32 | 33 | class EOShared: 34 | 35 | def __init__(self, virtual_path: str = None): 36 | """ """ 37 | self.shared_array_memory = None 38 | self.shared_metadata_memory = None 39 | self.virtual_path: str = None 40 | 41 | if virtual_path is not None: 42 | self._open_from_virtual_path(virtual_path = virtual_path) 43 | 44 | def _extract_from_vpath(self) -> tuple: 45 | """ 46 | Extract resource key and metada length from the virtual path 47 | """ 48 | split_v_path: list = self.virtual_path.split("/") 49 | resource_key: str = split_v_path[2] 50 | mtd_len: str = split_v_path[1] 51 | return resource_key, mtd_len 52 | 53 | def _open_from_virtual_path(self, virtual_path: str): 54 | """ """ 55 | 56 | self.virtual_path = virtual_path 57 | 58 | resource_key, mtd_len = self._extract_from_vpath() 59 | 60 | self.shared_array_memory = shared_memory.SharedMemory(name=resource_key, 61 | create=False) 62 | 63 | self.shared_metadata_memory = shared_memory.SharedMemory(name=resource_key + EOSHARED_MTD, 64 | create=False) 65 | 66 | def _build_virtual_path(self, key: str, mtd_len: str) -> None: 67 | """ """ 68 | self.virtual_path = EOSHARED_PREFIX + "/" + mtd_len + "/" + key 69 | 70 | def _create_shared_metadata(self, profile: dict, key: str): 71 | """ """ 72 | # Encode and compute the number of bytes of the metadata 73 | encoded_metadata = json.dumps(eoutils.rasterio_profile_to_dict(profile)).encode() 74 | mtd_size: int = len(encoded_metadata) 75 | self.shared_metadata_memory = shared_memory.SharedMemory(create=True, 76 | size=mtd_size, 77 | name=key + EOSHARED_MTD) 78 | self.shared_metadata_memory.buf[:] = encoded_metadata[:] 79 | 80 | # Create the virtual path to these shared resources 81 | self._build_virtual_path(mtd_len=str(mtd_size), key = key) 82 | 83 | def create_array(self, profile: dict): 84 | """ 85 | Allocate array 86 | """ 87 | # Shared key is made unique 88 | # this property is awesome since it allows the communication between parallel tasks 89 | resource_key: str = str(uuid.uuid4()) 90 | 91 | # Compute the number of bytes of this array 92 | d_size = numpy.dtype(profile["dtype"]).itemsize * profile["count"] * profile["height"] * profile["width"] 93 | 94 | # Create a shared memory instance of it 95 | # shared memory must remain open to keep the memory view 96 | self.shared_array_memory = shared_memory.SharedMemory(create=True, 97 | size=d_size, 98 | name=resource_key) 99 | 100 | big_array = numpy.ndarray(shape=(profile["count"] * profile["height"] * profile["width"]), 101 | dtype= numpy.dtype(profile["dtype"]), 102 | buffer=self.shared_array_memory.buf) 103 | 104 | big_array[:] = 0 105 | 106 | self._create_shared_metadata(profile = profile, key = resource_key) 107 | 108 | def create_from_raster_path(self, 109 | raster_path: str) -> str : 110 | 111 | """ Create a shared memory numpy array from a raster image """ 112 | with rasterio.open(raster_path, "r") as raster_dataset: 113 | 114 | # Shared key is made unique 115 | # this property is awesome since it allows the communication between parallel tasks 116 | resource_key: str = str(uuid.uuid4()) 117 | 118 | # Compute the number of bytes of this array 119 | d_size = numpy.dtype(raster_dataset.dtypes[0]).itemsize * raster_dataset.count * raster_dataset.height * raster_dataset.width 120 | 121 | # Create a shared memory instance of it 122 | # shared memory must remain open to keep the memory view 123 | self.shared_array_memory = shared_memory.SharedMemory(create=True, 124 | size=d_size, 125 | name=resource_key) 126 | 127 | 128 | big_array = numpy.ndarray(shape=(raster_dataset.count * raster_dataset.height * raster_dataset.width), 129 | dtype=raster_dataset.dtypes[0], 130 | buffer=self.shared_array_memory.buf) 131 | 132 | big_array[:] = raster_dataset.read().flatten()[:] 133 | 134 | self._create_shared_metadata(profile = raster_dataset.profile, key = resource_key) 135 | 136 | def get_profile(self) -> rasterio.DatasetReader.profile: 137 | """ 138 | Return a copy of the rasterio profile 139 | """ 140 | resource_key, mtd_len = self._extract_from_vpath() 141 | encoded_mtd = bytearray(int(mtd_len)) 142 | encoded_mtd[:] = self.shared_metadata_memory.buf[:] 143 | return copy.deepcopy(eoutils.dict_to_rasterio_profile(json.loads(encoded_mtd.decode()))) 144 | 145 | def get_array(self, 146 | tile: eoutils.MpTile = None) -> numpy.ndarray: 147 | 148 | """ 149 | Return a memory view of the array or a subset of it if a tile is given 150 | This has be done to be respect the dimension condition of the n_images_to_m_images filter. 151 | """ 152 | profile = self.get_profile() 153 | array_shape = (profile['count'], profile['height'], profile['width']) 154 | 155 | arr = numpy.ndarray(array_shape, 156 | dtype=profile['dtype'], 157 | buffer=self.shared_array_memory.buf) 158 | 159 | if tile is None: 160 | return arr 161 | else: 162 | start_y = tile.start_y - tile.top_margin 163 | end_y = tile.end_y + tile.bottom_margin + 1 164 | start_x = tile.start_x - tile.left_margin 165 | end_x = tile.end_x + tile.right_margin + 1 166 | return arr[:, start_y:end_y, start_x:end_x] 167 | 168 | def _update_profile(self, profile: dict) -> None: 169 | """ 170 | Update a shared metadata memory of an existing eoshared resource, 171 | normally it might not be called directly by the user because it is a dangerous operation... 172 | """ 173 | 174 | resource_key, mtd_len = self._extract_from_vpath() 175 | 176 | self._create_shared_metadata(profile = profile, key = resource_key) 177 | 178 | return self.virtual_path 179 | 180 | 181 | def _release_profile(self): 182 | """ 183 | Method used by EOScale only, normally it might not be called directly by the user 184 | """ 185 | if self.shared_metadata_memory is not None: 186 | self.shared_metadata_memory.close() 187 | self.shared_metadata_memory.unlink() 188 | self.shared_metadata_memory = None 189 | 190 | def close(self): 191 | """ 192 | A close does not mean release from memory. Must be called by a process once it has finished 193 | with this resource. 194 | """ 195 | if self.shared_array_memory is not None: 196 | self.shared_array_memory.close() 197 | 198 | if self.shared_metadata_memory is not None: 199 | self.shared_metadata_memory.close() 200 | 201 | def release(self): 202 | 203 | """ 204 | Definitely release the shared memory. 205 | """ 206 | if self.shared_array_memory is not None: 207 | self.shared_array_memory.close() 208 | self.shared_array_memory.unlink() 209 | self.shared_array_memory = None 210 | 211 | self._release_profile() 212 | 213 | -------------------------------------------------------------------------------- /bulldozer/eoscale/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | import rasterio 22 | import numpy 23 | from collections import namedtuple 24 | 25 | MpTile = namedtuple('MpTile', ["start_x", "start_y", "end_x", "end_y", "top_margin", "right_margin", "left_margin", "bottom_margin"]) 26 | 27 | JSON_NONE: str = "none" 28 | 29 | 30 | def rasterio_profile_to_dict(profile: rasterio.DatasetReader.profile) -> dict: 31 | """ 32 | Convert a rasterio profile to a serializable python dictionnary 33 | needed for storing in a chunk of memory that will be shared among 34 | processes 35 | """ 36 | metadata = dict() 37 | for key, value in profile.items(): 38 | if key == "crs": 39 | metadata['crs'] = profile['crs'].to_wkt() 40 | elif key == "transform": 41 | metadata['transform_1'] = profile['transform'][0] 42 | metadata['transform_2'] = profile['transform'][1] 43 | metadata['transform_3'] = profile['transform'][2] 44 | metadata['transform_4'] = profile['transform'][3] 45 | metadata['transform_5'] = profile['transform'][4] 46 | metadata['transform_6'] = profile['transform'][5] 47 | elif key == "nodata": 48 | if value is None: 49 | metadata[key] = JSON_NONE 50 | else: 51 | metadata[key] = value 52 | elif key == "dtype": 53 | if not isinstance(value, str): 54 | metadata[key] = numpy.dtype(value).name 55 | else: 56 | metadata[key] = value 57 | else: 58 | metadata[key] = value 59 | return metadata 60 | 61 | 62 | def dict_to_rasterio_profile(metadata: dict) -> rasterio.DatasetReader.profile : 63 | """ 64 | Convert a serializable dictionnary to a rasterio profile 65 | """ 66 | rasterio_profile = {} 67 | for key, value in metadata.items(): 68 | if key == "crs": 69 | rasterio_profile["crs"] = rasterio.crs.CRS.from_string(metadata['crs']) 70 | elif key == "transform_1": 71 | rasterio_profile['transform'] = rasterio.Affine(metadata['transform_1'], 72 | metadata['transform_2'], 73 | metadata['transform_3'], 74 | metadata['transform_4'], 75 | metadata['transform_5'], 76 | metadata['transform_6']) 77 | elif key.startswith("transform"): 78 | continue 79 | elif key == "nodata": 80 | if value == JSON_NONE: 81 | rasterio_profile[key] = None 82 | else: 83 | rasterio_profile[key] = value 84 | else: 85 | rasterio_profile[key] = value 86 | 87 | return rasterio_profile -------------------------------------------------------------------------------- /bulldozer/extraction/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/bulldozer/f55009c61542df42d5184db0599710ac62741c0a/bulldozer/extraction/__init__.py -------------------------------------------------------------------------------- /bulldozer/extraction/cython/c_springforce.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). 3 | This file is part of Bulldozer library 4 | All rights reserved. 5 | 6 | author: Pierre Lassalle (DNO/OT/IS) 7 | contact: pierre.lassalle@cnes.fr 8 | */ 9 | #include "c_springforce.h" 10 | 11 | namespace bulldozer { 12 | 13 | BulldozerFilters::BulldozerFilters() 14 | { 15 | } 16 | 17 | void BulldozerFilters::applyUniformFilter(float * input_data, 18 | float * output_data, 19 | unsigned int rows, 20 | unsigned int cols, 21 | float nodata, 22 | unsigned int filter_size) 23 | { 24 | // output_data is a buffer already allocated. 25 | 26 | // input_data is the input raster from which the uniform filter is applied 27 | // values are flattened, ie row = value / cols and col = value % cols 28 | for(unsigned int c = 0; c < rows * cols; c++) 29 | { 30 | if(input_data[c] != nodata) 31 | { 32 | output_data[c] = getUniformValue(c, input_data, rows, cols, filter_size, nodata); 33 | } 34 | else 35 | { 36 | output_data[c] = nodata; 37 | } 38 | } 39 | } 40 | 41 | inline float BulldozerFilters::getUniformValue(const unsigned int central_coord, 42 | float * input_data, 43 | const unsigned int rows, 44 | const unsigned int cols, 45 | const unsigned int filter_size, 46 | const float nodata) 47 | { 48 | unsigned int row = central_coord / cols; 49 | unsigned int col = central_coord % cols; 50 | unsigned int min_row = (row >= filter_size) ? row - filter_size : 0; 51 | unsigned int min_col = (col >= filter_size) ? col - filter_size : 0; 52 | unsigned int max_row = ( row + filter_size < rows ) ? row + filter_size : rows - 1; 53 | unsigned int max_col = (col + filter_size < cols ) ? col + filter_size : cols - 1; 54 | 55 | float num_valid = 0.f; 56 | 57 | float mean = 0.f; 58 | 59 | for(unsigned int r = min_row; r <= max_row; r++) 60 | { 61 | for(unsigned int c = min_col; c <= max_col; c++) 62 | { 63 | unsigned int coord = r * cols + c; 64 | 65 | if(input_data[coord] != nodata) 66 | { 67 | num_valid++; 68 | mean += input_data[coord]; 69 | } 70 | } 71 | } 72 | // num_valid must always be greater than 0 since we check before calling this method if 73 | // the center value is valid. 74 | return mean / num_valid; 75 | } 76 | 77 | 78 | } // end of namespace bulldozer 79 | -------------------------------------------------------------------------------- /bulldozer/extraction/cython/c_springforce.h: -------------------------------------------------------------------------------- 1 | #ifndef C_SPRING_FORCE_H 2 | #define C_SPRING_FORCE_H 3 | 4 | /* 5 | Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). 6 | This file is part of Bulldozer library 7 | All rights reserved. 8 | 9 | author: Pierre Lassalle (DNO/OT/IS) 10 | contact: pierre.lassalle@cnes.fr 11 | */ 12 | #include 13 | 14 | /* 15 | This class contains a list of useful filters for applying spring tension 16 | in the drap cloth algorithm while taking into account the no data value 17 | */ 18 | namespace bulldozer 19 | { 20 | 21 | class BulldozerFilters 22 | { 23 | public: 24 | 25 | BulldozerFilters(); 26 | 27 | void applyUniformFilter(float * input_data, 28 | float * output_data, 29 | unsigned int rows, 30 | unsigned int cols, 31 | float nodata, 32 | unsigned int filter_size); 33 | 34 | private: 35 | 36 | float getUniformValue(const unsigned int central_coord, 37 | float * input_data, 38 | const unsigned int rows, 39 | const unsigned int cols, 40 | const unsigned int filter_size, 41 | const float nodata); 42 | 43 | 44 | }; 45 | 46 | } // end of namespace bulldozer 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /bulldozer/extraction/cython/springforce.pyx: -------------------------------------------------------------------------------- 1 | # distutils: language = c++ 2 | import numpy as np 3 | from bulldozer.utils.helper import npAsContiguousArray 4 | 5 | # pxd section 6 | cdef extern from "c_springforce.cpp": 7 | pass 8 | 9 | # Declare the class with cdef 10 | cdef extern from "c_springforce.h" namespace "bulldozer": 11 | 12 | cdef cppclass BulldozerFilters: 13 | 14 | BulldozerFilters() except + 15 | 16 | void applyUniformFilter(float *, float *, unsigned int, unsigned int, float, unsigned int) 17 | # end pxd section 18 | 19 | cdef class PyBulldozerFilters: 20 | 21 | cdef BulldozerFilters c_bf # Hold a C++ instance wich we're wrapping 22 | 23 | def __cinit__(self): 24 | self.c_bf = BulldozerFilters() 25 | 26 | def run(self, np_dtm, filter_size, nodata): 27 | 28 | filter_size = filter_size // 2 29 | nb_rows: int = np_dtm.shape[0] 30 | nb_cols: int = np_dtm.shape[1] 31 | cdef float[::1] dtm_memview = npAsContiguousArray(np_dtm.flatten().astype(np.float32)) 32 | cdef float[::1] filtered_dtm_memview = npAsContiguousArray(np.zeros((nb_rows * nb_cols), dtype=np.float32)) 33 | self.c_bf.applyUniformFilter(&dtm_memview[0], &filtered_dtm_memview[0], nb_rows, nb_cols, nodata, filter_size) 34 | return np.asarray(filtered_dtm_memview).reshape(np_dtm.shape) 35 | -------------------------------------------------------------------------------- /bulldozer/extraction/drape_cloth.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import copy 3 | import numpy as np 4 | from tqdm import tqdm 5 | from rasterio import Affine 6 | 7 | import bulldozer.eoscale.manager as eom 8 | import bulldozer.eoscale.eo_executors as eoexe 9 | 10 | 11 | def next_power_of_2(x: int) -> int: 12 | """ 13 | This function returns the smallest power of 2 that is greater than or equal to a given non-negative integer x. 14 | 15 | Args: 16 | x : non negative integer. 17 | 18 | Returns: 19 | the corresponding power index power (2**index >= x). 20 | """ 21 | return 0 if x == 0 else (1 << (x-1).bit_length()).bit_length() - 1 22 | 23 | 24 | def get_max_pyramid_level(max_object_size_pixels: float) -> int : 25 | """ 26 | Given the max size of an object on the ground, 27 | this method computes the max level of the pyramid 28 | for drape cloth algorithm 29 | """ 30 | power = next_power_of_2(int(max_object_size_pixels)) 31 | 32 | # Take the closest power to the max object size 33 | if abs(2**(power-1) - max_object_size_pixels) < abs(2**power - max_object_size_pixels): 34 | power -= 1 35 | 36 | if power < 0 : 37 | power = 0 38 | return power 39 | 40 | 41 | def allocate_dezoom_dtm(level: int, 42 | dezoom_shape: tuple, 43 | eomanager: eom.EOContextManager) -> str: 44 | 45 | dezoomed_dtm_profile = { 46 | "count": 1, 47 | "height": dezoom_shape[1], 48 | "width": dezoom_shape[2], 49 | "dtype": np.float32 50 | } 51 | dtm_key = eomanager.create_image(profile=dezoomed_dtm_profile) 52 | return dtm_key 53 | 54 | 55 | def apply_first_tension(dtm_key: str, 56 | filled_dsm_key: str, 57 | ground_mask_key: str, 58 | eomanager: eom.EOContextManager, 59 | nb_levels: int, 60 | prevent_unhook_iter: int) -> None: 61 | 62 | dsm = eomanager.get_array(key=filled_dsm_key)[0, ::2**(nb_levels-1), ::2**(nb_levels-1)] 63 | predicted_anchors = eomanager.get_array(key=ground_mask_key)[0, ::2**(nb_levels-1), ::2**(nb_levels-1)] 64 | dtm = eomanager.get_array(key=dtm_key)[0, :, :] 65 | dtm[:, :] = dsm[:, :] 66 | snap_mask = predicted_anchors > 0 67 | 68 | # Prevent unhook from hills 69 | for i in tqdm(range(prevent_unhook_iter), desc="Prevent unhook from hills..."): 70 | dtm = scipy.ndimage.uniform_filter(dtm, output=dtm, size=3) 71 | dtm[snap_mask] = dsm[snap_mask] 72 | 73 | 74 | def upsample(dtm_key: str, 75 | filled_dsm_key: str, 76 | level: int, 77 | dezoom_profile: dict, 78 | eomanager: eom.EOContextManager) -> str: 79 | 80 | next_shape = eomanager.get_array(key=filled_dsm_key).shape 81 | next_dtm_key = allocate_dezoom_dtm(level=level, dezoom_shape=next_shape, eomanager=eomanager) 82 | prev_dtm = eomanager.get_array(key=dtm_key)[0, :, :] 83 | next_dtm = eomanager.get_array(key=next_dtm_key)[0, :, :] 84 | 85 | # Adjust the slicing for odd row count 86 | if next_dtm.shape[0] % 2 == 1: 87 | s0 = np.s_[:-1] 88 | else: 89 | s0 = np.s_[:] 90 | 91 | # Adjust the slicing for odd column count 92 | if next_dtm.shape[1] % 2 == 1: 93 | s1 = np.s_[:-1] 94 | else: 95 | s1 = np.s_[:] 96 | 97 | # Only fill upsampled value since we are working on the same shared memory 98 | next_dtm[::2, ::2] = prev_dtm[:, :] 99 | next_dtm[1::2, ::2] = prev_dtm[s0, :] 100 | next_dtm[::2, 1::2] = prev_dtm[:, s1] 101 | next_dtm[1::2, 1::2] = prev_dtm[s0, s1] 102 | 103 | # Can release the prev dtm buffer 104 | eomanager.release(key=dtm_key) 105 | 106 | return next_dtm_key 107 | 108 | 109 | def drape_cloth_filter(input_buffers: list, 110 | input_profiles: list, 111 | params: dict): 112 | """ """ 113 | num_outer_iterations: int = params['num_outer_iterations'] 114 | num_inner_iterations: int = params['num_inner_iterations'] 115 | step: float = params['step'] 116 | dtm = copy.deepcopy(input_buffers[0][0, :, :]) 117 | dsm = input_buffers[1][0, :, :] 118 | predicted_anchors = input_buffers[2][0, :, :] 119 | snap_mask = predicted_anchors > 0 120 | 121 | for i in range(num_outer_iterations): 122 | 123 | dtm += step 124 | 125 | for j in range(num_inner_iterations): 126 | 127 | # Snap dtm to anchors point 128 | dtm[snap_mask] = dsm[snap_mask] 129 | 130 | # handle DSM intersections 131 | np.minimum(dtm, dsm, out=dtm) 132 | 133 | # apply spring tension forces (blur the DTM) 134 | dtm = scipy.ndimage.uniform_filter(dtm, size=3) 135 | 136 | # One final intersection check 137 | dtm[snap_mask] = dsm[snap_mask] 138 | np.minimum(dtm, dsm, out=dtm) 139 | 140 | return dtm 141 | 142 | 143 | def drape_cloth_filter_gradient(input_buffers: list, 144 | input_profiles: list, 145 | params: dict): 146 | """ """ 147 | num_outer_iterations: int = params['num_outer_iterations'] 148 | num_inner_iterations: int = params['num_inner_iterations'] 149 | step_scale: float = params['step_scale'] 150 | nodata: int = params['nodata'] 151 | dtm = copy.deepcopy(input_buffers[0][0, :, :]) 152 | 153 | dsm = input_buffers[1][0, :, :] 154 | predicted_anchors = input_buffers[2][0, :, :] 155 | snap_mask = predicted_anchors > 0 156 | 157 | grad = np.abs(np.gradient(dtm)) 158 | 159 | step = np.maximum(grad[0, :, :], grad[1, :, :]) * step_scale 160 | step = scipy.ndimage.maximum_filter(step, 5) 161 | #bfilters = sf.PyBulldozerFilters() 162 | for i in range(num_outer_iterations): 163 | 164 | 165 | dtm += step 166 | 167 | for j in range(num_inner_iterations): 168 | # Snap dtm to anchors point 169 | dtm[snap_mask] = dsm[snap_mask] 170 | 171 | # handle DSM intersections 172 | np.minimum(dtm, dsm, out=dtm) 173 | 174 | # apply spring tension forces (blur the DTM) 175 | #dtm[input_buffers[0][0, :, :] == nodata] = np.nan 176 | dtm = scipy.ndimage.uniform_filter(dtm, size=3) 177 | #dtm = bfilters.run(dtm, 3, nodata) 178 | 179 | # One final intersection check 180 | dtm[snap_mask] = dsm[snap_mask] 181 | np.minimum(dtm, dsm, out=dtm) 182 | return dtm 183 | 184 | 185 | def drape_cloth_profiles(input_profiles: list, 186 | params: dict) -> dict: 187 | return input_profiles[0] 188 | 189 | 190 | def downsample_profile(profile, factor : float) : 191 | 192 | transform = profile['transform'] 193 | 194 | newprofile = profile.copy() 195 | dst_transform = Affine.translation(transform[2], transform[5]) * Affine.scale(transform[0]*factor, transform[4]*factor) 196 | 197 | newprofile.update({ 198 | 'transform': dst_transform, 199 | }) 200 | 201 | return newprofile 202 | 203 | 204 | def drape_cloth(filled_dsm_key: str, 205 | ground_mask_key: str, 206 | eomanager: eom.EOContextManager, 207 | max_object_size: float, 208 | prevent_unhook_iter: int, 209 | num_outer_iterations: int, 210 | num_inner_iterations: int, 211 | nodata: float) -> str: 212 | """ """ 213 | 214 | dsm_profile: dict = eomanager.get_profile(key=filled_dsm_key) 215 | dsm_resolution: float = dsm_profile["transform"][0] 216 | 217 | # Determine max object size in pixels 218 | max_object_size_pixels = max_object_size / dsm_resolution 219 | 220 | # Determine the dezoom factor wrt to max size of an object 221 | # on the ground. 222 | nb_levels = get_max_pyramid_level(max_object_size_pixels/2) + 1 223 | 224 | # Allocate memory for the max dezoomed dtm 225 | init_dtm_shape = eomanager.get_array(key=filled_dsm_key)[:, ::2**(nb_levels - 1), ::2**(nb_levels - 1)].shape 226 | dtm_key = allocate_dezoom_dtm(level=nb_levels - 1, 227 | dezoom_shape=init_dtm_shape, 228 | eomanager=eomanager) 229 | 230 | apply_first_tension(dtm_key=dtm_key, 231 | filled_dsm_key=filled_dsm_key, 232 | ground_mask_key=ground_mask_key, 233 | eomanager=eomanager, 234 | nb_levels=nb_levels, 235 | prevent_unhook_iter=prevent_unhook_iter) 236 | 237 | # Init classical parameters of drape cloth 238 | level = nb_levels - 1 239 | current_num_outer_iterations = num_outer_iterations 240 | 241 | while level >= 0: 242 | 243 | print(f"Process level {level} ...") 244 | 245 | # Create the memviews of the filled dsm map of this level 246 | 247 | current_dezoom_profile: dict = downsample_profile(profile=eomanager.get_profile(key=filled_dsm_key), 248 | factor=2**level) 249 | 250 | filled_dsm_memview = eomanager.create_memview( 251 | key=filled_dsm_key, 252 | arr_subset=eomanager.get_array(key=filled_dsm_key)[:, ::2**level, ::2**level], 253 | arr_subset_profile=current_dezoom_profile) 254 | 255 | ground_mask_key_memview = \ 256 | eomanager.create_memview(key=ground_mask_key, 257 | arr_subset=eomanager.get_array(key=ground_mask_key)[:, ::2**level, ::2**level], 258 | arr_subset_profile=current_dezoom_profile) 259 | 260 | if level < nb_levels - 1: 261 | dtm_key = upsample(dtm_key=dtm_key, 262 | filled_dsm_key=filled_dsm_memview, 263 | level=level, 264 | dezoom_profile=current_dezoom_profile, 265 | eomanager=eomanager) 266 | 267 | drape_cloth_parameters: dict = { 268 | 'num_outer_iterations': current_num_outer_iterations, 269 | 'num_inner_iterations': num_inner_iterations, 270 | 'nodata': nodata 271 | } 272 | 273 | drape_cloth_parameters['step_scale'] = 1. / (2 ** (nb_levels - level)) 274 | 275 | [new_dtm_key] = eoexe.n_images_to_m_images_filter( 276 | inputs=[dtm_key, filled_dsm_memview, ground_mask_key_memview], 277 | image_filter=drape_cloth_filter_gradient, 278 | generate_output_profiles=drape_cloth_profiles, 279 | filter_parameters=drape_cloth_parameters, 280 | stable_margin=int(current_num_outer_iterations * num_inner_iterations * (3 / 2)), # 3 correspond to filter size 281 | context_manager=eomanager, 282 | filter_desc="Drape cloth simulation...") 283 | 284 | eomanager.release(key=dtm_key) 285 | dtm_key = new_dtm_key 286 | 287 | level -= 1 288 | current_num_outer_iterations = max(1, int(num_outer_iterations / 2**(nb_levels - 1 - level))) 289 | 290 | # dtm_key contains the final dtm, we can save it to disk 291 | dtm_key = eomanager.update_profile(key=dtm_key, profile=eomanager.get_profile(key=filled_dsm_key)) 292 | 293 | return dtm_key -------------------------------------------------------------------------------- /bulldozer/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python 3 | # coding: utf8 4 | # 5 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 6 | # 7 | # This file is part of Bulldozer 8 | # (see https://github.com/CNES/bulldozer). 9 | # 10 | # Licensed under the Apache License, Version 2.0 (the "License"); 11 | # you may not use this file except in compliance with the License. 12 | # You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | # See the License for the specific language governing permissions and 20 | # limitations under the License. -------------------------------------------------------------------------------- /bulldozer/pipeline/bulldozer_parameters.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module contains the Bulldozer pipeline parameter structure and the dictionnary of default parameters. 23 | """ 24 | 25 | from bulldozer.utils.bulldozer_argparse import REQ_PARAM_KEY, OPT_PARAM_KEY, EXPERT_PARAM_KEY 26 | 27 | # This value is used if the provided nodata value is None or NaN 28 | DEFAULT_NODATA = -32768.0 29 | 30 | 31 | class BulldozerParam: 32 | """ 33 | Bulldozer pipeline parameter structure. 34 | """ 35 | 36 | def __init__(self, name: str, alias: str, label: str, description: str, 37 | param_type: type, default_value: object, value_label: str = None) -> None: 38 | """ 39 | BulldozerParam constructor. 40 | 41 | Args: 42 | name: parameter name (key in the yaml configuration file and str in the CLI). 43 | alias: parameter alias used for the CLI. 44 | label: parameter label suitable for display (without underscore, etc.). 45 | param_type: parameter type (used in QGIS plugin). 46 | description: complete parameter description displayed in helper. 47 | default_value: parameter default value. 48 | value_label: parameter value print in helper (e.g. ""). 49 | """ 50 | self.name = name 51 | self.alias = alias 52 | self.label = label 53 | self.description = description 54 | self.param_type = param_type 55 | self.default_value = default_value 56 | self.value_label = value_label 57 | 58 | def __str__(self) -> str: 59 | """ 60 | Human-friendly string description of BulldozerParam (method called by built-in print(), str(), and format() functions). 61 | 62 | Returns: 63 | corresponding string. 64 | """ 65 | return f"{self.name} {self.param_type} (default: {self.default_value})" 66 | 67 | def __repr__(self) -> str: 68 | """ 69 | Detailed string that can be used to recreate the BulldozerParam. 70 | 71 | Returns: 72 | corresponding string. 73 | """ 74 | return f"BulldozerParam(name=\"{self.name}\", alias=\"{self.alias}\", label=\"{self.label}\",description=\"{self.description}\", param_type={self.param_type}, default_value={self.default_value})" 75 | 76 | 77 | # This dict store all the Bulldozer parameters description and default values 78 | bulldozer_pipeline_params = { 79 | # Required parameters 80 | REQ_PARAM_KEY: [ 81 | BulldozerParam("dsm_path", "dsm", "Input DSM", "Input DSM path.", str, None, ""), 82 | BulldozerParam("output_dir", "out", "Output directory", "Output directory path.", str, None, "") 83 | ], 84 | # Options 85 | OPT_PARAM_KEY: [ 86 | BulldozerParam("generate_dhm", "dhm", "Generate DHM", "Generate the Digital Height Model (DHM=DSM-DTM).", bool, False), 87 | BulldozerParam("max_object_size", "max_size", "Max object size (m)", "Foreground max object size (in meter).", float, 16, ""), 88 | BulldozerParam("ground_mask_path", "ground", "Ground mask path", "Path to the binary ground classification mask.", str, None, ""), 89 | BulldozerParam("activate_ground_anchors", "anchors", "Activate ground anchors", "Activate ground anchor detection (ground pre-detection).", bool, False), 90 | BulldozerParam("nb_max_workers", "workers", "Number of workers", "Max number of CPU core to use.", int, None, ""), 91 | BulldozerParam("developer_mode", "dev", "Developper mode", "To keep the intermediate results.", bool, False) 92 | ], 93 | # Expert options: these parameters are considered as core settings and must be changed by users who are experts 94 | EXPERT_PARAM_KEY: [ 95 | BulldozerParam("reg_filtering_iter", "reg_it", "Number of regular mask filtering iterations", "Number of regular mask filtering iterations.", int, None, ""), 96 | BulldozerParam("dsm_z_accuracy", "dsm_z", "DSM altimetric accuracy (m)", "Altimetric height accuracy of the input DSM (m). If null, use the default value: 2*planimetric resolution.", float, None, ""), 97 | BulldozerParam("max_ground_slope", "max_slope", "Max ground slope (%%)", "Maximum slope of the observed landscape terrain (%%).", float, 20.0, ""), 98 | BulldozerParam("prevent_unhook_iter", "unhook_it", "Unhook iterations", "Number of unhook iterations.", int, 10, ""), 99 | BulldozerParam("num_outer_iter", "outer", "Number of outer iterations", "Number of gravity step iterations.", int, 25, ""), 100 | BulldozerParam("num_inner_iter", "inner", "Number of inner iterations", "Number of tension iterations.", int, 5, "") 101 | ] 102 | } 103 | -------------------------------------------------------------------------------- /bulldozer/postprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/bulldozer/f55009c61542df42d5184db0599710ac62741c0a/bulldozer/postprocessing/__init__.py -------------------------------------------------------------------------------- /bulldozer/postprocessing/fill_pits.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module is used to fill the remaining pits in the generated DTM. 23 | """ 24 | from typing import List 25 | import logging 26 | from copy import copy 27 | 28 | import numpy as np 29 | import scipy.ndimage as ndimage 30 | 31 | from bulldozer.utils.bulldozer_logger import BulldozerLogger 32 | from rasterio.fill import fillnodata 33 | import bulldozer.eoscale.manager as eom 34 | import bulldozer.eoscale.eo_executors as eoexe 35 | 36 | 37 | def fill_pits_filter(inputBuffers: list, 38 | input_profiles: list, 39 | params: dict) -> List[np.ndarray]: 40 | """ 41 | Perform pits removal and create pits detection mask. 42 | 43 | :param inputBuffers: DTM buffer 44 | :return: a List composed of the processed dtm without pits and the pits mask 45 | """ 46 | dtm = inputBuffers[0][0, :, :] 47 | pits_mask = np.zeros(dtm.shape, dtype=np.ubyte) 48 | 49 | border_mask = inputBuffers[1][0, :, :] 50 | 51 | dtm = fillnodata(dtm, mask=np.logical_not(border_mask), max_search_distance=params["search_distance"]) 52 | 53 | dtm_LF = ndimage.uniform_filter(dtm, size=params["filter_size"]) 54 | 55 | # Retrieves the high frequencies in the input DTM 56 | dtm_HF = dtm - dtm_LF 57 | 58 | # Tags the pits 59 | pits_mask[dtm_HF < 0.] = 1 60 | pits_mask[border_mask==1] = 0 61 | 62 | # fill pits 63 | dtm = np.where(pits_mask, dtm_LF, dtm) 64 | 65 | return [dtm, pits_mask] 66 | 67 | 68 | def fill_pits_profile(input_profiles: list, 69 | params: dict) -> dict: 70 | """ 71 | Defines filter outputs profiles 72 | """ 73 | msk_profile = copy(input_profiles[0]) 74 | msk_profile['dtype'] = np.uint8 75 | msk_profile['nodata'] = None 76 | return [input_profiles[0], msk_profile] 77 | 78 | #TODO - rename function + add @Runtime 79 | def run(dtm_key: str, 80 | border_nodata_key: str, 81 | eomanager: eom.EOContextManager): 82 | """ 83 | Performs the pit removal process using EOScale. 84 | 85 | :param dtm_key: the dtm to process key in the eo manager 86 | :param border_nodata_key: Border no data 87 | :return : The processed dtm and the pits mask keys 88 | """ 89 | resolution = eomanager.get_profile(dtm_key)['transform'][0] 90 | filter_size = 35.5 / resolution 91 | 92 | fill_pits_parameters: dict = { 93 | "filter_size": filter_size, 94 | "search_distance": 100 95 | } 96 | 97 | [filled_dtm_key, pits_mask_key] = \ 98 | eoexe.n_images_to_m_images_filter(inputs=[dtm_key, border_nodata_key], 99 | image_filter=fill_pits_filter, 100 | filter_parameters=fill_pits_parameters, 101 | generate_output_profiles=fill_pits_profile, 102 | context_manager=eomanager, 103 | stable_margin=int(filter_size/2), 104 | filter_desc="Pits removal processing...") 105 | 106 | eomanager.release(key=dtm_key) 107 | dtm_key = filled_dtm_key 108 | 109 | return dtm_key, pits_mask_key 110 | -------------------------------------------------------------------------------- /bulldozer/preprocessing/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. -------------------------------------------------------------------------------- /bulldozer/preprocessing/border_detection/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. -------------------------------------------------------------------------------- /bulldozer/preprocessing/border_detection/border_detector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module is used to detect border and inner nodata in the input DSM. 23 | """ 24 | import numpy as np 25 | # from scipy.spatial import ConvexHull 26 | from scipy.ndimage import zoom, binary_erosion, binary_fill_holes 27 | from skimage.morphology import convex_hull_image 28 | from skimage.draw import polygon 29 | 30 | import bulldozer.eoscale.manager as eom 31 | import bulldozer.eoscale.eo_executors as eoexe 32 | from bulldozer.utils.bulldozer_logger import Runtime 33 | import bulldozer.preprocessing.border as border 34 | 35 | 36 | def nodata_mask_profile(input_profile: list, 37 | params: dict) -> dict: 38 | """ 39 | This method is used in the main `detect_border_nodata` 40 | method to provide the output mask profile (binary profile). 41 | 42 | Args: 43 | input_profiles: input profile. 44 | params: extra parameters. 45 | 46 | Returns: 47 | updated profile. 48 | """ 49 | output_profile = input_profile[0] 50 | output_profile['dtype'] = np.ubyte 51 | output_profile['nodata'] = None 52 | return output_profile 53 | 54 | 55 | def border_nodata_filter(input_buffers: list, 56 | input_profiles: list, 57 | filter_parameters: dict) -> np.ndarray: 58 | """ 59 | This method is used in the main `detect_border_nodata` method. 60 | It calls the Cython method to extract border nodata along an axis (vertical or horizontal). 61 | 62 | Args: 63 | input_buffers: input DSM. 64 | input_profiles: DSM profile. 65 | filter_parameters: dictionary containing nodata value and the axis for the detection (True: vertical or False: horizontal). 66 | 67 | Returns: 68 | border nodata mask along specified axis. 69 | """ 70 | dsm = input_buffers[0] 71 | nodata = filter_parameters['nodata'] 72 | 73 | border_nodata = border.PyBorderNodata() 74 | 75 | if filter_parameters["doTranspose"]: 76 | # Vertical border nodata detection case 77 | border_nodata_mask = border_nodata.build_border_nodata_mask(dsm.T, nodata, True).astype(np.ubyte) 78 | return border_nodata_mask.T 79 | else: 80 | # Horizontal border nodata detection case 81 | return border_nodata.build_border_nodata_mask(dsm, nodata, False).astype(np.ubyte) 82 | 83 | 84 | 85 | 86 | def inner_nodata_filter(input_buffers: list, 87 | input_profiles: list, 88 | filter_parameters: dict) -> np.ndarray: 89 | """ 90 | This method is used in the main `detect_border_nodata` method. 91 | It calls the Cython method to extract inner nodata. 92 | 93 | Args: 94 | input_buffers: input DSM. 95 | input_profiles: DSM profile. 96 | filter_parameters: dictionary containing nodata value. 97 | 98 | Returns: 99 | inner nodata mask along specified axis. 100 | """ 101 | 102 | dsm = input_buffers[0] 103 | border_nodata_mask = input_buffers[1] 104 | nodata = filter_parameters['nodata'] 105 | 106 | inner_nodata_mask = np.logical_and(np.logical_not(border_nodata_mask), dsm == nodata) 107 | 108 | return inner_nodata_mask 109 | 110 | 111 | @Runtime 112 | def detect_border_nodata(dsm_key: str, 113 | nodata: float, 114 | eomanager: eom.EOContextManager) -> np.ndarray: 115 | """ 116 | This method returns the binary masks flagging the border and inner nodata. 117 | The border nodata correpond to the nodata points on the edges if the DSM is skewed and the inner nodata correspond to the other nodata points. 118 | 119 | Args: 120 | dsm_key: path to the input DSM. 121 | nodata: DSM nodata value (if nan, the nodata is set to default value: -32768.0). 122 | eomanager: eoscale context manager. 123 | 124 | Returns: 125 | border and inner nodata masks. 126 | """ 127 | # Horizontal border nodata detection 128 | border_nodata_parameters: dict = { 129 | 'nodata': nodata, 130 | 'doTranspose': False 131 | } 132 | [hor_border_nodata_mask_key] = eoexe.n_images_to_m_images_filter(inputs=[dsm_key], 133 | image_filter=border_nodata_filter, 134 | filter_parameters=border_nodata_parameters, 135 | generate_output_profiles=nodata_mask_profile, 136 | context_manager=eomanager, 137 | stable_margin=0, 138 | filter_desc="Horizontal nodata mask processing...", 139 | tile_mode=False) 140 | # Vertical border nodata detection 141 | border_nodata_parameters: dict = { 142 | 'nodata': nodata, 143 | 'doTranspose': True 144 | } 145 | [border_nodata_mask_key] = eoexe.n_images_to_m_images_filter(inputs=[dsm_key], 146 | image_filter=border_nodata_filter, 147 | filter_parameters=border_nodata_parameters, 148 | generate_output_profiles=nodata_mask_profile, 149 | context_manager=eomanager, 150 | stable_margin=0, 151 | filter_desc="Vertical nodata mask processing...", 152 | tile_mode=False, 153 | strip_along_lines=True) 154 | 155 | hor_mask = eomanager.get_array(key=hor_border_nodata_mask_key)[0] 156 | border_mask = eomanager.get_array(key=border_nodata_mask_key)[0] 157 | np.logical_and(hor_mask, border_mask, out=border_mask) 158 | 159 | eomanager.release(key=hor_border_nodata_mask_key) 160 | 161 | ### Filling the holes inside the border nodata mask 162 | border_mask = np.where(border_mask == 0, 1, 0).astype(np.uint8) 163 | binary_fill_holes(border_mask, output=border_mask) 164 | border_mask = np.where(border_mask == 0, 1, 0) 165 | new_border_mask = eomanager.get_array(key=border_nodata_mask_key)[0] 166 | new_border_mask[:] = border_mask 167 | 168 | # Inner nodata detection 169 | [inner_nodata_mask_key] = eoexe.n_images_to_m_images_filter(inputs=[dsm_key, border_nodata_mask_key], 170 | image_filter=inner_nodata_filter, 171 | filter_parameters=border_nodata_parameters, 172 | generate_output_profiles=nodata_mask_profile, 173 | context_manager=eomanager, 174 | stable_margin=0, 175 | filter_desc="Build Inner NoData Mask") 176 | 177 | 178 | return { 179 | "border_nodata_mask": border_nodata_mask_key, 180 | "inner_nodata_mask": inner_nodata_mask_key 181 | } 182 | -------------------------------------------------------------------------------- /bulldozer/preprocessing/border_detection/cython/border.pyx: -------------------------------------------------------------------------------- 1 | # distutils: language = c++ 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | import numpy as np 22 | from bulldozer.utils.helper import npAsContiguousArray 23 | 24 | # Begin PXD 25 | 26 | # Necessary to include the C++ code 27 | cdef extern from "c_border.cpp": 28 | pass 29 | 30 | # Declare the class with cdef 31 | cdef extern from "c_border.h" namespace "bulldozer": 32 | 33 | void buildBorderNodataMask(float *, 34 | unsigned char *, 35 | unsigned int, 36 | unsigned int, 37 | float) 38 | 39 | # End PXD 40 | 41 | cdef class PyBorderNodata: 42 | 43 | 44 | def __cinit__(self) -> None: 45 | """ 46 | Default constructor. 47 | """ 48 | pass 49 | 50 | 51 | def build_border_nodata_mask(self, 52 | dsm_strip : np.array, 53 | nodata_value : float, 54 | is_transposed: bool) -> np.array: 55 | """ 56 | This method detects the border nodata areas in the input DSM window. 57 | For the border nodata along vertical axis, transpose the input DSM window. 58 | 59 | Args: 60 | dsm_strip: part of the DSM analyzed. 61 | nodata_value: nodata value used in the input DSM. 62 | is_transposed: boolean flag indicating if the computation is vertical or horizontal. 63 | Returns: 64 | mask of the border nodata areas in the input DSM window. 65 | """ 66 | if is_transposed: 67 | first_index = 0 68 | second_index = 1 69 | else: 70 | first_index = 1 71 | second_index = 2 72 | 73 | cdef float[::1] dsm_memview = npAsContiguousArray(dsm_strip.ravel().astype(np.float32)) 74 | # Ouput mask that will be filled by the C++ part 75 | cdef unsigned char[::1] border_nodata_mask_memview = npAsContiguousArray(np.zeros((dsm_strip.shape[first_index] * dsm_strip.shape[second_index]), dtype=np.uint8)) 76 | # Border nodata detection 77 | buildBorderNodataMask(&dsm_memview[0], &border_nodata_mask_memview[0], dsm_strip.shape[first_index], dsm_strip.shape[second_index], nodata_value) 78 | # Reshape the output mask. From array to matrix corresponding to the input DSM strip shape 79 | return np.asarray(border_nodata_mask_memview).reshape(dsm_strip.shape[first_index], dsm_strip.shape[second_index]).astype(np.ubyte) -------------------------------------------------------------------------------- /bulldozer/preprocessing/border_detection/cython/c_border.cpp: -------------------------------------------------------------------------------- 1 | /*Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 2 | 3 | This file is part of Bulldozer 4 | (see https://github.com/CNES/bulldozer). 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License.*/ 17 | 18 | #include "c_border.h" 19 | 20 | namespace bulldozer { 21 | 22 | void buildBorderNodataMask(float * dsm, 23 | unsigned char * borderNodataMask, 24 | unsigned int nbRows, 25 | unsigned int nbCols, 26 | float nodataValue) { 27 | 28 | unsigned int col; 29 | for(unsigned int row = 0; row < nbRows; row++) { 30 | // extracts border nodata for the left side of the input DSM 31 | col = row * nbCols; 32 | while(col < ((row * nbCols)-1 + nbCols) && dsm[col] == nodataValue){ 33 | borderNodataMask[col] = true; 34 | col++; 35 | } 36 | // extracts border nodata for the right side of the input DSM 37 | col = row * nbCols + nbCols - 1; 38 | while(col > row * nbCols && dsm[col] == nodataValue){ 39 | borderNodataMask[col] = true; 40 | col--; 41 | } 42 | } 43 | } 44 | } -------------------------------------------------------------------------------- /bulldozer/preprocessing/border_detection/cython/c_border.h: -------------------------------------------------------------------------------- 1 | #ifndef BORDER_H 2 | #define BORDER_H 3 | 4 | /*Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | 6 | This file is part of Bulldozer 7 | (see https://github.com/CNES/bulldozer). 8 | 9 | Licensed under the Apache License, Version 2.0 (the "License"); 10 | you may not use this file except in compliance with the License. 11 | You may obtain a copy of the License at 12 | 13 | http://www.apache.org/licenses/LICENSE-2.0 14 | 15 | Unless required by applicable law or agreed to in writing, software 16 | distributed under the License is distributed on an "AS IS" BASIS, 17 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | See the License for the specific language governing permissions and 19 | limitations under the License.*/ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | /* 27 | This function is used to build a mask that flags the pixels associated to the border nodata of the input DSM. 28 | Those no data areas appears on the edges if the DSM is skewed or detoured (e.g. when part of the sea has been removed with a water mask). 29 | */ 30 | namespace bulldozer 31 | { 32 | void buildBorderNodataMask(float * dsm, 33 | unsigned char * borderNodataMask, 34 | unsigned int nbRows, 35 | unsigned int nbCols, 36 | float nodataValue); 37 | } // end of namespace bulldozer 38 | 39 | #endif -------------------------------------------------------------------------------- /bulldozer/preprocessing/dsm_filling/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. -------------------------------------------------------------------------------- /bulldozer/preprocessing/dsm_filling/cython/c_fill.cpp: -------------------------------------------------------------------------------- 1 | /*Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 2 | 3 | This file is part of Bulldozer 4 | (see https://github.com/CNES/bulldozer). 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License.*/ 17 | 18 | #include "c_fill.h" 19 | 20 | namespace bulldozer { 21 | 22 | void iterativeFilling(float * dsm, 23 | unsigned char * borderNodataMask, 24 | int dsmHeight, 25 | int dsmWidth, 26 | float nodataValue, 27 | int numIterations) { 28 | 29 | double diagWeight = 1 / sqrt(2); 30 | int corrected = 0, toCorrect = 0, nbPass = 0; 31 | bool hasNoData = true; 32 | int nbgoodNeighbor = 3; 33 | 34 | std::unique_ptr goods(new int[8]()); 35 | std::unique_ptr invMask(new unsigned char[dsmHeight * dsmWidth]); 36 | std::unique_ptr weights(new double[8]); 37 | 38 | for (int i = 0; i < dsmHeight; ++i) { 39 | for (int j = 0; j < dsmWidth; ++j) { 40 | int idx = i * dsmWidth + j; 41 | 42 | invMask[idx] = (dsm[idx] == nodataValue) ? 1 : 0; 43 | 44 | if (borderNodataMask && borderNodataMask[idx] == 1) { 45 | invMask[idx] = 0; 46 | } 47 | } 48 | } 49 | 50 | for (int k = 0; k < numIterations; ++k) { 51 | toCorrect = 0; 52 | corrected = 0; 53 | hasNoData = false; 54 | 55 | for (int i = 0; i < dsmHeight; ++i) { 56 | for (int j = 0; j < dsmWidth; ++j) { 57 | int idx = i * dsmWidth + j; 58 | 59 | if (invMask[idx] == 1) { 60 | hasNoData = true; 61 | ++toCorrect; 62 | 63 | // Neighborhood checks 64 | goods[0] = (i>0 && j>0) && (invMask[idx - dsmWidth - 1] == 0 && dsm[idx - dsmWidth - 1] != nodataValue); 65 | goods[1] = (i>0) && (invMask[idx - dsmWidth] == 0 && dsm[idx - dsmWidth] != nodataValue); 66 | goods[2] = (i>0 && j0) && (invMask[idx - 1] == 0 && dsm[idx - 1] != nodataValue); 68 | goods[4] = (j0 && i= nbgoodNeighbor) { 76 | ++corrected; 77 | 78 | weights[0] = goods[0] * diagWeight; 79 | weights[1] = goods[1]; 80 | weights[2] = goods[2] * diagWeight; 81 | weights[3] = goods[3]; 82 | weights[4] = goods[4]; 83 | weights[5] = goods[5] * diagWeight; 84 | weights[6] = goods[6]; 85 | weights[7] = goods[7] * diagWeight; 86 | 87 | double totalWeight = 0.0; 88 | double newValue = 0.0; 89 | 90 | if (goods[0]==1) newValue += weights[0] * dsm[idx - dsmWidth - 1]; 91 | if (goods[1]==1) newValue += weights[1] * dsm[idx - dsmWidth]; 92 | if (goods[2]==1) newValue += weights[2] * dsm[idx - dsmWidth + 1]; 93 | if (goods[3]==1) newValue += weights[3] * dsm[idx - 1]; 94 | if (goods[4]==1) newValue += weights[4] * dsm[idx + 1]; 95 | if (goods[5]==1) newValue += weights[5] * dsm[idx + dsmWidth - 1]; 96 | if (goods[6]==1) newValue += weights[6] * dsm[idx + dsmWidth]; 97 | if (goods[7]==1) newValue += weights[7] * dsm[idx + dsmWidth + 1]; 98 | 99 | totalWeight = weights[0] + weights[1] + weights[2] + weights[3] + weights[4] + weights[5] + weights[6] + weights[7]; 100 | dsm[idx] = newValue / totalWeight; 101 | 102 | invMask[idx] = 2; 103 | 104 | } 105 | } 106 | } 107 | } 108 | 109 | for (int i = 0; i < dsmHeight; ++i) { 110 | for (int j = 0; j < dsmWidth; ++j) { 111 | 112 | int idx = i * dsmWidth + j; 113 | 114 | if (invMask[idx] == 2) { 115 | invMask[idx] = 0; 116 | } 117 | } 118 | } 119 | 120 | if (!hasNoData || corrected == 0) { 121 | break; 122 | } 123 | 124 | ++nbPass; 125 | } 126 | 127 | } 128 | } // end of namespace bulldozer 129 | 130 | -------------------------------------------------------------------------------- /bulldozer/preprocessing/dsm_filling/cython/c_fill.h: -------------------------------------------------------------------------------- 1 | #ifndef C_FILL_H 2 | #define C_FILL_H 3 | 4 | /*Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | 6 | This file is part of Bulldozer 7 | (see https://github.com/CNES/bulldozer). 8 | 9 | Licensed under the Apache License, Version 2.0 (the "License"); 10 | you may not use this file except in compliance with the License. 11 | You may obtain a copy of the License at 12 | 13 | http://www.apache.org/licenses/LICENSE-2.0 14 | 15 | Unless required by applicable law or agreed to in writing, software 16 | distributed under the License is distributed on an "AS IS" BASIS, 17 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | See the License for the specific language governing permissions and 19 | limitations under the License.*/ 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | /* 26 | This function is used to fill the inner nodata of the input DSM. 27 | */ 28 | namespace bulldozer 29 | { 30 | void iterativeFilling(float * dsm, 31 | unsigned char * borderNodataMask, 32 | int dsmHeight, 33 | int dsmWidth, 34 | float nodataVal, 35 | int numIterations); 36 | 37 | } // end of namespace bulldozer 38 | 39 | #endif -------------------------------------------------------------------------------- /bulldozer/preprocessing/dsm_filling/cython/fill.pyx: -------------------------------------------------------------------------------- 1 | # distutils: language = c++ 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | import numpy as np 22 | from bulldozer.utils.helper import npAsContiguousArray 23 | 24 | # Begin PXD 25 | 26 | # Necessary to include the C++ code 27 | cdef extern from "c_fill.cpp": 28 | pass 29 | 30 | # Declare the class with cdef 31 | cdef extern from "c_fill.h" namespace "bulldozer": 32 | 33 | void iterativeFilling(float *, 34 | unsigned char *, 35 | int, 36 | int, 37 | float, 38 | int) 39 | 40 | 41 | # End PXD 42 | 43 | cdef class PyFill: 44 | 45 | def __cinit__(self) -> None: 46 | """ 47 | Default constructor. 48 | """ 49 | pass 50 | 51 | def iterative_filling(self, 52 | dsm_strip : np.array, 53 | nodata_value : float, 54 | nb_it : int, 55 | border_nodata_strip : np.array = None) -> np.array: 56 | """ 57 | This method fills the DSM 58 | 59 | Args: 60 | dsm_strip: part of the DSM analyzed. 61 | disturbance_strip: part of the disturbance mask analyzed. 62 | nodata_value: nodata value. 63 | Return: 64 | Filled DSM 65 | """ 66 | cdef float[::1] dsm_memview = npAsContiguousArray(dsm_strip.ravel().astype(np.float32)) 67 | cdef unsigned char[::1] border_nodata_mask_memview 68 | cdef unsigned char* border_nodata_mask_ptr = NULL # Initialize as NULL 69 | 70 | if border_nodata_strip is not None: # For the level 0 71 | border_nodata_mask_memview = npAsContiguousArray(border_nodata_strip.ravel().astype(np.uint8)) 72 | border_nodata_mask_ptr = &border_nodata_mask_memview[0] 73 | 74 | # Iterative Filling 75 | iterativeFilling(&dsm_memview[0], border_nodata_mask_ptr, dsm_strip.shape[0], dsm_strip.shape[1], nodata_value, nb_it) 76 | # Reshape the output DSM. From array to matrix corresponding to the input DSM strip shape 77 | return np.asarray(dsm_memview).reshape(dsm_strip.shape[0], dsm_strip.shape[1]).astype(np.float32) 78 | 79 | -------------------------------------------------------------------------------- /bulldozer/preprocessing/dsm_filling/dsm_filler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module is used to prefill the input DSM before the DTM extraction. 23 | """ 24 | import numpy as np 25 | import rasterio 26 | import logging 27 | import os 28 | import math 29 | 30 | from rasterio import Affine 31 | from scipy.ndimage import zoom, binary_erosion 32 | 33 | import bulldozer.eoscale.manager as eom 34 | import bulldozer.eoscale.eo_executors as eoexe 35 | import bulldozer.preprocessing.fill as fill 36 | from bulldozer.utils.bulldozer_logger import Runtime, BulldozerLogger 37 | 38 | 39 | def filled_dsm_profile(input_profiles: list, 40 | params: dict) -> dict: 41 | """ 42 | This method is used to provide the output filled dsm profile. 43 | 44 | Args: 45 | input_profiles: input profile. 46 | params: extra parameters. 47 | 48 | Returns: 49 | updated profile. 50 | """ 51 | return [input_profiles[0]] 52 | 53 | 54 | def fill_dsm_method(input_buffers: list, 55 | input_profiles: list, 56 | filter_parameters: dict) -> np.ndarray: 57 | """ 58 | This method is used in the main `fill_dsm_process`. 59 | It calls the Cython method to fill a DMS. 60 | 61 | Args: 62 | input_buffers: input DSM, input regular mask. 63 | input_profiles: DSM profile. 64 | filter_parameters: filter parameters. 65 | 66 | Returns: 67 | regular areas mask. 68 | """ 69 | fill_process = fill.PyFill() 70 | 71 | if filter_parameters["use_bordernodata_mask"]==True: 72 | # the input_buffers[0] corresponds to the input DSM raster, input_buffers[1] corresponds to the bordernodata mask 73 | dsm = fill_process.iterative_filling(dsm_strip=input_buffers[0][0, :, :], 74 | nodata_value=filter_parameters["nodata"], 75 | nb_it=filter_parameters["filling_iterations"], 76 | border_nodata_strip=input_buffers[1][0, :, :]) 77 | else: 78 | dsm = fill_process.iterative_filling(input_buffers[0][0, :, :], 79 | nodata_value=filter_parameters["nodata"], 80 | nb_it=filter_parameters["filling_iterations"]) 81 | 82 | return [dsm.astype(np.float32)] 83 | 84 | 85 | def downsample_profile(profile, factor : float) : 86 | 87 | transform = profile["transform"] 88 | 89 | newprofile = profile.copy() 90 | dst_transform = Affine.translation(transform[2], transform[5]) * Affine.scale(transform[0]*factor, transform[4]*factor) 91 | 92 | newprofile.update({ 93 | "transform": dst_transform, 94 | }) 95 | 96 | return newprofile 97 | 98 | 99 | @Runtime 100 | def fill_dsm(dsm_key: str, 101 | regular_key: str, 102 | border_nodata_key: str, 103 | nodata: float, 104 | max_object_size: int, 105 | eomanager: eom.EOContextManager, 106 | dev_mode: bool = False, 107 | dev_dir: str = "") -> dict: 108 | """ 109 | This fills the nodata of the input DSM for the following dtm extraction step. 110 | 111 | Args: 112 | dsm_key: input DSM. 113 | regular_key: regular mask. 114 | border_nodata_key: border nodata mask. 115 | nodata: DSM nodata value (if nan, the nodata is set to -32768). 116 | eomanager: eoscale context manager. 117 | dev_mode: if True, dev mode activated. 118 | dev_dir: path to save dev files. 119 | 120 | Returns: 121 | the filled DSM. 122 | """ 123 | 124 | if dev_mode: 125 | dev_dir = os.path.join(dev_dir, "filling_DSM") 126 | os.makedirs(dev_dir, exist_ok=True) 127 | 128 | filled_dsm = eomanager.get_array(key=dsm_key)[0] 129 | regular = eomanager.get_array(key=regular_key)[0] 130 | border_nodata = eomanager.get_array(key=border_nodata_key)[0] 131 | 132 | # We're also filling the irregular areas 133 | filled_dsm[regular==0] = nodata 134 | 135 | if dev_mode: 136 | profile = eomanager.get_profile(key=dsm_key).copy() 137 | profile['driver'] = 'GTiff' 138 | profile['interleave'] = 'band' 139 | profile['nodata'] = nodata 140 | filled_dsm_with_regular_path: str = os.path.join(dev_dir, "regular_dsm.tif") 141 | eomanager.write(key=dsm_key, img_path=filled_dsm_with_regular_path, profile=profile) 142 | 143 | dsm_resolution = eomanager.get_profile(key=dsm_key)["transform"][0] 144 | # Setting parameters for the DSM filling method 145 | regular_parameters: dict = { 146 | "nodata": nodata, 147 | # if max_object_size is less than 2+sqrt(2) overides the computed value to avoid information loss during dezoom 148 | "filling_iterations": int(np.max([int(2+np.sqrt(2)), np.floor((max_object_size / dsm_resolution) / 2)])), # Nb iterations = max_object_size (px) / 2 (allow to fill a hole between two points max_object_size apart) 149 | "use_bordernodata_mask": True 150 | } 151 | # if computed value for dezoom_factor is less than 2 overides the value with 2 to ensure a dezoom during the filling process 152 | dezoom_factor = int(np.max([2, np.floor(regular_parameters["filling_iterations"] * (2-np.sqrt(2)))])) # sqrt(2) to handle the diagonal neighbors 153 | nb_max_level = int(np.floor(math.log(np.min([filled_dsm.shape[0],filled_dsm.shape[1]]), dezoom_factor))) # limits the number of dezoom iterations 154 | BulldozerLogger.log("DSM filling parameters : filling_iterations={} / dezoom_factor={} / nb_max_level={}".format(regular_parameters["filling_iterations"], dezoom_factor,nb_max_level), logging.DEBUG) 155 | 156 | # Identifying the remaining inner no data areas 157 | remaining_nodata = (filled_dsm == nodata) & (border_nodata == 0) 158 | 159 | # if nodata areas are still in the DSM 160 | has_nodata = np.any(remaining_nodata) 161 | dezoom_level = 0 162 | downsample = True 163 | 164 | # Downsampling until there is nodata remaining or reaching max level 165 | while has_nodata and 0<=dezoom_level and dezoom_level<=nb_max_level: 166 | 167 | if dezoom_level==0: # When level is 0 168 | 169 | # First iterative filling for small no data areas 170 | regular_parameters["use_bordernodata_mask"] = True 171 | [dsm_key] = eoexe.n_images_to_m_images_filter(inputs=[dsm_key, border_nodata_key], 172 | image_filter=fill_dsm_method, 173 | filter_parameters=regular_parameters, 174 | generate_output_profiles=filled_dsm_profile, 175 | context_manager=eomanager, 176 | stable_margin=regular_parameters["filling_iterations"], 177 | filter_desc="Iterative filling DSM level 0") 178 | 179 | filled_dsm = eomanager.get_array(key=dsm_key)[0] 180 | border_nodata = eomanager.get_array(key=border_nodata_key)[0] 181 | 182 | # Identifying the remaining inner no data areas 183 | remaining_nodata = (filled_dsm == nodata) & (border_nodata == 0) 184 | 185 | if dev_mode: 186 | profile = eomanager.get_profile(key=dsm_key).copy() 187 | profile['driver'] = 'GTiff' 188 | profile['interleave'] = 'band' 189 | profile['nodata'] = nodata 190 | if downsample: 191 | filled_dsm_1stpass_path: str = os.path.join(dev_dir, "filled_dsm_downsample_level_0.tif") 192 | else: 193 | # Upsample case 194 | filled_dsm_1stpass_path: str = os.path.join(dev_dir, "filled_dsm_upsample_level_0.tif") 195 | eomanager.write(key=dsm_key, img_path=filled_dsm_1stpass_path, profile=profile) 196 | 197 | # if nodata areas are still in the DSM 198 | has_nodata = np.any(remaining_nodata) 199 | 200 | if downsample: 201 | dezoom_level+=1 202 | else: 203 | dezoom_level-=1 204 | 205 | else: # For every other level than 0 (with downsampling) 206 | 207 | regular_parameters["use_bordernodata_mask"] = False 208 | filled_dsm = eomanager.get_array(key=dsm_key)[0] 209 | 210 | # Putting nan values instead of no data for the sampling function 211 | filled_dsm[filled_dsm == nodata] = np.nan 212 | 213 | # Downsampling the DSM to fill the large nodata areas. the order is set to 1 because it's the only one that handle no data 214 | # The mode is set to nearest because it expands the image when zooming if the resampling factor is not proportional to the image size 215 | filled_dsm_downsampled = zoom(filled_dsm, 1/(dezoom_factor**dezoom_level), order=1, mode="nearest") 216 | # Putting back nodata values 217 | filled_dsm_downsampled = np.where(np.isnan(filled_dsm_downsampled), nodata, filled_dsm_downsampled) 218 | 219 | # Creating new profile for downsampled data 220 | downsampled_profile = downsample_profile(profile=eomanager.get_profile(key=dsm_key), factor=dezoom_factor**dezoom_level) 221 | downsampled_profile.update(width=np.shape(filled_dsm_downsampled)[1], height=np.shape(filled_dsm_downsampled)[0]) 222 | downsampled_filled_dsm_key = eomanager.create_image(downsampled_profile) 223 | 224 | filled_dsm_downsample = eomanager.get_array(key=downsampled_filled_dsm_key)[0] 225 | filled_dsm_downsample[:] = filled_dsm_downsampled 226 | 227 | if downsample: 228 | #TODO HOTFIX to remove: until we change eoscale we have to compute the tile size manually 229 | BulldozerLogger.log("DSM filling during downsampling step : level={} / computed_specific_tile_size={} / default_tile_size={}".format(dezoom_level, int(np.ceil(dsm_resolution*dezoom_factor**dezoom_level)), int(math.sqrt((filled_dsm.shape[0] * filled_dsm.shape[1]) // eomanager.nb_workers))), logging.DEBUG) 230 | 231 | else: 232 | # The number of iteration is set to the maximum at the current resolution (we consider the max distance to reach with current resolution considering the diagional of the image) 233 | regular_parameters["filling_iterations"] = int(np.floor(np.sqrt(filled_dsm.shape[0]**2+filled_dsm.shape[1]**2) // dezoom_factor**dezoom_level)) 234 | #TODO HOTFIX to remove: until we change eoscale we have to compute the tile size manually 235 | BulldozerLogger.log("DSM filling during upsampling step : level={} / computed_specific_tile_size={} / default_tile_size={} / filling_iterations={}".format(dezoom_level, int(np.ceil(dsm_resolution*dezoom_factor**dezoom_level)), int(math.sqrt((filled_dsm.shape[0] * filled_dsm.shape[1]) // eomanager.nb_workers)), regular_parameters["filling_iterations"]), logging.DEBUG) 236 | 237 | if np.ceil(dsm_resolution*dezoom_factor**dezoom_level) > math.sqrt((filled_dsm.shape[0] * filled_dsm.shape[1]) // eomanager.nb_workers): 238 | #TODO HOTFIX to remove: until we change eoscale we have to compute the tile size manually 239 | specific_tile_size = int(np.ceil(dsm_resolution*dezoom_factor**dezoom_level)) 240 | else: 241 | specific_tile_size = None 242 | 243 | # Iterative filling for the remaining no data areas 244 | [downsampled_filled_dsm_key] = eoexe.n_images_to_m_images_filter(inputs=[downsampled_filled_dsm_key], 245 | image_filter=fill_dsm_method, 246 | filter_parameters=regular_parameters, 247 | generate_output_profiles=filled_dsm_profile, 248 | context_manager=eomanager, 249 | stable_margin=regular_parameters["filling_iterations"], 250 | filter_desc="Iterative filling DSM level "+str(dezoom_level), 251 | specific_tile_size = specific_tile_size) 252 | 253 | filled_dsm_downsample = eomanager.get_array(key=downsampled_filled_dsm_key)[0] 254 | 255 | # Putting nan values instead of no data for the sampling function 256 | filled_dsm_downsample[filled_dsm_downsample == nodata] = np.nan 257 | 258 | if dev_mode: 259 | if downsample: 260 | filled_dsm_path: str = os.path.join(dev_dir, "filled_dsm_downsampled_level_"+str(dezoom_level)+".tif") 261 | else: 262 | # Upsample case 263 | filled_dsm_path: str = os.path.join(dev_dir, "filled_dsm_upsampled_level_"+str(dezoom_level)+".tif") 264 | eomanager.write(key=downsampled_filled_dsm_key, img_path=filled_dsm_path) 265 | 266 | # Merging the current level with the first one 267 | scale_y = filled_dsm.shape[0] / filled_dsm_downsample.shape[0] 268 | scale_x = filled_dsm.shape[1] / filled_dsm_downsample.shape[1] 269 | filled_dsm_resample = zoom(filled_dsm_downsample, (scale_y, scale_x), order=1, mode="nearest") 270 | 271 | eomanager.release(key=downsampled_filled_dsm_key) 272 | 273 | filled_dsm[:] = np.where(remaining_nodata == 1, filled_dsm_resample, filled_dsm) 274 | 275 | remaining_nodata = (np.isnan(filled_dsm)) & (border_nodata == 0) 276 | 277 | has_nodata = np.any(remaining_nodata) 278 | 279 | filled_dsm[:] = np.where(np.isnan(filled_dsm), nodata, filled_dsm) 280 | 281 | if downsample: 282 | dezoom_level+=1 283 | else: 284 | dezoom_level-=1 285 | 286 | if dezoom_level==nb_max_level: 287 | # For upsampling we prefer to start at the level just after the max_level 288 | dezoom_level-=2 289 | downsample=False 290 | 291 | # Set the border nodata to very high value in order to avoid underestimation of the DTM on the border 292 | #TODO change to another method? 293 | filled_dsm[border_nodata==1] = 9999 294 | 295 | return { 296 | "filled_dsm" : dsm_key 297 | } -------------------------------------------------------------------------------- /bulldozer/preprocessing/ground_detection/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. -------------------------------------------------------------------------------- /bulldozer/preprocessing/ground_detection/ground_anchors_detector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module is used to detect ground anchors points before the main DTM extraction step. 23 | """ 24 | import numpy as np 25 | import bulldozer.eoscale.manager as eom 26 | import bulldozer.eoscale.eo_executors as eoexe 27 | from bulldozer.utils.bulldozer_logger import Runtime 28 | 29 | 30 | def ground_anchors_profile(input_profiles: list, 31 | params: dict) -> dict: 32 | """ 33 | This method is used in the main `detect_ground_anchors` 34 | method to provide the output mask profile (binary profile). 35 | 36 | Args: 37 | input_profiles: input profile. 38 | params: extra parameters. 39 | 40 | Returns: 41 | updated profile. 42 | """ 43 | return input_profiles[2] 44 | 45 | 46 | def ground_anchors_filter(input_buffers: list, 47 | input_profiles: list, 48 | filter_parameters: dict) -> np.ndarray: 49 | """ 50 | This method is used in the main `detect_ground_anchors`. 51 | 52 | Args: 53 | input_buffers: input DSM. 54 | input_profiles: DSM profile. 55 | filter_parameters: dictionary containing the DTM altimetric uncertainty. 56 | 57 | Returns: 58 | ground anchors mask. 59 | """ 60 | inter_dtm = input_buffers[0][0, :, :] 61 | dsm = input_buffers[1][0, :, :] 62 | regular_mask = input_buffers[2][0, :, :] 63 | ground_anchors_mask = np.where(np.logical_and(np.absolute(inter_dtm-dsm) <= filter_parameters["dsm_z_accuracy"],regular_mask), 1, 0).astype(np.uint8) 64 | return ground_anchors_mask 65 | 66 | 67 | @Runtime 68 | def detect_ground_anchors(intermediate_dtm_key: str, 69 | dsm_key: str, 70 | regular_mask_key: str, 71 | dsm_z_accuracy: float, 72 | eomanager: eom.EOContextManager) -> dict: 73 | """ 74 | This method returns the binary mask flagging pre-detected ground areas location in the provided DSM. 75 | 76 | Args: 77 | intermediate_dtm_key: first estimation of the final DTM. 78 | dsm_key: input DSM. 79 | regular_mask_key: regular areas of the input DSM. 80 | dsm_z_accuracy: DSM altimetric resolution (in meter). 81 | eomanager: eoscale context manager. 82 | 83 | Returns: 84 | the regular areas mask. 85 | """ 86 | ground_anchors_parameters: dict = { 87 | "dsm_z_accuracy": dsm_z_accuracy 88 | } 89 | 90 | [ground_anchors_mask_key] = eoexe.n_images_to_m_images_filter(inputs=[intermediate_dtm_key, dsm_key, regular_mask_key], 91 | image_filter=ground_anchors_filter, 92 | filter_parameters=ground_anchors_parameters, 93 | generate_output_profiles=ground_anchors_profile, 94 | context_manager=eomanager, 95 | stable_margin=0, 96 | filter_desc="Ground anchors mask processing...") 97 | 98 | return { 99 | "ground_anchors_mask_key": ground_anchors_mask_key 100 | } 101 | -------------------------------------------------------------------------------- /bulldozer/preprocessing/regular_detection/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. -------------------------------------------------------------------------------- /bulldozer/preprocessing/regular_detection/cython/c_regular.cpp: -------------------------------------------------------------------------------- 1 | /*Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 2 | 3 | This file is part of Bulldozer 4 | (see https://github.com/CNES/bulldozer). 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License.*/ 17 | 18 | #include "c_regular.h" 19 | 20 | /*Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 21 | 22 | This file is part of Bulldozer 23 | (see https://github.com/CNES/bulldozer). 24 | 25 | Licensed under the Apache License, Version 2.0 (the "License"); 26 | you may not use this file except in compliance with the License. 27 | You may obtain a copy of the License at 28 | 29 | http://www.apache.org/licenses/LICENSE-2.0 30 | 31 | Unless required by applicable law or agreed to in writing, software 32 | distributed under the License is distributed on an "AS IS" BASIS, 33 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 34 | See the License for the specific language governing permissions and 35 | limitations under the License.*/ 36 | 37 | #include "c_regular.h" 38 | 39 | namespace bulldozer { 40 | 41 | void buildRegularMask(float * dsm, 42 | unsigned char * regularMask, 43 | unsigned int nbRows, 44 | unsigned int nbCols, 45 | float thresh, 46 | float nodataValue) { 47 | 48 | const long int x_size = nbCols; 49 | const long int y_size = nbRows; 50 | 51 | 52 | const int nbNeigbhors=8; 53 | std::ptrdiff_t v8_off[nbNeigbhors] = {-x_size-1, -x_size, -x_size+1, -1, +1, x_size-1, x_size, x_size+1 }; 54 | 55 | float sum; 56 | float used; 57 | std::ptrdiff_t pos; 58 | std::ptrdiff_t pos_off; 59 | 60 | 61 | for (long int y=0; y=0 && pos_off 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | /* 29 | This function is used to build a mask that flags the pixels considered as "regular" meaning not disturbed from an altimetric perspective. 30 | */ 31 | namespace bulldozer 32 | { 33 | void buildRegularMask(float * dsm, 34 | unsigned char * regularMask, 35 | unsigned int nbRows, 36 | unsigned int nbCols, 37 | float thresh, 38 | float nodataValue); 39 | 40 | } // end of namespace bulldozer 41 | 42 | #endif -------------------------------------------------------------------------------- /bulldozer/preprocessing/regular_detection/cython/regular.pyx: -------------------------------------------------------------------------------- 1 | # distutils: language = c++ 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | import numpy as np 22 | from bulldozer.utils.helper import npAsContiguousArray 23 | 24 | # Begin PXD 25 | 26 | # Necessary to include the C++ code 27 | cdef extern from "c_regular.cpp": 28 | pass 29 | 30 | # Declare the class with cdef 31 | cdef extern from "c_regular.h" namespace "bulldozer": 32 | 33 | void buildRegularMask(float * , 34 | unsigned char *, 35 | unsigned int, 36 | unsigned int, 37 | float, 38 | float) 39 | 40 | 41 | # End PXD 42 | 43 | cdef class PyRegularAreas: 44 | 45 | def __cinit__(self) -> None: 46 | """ 47 | Default constructor. 48 | """ 49 | pass 50 | 51 | def build_regular_mask(self, 52 | dsm_strip : np.array, 53 | slope_threshold : float, 54 | nodata_value : float) -> np.array: 55 | """ 56 | This method detects regular areas using average slope from the 8 neighbors. 57 | 58 | Args: 59 | dsm_strip: part of the DSM analyzed. 60 | slope_threshold: if the average slope is lower than this threshold then it's considered as a regular area. 61 | nodata_value: nodata value. 62 | Return: 63 | mask of the regular / disturbed areas. 64 | """ 65 | cdef float[::1] dsm_memview = npAsContiguousArray(dsm_strip.ravel().astype(np.float32)) 66 | # Ouput mask that will be filled by the C++ part 67 | cdef unsigned char[::1] regular_mask_memview = npAsContiguousArray(np.zeros((dsm_strip.shape[0] * dsm_strip.shape[1]), dtype=np.uint8)) 68 | # Regular detection 69 | buildRegularMask(&dsm_memview[0], ®ular_mask_memview[0], dsm_strip.shape[0], dsm_strip.shape[1], slope_threshold, nodata_value) 70 | # Reshape the output mask. From array to matrix corresponding to the input DSM strip shape 71 | return np.asarray(regular_mask_memview).reshape(dsm_strip.shape[0], dsm_strip.shape[1]).astype(np.uint8) 72 | 73 | -------------------------------------------------------------------------------- /bulldozer/preprocessing/regular_detection/regular_detector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module is used to extract the regular areas in the provided DSM. 23 | """ 24 | import numpy as np 25 | import rasterio 26 | import logging 27 | import os 28 | import bulldozer.eoscale.manager as eom 29 | import bulldozer.eoscale.eo_executors as eoexe 30 | import bulldozer.preprocessing.regular as regular 31 | from bulldozer.utils.bulldozer_logger import Runtime, BulldozerLogger 32 | from skimage.morphology import remove_small_objects 33 | from scipy.ndimage import binary_opening 34 | 35 | def regular_mask_profile(input_profiles: list, 36 | params: dict) -> dict: 37 | """ 38 | This method is used in the main `detect_regular_areas` 39 | method to provide the output mask profile (binary profile). 40 | 41 | Args: 42 | input_profiles: input profile. 43 | params: extra parameters. 44 | 45 | Returns: 46 | updated profile. 47 | """ 48 | output_profile = input_profiles[0] 49 | output_profile['dtype'] = np.ubyte 50 | output_profile['nodata'] = None 51 | return output_profile 52 | 53 | 54 | def regular_mask_filter(input_buffers: list, 55 | input_profiles: list, 56 | filter_parameters: dict) -> np.ndarray: 57 | """ 58 | This method is used in the main `detect_regular_areas`. 59 | It calls the Cython method to extract regular areas. 60 | 61 | Args: 62 | input_buffers: input DSM. 63 | input_profiles: DSM profile. 64 | filter_parameters: filter parameters. 65 | 66 | Returns: 67 | regular areas mask. 68 | """ 69 | reg_filter = regular.PyRegularAreas() 70 | # the input_buffers[0] corresponds to the input DSM raster 71 | reg_mask = reg_filter.build_regular_mask(input_buffers[0][0, :, :], 72 | slope_threshold=filter_parameters["regular_slope"], 73 | nodata_value=filter_parameters["nodata"]) 74 | return reg_mask.astype(np.ubyte) 75 | 76 | @Runtime 77 | def detect_regular_areas(dsm_key: str, 78 | regular_slope: float, 79 | nodata: float, 80 | max_object_size: int, 81 | eomanager: eom.EOContextManager, 82 | reg_filtering_iter: int = None, 83 | dev_mode: bool = False, 84 | dev_dir: str = "") -> dict: 85 | """ 86 | This method returns the binary mask flagging regular areas location in the provided DSM. 87 | 88 | Args: 89 | dsm_key: input DSM. 90 | regular_slope: maximum slope of a regular area. 91 | nodata: DSM nodata value (if nan, the nodata is set to -32768). 92 | max_object_size: foreground max object size (in meter). 93 | reg_filtering_iter: number of regular mask filtering iterations. 94 | eomanager: eoscale context manager. 95 | dev_mode: if True, dev mode activated 96 | dev_dir: path to save dev files 97 | 98 | Returns: 99 | the regular areas mask. 100 | """ 101 | regular_parameters: dict = { 102 | "regular_slope": regular_slope, 103 | "nodata": nodata 104 | } 105 | 106 | [regular_mask_key] = eoexe.n_images_to_m_images_filter(inputs=[dsm_key], 107 | image_filter=regular_mask_filter, 108 | filter_parameters=regular_parameters, 109 | generate_output_profiles=regular_mask_profile, 110 | context_manager=eomanager, 111 | stable_margin=1, 112 | filter_desc="Regular mask processing...") 113 | 114 | 115 | bin_regular_mask = eomanager.get_array(key=regular_mask_key)[0].astype(bool) 116 | 117 | if dev_mode: 118 | eomanager.write(key=regular_mask_key, img_path=os.path.join(dev_dir, "raw_regular_mask.tif"), binary=True) 119 | 120 | if reg_filtering_iter is not None: 121 | nb_iterations = reg_filtering_iter 122 | else: 123 | nb_iterations = int(np.max([1 ,max_object_size/2])) 124 | BulldozerLogger.log('\'reg_filtering_iter\' is not set or less than 1. Used default computed value: {}'.format(nb_iterations), logging.DEBUG) 125 | 126 | # This condition allows the user to desactivate the filtering (iterations=0 in binary_opening ends up to filtering until nothing change) 127 | if nb_iterations>=1: 128 | binary_opening(bin_regular_mask, iterations=nb_iterations, output=bin_regular_mask) 129 | 130 | regular_mask = eomanager.get_array(key=regular_mask_key)[0] 131 | regular_mask[:] = bin_regular_mask 132 | 133 | return { 134 | "regular_mask_key": regular_mask_key 135 | } -------------------------------------------------------------------------------- /bulldozer/scale/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/bulldozer/f55009c61542df42d5184db0599710ac62741c0a/bulldozer/scale/__init__.py -------------------------------------------------------------------------------- /bulldozer/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/bulldozer/f55009c61542df42d5184db0599710ac62741c0a/bulldozer/utils/__init__.py -------------------------------------------------------------------------------- /bulldozer/utils/bulldozer_argparse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module is used to display the helper of the Bulldozer command. 23 | """ 24 | 25 | import argparse 26 | from gettext import gettext as _ 27 | import sys as _sys 28 | 29 | REQ_PARAM_KEY = "REQUIRED" 30 | OPT_PARAM_KEY = "OPTIONAL" 31 | EXPERT_PARAM_KEY = "EXPERT OPTIONAL" 32 | 33 | 34 | class BulldozerHelpFormatter(argparse.RawTextHelpFormatter): 35 | """ 36 | Bulldozer formatter of help messages for command line options. 37 | Displays help message for execution with configuration file and CLI arguments. 38 | """ 39 | 40 | def __init__(self, **kwargs) -> None: 41 | """ 42 | BulldozerHelpFormatter constructor. 43 | """ 44 | argparse.HelpFormatter.__init__(self, max_help_position=55, **kwargs) 45 | 46 | 47 | def bulldozer_add_usage(self, positionals: list = [], 48 | optionals: list = [], commons: list = [], 49 | prefix: str = None) -> None: 50 | """ 51 | This method generates the usage description. 52 | 53 | Args: 54 | positionals: list of all positional arguments as actions (for bulldozer: only config file path expected). 55 | optionals: list of all Bulldozer optional arguments as actions. 56 | commons: list of all common optional arguments (for example: help, version, long usage). 57 | prefix: string to describe the usage. 58 | """ 59 | args = positionals, optionals, commons, prefix 60 | self._add_item(self._bulldozer_format_usage, args) 61 | 62 | 63 | def _bulldozer_format_usage(self, positionals: list, 64 | optionals: list, commons: list, 65 | prefix: str) -> str: 66 | """ 67 | This method defines the usage description format. 68 | 69 | Args: 70 | positionals: list of all positional arguments as actions (for bulldozer: only config file path expected). 71 | optionals: list of all Bulldozer optional arguments as actions. 72 | commons: list of all common optional arguments (for example: help, version, long usage). 73 | prefix: string to describe the usage. 74 | 75 | Returns: 76 | Usage description content as string. 77 | """ 78 | # Define prefixes 79 | if prefix is None: 80 | prefix = _("usage: ") 81 | 82 | # Define usages 83 | prog = "%(prog)s" % dict(prog=self._prog) 84 | 85 | # build full usage string 86 | format = self._format_actions_usage 87 | commons_usage = format(commons, []) 88 | usage = prog + " " + " ".join([param for param in [commons_usage] if param]) 89 | if optionals: 90 | # Formatter used for CLI with parameters usage 91 | for it, param in enumerate([arg+"] " for arg in format(optionals, []).split("] ") if arg]): 92 | # Skip a line every 3 parameters 93 | if param: 94 | if it % 3 == 0: 95 | # Add spacing before optionals parameters for alignement purpose 96 | usage+= "\n" + (len(prog)+len(prefix)+1)*" " + param 97 | else : 98 | usage+= param 99 | # Remove last "] " char 100 | usage = usage[:-2] 101 | # Add spacing before positionals parameters for alignement purpose 102 | usage+= "\n" + (len(prog)+len(prefix)+1)*" " + " ".join([param for param in [format(positionals, [])] if param]) 103 | else: 104 | # Formatter used for CLI with config file usage 105 | usage+= " " + " ".join([param for param in [format(positionals, [])] if param]) 106 | 107 | 108 | return "%s%s\n\n" % (prefix, usage) 109 | 110 | 111 | class BulldozerArgumentParser(argparse.ArgumentParser): 112 | """ 113 | Bulldozer arguments parser. 114 | """ 115 | 116 | def __init__(self, **kwargs): 117 | """ 118 | BulldozerHelpFormatter constructor. 119 | """ 120 | argparse.ArgumentParser.__init__(self, formatter_class=BulldozerHelpFormatter, **kwargs) 121 | 122 | 123 | def get_bulldozer_groups(self) -> tuple: 124 | """ 125 | This method returns the two groups of arguments containing the Bulldozer parameters (required and optional). 126 | 127 | Returns: 128 | cli_positionals: Arguments group for required Bulldozer parameters. 129 | cli_optionals: Arguments group for optional Bulldozer parameters. 130 | 131 | Raises: 132 | ValueError: if unknown group description is found. 133 | """ 134 | cli_positionals = [] 135 | cli_optionals = [] 136 | cli_experts_optionals = [] 137 | for action_group in self._action_groups: 138 | if action_group.title is None: # the group contains Bulldozer parameters 139 | if action_group.description == REQ_PARAM_KEY: 140 | cli_positionals = action_group 141 | elif action_group.description == OPT_PARAM_KEY: 142 | cli_optionals = action_group 143 | elif action_group.description == EXPERT_PARAM_KEY: 144 | cli_experts_optionals = action_group 145 | else: 146 | raise ValueError( 147 | f"Unknown group description {action_group.description}: expects {REQ_PARAM_KEY}, {OPT_PARAM_KEY} or {EXPERT_PARAM_KEY}" 148 | ) 149 | 150 | return cli_positionals, cli_optionals, cli_experts_optionals 151 | 152 | 153 | def print_help(self, file: str = None, long_help: bool = False, expert_mode: bool = False) -> None: 154 | """ 155 | This method generates the help message. 156 | 157 | Args: 158 | file: path to a file to save the output. 159 | long_help: whether print the full help or not (by default short help). 160 | export_mode: whether print the expert parameters or not (by default False). 161 | """ 162 | if file is None: 163 | file = _sys.stdout 164 | self._print_message(self.format_help(long_help, expert_mode), file) 165 | 166 | def format_usage(self) -> str: 167 | """ 168 | This method generates the Bulldozer global usage description. 169 | It contains both usages: using config file or using CLI arguments. 170 | 171 | Returns: 172 | The usage description for Bulldozer. 173 | """ 174 | return self.format_help(add_description=False, add_epilog=False) 175 | 176 | def format_help(self, long_help: bool = False, expert_mode: bool = False, 177 | add_description: bool = True, add_epilog: bool = True) -> str: 178 | """ 179 | This method generates the bulldozer global help message. 180 | It contains both execution methods: using a config file or using CLI arguments. 181 | 182 | Args: 183 | long_help: whether to display only usage description (False) or full arguments descriptions (True). 184 | export_mode: whether to display expert arguments or not. 185 | add_description: whether to add description or not at the beginning of the help message. 186 | add_epilog: whether to add epilog or not at the end of the help message. 187 | 188 | Returns: 189 | The help message for Bulldozer. 190 | """ 191 | # get groups corresponding to bulldozer parameters 192 | cli_pos_group, cli_opt_group, cli_expert_group = self.get_bulldozer_groups() 193 | 194 | # prepare input arguments for both config file and cli usages 195 | cli_positionals = cli_pos_group._group_actions # positional arguments for CLI 196 | # We set positionals to required (for visual display) 197 | for action in cli_positionals: 198 | action.required = True 199 | cli_optionals = cli_opt_group._group_actions # optionals arguments for CLI 200 | if expert_mode: 201 | cli_optionals += cli_expert_group._group_actions 202 | positionals = self._positionals._group_actions # positional arguments (only config file path) 203 | positionals[0].nargs = None # We set the nargs to None to highlight the requirement of this parameter 204 | 205 | # format 206 | formatter = self._get_formatter() 207 | 208 | # description 209 | if (add_description): 210 | formatter.add_text(self.description) 211 | 212 | # usage and help with config file 213 | formatter.bulldozer_add_usage(positionals, commons=self._optionals._group_actions, 214 | prefix=_("Usage with config file: ")) 215 | 216 | if long_help: 217 | # long help positional arguments 218 | formatter.start_section(self._positionals.title) 219 | formatter.add_text(self._positionals.description) 220 | formatter.add_arguments(self._positionals._group_actions) 221 | formatter.end_section() 222 | # long help optional arguments 223 | formatter.start_section(self._optionals.title) 224 | formatter.add_text(self._optionals.description) 225 | formatter.add_arguments(self._optionals._group_actions) 226 | formatter.end_section() 227 | 228 | # epilog 229 | if add_epilog: 230 | formatter.add_text("If extra arguments are provided, these will override the original values from the configuration file.") 231 | 232 | formatter.add_text("---------------------------------") 233 | 234 | # usage and help with cli arguments 235 | formatter.bulldozer_add_usage(cli_positionals, cli_optionals, self._optionals._group_actions, 236 | prefix=_("Usage with parameters: ")) 237 | 238 | if long_help: 239 | # long help positional arguments 240 | formatter.start_section("required arguments") 241 | formatter.add_arguments(cli_positionals) 242 | formatter.end_section() 243 | # long help optional arguments 244 | formatter.start_section(self._optionals.title) 245 | formatter.add_arguments(self._optionals._group_actions) 246 | formatter.end_section() 247 | formatter.start_section(None) # for visual spacing 248 | formatter.add_arguments(cli_optionals) 249 | formatter.end_section() 250 | 251 | # epilog 252 | if add_epilog: 253 | if long_help: 254 | #TODO uncomment when doc is online 255 | #formatter.add_text("For more details, consult https://bulldozer.readthedocs.io/") 256 | formatter.add_text("For more details, consult the documentation.") 257 | else: 258 | epilog = self.epilog.replace("prog", "%(prog)s" % dict(prog=formatter._prog)) 259 | formatter.add_text(epilog) 260 | 261 | # determine help from format above 262 | return formatter.format_help() 263 | -------------------------------------------------------------------------------- /bulldozer/utils/bulldozer_logger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module aims to centralize the use of the logger in Bulldozer. 23 | """ 24 | from __future__ import annotations 25 | import sys 26 | import os 27 | import getpass 28 | import platform 29 | import time 30 | import psutil 31 | import multiprocessing 32 | import logging 33 | import logging.config 34 | from bulldozer._version import __version__ 35 | 36 | class BulldozerLogger: 37 | """ 38 | Bulldozer logger singleton. Only used in the full pipeline mode (not for the standalone calls). 39 | """ 40 | __instance = None 41 | 42 | @staticmethod 43 | def getInstance(logger_file_path: str = None) -> BulldozerLogger: 44 | """ 45 | Return the logger or create it if the instance does not exist. 46 | 47 | Args: 48 | logger_file_path: path to the output logfile. 49 | 50 | Returns: 51 | the Bulldozer logger. 52 | """ 53 | if BulldozerLogger().__instance is None : 54 | 55 | # Create the Logger 56 | # Sub folders will inherits from the logger configuration, hence 57 | # we need to give the root package directory name of Bulldozer 58 | logger = logging.getLogger("bulldozer") 59 | logger.setLevel(logging.DEBUG) 60 | 61 | # create file handler which logs even debug messages 62 | fh = logging.FileHandler(filename=logger_file_path, mode='w') 63 | fh.setLevel(logging.DEBUG) 64 | 65 | LOG_FORMAT = '%(asctime)s [%(levelname)s] %(module)s - %(funcName)s (line %(lineno)d): %(message)s' 66 | logger_formatter = logging.Formatter(LOG_FORMAT, datefmt="%Y-%m-%dT%H:%M:%S") 67 | fh.setFormatter(logger_formatter) 68 | 69 | logger.addHandler(fh) 70 | 71 | sh = logging.StreamHandler(sys.stdout) 72 | sh.setLevel(logging.INFO) 73 | 74 | STREAM_FORMAT = '%(asctime)s [%(levelname)s] - %(message)s' 75 | logger_formatter = logging.Formatter(STREAM_FORMAT, datefmt="%H:%M:%S") 76 | sh.setFormatter(logger_formatter) 77 | 78 | logger.addHandler(sh) 79 | BulldozerLogger.__instance = logger 80 | 81 | BulldozerLogger.init_logger() 82 | 83 | return BulldozerLogger.__instance 84 | 85 | 86 | @staticmethod 87 | def log(msg : str, level : any) -> None: 88 | """ 89 | Bulldozer logger log function. 90 | The following logging levels are used: 91 | DEBUG 92 | INFO 93 | WARNING 94 | ERROR 95 | 96 | Args: 97 | msg: log message. 98 | level: crticity level. 99 | """ 100 | if BulldozerLogger.__instance is not None : 101 | if level == logging.DEBUG: 102 | BulldozerLogger.__instance.debug(msg) 103 | if level == logging.INFO: 104 | BulldozerLogger.__instance.info(msg) 105 | if level == logging.WARNING: 106 | BulldozerLogger.__instance.warning(msg) 107 | if level == logging.ERROR: 108 | BulldozerLogger.__instance.error(msg) 109 | 110 | @staticmethod 111 | def init_logger() -> None: 112 | """ 113 | This method store the environment state in the logfile. 114 | """ 115 | info={} 116 | try: 117 | # Node info 118 | try: 119 | info['user'] = getpass.getuser() 120 | except: 121 | info['user'] = 'unknown' 122 | try: 123 | info['node'] = platform.node() 124 | except: 125 | info['node'] = 'unknown' 126 | info['processor'] = platform.processor() 127 | info['cpu_count'] = multiprocessing.cpu_count() 128 | info['ram'] = str(round(psutil.virtual_memory().total / (1024 **3)))+" GB" 129 | 130 | # OS info 131 | info['system'] = platform.system() 132 | info['release'] = platform.release() 133 | info['os_version'] = platform.version() 134 | 135 | # Message format 136 | init = ("\n"+"#"*17+"\n# BULLDOZER #\n"+"#"*17+"\n# \n#\t- version: {}"+ 137 | "\n#\n# \n#\t - user: {}\n#\t - node: {}\n#\t - processor: {}\n#\t - CPU count: {}\n#\t - RAM: {}" 138 | "\n#\n# \n#\t - system: {}\n#\t - release: {}\n#\t - version: {}\n" 139 | +"#"*17).format(__version__, info['user'], info['node'], info['processor'], info['cpu_count'], info['ram'], 140 | info['system'], info['release'], info['os_version']) 141 | BulldozerLogger.log(init, logging.DEBUG) 142 | 143 | except Exception as e: 144 | BulldozerLogger.log("Error occured during logger init: \n" + str(e), logging.DEBUG) 145 | 146 | class Runtime: 147 | """ 148 | This class is used as decorator to monitor the runtime. 149 | """ 150 | 151 | def __init__(self, function) -> None: 152 | """ 153 | Decorator constructor. 154 | 155 | Args: 156 | function: the function to call. 157 | """ 158 | self.function = function 159 | 160 | def __call__(self, *args, **kwargs) -> Any: 161 | """ 162 | Log the start and end of the function with the associated runtime. 163 | 164 | Args: 165 | args: function arguments. 166 | kwargs: function key arguments. 167 | 168 | Returns: 169 | the function output. 170 | """ 171 | func_start = time.perf_counter() 172 | BulldozerLogger.log("{}: Starting...".format(self.function.__name__), logging.DEBUG) 173 | # Function run 174 | result = self.function(*args, **kwargs) 175 | func_end = time.perf_counter() 176 | BulldozerLogger.log("{}: Done (Runtime: {}s)".format(self.function.__name__, round(func_end-func_start,2)), logging.INFO) 177 | return result -------------------------------------------------------------------------------- /bulldozer/utils/config_parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | """ 21 | This module is used to retrieve the bulldozer parameters from an YAML configuration file. 22 | """ 23 | 24 | from yaml import safe_load,YAMLError 25 | import os.path 26 | import logging 27 | from bulldozer.utils.bulldozer_logger import BulldozerLogger 28 | 29 | class ConfigParser(object): 30 | """ 31 | Configuration file parser. Used to read the bulldozer parameters. 32 | """ 33 | 34 | def __init__(self, verbose : bool = False) -> None: 35 | """ 36 | Parser constructor. 37 | 38 | Args: 39 | verbose (default=False): increase output verbosity if true. 40 | """ 41 | if verbose: 42 | self.level=logging.DEBUG 43 | else: 44 | self.level=logging.INFO 45 | 46 | 47 | def read(self, path : str) -> dict: 48 | """ 49 | This method returns the dict containing the bulldozer parameters extracted from 50 | the input YAML configuration file. 51 | 52 | Args: 53 | path: path to the configuration file (expected YAML file). 54 | 55 | Returns: 56 | cfg: configuration parameters for bulldozer. 57 | 58 | Raises: 59 | ValueError: if bad input path is provided. 60 | FileNotFoundError: if the input file doesn't exist. 61 | YAMLError: if an error occured while reading the yaml file. 62 | """ 63 | # input file format check 64 | if not (isinstance(path, str) and (path.endswith('.yaml') or path.endswith('.yml'))) : 65 | BulldozerLogger.log('\'path\' argument should be a path to the YAML config file (here: {})'.format(path), logging.ERROR) 66 | raise ValueError() 67 | # input file existence check 68 | if not os.path.isfile(path): 69 | BulldozerLogger.log('The input config file \'{}\' doesn\'t exist'.format(path), logging.ERROR) 70 | raise FileNotFoundError() 71 | 72 | if self.level == logging.DEBUG: 73 | BulldozerLogger.log('Check input config file => Passed', self.level) 74 | 75 | with open(path, 'r') as stream: 76 | try: 77 | cfg = safe_load(stream) 78 | if self.level == logging.DEBUG: 79 | BulldozerLogger.log('Retrieved data: {}'.format(cfg), self.level) 80 | except YAMLError as e: 81 | BulldozerLogger.log('Exception occured while reading the configuration file: {}\nException: {}'.format(path, str(e)), logging.ERROR) 82 | raise YAMLError(str(e)) 83 | return cfg -------------------------------------------------------------------------------- /bulldozer/utils/helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | This module groups different generic methods used in Bulldozer. 23 | """ 24 | import rasterio 25 | import numpy as np 26 | from rasterio import Affine 27 | 28 | 29 | def npAsContiguousArray(arr: np.array) -> np.array: 30 | """ 31 | This method checks that the input array is contiguous. 32 | If not, returns the contiguous version of the input numpy array. 33 | 34 | Args: 35 | arr: input array. 36 | 37 | Returns: 38 | contiguous array usable in C++. 39 | """ 40 | if not arr.flags['C_CONTIGUOUS']: 41 | arr = np.ascontiguousarray(arr) 42 | return arr 43 | 44 | 45 | def downsample_profile(profile, factor: float): 46 | 47 | transform= profile['transform'] 48 | 49 | newprofile = profile.copy() 50 | dst_transform = Affine.translation(transform[2], transform[5]) * Affine.scale(transform[0]*factor, transform[4]*factor) 51 | 52 | newprofile.update({ 53 | 'transform': dst_transform, 54 | }) 55 | 56 | return newprofile -------------------------------------------------------------------------------- /conf/basic_conf_template.yaml: -------------------------------------------------------------------------------- 1 | #-------------------------# 2 | # Parameters # 3 | #-------------------------# 4 | # [Required] - Input DSM path (expected format: "//.<[tif/tiff]>") 5 | dsm_path : "input_dsm_path/dsm.tif" 6 | # [Required] - Output directory path (if the directory doesn't exist, create it) 7 | output_dir : "output_dir_path" 8 | 9 | #-------------------------# 10 | # Options # 11 | #-------------------------# 12 | # [Optional] - If True, generates the DHM (DSM - DTM) in the output directory 13 | generate_dhm : True 14 | # [Optional] - Foreground max object size (in meter) 15 | max_object_size : 16 16 | # [Optional] - If null, use the maximum number of available CPU core of your system 17 | nb_max_workers : 8 18 | -------------------------------------------------------------------------------- /conf/configuration_template.yaml: -------------------------------------------------------------------------------- 1 | #-------------------------# 2 | # Parameters # 3 | #-------------------------# 4 | # [Required] - Input DSM path (expected format: "//.<[tif/tiff]>") 5 | dsm_path : "input_dsm_path/dsm.tif" 6 | # [Required] - Output directory path (if the directory doesn't exist, create it) 7 | output_dir : "output_dir_path" 8 | 9 | #-------------------------# 10 | # Options # 11 | #-------------------------# 12 | # [Optional] - If True, generates the Digital Height Model (DHM=DSM-DTM) in the output directory 13 | generate_dhm : True 14 | # [Optional] - Foreground max object size (in meter) 15 | max_object_size : 16 16 | # [Optional] - Path to the binary ground classification mask (expected format: "//.<[tif/tiff]>") 17 | ground_mask_path : null 18 | # [Optional] - If True, activate ground anchor detection (ground pre-detection) 19 | activate_ground_anchors : False 20 | # [Optional] - Max number of CPU core to use. If null, use maximum number of available CPU core 21 | nb_max_workers : null 22 | # [Optional] - If True, keep the intermediate results 23 | developer_mode : False 24 | 25 | #-------------------------# 26 | # Expert options # 27 | #-------------------------# 28 | # /!\ Modify those data at your own risk (it is suggested to keep the default values) /!\ 29 | 30 | # [Optional] - Number of regular mask filtering iterations. If null, use the default value: max_object_size/4 31 | reg_filtering_iter: null 32 | # [Optional] - Altimetric height accuracy of the input DSM (m). If null, use the default value: 2*planimetric resolution 33 | dsm_z_accuracy: null 34 | # [Optional] - Maximum slope of the observed landscape terrain (%) 35 | max_ground_slope: 20.0 36 | # [Optional] - Number of unhook iterations 37 | prevent_unhook_iter : 10 38 | # [Optional] - Number of gravity step iterations 39 | num_outer_iter : 25 40 | # [Optional] - Number of tension iterations 41 | num_inner_iter : 5 -------------------------------------------------------------------------------- /docs/css/extra.css: -------------------------------------------------------------------------------- 1 | .wy-side-nav-search img { 2 | max-height: 150px; 3 | width: auto; 4 | } 5 | 6 | .wy-side-nav-search { 7 | background-color: #272525; 8 | } 9 | 10 | .wy-nav-side { 11 | background-color: #343131; 12 | } 13 | 14 | .wy-side-nav-search, .wy-menu-vertical a { 15 | color: white; 16 | } -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # 🚧 Work in Progress 🚧 2 | 3 | The **Bulldozer** documentation should be published with the version 1.2.0. -------------------------------------------------------------------------------- /docs/source/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/bulldozer/f55009c61542df42d5184db0599710ac62741c0a/docs/source/images/logo.png -------------------------------------------------------------------------------- /docs/source/images/logo_icon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/bulldozer/f55009c61542df42d5184db0599710ac62741c0a/docs/source/images/logo_icon.ico -------------------------------------------------------------------------------- /docs/source/images/logo_with_text.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/bulldozer/f55009c61542df42d5184db0599710ac62741c0a/docs/source/images/logo_with_text.png -------------------------------------------------------------------------------- /docs/source/images/result_overview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/bulldozer/f55009c61542df42d5184db0599710ac62741c0a/docs/source/images/result_overview.gif -------------------------------------------------------------------------------- /mkdocs.yaml: -------------------------------------------------------------------------------- 1 | site_name: Bulldozer 2 | 3 | theme: 4 | # Readthedocs template 5 | name: readthedocs 6 | logo: https://raw.githubusercontent.com/CNES/bulldozer/master/docs/source/images/logo.png 7 | favicon: source/images/logo_icon.ico 8 | custom_dir: docs 9 | 10 | extra_css: 11 | - css/extra.css 12 | markdown_extensions: 13 | - toc: 14 | permalink: true 15 | - admonition 16 | - tables 17 | - footnotes 18 | - codehilite 19 | 20 | nav: 21 | - Home: index.md 22 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2024 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | # pyproject.toml 23 | [build-system] 24 | requires = ["setuptools", "wheel", "Cython", "numpy"] 25 | build-backend = "setuptools.build_meta" 26 | 27 | # Avoid PyPy and Python < 3.8 wheel generation 28 | [tool.cibuildwheel] 29 | skip = ["pp*", "cp36-*", "cp37-*"] -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 2 | # 3 | # This file is part of Bulldozer 4 | # (see https://github.com/CNES/bulldozer). 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # Bulldozer setup configuration file 20 | 21 | # package setup main metadata 22 | [metadata] 23 | name = bulldozer-dtm 24 | author = CNES 25 | author_email = dimitri.lallement@cnes.fr 26 | description = Bulldozer is a DTM extraction tool requiring only a DSM as input 27 | url = https://github.com/CNES/bulldozer 28 | long_description = file: README.md 29 | long_description_content_type = text/markdown 30 | license = Apache V2.0 31 | license_files = LICENSE 32 | keywords= bulldozer,DTM,DSM,3D,Photogrammetry,Remote Sensing,LiDAR,CARS 33 | classifiers = 34 | Development Status :: 4 - Beta 35 | Intended Audience :: Developers 36 | Intended Audience :: End Users/Desktop 37 | Intended Audience :: Science/Research 38 | Topic :: Software Development :: Libraries :: Python Modules 39 | License :: OSI Approved :: Apache Software License 40 | Programming Language :: Python 41 | Programming Language :: Python :: 3.8 42 | Programming Language :: Cython 43 | 44 | [options] 45 | python_requires = >=3.8 46 | 47 | # Bulldozer packages dependencies 48 | install_requires = 49 | Cython >= 0.29.14 50 | numpy >= 1.22.2 51 | rasterio >= 1.2.10 52 | scipy >= 1.8.0 53 | scikit-image 54 | PyYAML 55 | tqdm 56 | argcomplete 57 | psutil 58 | pylint 59 | 60 | packages_dir = =bulldozer 61 | packages = find: 62 | 63 | [options.packages.find] 64 | where = bulldozer 65 | 66 | [options.extras_require] 67 | dev = 68 | pre-commit 69 | build 70 | pytest 71 | pytest-cov 72 | pytest-sugar # for prettier pytest 73 | mypy # Python typing 74 | 75 | docs = 76 | mkdocs 77 | 78 | notebook = 79 | matplotlib 80 | jupyterlab 81 | 82 | [options.entry_points] 83 | console_scripts = 84 | bulldozer = bulldozer.pipeline.bulldozer_pipeline:bulldozer_cli 85 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | from setuptools import setup, Extension, find_packages 22 | from distutils.util import convert_path 23 | from Cython.Build import cythonize 24 | import numpy 25 | 26 | extensions = [ 27 | Extension( "bulldozer.preprocessing.regular", ["bulldozer/preprocessing/regular_detection/cython/regular.pyx"]), 28 | Extension( "bulldozer.preprocessing.border", ["bulldozer/preprocessing/border_detection/cython/border.pyx"]), 29 | Extension( "bulldozer.preprocessing.fill", ["bulldozer/preprocessing/dsm_filling/cython/fill.pyx"], include_dirs=[numpy.get_include()]) 30 | ] 31 | 32 | compiler_directives = { "language_level": 3, "embedsignature": True} 33 | extensions = cythonize(extensions, compiler_directives=compiler_directives) 34 | 35 | main_ns = {} 36 | ver_path = convert_path("bulldozer/_version.py") 37 | with open(ver_path) as ver_file: 38 | exec(ver_file.read(), main_ns) 39 | 40 | try: 41 | setup( 42 | version = main_ns['__version__'], 43 | ext_modules=extensions, 44 | packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]) 45 | ) 46 | except Exception: 47 | print( 48 | "\n\nAn error occurred while building the project, " 49 | "please ensure you have the most updated version of setuptools, " 50 | "setuptools_scm and wheel with:\n" 51 | "\tpip install -U setuptools setuptools_scm wheel\n\n" 52 | ) 53 | raise 54 | 55 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. -------------------------------------------------------------------------------- /tests/utils/data/config_parser/parser_test.yaml: -------------------------------------------------------------------------------- 1 | # yaml file used to test the bulldozer config parser 2 | str_test : test 3 | int_test : 100 4 | boolean_test : True 5 | parent: 6 | child: 10.3 7 | child2: 13 8 | nan_test : nan 9 | none_test : null 10 | -------------------------------------------------------------------------------- /tests/utils/data/config_parser/wrong_syntax.yaml: -------------------------------------------------------------------------------- 1 | test: 1 2 | wrong_syntax -------------------------------------------------------------------------------- /tests/utils/test_config_parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2022-2025 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of Bulldozer 7 | # (see https://github.com/CNES/bulldozer). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | """ 22 | Test module for bulldozer/utils/config_parser.py 23 | """ 24 | import pytest 25 | import logging 26 | import os.path 27 | import numpy as np 28 | from pathlib import Path 29 | from yaml import YAMLError 30 | 31 | from bulldozer.utils.bulldozer_logger import BulldozerLogger 32 | from bulldozer.utils.config_parser import ConfigParser 33 | 34 | @pytest.fixture 35 | def setup() -> None: 36 | """ 37 | Setup function that will be provide to tests that require an input file 38 | """ 39 | parser = ConfigParser(False) 40 | path = Path(__file__).parent / "data/config_parser/" 41 | print("\nSetting up resources...") 42 | # Setting-up ressources 43 | yield {"parser": parser, "path": path} # Provide the data to the test 44 | # Teardown: Clean up resources (if any) after the test 45 | 46 | def test_format_check(setup) -> None: 47 | """ 48 | config_parser input file format checker test 49 | """ 50 | # Should raise exception because it's not an YAML file 51 | path = "test.txt" 52 | parser = setup["parser"] 53 | pytest.raises(ValueError, lambda: parser.read(path)) 54 | # Should raise exception for file not found but should pass the format check 55 | path = "test.yaml" 56 | pytest.raises(FileNotFoundError, lambda: parser.read(path)) 57 | # Should raise exception for file not found but should pass the format check 58 | path = "test.yml" 59 | pytest.raises(FileNotFoundError, lambda:parser.read(path)) 60 | 61 | def test_existence_check(setup) -> None: 62 | """ 63 | config_parser input file checker test 64 | """ 65 | parser = setup["parser"] 66 | # Should raise FileNotFoundException since the file doesn't exist 67 | path = str(setup["path"]) + "/test.yaml" 68 | pytest.raises(FileNotFoundError, lambda: parser.read(path)) 69 | 70 | # Shouldn't raise FileNotFoundException, if it raises an exception the unit test framework will flag this as an error 71 | path = str(setup["path"]) + "/parser_test.yaml" 72 | parser.read(path) 73 | 74 | def test_read(setup) -> None: 75 | """ 76 | config_parser read function test with several data types 77 | """ 78 | path = str(setup["path"]) + "/parser_test.yaml" 79 | cfg = setup["parser"].read(path) 80 | # Check data type 81 | assert isinstance(cfg, dict) 82 | 83 | # Check dict size (expected 6) 84 | assert len(cfg) == 6 85 | 86 | # Check string element read 87 | assert isinstance(cfg["str_test"], str) 88 | assert cfg["str_test"] == "test" 89 | 90 | # Check integer element read 91 | assert isinstance(cfg["int_test"], int) 92 | assert cfg["int_test"] == 100 93 | 94 | # Check boolean element read 95 | assert isinstance(cfg["boolean_test"], bool) 96 | assert cfg["boolean_test"] == True 97 | 98 | # Check float sub-element read 99 | assert isinstance(cfg["parent"], dict) 100 | assert isinstance(cfg["parent"]["child"], float) 101 | assert cfg["parent"]["child"] == 10.3 102 | assert cfg["parent"]["child2"] == 13 103 | 104 | # Check nan reading 105 | assert np.isnan(float(cfg["nan_test"])) 106 | assert cfg["none_test"] is None 107 | assert not cfg["none_test"] 108 | 109 | path = str(setup["path"]) + "/wrong_syntax.yaml" 110 | # Should raise YAMLError due to the wrong YAML data format in the file 111 | pytest.raises(YAMLError, lambda: setup["parser"].read(path)) 112 | 113 | 114 | def test_verbose() -> None: 115 | """ 116 | Verbosity level test 117 | """ 118 | non_verbose_parser = ConfigParser(verbose = False) 119 | # Check logging level value 120 | assert non_verbose_parser.level == logging.INFO 121 | 122 | verbose_parser = ConfigParser(verbose = True) 123 | # Check logging level value 124 | assert verbose_parser.level == logging.DEBUG --------------------------------------------------------------------------------