├── .coveragerc ├── .dockerignore ├── .gitattributes ├── .github └── workflows │ └── cars-ci.yml ├── .gitignore ├── .gitlab ├── issue_templates │ ├── Bug.md │ ├── Proposal.md │ ├── Refacto.md │ └── Release.md └── merge_request_templates │ └── MR.md ├── .pre-commit-config.yaml ├── .pylintrc ├── .readthedocs.yaml ├── AUTHORS.md ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── Makefile ├── NOTICE ├── README.md ├── cars ├── __init__.py ├── applications │ ├── __init__.py │ ├── application.py │ ├── application_constants.py │ ├── application_template.py │ ├── auxiliary_filling │ │ ├── __init__.py │ │ ├── auxiliary_filling.py │ │ ├── auxiliary_filling_from_sensors.py │ │ └── auxiliary_filling_tools.py │ ├── dem_generation │ │ ├── __init__.py │ │ ├── bulldozer_config │ │ │ └── base_config.yaml │ │ ├── dem_generation.py │ │ ├── dem_generation_constants.py │ │ ├── dem_generation_tools.py │ │ ├── dichotomic_generation.py │ │ └── rasterization.py │ ├── dense_match_filling │ │ ├── __init__.py │ │ ├── cpp │ │ │ ├── __init__.py │ │ │ ├── dense_match_filling_cpp.py │ │ │ ├── includes │ │ │ │ └── dense_match_filling.hpp │ │ │ ├── meson.build │ │ │ └── src │ │ │ │ ├── bindings.cpp │ │ │ │ └── dense_match_filling.cpp │ │ ├── dense_match_filling.py │ │ ├── fill_disp_constants.py │ │ ├── fill_disp_tools.py │ │ ├── plane.py │ │ └── zero_padding.py │ ├── dense_matching │ │ ├── __init__.py │ │ ├── census_mccnn_sgm.py │ │ ├── cpp │ │ │ ├── __init__.py │ │ │ ├── dense_matching_cpp.py │ │ │ ├── includes │ │ │ │ └── dense_matching.hpp │ │ │ ├── meson.build │ │ │ └── src │ │ │ │ ├── bindings.cpp │ │ │ │ └── dense_matching.cpp │ │ ├── dense_matching.py │ │ ├── dense_matching_constants.py │ │ ├── dense_matching_tools.py │ │ └── loaders │ │ │ ├── __init__.py │ │ │ ├── config_census_sgm_default.json │ │ │ ├── config_census_sgm_homogeneous.json │ │ │ ├── config_census_sgm_mountain_and_vegetation.json │ │ │ ├── config_census_sgm_shadow.json │ │ │ ├── config_census_sgm_sparse.json │ │ │ ├── config_census_sgm_urban.json │ │ │ ├── config_mccnn.json │ │ │ └── pandora_loader.py │ ├── dsm_filling │ │ ├── __init__.py │ │ ├── border_interpolation.py │ │ ├── bulldozer_config │ │ │ └── base_config.yaml │ │ ├── bulldozer_filling.py │ │ ├── dsm_filling.py │ │ └── exogenous_filling.py │ ├── grid_generation │ │ ├── __init__.py │ │ ├── epipolar_grid_generation.py │ │ ├── grid_constants.py │ │ ├── grid_correction.py │ │ ├── grid_generation.py │ │ └── grids.py │ ├── ground_truth_reprojection │ │ ├── __init__.py │ │ ├── direct_localization.py │ │ ├── ground_truth_reprojection.py │ │ └── ground_truth_reprojection_tools.py │ ├── hole_detection │ │ ├── __init__.py │ │ ├── cloud_to_bbox.py │ │ ├── hole_detection.py │ │ └── hole_detection_tools.py │ ├── point_cloud_denoising │ │ ├── __init__.py │ │ └── point_cloud_denoising.py │ ├── point_cloud_fusion │ │ ├── __init__.py │ │ ├── cloud_fusion_constants.py │ │ ├── mapping_to_terrain_tiles.py │ │ ├── pc_tif_tools.py │ │ ├── point_cloud_fusion.py │ │ └── point_cloud_tools.py │ ├── point_cloud_outlier_removal │ │ ├── __init__.py │ │ ├── outlier_removal_tools.py │ │ ├── pc_out_removal.py │ │ ├── point_removal_constants.py │ │ ├── small_components.py │ │ └── statistical.py │ ├── rasterization │ │ ├── __init__.py │ │ ├── point_cloud_rasterization.py │ │ ├── rasterization_constants.py │ │ ├── rasterization_tools.py │ │ └── simple_gaussian.py │ ├── resampling │ │ ├── __init__.py │ │ ├── bicubic_resampling.py │ │ ├── resampling.py │ │ ├── resampling_constants.py │ │ └── resampling_tools.py │ ├── sparse_matching │ │ ├── __init__.py │ │ ├── pandora_sparse_matching.py │ │ ├── sift.py │ │ ├── sparse_matching.py │ │ ├── sparse_matching_constants.py │ │ └── sparse_matching_tools.py │ └── triangulation │ │ ├── __init__.py │ │ ├── line_of_sight_intersection.py │ │ ├── triangulation.py │ │ ├── triangulation_constants.py │ │ └── triangulation_tools.py ├── bundleadjustment.py ├── cars.py ├── conf │ ├── __init__.py │ ├── geoid │ │ ├── egm96.grd │ │ └── egm96.grd.hdr │ ├── input_parameters.py │ └── mask_cst.py ├── core │ ├── __init__.py │ ├── cars_logging.py │ ├── constants.py │ ├── constants_disparity.py │ ├── datasets.py │ ├── geometry │ │ ├── __init__.py │ │ ├── abstract_geometry.py │ │ └── shareloc_geometry.py │ ├── inputs.py │ ├── outputs.py │ ├── preprocessing.py │ ├── projection.py │ ├── roi_tools.py │ ├── tiling.py │ └── utils.py ├── data_structures │ ├── __init__.py │ ├── cars_dataset.py │ ├── cars_dict.py │ ├── corresponding_tiles_tools.py │ ├── dataframe_converter.py │ └── format_transformation.py ├── devibrate.py ├── extractroi.py ├── orchestrator │ ├── __init__.py │ ├── achievement_tracker.py │ ├── cluster │ │ ├── __init__.py │ │ ├── abstract_cluster.py │ │ ├── abstract_dask_cluster.py │ │ ├── dask_cluster_tools.py │ │ ├── dask_config │ │ │ ├── README.md │ │ │ ├── dask.yaml │ │ │ ├── distributed.yaml │ │ │ ├── jobqueue.yaml │ │ │ └── reference_confs │ │ │ │ ├── dask-schema.yaml │ │ │ │ ├── dask.yaml │ │ │ │ ├── distributed-schema.yaml │ │ │ │ ├── distributed.yaml │ │ │ │ └── jobqueue.yaml │ │ ├── dask_jobqueue_utils.py │ │ ├── local_dask_cluster.py │ │ ├── log_wrapper.py │ │ ├── mp_cluster │ │ │ ├── __init__.py │ │ │ ├── mp_factorizer.py │ │ │ ├── mp_objects.py │ │ │ ├── mp_tools.py │ │ │ ├── mp_wrapper.py │ │ │ ├── multiprocessing_cluster.py │ │ │ └── multiprocessing_profiler.py │ │ ├── pbs_dask_cluster.py │ │ ├── sequential_cluster.py │ │ └── slurm_dask_cluster.py │ ├── orchestrator.py │ ├── orchestrator_constants.py │ ├── registry │ │ ├── __init__.py │ │ ├── abstract_registry.py │ │ ├── compute_registry.py │ │ ├── id_generator.py │ │ ├── replacer_registry.py │ │ ├── saver_registry.py │ │ └── unseen_registry.py │ └── tiles_profiler.py ├── pipelines │ ├── __init__.py │ ├── default │ │ ├── __init__.py │ │ └── default_pipeline.py │ ├── parameters │ │ ├── __init__.py │ │ ├── advanced_parameters.py │ │ ├── advanced_parameters_constants.py │ │ ├── depth_map_inputs.py │ │ ├── depth_map_inputs_constants.py │ │ ├── dsm_inputs.py │ │ ├── dsm_inputs_constants.py │ │ ├── output_constants.py │ │ ├── output_parameters.py │ │ ├── sensor_inputs.py │ │ └── sensor_inputs_constants.py │ ├── pipeline.py │ ├── pipeline_constants.py │ └── pipeline_template.py └── starter.py ├── meson.build ├── pyproject.toml ├── pytest.ini ├── setup.py ├── sonar-project.properties └── version.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = cars 3 | omit = 4 | # Omit tests 5 | /tests/* 6 | # Omit venv 7 | /venv*/ 8 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # exclude venv 2 | venv -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | docs/ export-ignore 2 | tests/ export-ignore 3 | tutorials/ export-ignore -------------------------------------------------------------------------------- /.github/workflows/cars-ci.yml: -------------------------------------------------------------------------------- 1 | name: CARS 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | name: Install and test CARS 8 | runs-on: ubuntu-latest 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | python-version: ["3.10"] 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Set up Python ${{ matrix.python-version }} 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: ${{ matrix.python-version }} 20 | - name: Install dependencies 21 | run: | 22 | python -m pip install --upgrade pip meson-python meson ninja setuptools_scm setuptools wheel pybind11 23 | pip install --upgrade cython numpy 24 | pip install --no-build-isolation --editable .[dev,docs,notebook,pandora_mccnn] 25 | pip list 26 | - name: Unit Tests 27 | run: | 28 | pytest -m "unit_tests" -o log_cli=true -o log_cli_level=INFO 29 | - name: Lint Tests 30 | run: | 31 | echo "Lint test" 32 | isort --check cars tests 33 | black --check cars tests 34 | flake8 cars tests 35 | pylint cars tests --rcfile=.pylintrc --output-format=parseable | tee pylint-report.txt # pipefail to propagate pylint exit code in bash 36 | - name: Notebooks Tests 37 | run: | 38 | pytest -m "notebook_tests" -o log_cli=true -o log_cli_level=INFO 39 | - name: End2end Tests 40 | run: | 41 | pytest -m "end2end_tests" -o log_cli=true -o log_cli_level=INFO 42 | 43 | build_wheels: 44 | name: Build wheels on ${{ matrix.os }} 45 | runs-on: ${{ matrix.os }} 46 | strategy: 47 | matrix: 48 | os: [ubuntu-latest, windows-latest] 49 | 50 | steps: 51 | - uses: actions/checkout@v4 52 | 53 | - uses: actions/setup-python@v5 54 | 55 | - name: Install cibuildwheel 56 | run: python -m pip install cibuildwheel==2.22.0 57 | 58 | - name: Build wheels 59 | run: python -m cibuildwheel --output-dir wheelhouse 60 | 61 | - uses: actions/upload-artifact@v4 62 | with: 63 | name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} 64 | path: ./wheelhouse/*.whl 65 | 66 | build_sdist: 67 | name: Build source distribution 68 | runs-on: ubuntu-latest 69 | steps: 70 | - uses: actions/checkout@v4 71 | 72 | - name: Build sdist 73 | run: pipx run build --sdist 74 | 75 | - uses: actions/upload-artifact@v4 76 | with: 77 | name: cibw-sdist 78 | path: dist/*.tar.gz 79 | 80 | upload_pypi: 81 | name: Publish package on pypi 82 | needs: [test, build_wheels, build_sdist] 83 | runs-on: ubuntu-latest 84 | environment: pypi 85 | permissions: 86 | id-token: write 87 | if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') 88 | steps: 89 | - uses: actions/download-artifact@v4 90 | with: 91 | pattern: cibw-* 92 | path: dist 93 | merge-multiple: true 94 | 95 | - uses: pypa/gh-action-pypi-publish@release/v1 96 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python files 2 | *.py[cod] 3 | *$py.class 4 | *.pyc 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Cache python 10 | __pycache__/ 11 | 12 | # Python egg metadata, regenerated from source files by setuptools 13 | /*.egg-info 14 | /.eggs/ 15 | develop-eggs/ 16 | wheels/ 17 | .installed.cfg 18 | *.egg 19 | 20 | # Setuptools build and distribution directories 21 | /dist/ 22 | /build/ 23 | sdist/ 24 | 25 | # Virtualenv 26 | /venv/ 27 | 28 | # Tox 29 | .tox 30 | 31 | # Pytest 32 | .pytest_cache/ 33 | pytest-report.xml 34 | 35 | # Python coverage output 36 | .coverage 37 | .coverage.* 38 | coverage.xml 39 | /htmlcov/ 40 | 41 | # pylint report 42 | pylint-report.txt 43 | 44 | # Debug file logging 45 | debug.log 46 | 47 | # Documentation build and API 48 | docs/build/ 49 | docs/_build/ 50 | docs/source/api_reference/ 51 | docs/source/apidoc/ 52 | 53 | # Installer logs 54 | pip-log.txt 55 | pip-delete-this-directory.txt 56 | 57 | # Code quality 58 | pylint-report.xml 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # dotenv 82 | .env 83 | 84 | # mkdocs documentation 85 | /site 86 | 87 | # mypy 88 | .mypy_cache/ 89 | 90 | # IDE settings 91 | .vscode/ 92 | .idea/ 93 | 94 | # Swap file 95 | *.swp 96 | 97 | # tutorial data files 98 | /tutorials/data_gizeh/ 99 | /tutorials/data_gizeh_small/ 100 | 101 | # qgis auxiliary files 102 | *tif.aux.xml -------------------------------------------------------------------------------- /.gitlab/issue_templates/Bug.md: -------------------------------------------------------------------------------- 1 | Préconisations générales: 2 | 1. Pensez à bien mettre un **titre** d'issue explicite par rapport au bug 3 | 2. Pensez à essayer au maximum d'aider le support (best effort) avec des informations les plus claires et précises 4 | 3. Mettre le bon label 5 | 4. Indiquer la criticité du bug : Urgent, Important, Moyen, Peu, Pas du tout. 6 | 7 | /label ~"Kind - Bug" 8 | 9 | ### Constat 10 | (ce qui permet de dire qu'il y a un bug) 11 | 12 | ### Contexte 13 | (ce qui permet de rejouer le bug) 14 | 15 | - Version utilisée (cars --version): 16 | 17 | - Contexte d'utilisation: module, cars-hal, source 18 | 19 | - Commande utilisée: 20 | 21 | #### Configuration et données utilisées 22 | 23 | *Copier le input.json(prepare) ou le content.json (compute_dsm) (si applicable)* 24 | 25 | *Pensez à vérifier l'accès par les autres à la donnée en entrée ou copiez dans* 26 | 27 | #### Environnement 28 | *Copier l'environnement python* 29 | 30 | ``` 31 | pip freeze 32 | ``` 33 | *Lister l'environnement modules* 34 | ``` 35 | module list 36 | ``` 37 | 38 | 39 | ### Pistes potentielles pouvant expliquer l'origine du bug 40 | 41 | 42 | ### Pistes potentielles pouvant corriger le bug 43 | -------------------------------------------------------------------------------- /.gitlab/issue_templates/Proposal.md: -------------------------------------------------------------------------------- 1 | 1. Pensez à bien mettre un titre d'issue explicite par rapport au bug 2 | 2. Pensez à essayer au maximum d'aider le support (non garanti) avec des informations les plus claires et précises 3 | 3. Pensez à mettre le label 4 | 5 | /label ~"Kind - Proposal" 6 | 7 | ### Constat 8 | (ce qui motive cette proposition) 9 | 10 | ### Proposition 11 | 12 | **Résumé** 13 | 14 | *Pensez à ajouter les labels associés à la proposition* 15 | 16 | 17 | **Détails techniques de la proposition si applicable** 18 | 19 | 20 | -------------------------------------------------------------------------------- /.gitlab/issue_templates/Refacto.md: -------------------------------------------------------------------------------- 1 | Préconisations générales: 2 | 1. Pensez à bien mettre un **titre** d'issue explicite par rapport au domaine de refactoring/reconception. 3 | 2. Pensez à essayer au maximum d'aider un développeur qui devra reprendre ce travail ou relire avec des informations synthétiques les plus claires et précises 4 | 5 | Il est important d'essayer de se mettre à la place de quelqu'un d'autre qui relira, ou dans la perspective d'une présentation à quelqu'un d'autre pour transférer le travail. 6 | 7 | /label ~"Kind - Refacto" 8 | 9 | ### Contexte 10 | Cette section donne le contexte général du refactoring prévu. 11 | 12 | ### Documentation API et fonctionnel 13 | 14 | Cette partie décrit le fonctionnement du module/domaine visé par l'issue. 15 | 16 | 1. décrire bien ce que fait le bloc logiciel à haut niveau avec API utilisateurs et internes (nom fonctions, paramètres, noms de structures de données (classe ou autre) explicites ) 17 | 2. décrire fonctionnellement les blocs avec le principe de base à haut niveau. 18 | 19 | Des schémas UML sont toujours bien pour des classes. 20 | 21 | Les objectifs: 22 | - qu'un utilisateur sache exactement ce que fait le module par l'API 23 | - qu'un autre développeur puisse comprendre/relire/reprendre le travail rapidement 24 | 25 | ### Plan de validation / tests 26 | 27 | Cette section décrit la manière de tester le domaine fonctionnel du refactoring prévu. 28 | 29 | En cohérence avec la documentation fonctionnel, cette section doit décrire **précisément** la façon de tester le module logiciel visé. 30 | 31 | Il est important de considérer: 32 | - les tests unitaires pour les fonctions de base 33 | - les tests fonctionnels du module : que doit on voir fonctionner pour que le module soit valide ? 34 | - considérer le temps de calcul et séparer si des tests sont trop lourds 35 | 36 | Utiliser les @pytest.mark pour organiser les tests suivant la découpe d'organisations des tests choisie. 37 | -------------------------------------------------------------------------------- /.gitlab/issue_templates/Release.md: -------------------------------------------------------------------------------- 1 | Release 2 | 3 | /label ~"Kind - Release" 4 | 5 | Liste de points à vérifier/faire pour la release en cours: 6 | 7 | - [ ] Vérifier issues milestone (mettre lien milestone avec %nom_milestone) 8 | - [ ] Finaliser Changelog de la version en cours: Vérifier en comparant avec les issues/MR du milestone de la version. 9 | - [ ] Mise du tag sur la version finale après dernieres MR. 10 | - [ ] Vérification Publication code sur github, read the docs, pypi 11 | - [ ] Dernière Vérification si installation, tests, ... (relance CI ou manuellement si pas automatique dans CI) 12 | - [ ] Tests et Upgrade cars-hal 13 | - [ ] Ajout dans module load si necessaire suivant projet 14 | - [ ] Génération image docker et Publication du docker (pas automatique pour l'instant) 15 | - [ ] Communication sur la release (si nécessaire mailing list ou autre) 16 | -------------------------------------------------------------------------------- /.gitlab/merge_request_templates/MR.md: -------------------------------------------------------------------------------- 1 | **A choisir dans le template les éléments à garder suivant le développement à faire.** 2 | 3 | Si la Merge Request est en cours de développement, merci d'ajouter le mot clé `WIP` ou `Draft` afin d'éviter la fusion de cette dernière. 4 | 5 | #### Résumé de la proposition 6 | 7 | 1. Que fait la MR ? (bien etre explicite sur le titre) 8 | 2. Liste des taches de la MR qui répondent à l'issue (partiellement ou globalement suivant si Closes ou Relates) 9 | 3. Etat du Reste à faire à reprendre pour finir l'issue 10 | 4. Lien vers l'issue source (Closes si la MR ferme l'issue, Relates si en fait une partie) 11 | 12 | A choisir: 13 | Closes #num_issue 14 | Relates #num_issue 15 | 16 | #### Détails techniques de l'implémentation 17 | 18 | Cette section décrit les solutions techniques que la MR met en oeuvre et qui répondent à la description fonctionnelle de l'issue associée. 19 | 20 | #### Stratégie de validation 21 | 22 | En lien avec le plan de validation de l'issue, il faut décrire la stratégie de validation dont s'occupe la MR. 23 | Au moins : 24 | - [ ] Tests Unitaires (obligatoire) 25 | - [ ] Tests Fonctionnels (intégration / interfaces avec d'autres outils) 26 | - [ ] Tests Performances 27 | 28 | Si besoin suivant l'issue/MR: 29 | - [ ] Tests Visuels ? (doc) 30 | - [ ] Tests Comparatifs ? (`feat` métier avec outil de référence) 31 | - dans ce cas citer les outils et leur paramétrage 32 | 33 | 34 | #### Validation de la MR 35 | 36 | Si la Merge Request est prête, merci de valider les étapes suivantes: 37 | - [ ] mise à jour de la documentation Sphinx et vérification du résultat. 38 | - [ ] tous les tests passent et la MR est couverte par la stratégie de validation 39 | - [ ] mise à jour du changelog Changelog.md 40 | - uniquement si la MR rempli l'un des objectifs suivants: 41 | - correction d'un bug 42 | - ajout d'une fonctionnalité métier 43 | - ajout d'une fonctionnalité à destination du grand public (communication) 44 | - suivre les recommandations de https://github.com/olivierlacan/keep-a-changelog/blob/master/CHANGELOG.md 45 | - inscrire la modification sous le titre `Unreleased` 46 | - [ ] S'assurer qu'il y a toutes les infos dans la MR pour qu'un autre développeur puisse relire facilement, ce qu'on s'attendrait à avoir soi même pour relire la MR (cf résumé ci dessus) 47 | 48 | #### Rappel Intégration Continue 49 | 50 | Pour relancer l'intégration continue merci de laisser le commentaire suivant : 51 | `Jenkins! Faster!! And you'd better make it work !` 52 | 53 | 54 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: local 3 | hooks: 4 | - id: isort 5 | name: isort 6 | stages: [pre-commit] 7 | language: system 8 | entry: isort 9 | types: [python] 10 | 11 | - id: black 12 | name: black 13 | stages: [pre-commit] 14 | language: system 15 | entry: black 16 | types: [python] 17 | 18 | - id: flake8 19 | name: flake8 20 | stages: [pre-commit] 21 | language: system 22 | entry: flake8 23 | types: [python] 24 | 25 | - id: pylint 26 | name: PyLint 27 | stages: [pre-commit] 28 | language: system 29 | entry: pylint --rcfile=.pylintrc 30 | files: \.py$ 31 | 32 | - id: sphinx-build 33 | name: Build Sphinx Doc 34 | stages: [pre-push] 35 | entry: make docs 36 | language: system 37 | files: ^(docs/.*|cars/.*)$ 38 | pass_filenames: False 39 | 40 | - id: jupyter-nb-clear-output 41 | name: jupyter-nb-clear-output 42 | files: \.ipynb$ 43 | stages: [pre-commit] 44 | language: system 45 | entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | version: 2 5 | 6 | # Set os required and python version (required) 7 | build: 8 | os: "ubuntu-22.04" 9 | tools: 10 | python: "3.10" 11 | 12 | # add projet [docs] extra url dependencies in sphinx RTD virtualenv 13 | python: 14 | install: 15 | - method: pip 16 | path: . 17 | extra_requirements: 18 | - docs 19 | 20 | # Build documentation in the docs/ directory with Sphinx 21 | sphinx: 22 | configuration: docs/source/conf.py -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Credits 2 | 3 | Copyright (c) 2022 Centre National d'Etudes Spatiales (CNES). 4 | 5 | CARS is licensed under permissive Apache 2 license (See LICENSE file). 6 | The copyright is kept CNES only for long term maintenance ease. 7 | 8 | See CONTRIBUTING.md for more details on Contributor License Agreement. 9 | 10 | This file keeps track of authors contributions. 11 | 12 | 13 | ## Development Lead 14 | 15 | * David Youssefi 16 | * Valentine Bellet 17 | * Yoann Steux 18 | * Mathis Roux 19 | * Cedric Traizet 20 | 21 | ## Contributors 22 | 23 | * Julien Michel 24 | * Emmanuelle Sarrazin 25 | * Emmanuel Dubois 26 | * Aurélie Emilien 27 | * Florian Douziech 28 | * Loïc Dumas 29 | * Marina Bertolino 30 | * Guillaume Pasero 31 | * Jonathan Guinet 32 | * Jasmin Siefert 33 | * Quentin Fardet 34 | * Teo Bouvard 35 | * Fabien Servoz 36 | * Natalia Jimenez Diaz 37 | * Roman Malinowski 38 | * Alice de Bardonneche-Richard 39 | * Tommy Calendini 40 | * Dimitri Lallement 41 | * Datafalk 42 | * Marian Rassat 43 | 44 | Update here with new contributors. 45 | 46 | ## Original Developers/Designers/Supporters 47 | 48 | * Julien Michel 49 | * David Youssefi 50 | * Emmanuelle Sarrazin 51 | * Myriam Cournet 52 | * Fabrice Buffe 53 | * Aurélie Emilien 54 | * Jean-Marc Delvit 55 | * Olivier Melet 56 | * Céline L'Helguen 57 | * Julien Bosman 58 | 59 | See [README References](README.md#documentation) 60 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # **CARS** **Contributing guide**. 2 | 3 | 1. [Bug report](#bug-report) 4 | 2. [Contributing workflow](#contributing-workflow) 5 | 3. [Contribution license agreement](#contribution-license-agreement) 6 | 4. [Coding guide](#coding-guide) 7 | 5. [Pylint pre-commit validation](#pylint-pre-commit-validation) 8 | 6. [Merge request acceptation process](#merge-request-acceptation-process) 9 | 10 | # Bug report 11 | 12 | Any proven or suspected malfunction should be traced in a bug report, the latter being an issue in the CARS github repository. 13 | 14 | **Don't hesitate to do so: It is best to open a bug report and quickly resolve it than to let a problem remains in the project.** 15 | **Notifying the potential bugs is the first way for contributing to a software.** 16 | 17 | In the problem description, be as accurate as possible. Include: 18 | * The procedure used to initialize the environment 19 | * The incriminated command line or python function 20 | * The content of the input and output configuration files (`content.json`) 21 | 22 | # Contributing workflow 23 | 24 | Any code modification requires a Merge Request. It is forbidden to push patches directly into master (this branch is protected). 25 | 26 | It is recommended to open your Merge Request as soon as possible in order to inform the developers of your ongoing work. 27 | Please add `WIP:` before your Merge Request title if your work is in progress: This prevents an accidental merge and informs the other developers of the unfinished state of your work. 28 | 29 | The Merge Request shall have a short description of the proposed changes. If it is relative to an issue, you can signal it by adding `Closes xx` where xx is the reference number of the issue. 30 | 31 | Likewise, if you work on a branch (which is recommended), prefix the branch's name by `xx-` in order to link it to the xx issue. 32 | 33 | CARS Classical workflow is : 34 | * Check Licence and sign [Contributor Licence Agreement](#contribution-license-agreement) (Individual or Corporate) 35 | * Create an issue (or begin from an existing one) 36 | * Create a Merge Request from the issue: a MR is created accordingly with "WIP:", "Closes xx" and associated "xx-name-issue" branch 37 | * CARS hacking code from a local working directory or from the forge (less possibilities) following [Developer manual](./docs/source/developer.rst) 38 | * Git add, commit and push from local working clone directory or from the forge directly 39 | * Follow [Conventional commits](https://www.conventionalcommits.org/) specifications for commit messages 40 | * Beware that quality pre-commit tools are installed in continuous integration with classical quality code tools (see [Developer manual](./docs/source/developer.rst)). 41 | * Launch the [tests](./docs/source/developer.rst) on your modifications (or don't forget to add ones). 42 | * When finished, change your Merge Request name (erase "WIP:" in title ) and ask `@cars` to review the code (see below Merge request acceptation process) 43 | 44 | 45 | # Contribution license agreement 46 | 47 | CARS requires that contributors sign out a [Contributor License 48 | Agreement](https://en.wikipedia.org/wiki/Contributor_License_Agreement). The 49 | purpose of this CLA is to ensure that the project has the necessary ownership or 50 | grants of rights over all contributions to allow them to distribute under the 51 | chosen license (Apache License Version 2.0) 52 | 53 | To accept your contribution, we need you to complete, sign and email to *cars [at] 54 | cnes [dot] fr* an [Individual Contributor Licensing 55 | Agreement](./docs/source/CLA/ICLA-CARS.doc) (ICLA) form and a 56 | [Corporate Contributor Licensing 57 | Agreement](./docs/source/CLA/CCLA-CARS.doc) (CCLA) form if you are 58 | contributing on behalf of your company or another entity which retains copyright 59 | for your contribution. 60 | 61 | The copyright owner (or owner's agent) must be mentioned in headers of all modified source files and also added to the [AUTHORS.md 62 | file](./AUTHORS.md). 63 | 64 | 65 | # Merge request acceptation process 66 | 67 | The Merge Request will be merged into master after being reviewed by CARS steering committee (core committers) composed of: 68 | * David Youssefi (CNES) 69 | * Emmanuelle Sarrazin (CNES) 70 | * Emmanuel Dubois (CNES) 71 | 72 | Only the members of this committee can merge into master. 73 | 74 | The checklist of a Merge Request acceptance is the following: 75 | * [ ] At least one code review has been done by members of the group above (who check among others the correct application of the rules listed in the [Coding guide](#coding-guide)). 76 | * [ ] All comments of the reviewers has been dealt with and are closed 77 | * [ ] The reviewers have signaled their approbation (thumb up) 78 | * [ ] No reviewer is against the Merge Request (thumb down) 79 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # hadolint ignore=DL3007 2 | FROM orfeotoolbox/otb:latest 3 | LABEL maintainer="CNES" 4 | 5 | # Dependencies packages 6 | # hadolint ignore=DL3008 7 | RUN apt-get update \ 8 | && apt-get install --no-install-recommends software-properties-common -y \ 9 | && add-apt-repository ppa:deadsnakes/ppa && rm -rf /var/lib/apt/lists/* 10 | 11 | # Python 3.10 12 | # hadolint ignore=DL3008 13 | RUN apt-get update && apt-get install --no-install-recommends -y --quiet git python3.10-dev \ 14 | && rm -rf /var/lib/apt/lists/* 15 | RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 16 | 17 | # Install dependancies 18 | SHELL ["/bin/bash", "-o", "pipefail", "-c"] 19 | # hadolint ignore=DL3013 20 | RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 \ 21 | && python3 -m pip install --no-cache-dir --upgrade requests \ 22 | && python3 -m pip install --no-cache-dir --no-binary fiona fiona \ 23 | && python3 -m pip install --no-cache-dir --no-binary rasterio rasterio 24 | 25 | # Install cars: from source or from pypi if version 26 | ARG version 27 | # hadolint ignore=DL3003,DL3013 28 | RUN if [ -z "$version" ] ; then git clone --depth 1 https://github.com/CNES/cars.git && cd cars && python3 -m pip install --no-cache-dir build && python3 -m build && python3 -m pip install --no-cache-dir dist/*.whl && cd - && rm -rf cars; \ 29 | else python3 -m pip install --no-cache-dir cars==$version; fi 30 | 31 | # Launch cars 32 | ENTRYPOINT ["cars"] 33 | CMD ["-h"] 34 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | prune docs 2 | prune tests 3 | prune tutorials -------------------------------------------------------------------------------- /cars/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | Cars module init file 23 | """ 24 | 25 | import os 26 | import sys 27 | from importlib.metadata import PackageNotFoundError, version 28 | 29 | try: 30 | __version__ = version("cars") 31 | except PackageNotFoundError: 32 | # package is not installed 33 | __version__ = "unknown" 34 | 35 | # Standard imports 36 | if sys.version_info < (3, 10): 37 | from importlib_metadata import entry_points 38 | else: 39 | from importlib.metadata import entry_points 40 | 41 | 42 | __author__ = "CNES" 43 | __email__ = "cars@cnes.fr" 44 | 45 | # Force the use of CARS dask configuration 46 | dask_config_path = os.path.join( 47 | os.path.dirname(__file__), "orchestrator", "cluster", "dask_config" 48 | ) 49 | if not os.path.isdir(dask_config_path): 50 | raise NotADirectoryError("Wrong dask config path") 51 | os.environ["DASK_CONFIG"] = str(dask_config_path) 52 | 53 | # Force monothread for child workers 54 | os.environ["PANDORA_NUMBA_PARALLEL"] = str(False) 55 | os.environ["PANDORA_NUMBA_CACHE"] = str(False) 56 | os.environ["SHARELOC_NUMBA_PARALLEL"] = str(False) 57 | os.environ["OMP_NUM_THREADS"] = "1" 58 | os.environ["MKL_NUM_THREADS"] = "1" 59 | os.environ["GDAL_NUM_THREADS"] = "1" 60 | 61 | # Limit GDAL cache per worker to 500MB 62 | os.environ["GDAL_CACHEMAX"] = "500" 63 | 64 | 65 | def import_plugins() -> None: 66 | """ 67 | Load all the registered entry points 68 | :return: None 69 | """ 70 | for entry_point in entry_points(group="cars.plugins"): 71 | entry_point.load() 72 | 73 | 74 | import_plugins() 75 | -------------------------------------------------------------------------------- /cars/applications/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS application module init file 23 | """ 24 | 25 | # Imports needed in order to register application for Application factory 26 | from . import auxiliary_filling # noqa: F401 27 | from . import dem_generation # noqa: F401 28 | from . import dense_match_filling # noqa: F401 29 | from . import dense_matching # noqa: F401 30 | from . import dsm_filling # noqa: F401 31 | from . import grid_generation # noqa: F401 32 | from . import ground_truth_reprojection # noqa: F401 33 | from . import hole_detection # noqa: F401 34 | from . import point_cloud_denoising # noqa: F401 35 | from . import point_cloud_fusion # noqa: F401 36 | from . import point_cloud_outlier_removal # noqa: F401 37 | from . import rasterization # noqa: F401 38 | from . import resampling # noqa: F401 39 | from . import sparse_matching # noqa: F401 40 | from . import triangulation # noqa: F401 41 | -------------------------------------------------------------------------------- /cars/applications/application.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | """ 23 | This module contains class application factory. 24 | """ 25 | 26 | # Standard imports 27 | import logging 28 | 29 | # CARS imports 30 | from cars.conf.input_parameters import ConfigType 31 | 32 | 33 | class Application: 34 | """ 35 | Application factory: 36 | A class designed for registered all available Cars application and 37 | instantiate when needed. 38 | 39 | """ 40 | 41 | # Dict (application_name:str, class: object) containing registered 42 | # applications 43 | available_applications = {} 44 | 45 | def __new__( 46 | cls, 47 | app_name: str, 48 | cfg: ConfigType = None, 49 | ): 50 | """ 51 | Return the instance of application associated with the application 52 | name given as parameter 53 | 54 | :param app_name: name of the application. 55 | :type app_name: str 56 | :param cfg: configuration {'matching_cost_method': value} 57 | :type cfg: dictionary 58 | """ 59 | 60 | return cls.create_app(app_name, cfg) 61 | 62 | @classmethod 63 | def create_app(cls, name: str, cfg: ConfigType): 64 | """Factory command to create the application 65 | Return the instance of application associated with the application 66 | name given as parameter 67 | 68 | :param app_name: name of the application. 69 | :type app_name: str 70 | :param cfg: configuration {'matching_cost_method': value} 71 | :type cfg: dictionary 72 | """ 73 | app = None 74 | 75 | try: 76 | app_class = cls.available_applications[name] 77 | except KeyError: 78 | logging.error("No application named {0} supported".format(name)) 79 | return None 80 | app = app_class(conf=cfg) 81 | return app 82 | 83 | @classmethod 84 | def print_applications(cls): 85 | """ 86 | Print all registered applications 87 | """ 88 | 89 | for app_name in cls.available_applications: 90 | print(app_name) 91 | 92 | @classmethod 93 | def register(cls, app_name: str): 94 | """ 95 | Allows to register the application with its name 96 | :param app_name: the application to be registered 97 | :type app_name: string 98 | """ 99 | 100 | def decorator(app): 101 | """ 102 | Registers the class in the available methods 103 | :param app: the app class to be registered 104 | :type app: object 105 | """ 106 | cls.available_applications[app_name] = app 107 | return app 108 | 109 | return decorator 110 | -------------------------------------------------------------------------------- /cars/applications/application_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | application_constants contains constants used in application module 23 | """ 24 | 25 | 26 | APPLICATION_TAG = "applications" 27 | 28 | # common parameters 29 | SAVE_INTERMEDIATE_DATA = "save_intermediate_data" 30 | -------------------------------------------------------------------------------- /cars/applications/application_template.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | """ 23 | This module contains class application template for 24 | templating the application concept. 25 | Useful for shared parameters and functions in all applications 26 | Beware: argument-differ is activated in pylintrc for run parameters different 27 | in sub application classes 28 | """ 29 | 30 | # Standard imports 31 | import logging 32 | import pprint 33 | from abc import ABCMeta, abstractmethod 34 | 35 | # CARS imports 36 | from cars.conf.input_parameters import ConfigType 37 | 38 | 39 | class ApplicationTemplate(metaclass=ABCMeta): 40 | """ 41 | Class for general specification of an application 42 | Empty for the moment because there is no any common method or function 43 | """ 44 | 45 | # Define abstract attributes 46 | used_config: ConfigType 47 | 48 | @abstractmethod 49 | def run(self, *args, **kwargs): 50 | """ 51 | Generic run() function to be defined in subclasses 52 | """ 53 | 54 | def __init__(self, conf=None): # pylint: disable=W0613 55 | """ 56 | Init function of ApplicationTemplate 57 | 58 | :param conf: configuration for application 59 | 60 | """ 61 | # Check conf 62 | try: 63 | self.used_config = self.check_conf(conf) 64 | except Exception: 65 | logging.error( 66 | "The {} application checking has been failed!".format( 67 | self.__class__.__bases__[0].__name__ 68 | ) 69 | ) 70 | raise 71 | 72 | @abstractmethod 73 | def check_conf(self, conf): 74 | """ 75 | Check configuration 76 | 77 | :param conf: configuration to check 78 | :type conf: dict 79 | 80 | :return: overloaded configuration 81 | :rtype: dict 82 | 83 | """ 84 | 85 | def print_config(self): 86 | """ 87 | Print used application configuration 88 | 89 | """ 90 | pretty_printer = pprint.PrettyPrinter(indent=4) 91 | 92 | try: 93 | pretty_printer.pprint(self.used_config) 94 | except Exception: 95 | logging.error("self.used_config not filled by application") 96 | 97 | def get_conf(self): 98 | """ 99 | Get used conf 100 | 101 | :return: used conf 102 | """ 103 | 104 | return self.used_config 105 | -------------------------------------------------------------------------------- /cars/applications/auxiliary_filling/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2024 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS auxiliary filling module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | import cars.applications.auxiliary_filling.auxiliary_filling 27 | from cars.applications.auxiliary_filling.auxiliary_filling_from_sensors import ( 28 | AuxiliaryFilling, 29 | ) 30 | -------------------------------------------------------------------------------- /cars/applications/auxiliary_filling/auxiliary_filling.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2024 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the abstract AuxiliaryFilling application class. 23 | """ 24 | 25 | import logging 26 | from abc import ABCMeta, abstractmethod 27 | from typing import Dict 28 | 29 | from cars.applications.application import Application 30 | from cars.applications.application_template import ApplicationTemplate 31 | 32 | 33 | @Application.register("auxiliary_filling") 34 | class AuxiliaryFilling(ApplicationTemplate, metaclass=ABCMeta): 35 | """ 36 | AuxiliaryFilling abstract class 37 | """ 38 | 39 | available_applications: Dict = {} 40 | default_application = "auxiliary_filling_from_sensors" 41 | 42 | def __new__(cls, conf=None): # pylint: disable=W0613 43 | """ 44 | Return the required application 45 | :raises: 46 | - KeyError when the required application is not registered 47 | 48 | :param conf: configuration for auxiliary_filling 49 | :return: an application_to_use object 50 | """ 51 | 52 | auxiliary_filling_method = cls.default_application 53 | 54 | if bool(conf) is False or "method" not in conf: 55 | logging.info( 56 | "Auxiliary filling method not specified, " 57 | "default {} is used".format(auxiliary_filling_method) 58 | ) 59 | else: 60 | auxiliary_filling_method = conf["method"] 61 | 62 | if auxiliary_filling_method not in cls.available_applications: 63 | logging.error( 64 | "No auxiliary_filling application named {} registered".format( 65 | auxiliary_filling_method 66 | ) 67 | ) 68 | raise KeyError( 69 | "No auxiliary_filling application named {} registered".format( 70 | auxiliary_filling_method 71 | ) 72 | ) 73 | 74 | logging.info( 75 | "The AuxiliaryFilling({}) application will be used".format( 76 | auxiliary_filling_method 77 | ) 78 | ) 79 | 80 | return super(AuxiliaryFilling, cls).__new__( 81 | cls.available_applications[auxiliary_filling_method] 82 | ) 83 | 84 | def __init_subclass__(cls, short_name, **kwargs): # pylint: disable=E0302 85 | super().__init_subclass__(**kwargs) 86 | cls.available_applications[short_name] = cls 87 | 88 | @abstractmethod 89 | def run( 90 | self, dsm_file, color_file, classif_file, dump_dir, orchestrator=None 91 | ): 92 | """ 93 | Run Auxiliary filling 94 | """ 95 | -------------------------------------------------------------------------------- /cars/applications/dem_generation/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS dem generation init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.dem_generation.dem_generation import DemGeneration 27 | 28 | from . import dichotomic_generation, rasterization 29 | -------------------------------------------------------------------------------- /cars/applications/dem_generation/bulldozer_config/base_config.yaml: -------------------------------------------------------------------------------- 1 | #-------------------------# 2 | # Parameters # 3 | #-------------------------# 4 | # [Required] - Input DSM path (expected format: "//.<[tif/tiff]>") 5 | dsm_path : "input_dsm_path/dsm.tif" 6 | # [Required] - Output directory path (if the directory doesn't exist, create it) 7 | output_dir : "output_dir_path" 8 | 9 | #-------------------------# 10 | # Options # 11 | #-------------------------# 12 | # [Optional] - If True, generates the Digital Height Model (DHM=DSM-DTM) in the output directory 13 | generate_dhm : True 14 | # [Optional] - Foreground max object size (in meter) 15 | max_object_size : 16 16 | # [Optional] - Path to the binary ground classification mask (expected format: "//.<[tif/tiff]>") 17 | ground_mask_path : null 18 | # [Optional] - If True, activate ground anchor detection (ground pre-detection) 19 | activate_ground_anchors : False 20 | # [Optional] - Max number of CPU core to use. If null, use maximum number of available CPU core 21 | nb_max_workers : null 22 | # [Optional] - If True, keep the intermediate results 23 | developer_mode : False 24 | 25 | #-------------------------# 26 | # Expert options # 27 | #-------------------------# 28 | # /!\ Modify those data at your own risk (it is suggested to keep the default values) /!\ 29 | 30 | # [Optional] - Number of regular mask filtering iterations. If null, use the default value: max_object_size/4 31 | reg_filtering_iter: null 32 | # [Optional] - Altimetric height accuracy of the input DSM (m). If null, use the default value: 2*planimetric resolution 33 | dsm_z_accuracy: null 34 | # [Optional] - Maximum slope of the observed landscape terrain (%) 35 | max_ground_slope: 20.0 36 | # [Optional] - Number of unhook iterations 37 | prevent_unhook_iter : 10 38 | # [Optional] - Number of gravity step iterations 39 | num_outer_iter : 25 40 | # [Optional] - Number of tension iterations 41 | num_inner_iter : 5 42 | 43 | -------------------------------------------------------------------------------- /cars/applications/dem_generation/dem_generation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | This module contains the abstract dem_generation application class. 23 | """ 24 | import logging 25 | from abc import ABCMeta, abstractmethod 26 | from typing import Dict 27 | 28 | from cars.applications.application import Application 29 | from cars.applications.application_template import ApplicationTemplate 30 | 31 | 32 | @Application.register("dem_generation") 33 | class DemGeneration(ApplicationTemplate, metaclass=ABCMeta): 34 | """ 35 | DemGeneration 36 | """ 37 | 38 | available_applications: Dict = {} 39 | default_application = "bulldozer_on_raster" 40 | 41 | def __new__(cls, conf=None): # pylint: disable=W0613 42 | """ 43 | Return the required application 44 | :raises: 45 | - KeyError when the required application is not registered 46 | 47 | :param orchestrator: orchestrator used 48 | :param conf: configuration for resampling 49 | :return: an application_to_use object 50 | """ 51 | 52 | dem_generation_method = cls.default_application 53 | if bool(conf) is False or "method" not in conf: 54 | logging.info( 55 | "MntGeneration method not specified, default" 56 | " {} is used".format(dem_generation_method) 57 | ) 58 | else: 59 | dem_generation_method = conf["method"] 60 | 61 | if dem_generation_method not in cls.available_applications: 62 | logging.error( 63 | "No dem_generation application named {} registered".format( 64 | dem_generation_method 65 | ) 66 | ) 67 | raise KeyError( 68 | "No dem_generation application named {} registered".format( 69 | dem_generation_method 70 | ) 71 | ) 72 | 73 | logging.info( 74 | "The DemGeneration {} application will be used".format( 75 | dem_generation_method 76 | ) 77 | ) 78 | 79 | return super(DemGeneration, cls).__new__( 80 | cls.available_applications[dem_generation_method] 81 | ) 82 | 83 | def __init_subclass__(cls, short_name, **kwargs): # pylint: disable=E0302 84 | super().__init_subclass__(**kwargs) 85 | cls.available_applications[short_name] = cls 86 | 87 | def __init__(self, conf=None): 88 | """ 89 | Init function of MntGeneration 90 | 91 | :param conf: configuration 92 | :return: an application_to_use object 93 | """ 94 | 95 | super().__init__(conf=conf) 96 | 97 | @abstractmethod 98 | def run(self, triangulated_matches_list, output_dir): 99 | """ 100 | Run dem generation using matches 101 | 102 | :param triangulated_matches_list: list of triangulated matches 103 | positions must be in a metric system 104 | :type triangulated_matches_list: list(pandas.Dataframe) 105 | :param output_dir: directory to save dem 106 | :type output_dir: str 107 | 108 | :return: dem data computed with mean, min and max. 109 | dem is also saved in disk, and paths are available in attributes. 110 | (DEM_MEDIAN_PATH, DEM_MIN_PATH, DEM_MAX_PATH) 111 | :rtype: CarsDataset 112 | """ 113 | -------------------------------------------------------------------------------- /cars/applications/dem_generation/dem_generation_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the dem generation constants. 23 | """ 24 | 25 | 26 | DEM_MEDIAN = "dem_median" 27 | DEM_MIN = "dem_min" 28 | DEM_MAX = "dem_max" 29 | 30 | DEM_MEDIAN_PATH = "dem_median_path" 31 | DEM_MIN_PATH = "dem_min_path" 32 | DEM_MAX_PATH = "dem_max_path" 33 | -------------------------------------------------------------------------------- /cars/applications/dem_generation/dem_generation_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains tools for the dem generation 23 | """ 24 | 25 | import contextlib 26 | import logging 27 | import os 28 | 29 | # Third party imports 30 | import xdem 31 | from rasterio.coords import BoundingBox 32 | 33 | 34 | def fit_initial_elevation_on_dem_median( 35 | dem_to_fit_path: str, dem_ref_path: str, dem_out_path: str 36 | ): 37 | """ 38 | Coregistrates the two DEMs given then saves the result. 39 | The initial elevation will be cropped to reduce computation costs. 40 | Returns the transformation applied. 41 | 42 | :param dem_to_fit_path: Path to the dem to be fitted 43 | :type dem_to_fit_path: str 44 | :param dem_ref_path: Path to the dem to fit onto 45 | :type dem_ref_path: str 46 | :param dem_out_path: Path to save the resulting dem into 47 | :type dem_out_path: str 48 | 49 | :return: coregistration transformation applied 50 | :rtype: dict 51 | """ 52 | # suppress all outputs of xdem 53 | with open(os.devnull, "w", encoding="utf8") as devnull: 54 | with ( 55 | contextlib.redirect_stdout(devnull), 56 | contextlib.redirect_stderr(devnull), 57 | ): 58 | 59 | # load DEMs 60 | dem_to_fit = xdem.DEM(dem_to_fit_path) 61 | dem_ref = xdem.DEM(dem_ref_path) 62 | 63 | # get the crs needed to reproject the data 64 | crs_out = dem_ref.crs 65 | crs_metric = dem_ref.get_metric_crs() 66 | 67 | # Crop dem_to_fit with dem_ref to reduce 68 | # computation costs. 69 | bbox = dem_ref.bounds 70 | bbox = add_margin(bbox) 71 | dem_to_fit = dem_to_fit.crop(bbox).reproject(crs=crs_metric) 72 | # Reproject dem_ref to dem_to_fit resolution to reduce 73 | # computation costs 74 | dem_ref = dem_ref.reproject(dem_to_fit) 75 | bbox = dem_ref.bounds 76 | bbox = add_margin(bbox) 77 | 78 | coreg_pipeline = xdem.coreg.NuthKaab() 79 | 80 | try: 81 | # fit dem_to_fit onto dem_ref, crop it, then reproject it 82 | # set a random state to always get the same results 83 | fit_dem = ( 84 | coreg_pipeline.fit_and_apply( 85 | dem_ref, dem_to_fit, random_state=0 86 | ) 87 | .crop(bbox) 88 | .reproject(crs=crs_out) 89 | ) 90 | # save the results 91 | fit_dem.save(dem_out_path) 92 | coreg_offsets = coreg_pipeline.meta["outputs"]["affine"] 93 | except (ValueError, AssertionError, TypeError): 94 | logging.warning( 95 | "xDEM coregistration failed. This can happen when sensor " 96 | "images are too small. No shift will be applied on DEM" 97 | ) 98 | coreg_offsets = None 99 | 100 | return coreg_offsets 101 | 102 | 103 | def add_margin(bbox, ratio=1): 104 | """ 105 | Add margin to a bounding box 106 | :param bbox: input bounding box 107 | :type bbox: rasterio.coords.BoundingBox 108 | :param ratio: factor of bbox size to add to each side of bbox 109 | :type ratio: float 110 | 111 | :return: bounding box with margins 112 | :rtype: rasterio.coords.BoundingBox 113 | """ 114 | try: 115 | assert bbox.left < bbox.right 116 | assert bbox.bottom < bbox.top 117 | width = bbox.right - bbox.left 118 | height = bbox.top - bbox.bottom 119 | new_left = bbox.left - ratio * width 120 | new_right = bbox.right + ratio * width 121 | new_bottom = bbox.bottom - ratio * height 122 | new_top = bbox.top + ratio * height 123 | new_bbox = BoundingBox(new_left, new_bottom, new_right, new_top) 124 | except AssertionError: 125 | logging.warning("Bounding box {} cannot be read".format(bbox)) 126 | new_bbox = bbox 127 | return new_bbox 128 | -------------------------------------------------------------------------------- /cars/applications/dense_match_filling/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core fill disparity map module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.dense_match_filling.dense_match_filling import ( 27 | DenseMatchFilling, 28 | ) 29 | 30 | from . import plane, zero_padding 31 | -------------------------------------------------------------------------------- /cars/applications/dense_match_filling/cpp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/cars/d86b7491ac1e7ea3627535bd571ea65df3933073/cars/applications/dense_match_filling/cpp/__init__.py -------------------------------------------------------------------------------- /cars/applications/dense_match_filling/cpp/dense_match_filling_cpp.py: -------------------------------------------------------------------------------- 1 | """ 2 | this module contains the headers of the dense_match_filling_cpp module. 3 | """ 4 | 5 | # pylint: skip-file 6 | 7 | from typing import Tuple 8 | 9 | import numpy as np 10 | 11 | 12 | def fill_disp_pandora( 13 | disp: np.ndarray, msk_fill_disp: np.ndarray, nb_directions: int 14 | ) -> Tuple[np.ndarray, np.ndarray]: 15 | """ 16 | Interpolation of the left disparity map to fill holes. 17 | Interpolate invalid pixel by finding the nearest correct pixels in 18 | 8/16 different directions and use the median of their disparities. 19 | ?bontar, J., & LeCun, Y. (2016). Stereo matching by training 20 | a convolutional neural network to compare image 21 | patches. The journal of machine learning research, 17(1), 2287-2318. 22 | HIRSCHMULLER, Heiko. Stereo processing by semiglobal matching 23 | and mutual information. 24 | IEEE Transactions on pattern analysis and machine intelligence, 25 | 2007, vol. 30, no 2, p. 328-341. 26 | 27 | Copied/adapted fct from pandora/validation/interpolated_disparity.py 28 | 29 | :param disp: disparity map 30 | :type disp: 2D np.array (row, col) 31 | :param msk_fill_disp: validity mask 32 | :type msk_fill_disp: 2D np.array (row, col) 33 | :param nb_directions: nb directions to explore 34 | :type nb_directions: integer 35 | 36 | :return: the interpolate left disparity map, 37 | with the validity mask update : 38 | :rtype: tuple(2D np.array (row, col), 2D np.array (row, col)) 39 | """ 40 | return None, None 41 | 42 | 43 | def find_valid_neighbors( 44 | dirs: np.ndarray, 45 | disp: np.ndarray, 46 | valid: np.ndarray, 47 | row: int, 48 | col: int, 49 | nb_directions: int, 50 | ): 51 | """ 52 | Find valid neighbors along directions 53 | 54 | Copied/adapted fct from pandora/validation/interpolated_disparity.py 55 | 56 | :param dirs: directions 57 | :type dirs: 2D np.array (row, col) 58 | :param disp: disparity map 59 | :type disp: 2D np.array (row, col) 60 | :param valid: validity mask 61 | :type valid: 2D np.array (row, col) 62 | :param row: row current value 63 | :type row: int 64 | :param col: col current value 65 | :type col: int 66 | :param nb_directions: nb directions to explore 67 | :type nb_directions: int 68 | 69 | :return: valid neighbors 70 | :rtype: 2D np.array 71 | """ 72 | ... 73 | -------------------------------------------------------------------------------- /cars/applications/dense_match_filling/cpp/includes/dense_match_filling.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DENSE_MATCH_FILLING_HPP 2 | #define DENSE_MATCH_FILLING_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace py = pybind11; 9 | 10 | /** 11 | * @brief Interpolate the left disparity map to fill holes. 12 | * Invalid pixels are interpolated by finding the nearest valid pixels in 13 | * 8 or 16 directions and taking the median of their disparities. 14 | * 15 | * @param disp Disparity map (2D array: row, col) 16 | * @param msk_fill_disp Validity mask (2D array: row, col) 17 | * @param nb_directions Number of directions to explore (8 or 16) 18 | * @return Tuple containing the interpolated disparity map and updated validity mask 19 | */ 20 | std::pair, py::array_t> fill_disp_pandora( 21 | py::array_t disp, 22 | py::array_t msk_fill_disp, 23 | int nb_directions 24 | ); 25 | 26 | /** 27 | * @brief Find valid neighbors along specified directions. 28 | * 29 | * @param dirs Directions to explore (2D array: direction x [row_offset, col_offset]) 30 | * @param disp Disparity map (2D array: row, col) 31 | * @param valid Validity mask (2D array: row, col) 32 | * @param row Current row index 33 | * @param col Current column index 34 | * @param nb_directions Number of directions to explore 35 | * @return Array of valid neighbors’ disparities along each direction 36 | */ 37 | py::array_t find_valid_neighbors( 38 | py::array_t dirs, 39 | py::array_t disp, 40 | py::array_t valid, 41 | int row, 42 | int col, 43 | int nb_directions 44 | ); 45 | 46 | #endif // DENSE_MATCH_FILLING_HPP 47 | -------------------------------------------------------------------------------- /cars/applications/dense_match_filling/cpp/meson.build: -------------------------------------------------------------------------------- 1 | 2 | py.extension_module( 3 | 'dense_match_filling_cpp', 4 | ['src/bindings.cpp', 'src/dense_match_filling.cpp'], 5 | include_directories: ['includes'], 6 | subdir: 'cars/applications/dense_match_filling/cpp', 7 | install: true, 8 | dependencies : [pybind11_dep], 9 | ) 10 | -------------------------------------------------------------------------------- /cars/applications/dense_match_filling/cpp/src/bindings.cpp: -------------------------------------------------------------------------------- 1 | #include "dense_match_filling.hpp" 2 | #include 3 | 4 | namespace py = pybind11; 5 | 6 | PYBIND11_MODULE(dense_match_filling_cpp, m) { 7 | m.doc() = "cars's pybind11 dense match filling module"; 8 | 9 | m.def("fill_disp_pandora", &fill_disp_pandora, ""); 10 | m.def("find_valid_neighbors", &find_valid_neighbors, ""); 11 | } -------------------------------------------------------------------------------- /cars/applications/dense_match_filling/cpp/src/dense_match_filling.cpp: -------------------------------------------------------------------------------- 1 | #include "dense_match_filling.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace py = pybind11; 11 | 12 | std::pair, py::array_t> fill_disp_pandora( 13 | py::array_t disp, 14 | py::array_t msk_fill_disp, 15 | int nb_directions 16 | ) { 17 | auto r_disp = disp.unchecked<2>(); 18 | auto r_msk_fill_disp = msk_fill_disp.unchecked<2>(); 19 | 20 | size_t nrow = r_disp.shape(0); 21 | size_t ncol = r_disp.shape(1); 22 | 23 | py::array_t out_disp({nrow, ncol}); 24 | py::array_t out_msk({nrow, ncol}); 25 | 26 | auto rw_out_disp = out_disp.mutable_unchecked<2>(); 27 | auto rw_out_msk = out_msk.mutable_unchecked<2>(); 28 | 29 | py::array_t dirs; 30 | if (nb_directions == 8) { 31 | std::vector data = { 32 | 0.0f, 1.0f, 33 | -1.0f, 1.0f, 34 | -1.0f, 0.0f, 35 | -1.0f, -1.0f, 36 | 0.0f, -1.0f, 37 | 1.0f, -1.0f, 38 | 1.0f, 0.0f, 39 | 1.0f, 1.0f 40 | }; 41 | 42 | dirs = py::array({8, 2}, data.data()); 43 | } else if (nb_directions == 16) { 44 | std::vector data = { 45 | 0.0f, 1.0f, -0.5f, 1.0f, 46 | -1.0f, 1.0f, -1.0f, 0.5f, 47 | -1.0f, 0.0f, -1.0f, -0.5f, 48 | -1.0f, -1.0f, -0.5f, -1.0f, 49 | 0.0f, -1.0f, 0.5f, -1.0f, 50 | 1.0f, -1.0f, 1.0f, -0.5f, 51 | 1.0f, 0.0f, 1.0f, 0.5f, 52 | 1.0f, 1.0f, 0.5f, 1.0f 53 | }; 54 | 55 | dirs = py::array({16, 2}, data.data()); 56 | } else { 57 | throw std::invalid_argument("nb_directions must be 8 or 16"); 58 | } 59 | 60 | for (size_t row = 0; row < nrow; ++row) { 61 | for (size_t col = 0; col < ncol; ++col) { 62 | if (r_msk_fill_disp(row, col)) { 63 | auto valid_neighbors = find_valid_neighbors( 64 | dirs, disp, msk_fill_disp, 65 | static_cast(row), static_cast(col), 66 | nb_directions 67 | ).unchecked<1>(); 68 | 69 | std::vector valid_values; 70 | for (size_t i = 0; i < valid_neighbors.shape(0); ++i) { 71 | if (!std::isnan(valid_neighbors(i))) { 72 | valid_values.push_back(valid_neighbors(i)); 73 | } 74 | } 75 | 76 | 77 | if (valid_values.empty()) { 78 | rw_out_disp(row, col) = std::nanf(""); 79 | } 80 | else { 81 | std::sort(valid_values.begin(), valid_values.end()); 82 | if (valid_values.size()%2==1) { 83 | rw_out_disp(row, col) = valid_values[valid_values.size()/2]; 84 | } else { 85 | rw_out_disp(row, col) = valid_values[valid_values.size()/2-1] 86 | + valid_values[valid_values.size()/2]; 87 | rw_out_disp(row, col) /= 2.f; 88 | } 89 | } 90 | rw_out_msk(row, col) = false; 91 | } else { 92 | rw_out_disp(row, col) = r_disp(row, col); 93 | rw_out_msk(row, col) = r_msk_fill_disp(row, col); 94 | } 95 | } 96 | } 97 | 98 | return {out_disp, out_msk}; 99 | } 100 | 101 | py::array_t find_valid_neighbors( 102 | py::array_t dirs, 103 | py::array_t disp, 104 | py::array_t valid, 105 | int row, 106 | int col, 107 | int nb_directions 108 | ) { 109 | auto r_dirs = dirs.unchecked<2>(); 110 | auto r_disp = disp.unchecked<2>(); 111 | auto r_valid = valid.unchecked<2>(); 112 | 113 | size_t nrow = r_disp.shape(0); 114 | size_t ncol = r_disp.shape(1); 115 | size_t max_path_length = std::max(nrow, ncol); 116 | 117 | py::array_t valid_neighbors(nb_directions); 118 | auto rw_valid_neighbors = valid_neighbors.mutable_unchecked<1>(); 119 | 120 | for (int direction = 0; direction < nb_directions; ++direction) { 121 | 122 | rw_valid_neighbors(direction) = 0.f; 123 | 124 | for (size_t i = 1; i < max_path_length; ++i) { 125 | int tmp_row = row + static_cast(r_dirs(direction, 0) * static_cast(i)); 126 | int tmp_col = col + static_cast(r_dirs(direction, 1) * static_cast(i)); 127 | 128 | if (tmp_row < 0 || tmp_row >= static_cast(nrow) || 129 | tmp_col < 0 || tmp_col >= static_cast(ncol)) { 130 | rw_valid_neighbors(direction) = std::nanf(""); 131 | break; 132 | } 133 | 134 | if (!r_valid(tmp_row, tmp_col) && r_disp(tmp_row, tmp_col) != 0) { 135 | rw_valid_neighbors(direction) = r_disp(tmp_row, tmp_col); 136 | break; 137 | } 138 | } 139 | } 140 | 141 | return valid_neighbors; 142 | } 143 | -------------------------------------------------------------------------------- /cars/applications/dense_match_filling/fill_disp_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the constants of fill_disp. 23 | """ 24 | 25 | 26 | # USED VARIABLES 27 | 28 | FILL_DISP_WITH_PLAN_RUN_TAG = "fill_disparity_with_plan" 29 | FILL_DISP_WITH_ZEROS_RUN_TAG = "fill_disparity_with_zeros" 30 | 31 | # PARAMS 32 | METHOD = "method" # default : 'plane' 33 | INTERP_TYPE = "interpolation_type" 34 | INTERP_METHOD = "interpolation_method" 35 | MAX_DIST = "max_search_distance" 36 | SMOOTH_IT = "smoothing_iterations" 37 | IGNORE_NODATA = "ignore_nodata_at_disp_mask_borders" 38 | IGNORE_ZERO = "ignore_zero_fill_disp_mask_values" 39 | IGNORE_EXTREMA = "ignore_extrema_disp_values" 40 | -------------------------------------------------------------------------------- /cars/applications/dense_matching/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core dense matching module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.dense_matching.dense_matching import DenseMatching 27 | 28 | from . import census_mccnn_sgm 29 | -------------------------------------------------------------------------------- /cars/applications/dense_matching/cpp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/cars/d86b7491ac1e7ea3627535bd571ea65df3933073/cars/applications/dense_matching/cpp/__init__.py -------------------------------------------------------------------------------- /cars/applications/dense_matching/cpp/dense_matching_cpp.py: -------------------------------------------------------------------------------- 1 | """ 2 | this module contains the headers of the dense_matching_cpp module. 3 | """ 4 | 5 | # pylint: skip-file 6 | 7 | 8 | def estimate_right_classif_on_left( 9 | right_classif, disp_map, disp_mask, disp_min, disp_max 10 | ): 11 | """ 12 | Estimate right classif on left image 13 | 14 | :param right_classif: right classification 15 | :type right_classif: np ndarray 16 | :param disp_map: disparity map 17 | :type disp_map: np ndarray 18 | :param disp_mask: disparity mask 19 | :type disp_mask: np ndarray 20 | :param disp_min: disparity min 21 | :type disp_min: int 22 | :param disp_max: disparity max 23 | :type disp_max: int 24 | 25 | :return: right classif on left image 26 | :rtype: np nadarray 27 | """ 28 | ... 29 | 30 | 31 | def mask_left_classif_from_right_mask( 32 | left_classif, right_mask, disp_min, disp_max 33 | ): 34 | """ 35 | Mask left classif with right mask. 36 | 37 | :param left_classif: right classification 38 | :type left_classif: np ndarray 39 | :param right_mask: right mask 40 | :type right_mask: np ndarray 41 | :param disp_min: disparity min 42 | :type disp_min: np.array type int 43 | :param disp_max: disparity max 44 | :type disp_max: np.array type int 45 | 46 | :return: masked left classif 47 | :rtype: np nadarray 48 | """ 49 | ... 50 | 51 | 52 | def estimate_right_grid_disp_int(disp_min_grid, disp_max_grid): 53 | """ 54 | Estimate right grid min and max for int inputs. 55 | Correspond to the range of pixels that can be correlated 56 | from left -> right. 57 | If no left pixels can be associated to right, use global values 58 | 59 | :param disp_min_grid: left disp min grid 60 | :type disp_min_grid: numpy ndarray 61 | :param disp_max_grid: left disp max grid 62 | :type disp_max_grid: numpy ndarray 63 | 64 | :return: disp_min_right_grid, disp_max_right_grid 65 | :rtype: numpy ndarray, numpy ndarray 66 | """ 67 | return None, None 68 | 69 | 70 | def estimate_right_grid_disp_float(disp_min_grid, disp_max_grid): 71 | """ 72 | Estimate right grid min and max for float inputs. 73 | Correspond to the range of pixels that can be correlated 74 | from left -> right. 75 | If no left pixels can be associated to right, use global values 76 | 77 | :param disp_min_grid: left disp min grid 78 | :type disp_min_grid: numpy ndarray 79 | :param disp_max_grid: left disp max grid 80 | :type disp_max_grid: numpy ndarray 81 | 82 | :return: disp_min_right_grid, disp_max_right_grid 83 | :rtype: numpy ndarray, numpy ndarray 84 | """ 85 | return None, None 86 | -------------------------------------------------------------------------------- /cars/applications/dense_matching/cpp/includes/dense_matching.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DENSE_MATCHING_HPP 2 | #define DENSE_MATCHING_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace py = pybind11; 9 | 10 | /** 11 | * @brief Estimate right classif on left image 12 | * 13 | * @param right_classif right classification 14 | * @param disp_map disparity map 15 | * @param disp_mask disparity mask 16 | * @param disp_min disparity min 17 | * @param disp_max disparity max 18 | * @return right classif on left image 19 | */ 20 | py::array_t estimate_right_classif_on_left( 21 | py::array_t right_classif, 22 | py::array_t disp_map, 23 | std::optional> disp_mask, 24 | int disp_min, 25 | int disp_max 26 | ); 27 | 28 | /** 29 | * @brief Mask left classif with right mask 30 | * 31 | * @param left_classif left classification 32 | * @param right_mask right mask 33 | * @param disp_min disparity min 34 | * @param disp_max disparity max 35 | * @return masked left classif 36 | */ 37 | py::array_t mask_left_classif_from_right_mask( 38 | py::array_t left_classif, 39 | py::array_t right_mask, 40 | py::array_t disp_min, 41 | py::array_t disp_max 42 | ); 43 | 44 | /** 45 | * @brief Estimate right grid disparities from left grid disparities 46 | * 47 | * @param disp_min_grid left disparity minimum grid 48 | * @param disp_max_grid left disparity maximum grid 49 | * @return pair of right grid minimum and maximum disparities 50 | */ 51 | template std::pair, py::array_t> estimate_right_grid_disp( 52 | py::array_t disp_min_grid, 53 | py::array_t disp_max_grid 54 | ); 55 | 56 | #endif // DENSE_MATCHING_HPP 57 | -------------------------------------------------------------------------------- /cars/applications/dense_matching/cpp/meson.build: -------------------------------------------------------------------------------- 1 | 2 | py.extension_module( 3 | 'dense_matching_cpp', 4 | ['src/bindings.cpp', 'src/dense_matching.cpp'], 5 | include_directories: ['includes'], 6 | subdir: 'cars/applications/dense_matching/cpp', 7 | install: true, 8 | dependencies : [pybind11_dep], 9 | ) 10 | -------------------------------------------------------------------------------- /cars/applications/dense_matching/cpp/src/bindings.cpp: -------------------------------------------------------------------------------- 1 | #include "dense_matching.hpp" 2 | #include 3 | 4 | namespace py = pybind11; 5 | 6 | PYBIND11_MODULE(dense_matching_cpp, m) { 7 | m.doc() = "cars's pybind11 dense matching module"; // optional module docstring 8 | 9 | m.def("estimate_right_classif_on_left", &estimate_right_classif_on_left, ""); 10 | m.def("mask_left_classif_from_right_mask", &mask_left_classif_from_right_mask, ""); 11 | m.def("estimate_right_grid_disp_int", &estimate_right_grid_disp, ""); 12 | m.def("estimate_right_grid_disp_float", &estimate_right_grid_disp, ""); 13 | } -------------------------------------------------------------------------------- /cars/applications/dense_matching/dense_matching_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the constants of dense_matching. 23 | """ 24 | from pandora import constants as pandora_cst 25 | 26 | from cars.core import constants_disparity as disp_cst 27 | 28 | # USED VARIABLES 29 | 30 | 31 | DENSE_MATCHING_RUN_TAG = "dense_matching" 32 | 33 | # grids disp 34 | DISP_MIN_GRID = "disp_min_grid" 35 | DISP_MAX_GRID = "disp_max_grid" 36 | 37 | 38 | # PARAMS 39 | METHOD = "method" 40 | MIN_EPI_TILE_SIZE = "min_epi_tile_size" 41 | MAX_EPI_TILE_SIZE = "max_epi_tile_size" 42 | EPI_TILE_MARGIN_IN_PERCENT = "epipolar_tile_margin_in_percent" 43 | MIN_ELEVATION_OFFSET = "min_elevation_offset" 44 | MAX_ELEVATION_OFFSET = "max_elevation_offset" 45 | 46 | # ABRIDGED PANDORA CONSTANTS 47 | IN_VALIDITY_MASK_LEFT = "IN_VALIDITY_MASK_LEFT" 48 | IN_VALIDITY_MASK_RIGHT = "IN_VALIDITY_MASK_RIGHT" 49 | RIGHT_INCOMPLETE_DISPARITY_RANGE = "RIGHT_INCOMPLETE_DISPARITY_RANGE" 50 | STOPPED_INTERPOLATION = "STOPPED_INTERPOLATION" 51 | FILLED_OCCLUSION = "FILLED_OCCLUSION" 52 | FILLED_MISMATCH = "FILLED_MISMATCH" 53 | LEFT_NODATA_OR_BORDER = "LEFT_NODATA_OR_BORDER" 54 | RIGHT_NODATA_OR_DISPARITY_RANGE_MISSING = ( 55 | "RIGHT_NODATA_OR_DISPARITY_RANGE_MISSING" 56 | ) 57 | OCCLUSION = "OCCLUSION" 58 | MISMATCH = "MISMATCH" 59 | 60 | 61 | def get_cst(key): 62 | """ 63 | get pandora constant from abridged key 64 | 65 | :param key: abridged key of pandora mask 66 | 67 | Returns: 68 | _type_: pandora mask constant 69 | """ 70 | return pandora_cst.__dict__.get("PANDORA_MSK_PIXEL_" + key) 71 | 72 | 73 | # CORRESPONDING MSK TABLE PANDORA CARS 74 | MASK_HASH_TABLE = { 75 | disp_cst.MASKED_REF: get_cst(IN_VALIDITY_MASK_LEFT), 76 | disp_cst.MASKED_SEC: get_cst(IN_VALIDITY_MASK_RIGHT), 77 | disp_cst.INCOMPLETE_DISP: get_cst(RIGHT_INCOMPLETE_DISPARITY_RANGE), 78 | disp_cst.STOPPED_INTERP: get_cst(STOPPED_INTERPOLATION), 79 | disp_cst.FILLED_OCCLUSION: get_cst(FILLED_OCCLUSION), 80 | disp_cst.FILLED_FALSE_MATCH: get_cst(FILLED_MISMATCH), 81 | disp_cst.INVALID_REF: get_cst(LEFT_NODATA_OR_BORDER), 82 | disp_cst.INVALID_SEC: get_cst(RIGHT_NODATA_OR_DISPARITY_RANGE_MISSING), 83 | disp_cst.OCCLUSION: get_cst(OCCLUSION), 84 | disp_cst.FALSE_MATCH: get_cst(MISMATCH), 85 | } 86 | -------------------------------------------------------------------------------- /cars/applications/dense_matching/loaders/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core dense matching module init file 23 | """ 24 | -------------------------------------------------------------------------------- /cars/applications/dense_matching/loaders/config_census_sgm_default.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": {}, 3 | "pipeline": { 4 | "matching_cost": { 5 | "matching_cost_method": "census", 6 | "window_size": 5, 7 | "subpix": 1 8 | }, 9 | "optimization": { 10 | "optimization_method": "sgm", 11 | "overcounting": false, 12 | "penalty": { 13 | "P1": 8, 14 | "P2": 32, 15 | "p2_method": "constant", 16 | "penalty_method": "sgm_penalty" 17 | } 18 | }, 19 | "disparity": { 20 | "disparity_method": "wta", 21 | "invalid_disparity": "NaN" 22 | }, 23 | "refinement": { 24 | "refinement_method": "vfit" 25 | }, 26 | "filter": { 27 | "filter_method": "median", 28 | "filter_size": 3 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /cars/applications/dense_matching/loaders/config_census_sgm_homogeneous.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": {}, 3 | "pipeline": { 4 | "matching_cost": { 5 | "matching_cost_method": "census", 6 | "window_size": 11, 7 | "subpix": 1 8 | }, 9 | "optimization": { 10 | "optimization_method": "sgm", 11 | "overcounting": true, 12 | "penalty": { 13 | "P1": 72, 14 | "P2": 309, 15 | "penalty_method": "sgm_penalty" 16 | } 17 | }, 18 | "disparity": { 19 | "disparity_method": "wta", 20 | "invalid_disparity": "NaN" 21 | }, 22 | "refinement": { 23 | "refinement_method": "vfit" 24 | }, 25 | "filter": { 26 | "filter_method": "median", 27 | "filter_size": 3 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /cars/applications/dense_matching/loaders/config_census_sgm_mountain_and_vegetation.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": {}, 3 | "pipeline": { 4 | "matching_cost": { 5 | "matching_cost_method": "census", 6 | "window_size": 11, 7 | "subpix": 1 8 | }, 9 | "optimization": { 10 | "optimization_method": "sgm", 11 | "overcounting": true, 12 | "penalty": { 13 | "P1": 38, 14 | "P2": 464, 15 | "penalty_method": "sgm_penalty" 16 | } 17 | }, 18 | "disparity": { 19 | "disparity_method": "wta", 20 | "invalid_disparity": "NaN" 21 | }, 22 | "refinement": { 23 | "refinement_method": "vfit" 24 | }, 25 | "filter": { 26 | "filter_method": "median", 27 | "filter_size": 3 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /cars/applications/dense_matching/loaders/config_census_sgm_shadow.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": {}, 3 | "pipeline": { 4 | "matching_cost": { 5 | "matching_cost_method": "census", 6 | "window_size": 11, 7 | "subpix": 1 8 | }, 9 | "optimization": { 10 | "optimization_method": "sgm", 11 | "overcounting": true, 12 | "penalty": { 13 | "P1": 20, 14 | "P2": 160, 15 | "penalty_method": "sgm_penalty" 16 | } 17 | }, 18 | "disparity": { 19 | "disparity_method": "wta", 20 | "invalid_disparity": "NaN" 21 | }, 22 | "refinement": { 23 | "refinement_method": "vfit" 24 | }, 25 | "filter": { 26 | "filter_method": "median", 27 | "filter_size": 3 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /cars/applications/dense_matching/loaders/config_census_sgm_sparse.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": {}, 3 | "pipeline": { 4 | "matching_cost": { 5 | "matching_cost_method": "census", 6 | "window_size": 11, 7 | "subpix": 1 8 | }, 9 | "optimization": { 10 | "optimization_method": "sgm", 11 | "overcounting": true, 12 | "penalty": { 13 | "penalty_method": "sgm_penalty" 14 | } 15 | }, 16 | "cost_volume_confidence.risk":{ 17 | "confidence_method": "risk" 18 | }, 19 | "cost_volume_confidence.intervals":{ 20 | "confidence_method": "interval_bounds" 21 | }, 22 | 23 | "disparity": { 24 | "disparity_method": "wta", 25 | "invalid_disparity": "NaN" 26 | }, 27 | "refinement": { 28 | "refinement_method": "vfit" 29 | }, 30 | "filter": { 31 | "filter_method": "median", 32 | "filter_size": 3 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /cars/applications/dense_matching/loaders/config_census_sgm_urban.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": {}, 3 | "pipeline": { 4 | "matching_cost": { 5 | "matching_cost_method": "census", 6 | "window_size": 11, 7 | "subpix": 1 8 | }, 9 | "optimization": { 10 | "optimization_method": "sgm", 11 | "overcounting": true, 12 | "penalty": { 13 | "P1": 20, 14 | "P2": 80, 15 | "penalty_method": "sgm_penalty" 16 | } 17 | }, 18 | "disparity": { 19 | "disparity_method": "wta", 20 | "invalid_disparity": "NaN" 21 | }, 22 | "refinement": { 23 | "refinement_method": "vfit" 24 | }, 25 | "filter": { 26 | "filter_method": "median", 27 | "filter_size": 3 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /cars/applications/dense_matching/loaders/config_mccnn.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": {}, 3 | "pipeline": { 4 | "matching_cost" : { 5 | "matching_cost_method": "mc_cnn", 6 | "subpix": 1, 7 | "window_size": 11 8 | }, 9 | "optimization" : { 10 | "optimization_method": "sgm", 11 | "overcounting": false, 12 | "penalty": { 13 | "penalty_method": "mc_cnn_fast_penalty" 14 | } 15 | }, 16 | "disparity": { 17 | "disparity_method": "wta", 18 | "invalid_disparity": "NaN" 19 | }, 20 | "refinement": { 21 | "refinement_method": "vfit" 22 | }, 23 | "filter" : { 24 | "filter_method": "median", 25 | "filter_size": 3 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /cars/applications/dsm_filling/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS dsm filling module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.dsm_filling.dsm_filling import DsmFilling 27 | 28 | from . import border_interpolation, bulldozer_filling, exogenous_filling 29 | -------------------------------------------------------------------------------- /cars/applications/dsm_filling/bulldozer_config/base_config.yaml: -------------------------------------------------------------------------------- 1 | # both paths will be overwritten in the filling application 2 | dsm_path: "path/to/dsm" 3 | output_dir: "path/to/output" 4 | 5 | # everything else will be used directly as-is 6 | 7 | #-------------------------# 8 | # Options # 9 | #-------------------------# 10 | 11 | 12 | max_object_size : 16 13 | # If True, keep the intermediate results 14 | developer_mode : False 15 | 16 | 17 | #-------------------------# 18 | # Advanced settings # 19 | #-------------------------# 20 | # [Optional] - Altimetric height accuracy of the input DSM (m). If null, use the default value: 2*planimetric resolution 21 | dsm_z_accuracy: null 22 | # [Optional] - Maximum slope of the observed landscape terrain (%) 23 | # former slope_threshold 24 | max_ground_slope: 2.0 25 | # [Optional] - If True, activate the ground pre-detection 26 | activate_ground_anchors : False 27 | # [Optional] - If True, keep the intermediate results 28 | developer_mode : False 29 | 30 | 31 | #-------------------------# 32 | # Bulldozer core settings # 33 | #-------------------------# 34 | # /!\ Modify those data at your own risk (it is suggested to keep the default values) /!\ 35 | 36 | # [Optional] - DtmExtraction parameters 37 | # former uniform_filter_size 38 | cloth_tension_force : 3 39 | prevent_unhook_iter : 10 40 | num_outer_iter : 50 41 | num_inner_iter : 10 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /cars/applications/dsm_filling/dsm_filling.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the abstract dsm filling application class. 23 | """ 24 | import logging 25 | from abc import ABCMeta, abstractmethod 26 | from typing import Dict 27 | 28 | from cars.applications.application import Application 29 | from cars.applications.application_template import ApplicationTemplate 30 | 31 | 32 | @Application.register("dsm_filling") 33 | class DsmFilling(ApplicationTemplate, metaclass=ABCMeta): 34 | """ 35 | DsmFilling 36 | """ 37 | 38 | available_applications: Dict = {} 39 | default_application = "bulldozer" 40 | 41 | def __new__(cls, conf=None): # pylint: disable=W0613 42 | """ 43 | Return the required application 44 | :raises: 45 | - KeyError when the required application is not registered 46 | 47 | :param orchestrator: orchestrator used 48 | :param conf: configuration for filling 49 | :return: an application_to_use object 50 | """ 51 | 52 | dsm_filling_method = cls.default_application 53 | if bool(conf) is False: 54 | logging.info( 55 | "dsm_filling method not specified, default" 56 | " {} is used".format(dsm_filling_method) 57 | ) 58 | else: 59 | dsm_filling_method = conf["method"] 60 | 61 | if dsm_filling_method not in cls.available_applications: 62 | logging.error( 63 | "No dsm_filling application named {} registered".format( 64 | dsm_filling_method 65 | ) 66 | ) 67 | raise KeyError( 68 | "No dsm_filling application named {} registered".format( 69 | dsm_filling_method 70 | ) 71 | ) 72 | 73 | logging.info( 74 | "The DsmFilling {} application will be used".format( 75 | dsm_filling_method 76 | ) 77 | ) 78 | 79 | return super(DsmFilling, cls).__new__( 80 | cls.available_applications[dsm_filling_method] 81 | ) 82 | 83 | def __init_subclass__(cls, short_name, **kwargs): # pylint: disable=E0302 84 | super().__init_subclass__(**kwargs) 85 | cls.available_applications[short_name] = cls 86 | 87 | def __init__(self, conf=None): 88 | """ 89 | Init function of DSM Filling 90 | 91 | :param conf: configuration 92 | :return: an application_to_use object 93 | """ 94 | 95 | super().__init__(conf=conf) 96 | 97 | @abstractmethod 98 | def run(self): 99 | """ 100 | Run dsm filling using initial elevation and the current dsm 101 | """ 102 | -------------------------------------------------------------------------------- /cars/applications/grid_generation/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS grid_generation module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.grid_generation.grid_generation import GridGeneration 27 | 28 | from . import epipolar_grid_generation 29 | -------------------------------------------------------------------------------- /cars/applications/grid_generation/grid_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | grid_constants contains all the constants used in grid_generation module 23 | """ 24 | 25 | 26 | GRID_GENERATION_RUN_TAG = "grid_generation" 27 | 28 | # Run infos 29 | 30 | GRID_CORRECTION_TAG = "grid_correction" 31 | CORRECTED_MATCHES_TAG = "corrected_filtered_matches" 32 | 33 | EPIPOLAR_SIZE_X_TAG = "epipolar_size_x" 34 | EPIPOLAR_SIZE_Y_TAG = "epipolar_size_y" 35 | EPIPOLAR_ORIGIN_X_TAG = "epipolar_origin_x" 36 | EPIPOLAR_ORIGIN_Y_TAG = "epipolar_origin_y" 37 | EPIPOLAR_SPACING_X_TAG = "epipolar_spacing_x" 38 | EPIPOLAR_SPACING_Y_TAG = "epipolar_spacing_y" 39 | 40 | DISP_TO_ALT_RATIO_TAG = "disp_to_alt_ratio" 41 | LEFT_AZIMUTH_ANGLE_TAG = "left_azimuth_angle" 42 | LEFT_ELEVATION_ANGLE_TAG = "left_elevation_angle" 43 | RIGHT_AZIMUTH_ANGLE_TAG = "right_azimuth_angle" 44 | RIGHT_ELEVATION_ANGLE_TAG = "right_elevation_angle" 45 | CONVERGENCE_ANGLE_TAG = "convergence_angle" 46 | -------------------------------------------------------------------------------- /cars/applications/grid_generation/grid_generation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the abstract grid generation application class. 23 | """ 24 | import logging 25 | from abc import ABCMeta, abstractmethod 26 | from typing import Dict 27 | 28 | from cars.applications.application import Application 29 | from cars.applications.application_template import ApplicationTemplate 30 | 31 | 32 | @Application.register("grid_generation") 33 | class GridGeneration(ApplicationTemplate, metaclass=ABCMeta): 34 | """ 35 | AbstractGridGeneration 36 | """ 37 | 38 | available_applications: Dict = {} 39 | default_application = "epipolar" 40 | 41 | def __new__(cls, conf=None): # pylint: disable=W0613 42 | """ 43 | Return the required application 44 | :raises: 45 | - KeyError when the required application is not registered 46 | 47 | :param conf: configuration for grid generation 48 | :return: a application_to_use object 49 | """ 50 | 51 | grid_method = cls.default_application 52 | 53 | if bool(conf) is False or "method" not in conf: 54 | logging.info( 55 | "Grid generation method not specified, default " 56 | " {} is used".format(grid_method) 57 | ) 58 | else: 59 | grid_method = conf.get("method", cls.default_application) 60 | 61 | if grid_method not in cls.available_applications: 62 | logging.error( 63 | "No GridGeneration application named {} registered".format( 64 | grid_method 65 | ) 66 | ) 67 | raise KeyError( 68 | "No GridGeneration application named {} registered".format( 69 | grid_method 70 | ) 71 | ) 72 | 73 | logging.info( 74 | "The GridGeneration({}) application will be used".format( 75 | grid_method 76 | ) 77 | ) 78 | 79 | return super(GridGeneration, cls).__new__( 80 | cls.available_applications[grid_method] 81 | ) 82 | 83 | def __init_subclass__(cls, short_name, **kwargs): # pylint: disable=E0302 84 | super().__init_subclass__(**kwargs) 85 | cls.available_applications[short_name] = cls 86 | 87 | def __init__(self, conf=None): 88 | """ 89 | Init function of GridGeneration 90 | 91 | :param conf: configuration 92 | :return: an application_to_use object 93 | """ 94 | 95 | super().__init__(conf=conf) 96 | 97 | @abstractmethod 98 | def get_save_grids(self): 99 | """ 100 | Get whether the grid will be saved 101 | 102 | :return: true is grid saving is activated 103 | :rtype: bool 104 | """ 105 | 106 | @abstractmethod 107 | def run( 108 | self, 109 | image_left, 110 | image_right, 111 | orchestrator=None, 112 | pair_folder=None, 113 | srtm_dir=None, 114 | default_alt=None, 115 | geoid_path=None, 116 | pair_key="PAIR_0", 117 | ): 118 | """ 119 | Run EpipolarGridGeneration application 120 | 121 | Create left and right grid CarsDataset filled with xarray.Dataset , 122 | corresponding to left and right epipolar grids. 123 | 124 | :param image_left: left image 125 | :type image_left: dict 126 | :param image_right: right image 127 | :type image_right: dict 128 | :param pair_folder: folder used for current pair 129 | :type pair_folder: str 130 | :param orchestrator: orchestrator used 131 | :param srtm_dir: srtm directory 132 | :type srtm_dir: str 133 | :param default_alt: default altitude 134 | :type default_alt: float 135 | :param geoid_path: geoid path 136 | :type geoid_path: str 137 | :param pair_key: pair configuration id 138 | :type pair_key: str 139 | 140 | :return: left grid, right grid 141 | :rtype: Tuple(CarsDataset, CarsDataset) 142 | """ 143 | -------------------------------------------------------------------------------- /cars/applications/ground_truth_reprojection/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2024 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core epipolar matches ground truth generation module init file 23 | """ 24 | # flake8: noqa: F401, E501 25 | 26 | from cars.applications.ground_truth_reprojection import ( 27 | ground_truth_reprojection, 28 | ) 29 | 30 | from . import direct_localization 31 | -------------------------------------------------------------------------------- /cars/applications/ground_truth_reprojection/ground_truth_reprojection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2024 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | This module contains the abstract ground_truth_reprojection 23 | application class. 24 | """ 25 | 26 | import logging 27 | from abc import ABCMeta, abstractmethod 28 | from typing import Dict 29 | 30 | from cars.applications.application import Application 31 | from cars.applications.application_template import ApplicationTemplate 32 | 33 | 34 | @Application.register("ground_truth_reprojection") 35 | class GroundTruthReprojection(ApplicationTemplate, metaclass=ABCMeta): 36 | """ 37 | Epipolar matches ground truth computation 38 | 39 | """ 40 | 41 | available_applications: Dict = {} 42 | default_application = "direct_loc" 43 | 44 | def __new__(cls, conf=None): # pylint: disable=W0613 45 | """ 46 | Return the required application 47 | :raises: KeyError when the required application is not registered 48 | :param conf: configuration for grid generation 49 | :return: an application_to_use object 50 | 51 | """ 52 | 53 | ground_truth_computation_method = cls.default_application 54 | 55 | if bool(conf) is False or "method" not in conf: 56 | logging.info( 57 | "Ground truth reprojection method not specified" 58 | ", default {} is used".format(ground_truth_computation_method) 59 | ) 60 | else: 61 | ground_truth_computation_method = conf.get( 62 | "method", cls.default_application 63 | ) 64 | 65 | if ground_truth_computation_method not in cls.available_applications: 66 | logging.error( 67 | "No GroundTruthReprojection application named {} " 68 | "registered".format(ground_truth_computation_method) 69 | ) 70 | raise KeyError( 71 | "No GroundTruthReprojection application named {} " 72 | "registered".format(ground_truth_computation_method) 73 | ) 74 | 75 | logging.info( 76 | "The GroundTruthReprojection({}) application will be " 77 | "used".format(ground_truth_computation_method) 78 | ) 79 | 80 | return super(GroundTruthReprojection, cls).__new__( 81 | cls.available_applications[ground_truth_computation_method] 82 | ) 83 | 84 | def __init_subclass__(cls, short_name, **kwargs): # pylint: disable=E0302 85 | super().__init_subclass__(**kwargs) 86 | cls.available_applications[short_name] = cls 87 | 88 | def __init__(self, conf=None): 89 | """ 90 | Init function of Epipolar matches ground truth computation 91 | 92 | :param conf: configuration 93 | :return: an application_to_use object 94 | 95 | """ 96 | 97 | super().__init__(conf=conf) 98 | 99 | @abstractmethod 100 | def run( 101 | self, 102 | sensor_left, 103 | sensor_right, 104 | grid_left, 105 | grid_right, 106 | geom_plugin, 107 | geom_plugin_dem_median, 108 | disp_to_alt_ratio, 109 | auxiliary_values, 110 | auxiliary_interp, 111 | orchestrator=None, 112 | pair_folder=None, 113 | ): # noqa: C901 114 | """ 115 | Compute disparity maps from a DSM. This function will be run 116 | as a delayed task. If user want to correctly save dataset, the user must 117 | provide saving_info_left and right. See cars_dataset.fill_dataset. 118 | 119 | :param geom_plugin_dem_median: Geometry plugin with dem median 120 | :type geom_plugin_dem_median: geometry_plugin 121 | :param sensor_left: Tiled sensor left image. 122 | Dict must contain keys: "image", "color", "geomodel", 123 | "no_data", "mask". Paths must be absolute. 124 | :type sensor_left: CarsDataset 125 | :param grid_left: Grid left. 126 | :type grid_left: CarsDataset 127 | :param geom_plugin: Geometry plugin with user's DSM used to generate 128 | epipolar grids. 129 | :type geom_plugin: GeometryPlugin 130 | :param disp_to_alt_ratio: Disp to altitude ratio used for 131 | performance map. 132 | :type disp_to_alt_ratio: float 133 | :param orchestrator: Orchestrator used. 134 | :type orchestrator: Any 135 | :param pair_folder: Folder used for current pair. 136 | :type pair_folder: str 137 | """ 138 | -------------------------------------------------------------------------------- /cars/applications/hole_detection/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS holes detection module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.hole_detection.hole_detection import HoleDetection 27 | 28 | from . import cloud_to_bbox 29 | -------------------------------------------------------------------------------- /cars/applications/hole_detection/hole_detection.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS hole detection module init file 23 | """ 24 | 25 | import logging 26 | from abc import ABCMeta, abstractmethod 27 | from typing import Dict 28 | 29 | from cars.applications.application import Application 30 | from cars.applications.application_template import ApplicationTemplate 31 | 32 | 33 | @Application.register("hole_detection") 34 | class HoleDetection(ApplicationTemplate, metaclass=ABCMeta): 35 | """ 36 | HoleDetection 37 | """ 38 | 39 | available_applications: Dict = {} 40 | default_application = "cloud_to_bbox" 41 | 42 | def __new__(cls, conf=None): # pylint: disable=W0613 43 | """ 44 | Return the required application 45 | :raises: 46 | - KeyError when the required application is not registered 47 | 48 | :param conf: configuration for hole_detection 49 | :return: a application_to_use object 50 | """ 51 | 52 | hole_detection_method = cls.default_application 53 | if bool(conf) is False or "method" not in conf: 54 | logging.info( 55 | "Holes Detection method not specified, " 56 | "default {} is used".format(hole_detection_method) 57 | ) 58 | else: 59 | hole_detection_method = conf["method"] 60 | 61 | if hole_detection_method not in cls.available_applications: 62 | logging.error( 63 | "No hole_detection application named {} registered".format( 64 | hole_detection_method 65 | ) 66 | ) 67 | raise KeyError( 68 | "No hole_detection application named {} registered".format( 69 | hole_detection_method 70 | ) 71 | ) 72 | 73 | logging.info( 74 | "The HoleDetection({}) application will be used".format( 75 | hole_detection_method 76 | ) 77 | ) 78 | 79 | return super(HoleDetection, cls).__new__( 80 | cls.available_applications[hole_detection_method] 81 | ) 82 | 83 | def __init_subclass__(cls, short_name, **kwargs): # pylint: disable=E0302 84 | super().__init_subclass__(**kwargs) 85 | cls.available_applications[short_name] = cls 86 | 87 | @abstractmethod 88 | def run( 89 | self, 90 | epipolar_images_left, 91 | epipolar_images_right, 92 | is_activated=True, 93 | margin=0, 94 | mask_holes_to_fill_left=None, 95 | mask_holes_to_fill_right=None, 96 | orchestrator=None, 97 | pair_folder=None, 98 | pair_key="PAIR_0", 99 | ): 100 | """ 101 | Run Refill application using plane method. 102 | 103 | :param epipolar_images_left: left epipolar image 104 | :type epipolar_images_left: CarsDataset 105 | :param epipolar_images_right: right epipolar image 106 | :type epipolar_images_right: CarsDataset 107 | :param is_activated: activate application 108 | :type is_activated: bool 109 | :param margin: margin to use 110 | :type margin: int 111 | :param mask_holes_to_fill_left: mask classes to use 112 | :type mask_holes_to_fill_left: list(int) 113 | :param mask_holes_to_fill_right: mask classes to use 114 | :type mask_holes_to_fill_right: list(int) 115 | :param orchestrator: orchestrator used 116 | :type orchestrator: Orchestrator 117 | :param pair_folder: folder used for current pair 118 | :type pair_folder: str 119 | :param pair_key: pair id 120 | :type pair_key: str 121 | 122 | :return: left holes, right holes 123 | :rtype: Tuple(CarsDataset, CarsDataset) 124 | 125 | """ 126 | -------------------------------------------------------------------------------- /cars/applications/point_cloud_denoising/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS holes detection module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | import cars.applications.point_cloud_denoising.point_cloud_denoising 27 | from cars.applications.point_cloud_denoising.point_cloud_denoising import ( 28 | PCDenoising, 29 | ) 30 | -------------------------------------------------------------------------------- /cars/applications/point_cloud_fusion/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core cloud fusion module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.point_cloud_fusion.point_cloud_fusion import ( 27 | PointCloudFusion, 28 | ) 29 | 30 | from . import mapping_to_terrain_tiles 31 | -------------------------------------------------------------------------------- /cars/applications/point_cloud_fusion/cloud_fusion_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | file contains all the constants used in cloud fusion module 23 | """ 24 | 25 | 26 | CLOUD_FUSION_RUN_TAG = "cloud_fusion" 27 | 28 | # Params 29 | METHOD = "method" 30 | TERRAIN_TILE_SIZE = "terrain_tile_size" 31 | RESOLUTION = "resolution" 32 | 33 | # Run infos 34 | EPSG_TAG = "epsg" 35 | MARGINS_TAG = "margins" 36 | NUMBER_TERRAIN_TILES = "number_terrain_tiles" 37 | XSIZE = "x_size" 38 | YSIZE = "y_size" 39 | BOUNDS = "bounds" 40 | -------------------------------------------------------------------------------- /cars/applications/point_cloud_outlier_removal/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core cloud outlier removing module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.point_cloud_outlier_removal.pc_out_removal import ( 27 | PointCloudOutlierRemoval, 28 | ) 29 | 30 | from . import small_components, statistical 31 | -------------------------------------------------------------------------------- /cars/applications/point_cloud_outlier_removal/point_removal_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | file contains all the constants used in cloud fusion module 23 | """ 24 | 25 | 26 | CLOUD_OUTLIER_REMOVAL_RUN_TAG = "outlier_removal" 27 | 28 | # Params 29 | METHOD = "method" 30 | 31 | 32 | # small components 33 | SMALL_COMPONENT_FILTER = "small_components_filter_activated" 34 | SC_ON_GROUND_MARGIN = "on_ground_margin" 35 | SC_CONNECTION_DISTANCE = "connection_distance" 36 | SC_NB_POINTS_THRESHOLD = "nb_points_threshold" 37 | SC_CLUSTERS_DISTANCES_THRESHOLD = "clusters_distance_threshold" 38 | 39 | 40 | # statistical outlier 41 | STATISTICAL_OUTLIER = "statistical_outliers_filter_activated" 42 | SO_K = "k" 43 | SO_STD_DEV_FACTOR = "std_dev_factor" 44 | -------------------------------------------------------------------------------- /cars/applications/rasterization/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core rasterization module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.rasterization.point_cloud_rasterization import ( 27 | PointCloudRasterization, 28 | ) 29 | 30 | from . import simple_gaussian 31 | -------------------------------------------------------------------------------- /cars/applications/rasterization/rasterization_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | file contains all the constants used in rasterization module 23 | """ 24 | 25 | RASTERIZATION_RUN_TAG = "rasterization" 26 | 27 | # Params 28 | METHOD = "method" 29 | DSM_RADIUS = "dsm_radius" 30 | SIGMA = "sigma" 31 | GRID_POINTS_DIVISION_FACTOR = "grid_points_division_factor" 32 | RESOLUTION = "resolution" 33 | 34 | 35 | # Run infos 36 | EPSG_TAG = "epsg" 37 | DSM_NO_DATA_TAG = "dsm_no_data" 38 | COLOR_NO_DATA_TAG = "color_no_data" 39 | -------------------------------------------------------------------------------- /cars/applications/resampling/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core resampling module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.resampling.resampling import Resampling 27 | 28 | from . import bicubic_resampling 29 | -------------------------------------------------------------------------------- /cars/applications/resampling/resampling_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the constants of resampling. 23 | """ 24 | 25 | 26 | # USED VARIABLES 27 | 28 | 29 | RESAMPLING_RUN_TAG = "resampling" 30 | 31 | 32 | # PARAMS 33 | METHOD = "method" 34 | EPI_TILE_SIZE = "epi_tile_size" 35 | 36 | # INFOS 37 | -------------------------------------------------------------------------------- /cars/applications/sparse_matching/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core matching module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | from cars.applications.sparse_matching.sparse_matching import SparseMatching 27 | 28 | from . import pandora_sparse_matching, sift 29 | -------------------------------------------------------------------------------- /cars/applications/sparse_matching/sparse_matching_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the constants of sparse matching. 23 | """ 24 | 25 | 26 | # USED VARIABLES 27 | 28 | 29 | SPARSE_MATCHING_RUN_TAG = "sparse_matching" 30 | 31 | 32 | # INFOS 33 | 34 | # Sparse matching PARAMS 35 | DISPARITY_MARGIN_TAG = "disparity_margin" 36 | ELEVATION_DELTA_LOWER_BOUND = "elevation_delta_lower_bound" 37 | ELEVATION_DELTA_UPPER_BOUND = "elevation_delta_upper_bound" 38 | EPIPOLAR_ERROR_UPPER_BOUND = "epipolar_error_upper_bound" 39 | EPIPOLAR_ERROR_MAXIMUM_BIAS = "epipolar_error_maximum_bias" 40 | 41 | SIFT_THRESH_HOLD = "sift_matching_threshold" 42 | SIFT_N_OCTAVE = "sift_n_octave" 43 | SIFT_N_SCALE_PER_OCTAVE = "sift_n_scale_per_octave" 44 | SIFT_PEAK_THRESHOLD = "sift_peak_threshold" 45 | SIFT_EDGE_THRESHOLD = "sift_edge_threshold" 46 | SIFT_MAGNIFICATION = "sift_magnification" 47 | SIFT_BACK_MATCHING = "sift_back_matching" 48 | 49 | # Sparse matching RUN 50 | DISP_LOWER_BOUND = "disp_lower_bound" 51 | DISP_UPPER_BOUND = "disp_upper_bound" 52 | 53 | 54 | # disparity range computation 55 | DISPARITY_RANGE_COMPUTATION_TAG = "disparity_range_computation" 56 | MINIMUM_DISPARITY_TAG = "minimum_disparity" 57 | MAXIMUM_DISPARITY_TAG = "maximum_disparity" 58 | MATCHES_TAG = "matches" 59 | DISPARITY_MARGIN_PARAM_TAG = "disparity_margin_param" 60 | 61 | # Matches filtering 62 | METHOD = "method" 63 | MATCH_FILTERING_TAG = "match_filtering" 64 | NUMBER_MATCHES_TAG = "number_matches" 65 | RAW_NUMBER_MATCHES_TAG = "raw_number_matches" 66 | BEFORE_CORRECTION_EPI_ERROR_MEAN = "before_correction_epi_error_mean" 67 | BEFORE_CORRECTION_EPI_ERROR_STD = "before_correction_epi_error_std" 68 | BEFORE_CORRECTION_EPI_ERROR_MAX = "before_correction_epi_error_max" 69 | -------------------------------------------------------------------------------- /cars/applications/triangulation/__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-module-docstring 2 | # flake8: noqa 3 | # 4 | # !/usr/bin/env python 5 | # coding: utf8 6 | # 7 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 8 | # 9 | # This file is part of CARS 10 | # (see https://github.com/CNES/cars). 11 | # 12 | # Licensed under the Apache License, Version 2.0 (the "License"); 13 | # you may not use this file except in compliance with the License. 14 | # You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | # See the License for the specific language governing permissions and 22 | # limitations under the License. 23 | # 24 | """ 25 | CARS core triangulation module init file 26 | """ 27 | 28 | from cars.applications.triangulation.triangulation import Triangulation 29 | 30 | from . import line_of_sight_intersection 31 | -------------------------------------------------------------------------------- /cars/applications/triangulation/triangulation_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the constants of triangulation. 23 | """ 24 | 25 | 26 | # USED VARIABLES 27 | 28 | 29 | TRIANGULATION_RUN_TAG = "triangulation" 30 | 31 | 32 | # PARAMS 33 | METHOD = "method" 34 | SNAP_TO_IMG1 = "snap_to_img1" 35 | GEOMETRY_PLUGIN = "geometry_plugin" 36 | 37 | # INFOS 38 | ALT_REFERENCE_TAG = "alt_reference" 39 | -------------------------------------------------------------------------------- /cars/conf/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS conf module init file 23 | """ 24 | -------------------------------------------------------------------------------- /cars/conf/geoid/egm96.grd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/cars/d86b7491ac1e7ea3627535bd571ea65df3933073/cars/conf/geoid/egm96.grd -------------------------------------------------------------------------------- /cars/conf/geoid/egm96.grd.hdr: -------------------------------------------------------------------------------- 1 | ENVI 2 | samples = 1441 3 | lines = 721 4 | bands = 1 5 | header offset = 24 6 | file type = ENVI Standard 7 | data type = 4 8 | interleave = bsq 9 | sensor type = Unknown 10 | byte order = 1 11 | wavelength units = Unknown 12 | map info = {Geographic Lat/Lon, 1, 1,-0.125, 90.125, 0.25, 0.25,WGS-84} 13 | coordinate system string = {GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]} 14 | band names = { 15 | Band 1} 16 | -------------------------------------------------------------------------------- /cars/conf/mask_cst.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS mask module 23 | """ 24 | 25 | # Standard imports 26 | 27 | 28 | # CARS imports 29 | 30 | # Specific values 31 | # 0 = valid pixels 32 | # 255 = value used as no data during the rectification in the epipolar geometry 33 | VALID_VALUE = 0 34 | NO_DATA_IN_EPIPOLAR_RECTIFICATION = 255 35 | PROTECTED_VALUES = [NO_DATA_IN_EPIPOLAR_RECTIFICATION] 36 | -------------------------------------------------------------------------------- /cars/core/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS core module init file 23 | """ 24 | -------------------------------------------------------------------------------- /cars/core/constants_disparity.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS Disparity Constants module 23 | """ 24 | 25 | # disparity map 26 | MAP = "disp" 27 | CONFIDENCE = "confidence" 28 | CONFIDENCE_KEY = "cost_volume_confidence" 29 | INTERVAL = "interval_bounds" 30 | INTERVAL_INF = "confidence_from_interval_bounds_inf" 31 | INTERVAL_SUP = "confidence_from_interval_bounds_sup" 32 | EPI_DISP_MIN_GRID = "disp_min_grid" 33 | EPI_DISP_MAX_GRID = "disp_max_grid" 34 | 35 | # disparity mask 36 | VALID = "disp_msk" 37 | FILLING = "filling" 38 | INVALID_REF = "msk_invalid_ref" 39 | INVALID_SEC = "msk_invalid_sec" 40 | MASKED_REF = "msk_masked_ref" 41 | MASKED_SEC = "msk_masked_sec" 42 | OCCLUSION = "msk_occlusion" 43 | FALSE_MATCH = "msk_false_match" 44 | INCOMPLETE_DISP = "msk_incomplete_disp" 45 | STOPPED_INTERP = "msk_stopped_interp" 46 | FILLED_OCCLUSION = "msk_filled_occlusion" 47 | FILLED_FALSE_MATCH = "msk_filled_false_match" 48 | INSIDE_SEC_ROI = "msk_inside_sec_roi" 49 | DISP_TO_0 = "msk_disp_to_0" 50 | -------------------------------------------------------------------------------- /cars/core/datasets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS Datasets module 23 | """ 24 | 25 | # Standard imports 26 | from typing import List 27 | 28 | # Third party imports 29 | import numpy as np 30 | import rasterio as rio 31 | import xarray as xr 32 | 33 | # CARS imports 34 | from cars.core import constants as cst 35 | 36 | # TODO: refacto constants: define constants here as only concerning datasets 37 | 38 | 39 | def create_im_dataset( 40 | img: np.ndarray, 41 | region: List[int], 42 | largest_size: List[int], 43 | img_path: str = None, 44 | band_coords: str = None, 45 | msk: np.ndarray = None, 46 | ) -> xr.Dataset: 47 | """ 48 | Create image dataset as used in cars. 49 | 50 | :param img: image as a numpy array 51 | :param region: region as list [xmin ymin xmax ymax] 52 | :param largest_size: whole image size 53 | :param img_path: path to image 54 | :param band_type: set to band coord names (cst.BAND_IM or BAND_CLASSIF) 55 | to add band description in the dataset 56 | :param msk: image mask as a numpy array (default None) 57 | :return: The image dataset as used in cars 58 | """ 59 | nb_bands = img.shape[0] 60 | 61 | # Get georef and transform 62 | img_crs = None 63 | img_transform = None 64 | descriptions = None 65 | if img_path is not None: 66 | with rio.open(img_path) as img_srs: 67 | img_crs = img_srs.profile["crs"] 68 | img_transform = img_srs.profile["transform"] 69 | descriptions = list(img_srs.descriptions) 70 | 71 | if img_crs is None: 72 | img_crs = "None" 73 | if img_transform is None: 74 | img_transform = "None" 75 | 76 | # Add band dimension if needed 77 | if band_coords or nb_bands > 1: 78 | # Reorder dimensions in color dataset in order that the first dimension 79 | # is band. 80 | if band_coords == cst.BAND_IM: 81 | if descriptions is None or None in descriptions: 82 | if nb_bands > 4: 83 | raise RuntimeError("Not implemented case") 84 | default_band = ["R", "G", "B", "N"] 85 | descriptions = default_band[:nb_bands] 86 | 87 | dataset = xr.Dataset( 88 | { 89 | cst.EPI_IMAGE: ( 90 | [band_coords, cst.ROW, cst.COL], 91 | img, 92 | ) 93 | }, 94 | coords={ 95 | band_coords: descriptions, 96 | cst.ROW: np.array(range(region[1], region[3])), 97 | cst.COL: np.array(range(region[0], region[2])), 98 | }, 99 | ) 100 | else: 101 | if descriptions is None or None in descriptions: 102 | descriptions = None 103 | dataset = xr.Dataset( 104 | {cst.EPI_IMAGE: ([cst.ROW, cst.COL], img[0, ...])}, 105 | coords={ 106 | cst.ROW: np.array(range(region[1], region[3])), 107 | cst.COL: np.array(range(region[0], region[2])), 108 | }, 109 | ) 110 | 111 | if msk is not None: 112 | dataset[cst.EPI_MSK] = xr.DataArray( 113 | msk[0, ...].astype(np.int16), dims=[cst.ROW, cst.COL] 114 | ) 115 | 116 | dataset.attrs[cst.EPI_VALID_PIXELS] = 0 117 | dataset.attrs[cst.EPI_NO_DATA_MASK] = 255 118 | dataset.attrs[cst.EPI_FULL_SIZE] = largest_size 119 | dataset.attrs[cst.EPI_CRS] = img_crs 120 | dataset.attrs[cst.EPI_TRANSFORM] = img_transform 121 | dataset.attrs["region"] = np.array(region) 122 | if descriptions is not None: 123 | dataset.attrs[cst.BAND_NAMES] = descriptions 124 | return dataset 125 | 126 | 127 | def get_color_bands(dataset): 128 | """ 129 | Get band names list from the cardataset color 130 | 131 | :param dataset: carsdataset with the color data 132 | :type dataset: CarsDataset 133 | :param key: dataset color data key 134 | :param key: string 135 | 136 | :return: list of color band names 137 | """ 138 | band_im = None 139 | if cst.BAND_NAMES in dataset.attrs.keys(): 140 | band_im = dataset.attrs[cst.BAND_NAMES] 141 | 142 | return band_im 143 | -------------------------------------------------------------------------------- /cars/core/geometry/__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-module-docstring 2 | # flake8: noqa 3 | #!/usr/bin/env python 4 | # coding: utf8 5 | # 6 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 7 | # 8 | # This file is part of CARS 9 | # (see https://github.com/CNES/cars). 10 | # 11 | # Licensed under the Apache License, Version 2.0 (the "License"); 12 | # you may not use this file except in compliance with the License. 13 | # You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, software 18 | # distributed under the License is distributed on an "AS IS" BASIS, 19 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 20 | # See the License for the specific language governing permissions and 21 | # limitations under the License. 22 | # 23 | """ 24 | Init file of geometry module 25 | """ 26 | 27 | from . import abstract_geometry, shareloc_geometry 28 | -------------------------------------------------------------------------------- /cars/data_structures/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS data_structures module init file 23 | """ 24 | -------------------------------------------------------------------------------- /cars/data_structures/cars_dict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | cars_dict module: 23 | 24 | """ 25 | 26 | 27 | class CarsDict: 28 | """ 29 | CarsDict. 30 | 31 | Internal CARS structure for dict representation 32 | """ 33 | 34 | def __init__(self, dict_data, attributes=None): 35 | """ 36 | Init function of CarsDict. 37 | If a path is provided, restore CarsDataset saved on disk. 38 | 39 | :param dict_data: dictrionary to store 40 | :type dict_data: dict 41 | :param attributes: attributes 42 | :type attributes: dict 43 | 44 | """ 45 | 46 | self.data = dict_data 47 | self.attrs = attributes 48 | if self.attrs is None: 49 | self.attrs = {} 50 | 51 | def __repr__(self): 52 | """ 53 | Repr function 54 | :return: printable self CarsDataset 55 | """ 56 | return self.custom_print() 57 | 58 | def __str__(self): 59 | """ 60 | Str function 61 | :return: printable self CarsDataset 62 | """ 63 | return self.custom_print() 64 | 65 | def custom_print(self): 66 | """ 67 | Return string of self 68 | :return: printable self 69 | """ 70 | 71 | res = str(self.__class__) + ": \n" "data: " + str( 72 | self.data 73 | ) + "\n" + "attrs: " + str(self.attrs) 74 | return res 75 | -------------------------------------------------------------------------------- /cars/orchestrator/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | Cars orchestrator module init file 23 | """ 24 | -------------------------------------------------------------------------------- /cars/orchestrator/achievement_tracker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the achievement tracker 23 | """ 24 | 25 | import logging 26 | 27 | import numpy as np 28 | 29 | from cars.orchestrator.registry.abstract_registry import ( 30 | AbstractCarsDatasetRegistry, 31 | ) 32 | 33 | 34 | class AchievementTracker: 35 | """ 36 | AchievementTracker 37 | """ 38 | 39 | def __init__(self): 40 | """ 41 | Init function of AchievementTracker 42 | 43 | """ 44 | self.tracked_cars_ds = [] 45 | self.cars_ds_ids = [] 46 | self.achievement = [] 47 | 48 | def track(self, cars_ds, cars_ds_id): 49 | """ 50 | Track cars dataset 51 | 52 | :param cars_ds: cars dataset to track 53 | :type cars_ds: CarsDataset 54 | :param cars_ds_id: id of cars dataset 55 | :type cars_ds_id: int 56 | """ 57 | 58 | if cars_ds not in self.tracked_cars_ds: 59 | self.tracked_cars_ds.append(cars_ds) 60 | self.cars_ds_ids.append(cars_ds_id) 61 | self.achievement.append(np.zeros(cars_ds.shape, dtype=bool)) 62 | 63 | def add_tile(self, tile): 64 | """ 65 | Add finished tile 66 | 67 | :param tile: finished tile 68 | :type tile: xarray Dataset or Pandas Dataframe 69 | """ 70 | 71 | try: 72 | self._add_tile(tile) 73 | except RuntimeError: 74 | logging.error("Error getting id in Achiement Tracker") 75 | 76 | def _add_tile(self, tile): 77 | """ 78 | Add finished tile 79 | 80 | :param tile: finished tile 81 | :type tile: xarray Dataset or Pandas Dataframe 82 | """ 83 | 84 | # Get cars dataset id 85 | cars_ds_id = AbstractCarsDatasetRegistry.get_future_cars_dataset_id( 86 | tile 87 | ) 88 | if cars_ds_id is None: 89 | raise RuntimeError("No id in data") 90 | if cars_ds_id not in self.cars_ds_ids: 91 | raise RuntimeError("Cars ds not registered") 92 | index = self.cars_ds_ids.index(cars_ds_id) 93 | 94 | # Get position 95 | row, col = AbstractCarsDatasetRegistry.get_future_cars_dataset_position( 96 | tile 97 | ) 98 | if None in (row, col): 99 | logging.error("None in row, col in achievement tracker") 100 | else: 101 | # update 102 | self.achievement[index][row, col] = 1 103 | 104 | def get_remaining_tiles(self): 105 | """ 106 | Get remaining tiles to compute 107 | 108 | :return: remaining tiles 109 | :rtype: list(delayed) 110 | """ 111 | 112 | tiles = [] 113 | 114 | for cars_ds, achievement in zip( # noqa: B905 115 | self.tracked_cars_ds, self.achievement 116 | ): 117 | for row in range(cars_ds.shape[0]): 118 | for col in range(cars_ds.shape[1]): 119 | if ( 120 | not achievement[row, col] 121 | and cars_ds[row, col] is not None 122 | ): 123 | tiles.append(cars_ds[row, col]) 124 | 125 | return tiles 126 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS cluster module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | # CARS imports 27 | from cars.orchestrator.cluster.abstract_cluster import AbstractCluster 28 | 29 | from . import ( 30 | abstract_dask_cluster, 31 | dask_cluster_tools, 32 | local_dask_cluster, 33 | mp_cluster, 34 | pbs_dask_cluster, 35 | sequential_cluster, 36 | slurm_dask_cluster, 37 | ) 38 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/dask_cluster_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | Contains functions cluster conf checker 23 | """ 24 | import time 25 | 26 | from json_checker import And, Checker, Or 27 | 28 | 29 | def create_checker_schema(conf): 30 | """ 31 | Create checker shema that it can be overloaded 32 | 33 | :param conf: configuration to check 34 | :type conf: dict 35 | 36 | :return: overloaded configuration 37 | :rtype: dict 38 | 39 | """ 40 | 41 | # init conf 42 | if conf is not None: 43 | overloaded_conf = conf.copy() 44 | else: 45 | conf = {} 46 | overloaded_conf = {} 47 | 48 | # Overload conf 49 | overloaded_conf["mode"] = conf.get("mode", "unknowed_dask") 50 | overloaded_conf["use_memory_logger"] = conf.get("use_memory_logger", False) 51 | overloaded_conf["nb_workers"] = conf.get("nb_workers", 2) 52 | overloaded_conf["task_timeout"] = conf.get("task_timeout", 600) 53 | overloaded_conf["max_ram_per_worker"] = conf.get("max_ram_per_worker", 2000) 54 | overloaded_conf["walltime"] = conf.get("walltime", "00:59:00") 55 | overloaded_conf["config_name"] = conf.get("config_name", "unknown") 56 | overloaded_conf["activate_dashboard"] = conf.get( 57 | "activate_dashboard", False 58 | ) 59 | overloaded_conf["python"] = conf.get("python", None) 60 | overloaded_conf["profiling"] = conf.get("profiling", {}) 61 | 62 | cluster_schema = { 63 | "mode": str, 64 | "use_memory_logger": bool, 65 | "nb_workers": And(int, lambda x: x > 0), 66 | "task_timeout": And(int, lambda x: x > 0), 67 | "max_ram_per_worker": And(Or(float, int), lambda x: x > 0), 68 | "walltime": str, 69 | "config_name": str, 70 | "activate_dashboard": bool, 71 | "profiling": dict, 72 | "python": Or(None, str), 73 | } 74 | 75 | return overloaded_conf, cluster_schema 76 | 77 | 78 | def check_configuration(overloaded_conf, cluster_schema): 79 | """ 80 | Check configuration from overload conf and cluster schema 81 | 82 | 83 | :param conf: overloaded_conf to check 84 | :type conf: dict 85 | :param conf: cluster_schema checking rules 86 | :type conf: dict 87 | 88 | :return: overloaded configuration 89 | :rtype: dict 90 | 91 | """ 92 | # Check conf 93 | checker = Checker(cluster_schema) 94 | checker.validate(overloaded_conf) 95 | 96 | # Check walltime format 97 | walltime = overloaded_conf["walltime"] 98 | try: 99 | time.strptime(walltime, "%H:%M:%S") 100 | except ValueError as err: 101 | raise ValueError("Walltime should be formatted as HH:MM:SS") from err 102 | 103 | return overloaded_conf 104 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/dask_config/dask.yaml: -------------------------------------------------------------------------------- 1 | temporary-directory: null # Directory for local disk like /tmp, /scratch, or /local. null -> local dask-worker-space directory. 2 | 3 | dataframe: 4 | shuffle: 5 | compression: null # compression for on disk-shuffling. Partd supports ZLib, BZ2, SNAPPY, BLOSC 6 | 7 | array: 8 | svg: 9 | size: 120 # pixels for jupyter notebook array rendering 10 | slicing: 11 | split-large-chunks: null # How to handle large output chunks in slicing. Warns by default. 12 | 13 | optimization: 14 | fuse: # Options for Dask's task fusion optimizations 15 | active: true 16 | ave-width: 1 # Upper limit for width, where width = num_nodes / height 17 | max-width: null # 1.5 + ave_width * log(ave_width + 1) 18 | max-height: .inf # Fuse all possibilities 19 | max-depth-new-edges: null # ave_width * 1.5 20 | subgraphs: null # true for dask.dataframe, false for everything else. Set to null to let the default optimizer of individual dask collections decide. 21 | rename-keys: true # Set to true to rename the fused keys with `default_fused_keys_renamer`. Can be costly. 22 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/dask_config/distributed.yaml: -------------------------------------------------------------------------------- 1 | distributed: 2 | version: 2 3 | logging: # Logging dask distributed module conf 4 | distributed: warning 5 | distributed.client: warning 6 | distributed.worker: warning 7 | distributed.scheduler: warning 8 | distributed.nanny: warning 9 | distributed.core: warning 10 | bokeh: critical 11 | # http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr 12 | tornado: critical 13 | tornado.application: error 14 | 15 | scheduler: 16 | blocked-handlers: [] # message operation list to block from scheduler to worker 17 | allowed-failures: 10 # number of retries before a task is considered bad 18 | bandwidth: 100000000 # 100 MB/s estimated worker-worker bandwidth 19 | work-stealing: True # workers should steal tasks from each other 20 | 21 | worker: 22 | blocked-handlers: [] # message operation list to block from scheduler to worker 23 | multiprocessing-method: forkserver # can be fork, spawn or forkserver multiprocessing method 24 | connections: # Maximum concurrent connections for data 25 | outgoing: 50 # This helps to control network saturation 26 | incoming: 50 27 | validate: True # Check worker state at every step for debugging 28 | lifetime: 29 | duration: null 30 | stagger: 0 # Random amount of time by which to stagger lifetime. Avoid kill at the same lifetime 31 | restart: False # Do we ressurrect the worker after the lifetime deadline? 32 | profile: 33 | interval: 10ms # Time between statistical profiling queries 34 | cycle: 1000ms # Time between starting new profile 35 | memory: 36 | target: False # target fraction to stay below 37 | spill: False # fraction at which we spill to disk 38 | pause: False # fraction at which we pause worker threads 39 | terminate: False # fraction at which we terminate the worker 40 | comm: 41 | retry: # some operations (such as gathering data) are subject to re-tries with the below parameters 42 | count: 10 # the maximum retry attempts. 0 disables re-trying. 43 | compression: auto # Compression for communication; Default: auto 44 | default-scheme: tcp # default scheme (can be tls for secure comm) 45 | socket-backlog: 2048 # default value, must be large enough for data between workers 46 | 47 | timeouts: 48 | connect: 60s # time before connecting fails (default: 10s) 49 | tcp: 120s # time before calling an unresponsive connection dead (default: 30s) 50 | 51 | require-encryption: False # Whether to require encryption on non-local comms 52 | 53 | ################### 54 | # Bokeh dashboard # 55 | ################### 56 | 57 | dashboard: 58 | export-tool: False # Deactivate bokeh dashboard for performance 59 | 60 | ################## 61 | # Administrative # 62 | ################## 63 | 64 | admin: 65 | tick: 66 | interval: 20ms # time between event loop health checks 67 | limit: 1s # time allowed before triggering a warning 68 | log-format: '%(asctime)s :: %(name)s - %(levelname)s - %(message)s' 69 | pdb-on-err: False # enter debug mode on scheduling error 70 | low-level-log-length: 100000 # How long should we keep the transition log in memory (default length 100000 (bytes?)) 71 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/dask_config/jobqueue.yaml: -------------------------------------------------------------------------------- 1 | jobqueue: 2 | pbs: 3 | name: cars-dask-worker # Name of dask worker (set to PBS JOB NAME) 4 | 5 | # Dask worker options 6 | cores: null # Total number of cores per job 7 | memory: null # Total amount of memory per job 8 | processes: null # Number of Python processes per job 9 | 10 | interface: null # Network interface to use like eth0 or ib0 11 | death-timeout: 3000 # Number of seconds to wait if a worker can not find a scheduler 12 | local-directory: null # Location of fast local storage like /scratch or $TMPDIR 13 | extra: [] # Additional arguments to pass to `dask-worker` 14 | 15 | # PBS resource manager options 16 | shebang: "#!/usr/bin/env bash" # Path to desired interpreter for your batch submission script. 17 | queue: null # PBS queue by default. Take the one specified in PBSCluster() first 18 | project: null # PBS project name in ACCOUNT information 19 | walltime: '00:59:00' # PBS walltime by default. 20 | env-extra: [] # PBS environnements to pass to workers 21 | resource-spec: null # Resource string for PBS cpu and memory by default 22 | job-extra: [] # ? 23 | log-directory: null # Log directory by default 24 | 25 | # Scheduler options 26 | scheduler-options: {} # Used to pass additional arguments to Dask Scheduler. 27 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/dask_config/reference_confs/dask.yaml: -------------------------------------------------------------------------------- 1 | temporary-directory: null # Directory for local disk like /tmp, /scratch, or /local 2 | 3 | tokenize: 4 | ensure-deterministic: false # If true, tokenize will error instead of falling back to uuids 5 | 6 | dataframe: 7 | shuffle-compression: null # compression for on disk-shuffling. Partd supports ZLib, BZ2, SNAPPY, BLOSC 8 | parquet: 9 | metadata-task-size-local: 512 # Number of files per local metadata-processing task 10 | metadata-task-size-remote: 16 # Number of files per remote metadata-processing task 11 | 12 | array: 13 | svg: 14 | size: 120 # pixels 15 | slicing: 16 | split-large-chunks: null # How to handle large output chunks in slicing. Warns by default. 17 | 18 | optimization: 19 | fuse: 20 | active: null # Treat as false for dask.dataframe, true for everything else 21 | ave-width: 1 22 | max-width: null # 1.5 + ave_width * log(ave_width + 1) 23 | max-height: .inf 24 | max-depth-new-edges: null # ave_width * 1.5 25 | subgraphs: null # true for dask.dataframe, false for everything else 26 | rename-keys: true 27 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/local_dask_cluster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | Contains functions for local dask Cluster 23 | """ 24 | 25 | # Standard imports 26 | import logging 27 | 28 | # Third party imports 29 | from dask.distributed import Client, LocalCluster 30 | 31 | # CARS imports 32 | from cars.orchestrator.cluster import abstract_cluster, abstract_dask_cluster 33 | from cars.orchestrator.cluster.dask_cluster_tools import ( 34 | check_configuration, 35 | create_checker_schema, 36 | ) 37 | 38 | 39 | @abstract_cluster.AbstractCluster.register_subclass("local_dask") 40 | class LocalDaskCluster(abstract_dask_cluster.AbstractDaskCluster): 41 | """ 42 | LocalDaskCluster 43 | """ 44 | 45 | def check_conf(self, conf): 46 | """ 47 | Check configuration 48 | 49 | :param conf: configuration to check 50 | :type conf: dict 51 | 52 | :return: overloaded configuration 53 | :rtype: dict 54 | 55 | """ 56 | 57 | return check_configuration(*create_checker_schema(conf)) 58 | 59 | def start_dask_cluster(self): 60 | """ 61 | Start dask cluster 62 | """ 63 | 64 | return start_local_cluster( 65 | self.nb_workers, activate_dashboard=self.activate_dashboard 66 | ) 67 | 68 | def cleanup(self): 69 | """ 70 | Cleanup cluster 71 | 72 | """ 73 | stop_local_cluster(self.cluster, self.client) 74 | logging.info("Dask cluster closed") 75 | 76 | 77 | def start_local_cluster(nb_workers, timeout=600, activate_dashboard=False): 78 | """ 79 | Start a local cluster 80 | 81 | :param nb_workers: Number of dask workers 82 | :type nb_workers: int 83 | :param timeout: Connection timeout 84 | :type timeout: int 85 | :return: Local cluster and Dask client 86 | :rtype: (dask.distributed.LocalCluster, dask.distributed.Client) tuple 87 | """ 88 | logging.info("Local cluster with {} workers started".format(nb_workers)) 89 | 90 | if activate_dashboard: 91 | dashboard_address = ":0" 92 | else: 93 | dashboard_address = None 94 | 95 | cluster = LocalCluster( 96 | n_workers=nb_workers, 97 | threads_per_worker=1, 98 | dashboard_address=dashboard_address, 99 | ) 100 | 101 | client = Client(cluster, timeout=timeout) 102 | return cluster, client 103 | 104 | 105 | def stop_local_cluster(cluster, client): 106 | """ 107 | Stop a local cluster 108 | 109 | :param cluster: Local cluster 110 | :type cluster: dask.distributed.LocalCluster 111 | :param client: Dask client 112 | :type client: dask.distributed.Client 113 | """ 114 | client.close() 115 | cluster.close() 116 | logging.info("Local cluster correctly stopped") 117 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/mp_cluster/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS multiprocessing cluster module init file 23 | """ 24 | # flake8: noqa: F401 25 | 26 | 27 | from . import multiprocessing_cluster 28 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/mp_cluster/mp_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | Contains tools for multiprocessing 23 | """ 24 | 25 | 26 | def replace_data(list_or_dict, func_to_apply, *func_args): 27 | """ 28 | Replace MpJob in list or dict by their real data 29 | (can deal with FactorizedObject) 30 | 31 | :param list_or_dict: list or dict of data or mp_objects.FactorizedObject 32 | :param func_to_apply: function to apply 33 | :param func_args: function arguments 34 | 35 | :return: list or dict with real data 36 | :rtype: list, tuple, dict, mp_objects.FactorizedObject 37 | """ 38 | if ( 39 | isinstance(list_or_dict, (list, tuple)) 40 | and len(list_or_dict) == 1 41 | and type(list_or_dict[0]).__name__ == "FactorizedObject" 42 | ): 43 | # list_or_dict is a single FactorizedObject 44 | factorized_object = list_or_dict[0] 45 | args = factorized_object.get_args() 46 | args = replace_data_rec(args, func_to_apply, *func_args) 47 | kwargs = factorized_object.get_kwargs() 48 | kwargs = replace_data_rec(kwargs, func_to_apply, *func_args) 49 | 50 | factorized_object.set_args(args) 51 | factorized_object.set_kwargs(kwargs) 52 | return [factorized_object] 53 | 54 | return replace_data_rec(list_or_dict, func_to_apply, *func_args) 55 | 56 | 57 | def replace_data_rec(list_or_dict, func_to_apply, *func_args): 58 | """ 59 | Replace MpJob in list or dict by their real data recursively 60 | 61 | :param list_or_dict: list or dict of data 62 | :param func_to_apply: function to apply 63 | :param func_args: function arguments 64 | 65 | :return: list or dict with real data 66 | :rtype: list, tuple, dict 67 | """ 68 | 69 | if isinstance(list_or_dict, (list, tuple)): 70 | res = [] 71 | for arg in list_or_dict: 72 | if isinstance(arg, (list, tuple, dict)): 73 | res.append(replace_data_rec(arg, func_to_apply, *func_args)) 74 | else: 75 | res.append(func_to_apply(arg, *func_args)) 76 | if isinstance(list_or_dict, tuple): 77 | res = tuple(res) 78 | 79 | elif isinstance(list_or_dict, dict): 80 | res = {} 81 | for key, value in list_or_dict.items(): 82 | if isinstance(value, (list, dict, tuple)): 83 | res[key] = replace_data_rec(value, func_to_apply, *func_args) 84 | else: 85 | res[key] = func_to_apply(value, *func_args) 86 | 87 | else: 88 | raise TypeError( 89 | "Function only support list or dict or tuple, " 90 | "but type is {}".format(list_or_dict) 91 | ) 92 | 93 | return res 94 | -------------------------------------------------------------------------------- /cars/orchestrator/cluster/sequential_cluster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | Contains functions for sequential cluster 23 | """ 24 | 25 | # Standard imports 26 | 27 | # Third party imports 28 | from json_checker import Checker, Or 29 | 30 | # CARS imports 31 | from cars.orchestrator.cluster import abstract_cluster 32 | 33 | 34 | @abstract_cluster.AbstractCluster.register_subclass("sequential") 35 | class SequentialCluster(abstract_cluster.AbstractCluster): 36 | """ 37 | SequentialCluster 38 | """ 39 | 40 | def __init__(self, conf_cluster, out_dir, launch_worker=True): 41 | """ 42 | Init function of SequentialCluster 43 | 44 | :param conf_cluster: configuration for cluster 45 | 46 | """ 47 | # call parent init 48 | super().__init__(conf_cluster, out_dir, launch_worker=launch_worker) 49 | 50 | # retrieve parameters 51 | self.profiling = self.checked_conf_cluster["profiling"] 52 | self.out_dir = out_dir 53 | self.launch_worker = launch_worker 54 | 55 | def check_conf(self, conf): 56 | """ 57 | Check configuration 58 | 59 | :param conf: configuration to check 60 | :type conf: dict 61 | 62 | :return: overloaded configuration 63 | :rtype: dict 64 | 65 | """ 66 | 67 | # init conf 68 | if conf is not None: 69 | overloaded_conf = conf.copy() 70 | else: 71 | conf = {} 72 | overloaded_conf = {} 73 | 74 | # Overload conf 75 | overloaded_conf["mode"] = conf.get("mode", "sequential") 76 | overloaded_conf["max_ram_per_worker"] = conf.get( 77 | "max_ram_per_worker", 2000 78 | ) 79 | overloaded_conf["profiling"] = conf.get("profiling", {}) 80 | 81 | cluster_schema = { 82 | "mode": str, 83 | "max_ram_per_worker": Or(float, int), 84 | "profiling": dict, 85 | } 86 | 87 | # Check conf 88 | checker = Checker(cluster_schema) 89 | checker.validate(overloaded_conf) 90 | 91 | return overloaded_conf 92 | 93 | def get_delayed_type(self): 94 | """ 95 | Get delayed type 96 | """ 97 | return object 98 | 99 | def cleanup(self): 100 | """ 101 | Cleanup cluster 102 | 103 | """ 104 | 105 | def create_task_wrapped(self, func, nout=2): 106 | """ 107 | Create task 108 | 109 | :param func: function 110 | :param nout: number of outputs 111 | """ 112 | return func 113 | 114 | def start_tasks(self, task_list): 115 | """ 116 | Start all tasks 117 | 118 | :param task_list: task list 119 | """ 120 | return task_list 121 | 122 | def scatter(self, data, broadcast=True): # pylint: disable=W0613 123 | """ 124 | Distribute data through workers 125 | 126 | :param data: task data 127 | """ 128 | return data 129 | 130 | def future_iterator(self, future_list, timeout=None): 131 | """ 132 | Start all tasks 133 | 134 | :param future_list: future_list list 135 | """ 136 | 137 | for future in future_list: 138 | yield future 139 | -------------------------------------------------------------------------------- /cars/orchestrator/orchestrator_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the orchestrator constants 23 | """ 24 | 25 | 26 | CARS_DATASET_KEY = "cars_ds_key" 27 | SAVING_INFO = "saving_info" 28 | CARS_DS_ROW = "cars_ds_row" 29 | CARS_DS_COL = "cars_ds_col" 30 | -------------------------------------------------------------------------------- /cars/orchestrator/registry/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | Cars registry module init file 23 | """ 24 | -------------------------------------------------------------------------------- /cars/orchestrator/registry/abstract_registry.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | This module contains the abstract registry class 23 | """ 24 | 25 | import logging 26 | from abc import abstractmethod 27 | 28 | # CARS imports 29 | from cars.orchestrator.orchestrator_constants import ( 30 | CARS_DATASET_KEY, 31 | CARS_DS_COL, 32 | CARS_DS_ROW, 33 | SAVING_INFO, 34 | ) 35 | 36 | 37 | class AbstractCarsDatasetRegistry: 38 | """ 39 | AbstractCarsDatasetRegistry 40 | This is the abstract class of registries, managing delayed CarsDatasets 41 | 42 | """ 43 | 44 | def __init__(self, id_generator): 45 | """ 46 | Init function of AbstractCarsDatasetRegistry 47 | 48 | :param id_generator: id generator 49 | :type id_generator: IdGenerator 50 | 51 | """ 52 | 53 | self.id_generator = id_generator 54 | self.id_generator.add_registry(self) 55 | 56 | @abstractmethod 57 | def cars_dataset_in_registry(self, cars_ds): 58 | """ 59 | Check if a CarsDataset is already registered, return id if exists 60 | 61 | :param cars_ds: cars dataset 62 | :type cars_ds: CarsDataset 63 | 64 | :return : True if in registry, if of cars dataset 65 | :rtype : Tuple(bool, int) 66 | """ 67 | 68 | @abstractmethod 69 | def get_cars_datasets_list(self): 70 | """ 71 | Get a list of registered CarsDataset 72 | 73 | :return list of CarsDataset 74 | :rtype: list(CarsDataset) 75 | """ 76 | 77 | @abstractmethod 78 | def get_cars_ds(self, future_result): 79 | """ 80 | Get a list of registered CarsDataset 81 | 82 | :param future_result: object to get cars dataset from 83 | 84 | :return corresponding CarsDataset 85 | :rtype: CarsDataset 86 | """ 87 | 88 | @staticmethod 89 | def get_future_cars_dataset_id(future_result): 90 | """ 91 | Get cars dataset id for current future result 92 | 93 | :param future_result: future result: 94 | :type future_result: xr.Dataset or pd.DataFrame 95 | 96 | :return cars dataset id 97 | :rtype : int 98 | """ 99 | 100 | cars_ds_id = None 101 | 102 | if isinstance(future_result, dict): 103 | attributes_info_dict = future_result 104 | else: 105 | attributes_info_dict = future_result.attrs 106 | 107 | if SAVING_INFO in attributes_info_dict: 108 | if CARS_DATASET_KEY in attributes_info_dict[SAVING_INFO]: 109 | cars_ds_id = attributes_info_dict[SAVING_INFO][CARS_DATASET_KEY] 110 | 111 | return cars_ds_id 112 | 113 | @staticmethod 114 | def get_future_cars_dataset_position(future_result): 115 | """ 116 | Get cars dataset positions for current future result 117 | 118 | :param future_result: future result: 119 | :type future_result: xr.Dataset or pd.DataFrame 120 | 121 | :return cars dataset id 122 | :rtype : tuple(int) 123 | """ 124 | 125 | cars_ds_row = None 126 | cars_ds_col = None 127 | 128 | if isinstance(future_result, dict): 129 | attributes_info_dict = future_result 130 | else: 131 | attributes_info_dict = future_result.attrs 132 | 133 | if SAVING_INFO in attributes_info_dict: 134 | if CARS_DS_ROW in attributes_info_dict[SAVING_INFO]: 135 | cars_ds_row = attributes_info_dict[SAVING_INFO][CARS_DS_ROW] 136 | else: 137 | logging.debug("No row given in object") 138 | if CARS_DS_COL in attributes_info_dict[SAVING_INFO]: 139 | cars_ds_col = attributes_info_dict[SAVING_INFO][CARS_DS_COL] 140 | else: 141 | logging.debug("No col given in object") 142 | 143 | return cars_ds_row, cars_ds_col 144 | -------------------------------------------------------------------------------- /cars/orchestrator/registry/compute_registry.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | This module contains the replacer registry class 23 | """ 24 | 25 | # CARS imports 26 | from cars.orchestrator.registry.abstract_registry import ( 27 | AbstractCarsDatasetRegistry, 28 | ) 29 | 30 | 31 | class CarsDatasetRegistryCompute(AbstractCarsDatasetRegistry): 32 | """ 33 | CarsDatasetRegistryCompute 34 | This registry manages the computation of arriving future results 35 | into corresponding CarsDataset 36 | """ 37 | 38 | def __init__(self, id_generator): 39 | """ 40 | Init function of CarsDatasetRegistryCompute 41 | 42 | :param id_generator: id generator 43 | :type id_generator: IdGenerator 44 | 45 | """ 46 | super().__init__(id_generator) 47 | self.registered_cars_datasets = [] 48 | self.cars_ds_ids = [] 49 | 50 | def cars_dataset_in_registry(self, cars_ds): 51 | """ 52 | Check if a CarsDataset is already registered, return id if exists 53 | 54 | :param cars_ds: cars dataset 55 | :type cars_ds: CarsDataset 56 | 57 | :return : True if in registry, if of cars dataset 58 | :rtype : Tuple(bool, int) 59 | """ 60 | cars_ds_id = None 61 | in_registry = False 62 | 63 | if cars_ds in self.registered_cars_datasets: 64 | in_registry = True 65 | cars_ds_id = self.cars_ds_ids[ 66 | self.registered_cars_datasets.index(cars_ds) 67 | ] 68 | 69 | return in_registry, cars_ds_id 70 | 71 | def get_cars_datasets_list(self): 72 | """ 73 | Get a list of registered CarsDataset 74 | 75 | :return list of CarsDataset 76 | :rtype: list(CarsDataset) 77 | """ 78 | 79 | return self.registered_cars_datasets 80 | 81 | def add_cars_ds_to_compute(self, cars_ds): 82 | """ 83 | Add cars dataset to registry 84 | 85 | :param cars_ds: cars dataset 86 | :type cars_ds: CarsDataset 87 | 88 | """ 89 | 90 | # Generate_id 91 | new_id = self.id_generator.get_new_id(cars_ds) 92 | self.cars_ds_ids.append(new_id) 93 | self.registered_cars_datasets.append(cars_ds) 94 | 95 | def get_cars_ds(self, future_result): 96 | """ 97 | Get a list of registered CarsDataset 98 | 99 | :param future_result: object to get cars dataset from 100 | 101 | :return corresponding CarsDataset 102 | :rtype: CarsDataset 103 | """ 104 | raise RuntimeError( 105 | "get_cars_ds shoud not be used in CarsDatasetRegistryCompute" 106 | ) 107 | -------------------------------------------------------------------------------- /cars/orchestrator/registry/id_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the abstract id generator class 23 | """ 24 | 25 | from cars.orchestrator.orchestrator_constants import CARS_DATASET_KEY 26 | 27 | # CARS imports 28 | from cars.orchestrator.registry.unseen_registry import CarsDatasetRegistryUnseen 29 | 30 | 31 | class IdGenerator: 32 | """ 33 | IdGenerator 34 | Creates 35 | """ 36 | 37 | def __init__(self): 38 | """ 39 | Init function of IdGenerator 40 | 41 | """ 42 | self.current_id = 0 43 | self.registries = [] 44 | 45 | # Create this registry if user gets saving infos 46 | # before telling orchestrator 47 | # he wants to save it or replace it 48 | self.unseen_registry = CarsDatasetRegistryUnseen(self) 49 | # self.unseen_registry is now in self.registries, seen Abstract init 50 | 51 | def add_registry(self, registry): 52 | """ 53 | Add registry to self 54 | 55 | :param registry: registry 56 | :type registry: AbstractCarsDatasetRegistry 57 | """ 58 | 59 | self.registries.append(registry) 60 | 61 | def get_new_id(self, cars_ds): 62 | """ 63 | Generate new id 64 | 65 | :param cars_ds: cars dataset 66 | :type cars_ds: CarsDataset 67 | 68 | :return id 69 | :rtype: int 70 | """ 71 | 72 | # Check if dataset already registered 73 | registered = False 74 | allready_registered_id = None 75 | for registry in self.registries: 76 | ( 77 | is_registered, 78 | allready_registered_id, 79 | ) = registry.cars_dataset_in_registry(cars_ds) 80 | if is_registered: 81 | registered = True 82 | returned_id = allready_registered_id 83 | 84 | if not registered: 85 | returned_id = self.current_id 86 | self.current_id += 1 87 | 88 | return returned_id 89 | 90 | def get_saving_infos(self, cars_ds): 91 | """ 92 | Get saving infos 93 | 94 | :param cars_ds: cars dataset 95 | :type cars_ds: CarsDataset 96 | 97 | :return saving infos 98 | :rtype: dict 99 | """ 100 | 101 | obj_id = None 102 | for registry in self.registries: 103 | ( 104 | is_registered, 105 | allready_registered_id, 106 | ) = registry.cars_dataset_in_registry(cars_ds) 107 | if is_registered: 108 | obj_id = allready_registered_id 109 | 110 | # add cars_ds to other register to create id 111 | obj_id = self.unseen_registry.add_cars_ds_to_unseen(cars_ds) 112 | 113 | infos = {CARS_DATASET_KEY: obj_id} 114 | 115 | return infos 116 | -------------------------------------------------------------------------------- /cars/orchestrator/registry/unseen_registry.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | This module contains the unseen registry class 23 | """ 24 | 25 | # CARS imports 26 | from cars.orchestrator.registry.abstract_registry import ( 27 | AbstractCarsDatasetRegistry, 28 | ) 29 | from cars.orchestrator.registry.replacer_registry import ( 30 | SingleCarsDatasetReplacer, 31 | ) 32 | 33 | 34 | class CarsDatasetRegistryUnseen(AbstractCarsDatasetRegistry): 35 | """ 36 | CarsDatasetRegistryUnseen 37 | This registry manages the unseen CarsDataset, that might be needed 38 | to get infos 39 | """ 40 | 41 | def __init__(self, id_generator): 42 | """ 43 | Init function of CarsDatasetRegistryUnseen 44 | 45 | :param id_generator: id generator 46 | :type id_generator: IdGenerator 47 | 48 | """ 49 | super().__init__(id_generator) 50 | self.registered_cars_datasets_unseen = [] 51 | 52 | def get_cars_ds(self, future_result): 53 | """ 54 | Get a list of registered CarsDataset 55 | 56 | :param obj: object to get cars dataset from 57 | 58 | :return corresponding CarsDataset 59 | :rtype: CarsDataset 60 | """ 61 | 62 | return None 63 | 64 | def cars_dataset_in_registry(self, cars_ds): 65 | """ 66 | Check if a CarsDataset is already registered, return id if exists 67 | 68 | :param cars_ds: cars dataset 69 | :type cars_ds: CarsDataset 70 | 71 | :return : True if in registry, if of cars dataset 72 | :rtype : Tuple(bool, int) 73 | """ 74 | 75 | in_registry = False 76 | registered_id = None 77 | for obj in self.registered_cars_datasets_unseen: 78 | if cars_ds == obj.cars_ds: 79 | in_registry = True 80 | registered_id = obj.obj_id 81 | break 82 | 83 | return in_registry, registered_id 84 | 85 | def add_cars_ds_to_unseen(self, cars_ds): 86 | """ 87 | Add cars dataset to unseen registry, and 88 | get corresponding id 89 | 90 | :param cars_ds: cars dataset 91 | :type cars_ds: CarsDataset 92 | 93 | :return : id 94 | :rtype : int 95 | """ 96 | 97 | # Generate_id 98 | new_id = self.id_generator.get_new_id(cars_ds) 99 | # create CarsDataset replacer (same storage) 100 | unseen_obj = SingleCarsDatasetReplacer(cars_ds, new_id) 101 | self.registered_cars_datasets_unseen.append(unseen_obj) 102 | 103 | return new_id 104 | 105 | def get_cars_datasets_list(self): 106 | """ 107 | Get a list of registered CarsDataset 108 | 109 | :return list of CarsDataset 110 | :rtype: list(CarsDataset) 111 | 112 | """ 113 | cars_ds_list = [] 114 | 115 | for cars_ds_saver in self.registered_cars_datasets_unseen: 116 | cars_ds_list.append(cars_ds_saver.cars_ds) 117 | 118 | return cars_ds_list 119 | -------------------------------------------------------------------------------- /cars/pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS pipelines module init file 23 | """ 24 | 25 | # Imports needed in order to register pipeline for Pipeline factory 26 | from . import default # noqa: F401 27 | -------------------------------------------------------------------------------- /cars/pipelines/default/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS default pipeline module init file 23 | """ 24 | 25 | # Cars imports 26 | from cars.pipelines.default import default_pipeline # noqa: F401 27 | -------------------------------------------------------------------------------- /cars/pipelines/parameters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CNES/cars/d86b7491ac1e7ea3627535bd571ea65df3933073/cars/pipelines/parameters/__init__.py -------------------------------------------------------------------------------- /cars/pipelines/parameters/advanced_parameters_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | """ 23 | This module contains the advanced parameter definitions 24 | """ 25 | 26 | SAVE_INTERMEDIATE_DATA = "save_intermediate_data" 27 | PHASING = "phasing" 28 | DEBUG_WITH_ROI = "debug_with_roi" 29 | 30 | USE_EPIPOLAR_A_PRIORI = "use_epipolar_a_priori" 31 | EPIPOLAR_A_PRIORI = "epipolar_a_priori" 32 | GROUND_TRUTH_DSM = "ground_truth_dsm" 33 | 34 | 35 | MERGING = "merging" 36 | DSM_MERGING_TILE_SIZE = "dsm_merging_tile_size" 37 | 38 | # inner epipolar a priori constants 39 | GRID_CORRECTION = "grid_correction" 40 | DISPARITY_RANGE = "disparity_range" 41 | 42 | TERRAIN_A_PRIORI = "terrain_a_priori" 43 | 44 | DEM_MEDIAN = "dem_median" 45 | DEM_MIN = "dem_min" 46 | DEM_MAX = "dem_max" 47 | ALTITUDE_DELTA_MAX = "altitude_delta_max" 48 | ALTITUDE_DELTA_MIN = "altitude_delta_min" 49 | 50 | # ground truth dsm 51 | INPUT_GROUND_TRUTH_DSM = "dsm" 52 | INPUT_CLASSIFICATION = "classification" 53 | INPUT_AUX_PATH = "auxiliary_data" 54 | INPUT_AUX_INTERP = "auxiliary_data_interpolation" 55 | INPUT_GEOID = "geoid" 56 | INPUT_EPSG = "epsg" 57 | 58 | PERFORMANCE_MAP_CLASSES = "performance_map_classes" 59 | 60 | GEOMETRY_PLUGIN = "geometry_plugin" 61 | 62 | PIPELINE = "pipeline" 63 | -------------------------------------------------------------------------------- /cars/pipelines/parameters/depth_map_inputs_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS point cloud to full resolution dsm pipeline constants file 23 | """ 24 | 25 | DEPTH_MAPS = "depth_maps" 26 | -------------------------------------------------------------------------------- /cars/pipelines/parameters/dsm_inputs_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS phased dsms to full resolution dsm pipeline constants file 23 | """ 24 | 25 | DSMS = "dsms" 26 | -------------------------------------------------------------------------------- /cars/pipelines/parameters/output_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | """ 23 | This module contains the output constants 24 | """ 25 | 26 | # Pipeline output keys 27 | OUT_DIRECTORY = "directory" 28 | PRODUCT_LEVEL = "product_level" 29 | INFO_FILENAME = "metadata.json" 30 | OUT_GEOID = "geoid" 31 | EPSG = "epsg" 32 | RESOLUTION = "resolution" 33 | SAVE_BY_PAIR = "save_by_pair" 34 | AUXILIARY = "auxiliary" 35 | 36 | # Auxiliary keys 37 | AUX_COLOR = "color" 38 | AUX_WEIGHTS = "weights" 39 | AUX_MASK = "mask" 40 | AUX_CLASSIFICATION = "classification" 41 | AUX_PERFORMANCE_MAP = "performance_map" 42 | AUX_FILLING = "filling" 43 | AUX_CONTRIBUTING_PAIR = "contributing_pair" 44 | AUX_AMBIGUITY = "ambiguity" 45 | 46 | 47 | # Output tree constants 48 | DSM_DIRECTORY = "dsm" 49 | -------------------------------------------------------------------------------- /cars/pipelines/parameters/sensor_inputs_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the constants used in sensor_to_full_resolution 23 | pipeline. 24 | """ 25 | 26 | # Sensor input 27 | 28 | SENSORS = "sensors" 29 | PAIRING = "pairing" 30 | 31 | INITIAL_ELEVATION = "initial_elevation" 32 | 33 | CHECK_INPUTS = "check_inputs" 34 | ROI = "roi" 35 | GEOID = "geoid" 36 | DEM_PATH = "dem" 37 | ALTITUDE_DELTA_MIN = "altitude_delta_min" 38 | ALTITUDE_DELTA_MAX = "altitude_delta_max" 39 | 40 | INPUT_IMG = "image" 41 | INPUT_MSK = "mask" 42 | INPUT_CLASSIFICATION = "classification" 43 | INPUT_GEO_MODEL = "geomodel" 44 | INPUT_GEO_MODEL_TYPE = "geomodel_type" 45 | INPUT_GEO_MODEL_FILTER = "geomodel_filters" 46 | INPUT_NODATA = "no_data" 47 | INPUT_COLOR = "color" 48 | 49 | CARS_DEFAULT_ALT = 0 # Default altitude used in cars pipelines 50 | -------------------------------------------------------------------------------- /cars/pipelines/pipeline.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | """ 23 | This module contains class pipeline factory. 24 | """ 25 | 26 | import logging 27 | from typing import Dict, Union 28 | 29 | 30 | class Pipeline: 31 | """ 32 | Pipeline factory: 33 | A class designed for registered all available Cars Pipeline and 34 | instantiate when needed. 35 | 36 | """ 37 | 38 | # Dict (pipeline_name:str, class: object) containing registered 39 | # pipelines 40 | available_pipeline = {} 41 | 42 | def __new__( 43 | cls, 44 | pipeline_name: str, 45 | cfg: Dict[str, Union[str, int]], 46 | config_json_dir, 47 | ): 48 | """ 49 | Return the instance of pipeline associated with the pipeline 50 | name given as parameter 51 | 52 | :param pipeline_name: name of the pipeline. 53 | :type pipeline_name: str 54 | :param cfg: configuration {'matching_cost_method': value} 55 | :type cfg: dictionary 56 | :param config_json_dir: path to dir containing json 57 | :type config_json_dir: str 58 | """ 59 | 60 | return cls.create_pipeline(pipeline_name, cfg, config_json_dir) 61 | 62 | @classmethod 63 | def create_pipeline( 64 | cls, name: str, cfg: Dict[str, Union[str, int]], config_json_dir 65 | ): 66 | """Factory command to create the pipeline 67 | Return the instance of pipeline associated with the pipeline 68 | name given as parameter 69 | 70 | :param pipeline_name: name of the pipeline. 71 | :type pipeline_name: str 72 | :param cfg: cars input configuration 73 | :type cfg: dictionary 74 | :param config_json_dir: path to dir containing json 75 | :type config_json_dir: str 76 | """ 77 | 78 | pipeline = None 79 | try: 80 | pipeline_class = cls.available_pipeline[name] 81 | 82 | except KeyError as kerr: 83 | logging.error("No pipeline named {0} supported".format(name)) 84 | raise NameError( 85 | "No pipeline named {0} supported".format(name) 86 | ) from kerr 87 | 88 | pipeline = pipeline_class(cfg, config_json_dir) 89 | 90 | return pipeline 91 | 92 | @classmethod 93 | def print_available_pipelines(cls): 94 | """ 95 | Print all registered pipelines 96 | """ 97 | 98 | for pipeline_name in cls.available_pipeline: 99 | print(pipeline_name) 100 | 101 | @classmethod 102 | def register(cls, *pipeline_names: str): 103 | """ 104 | Allows to register the pipeline with its name 105 | :param pipeline_name: the pipelines to be registered 106 | :type pipeline_name: string 107 | """ 108 | 109 | def decorator(app): 110 | """ 111 | Registers the class in the available methods 112 | :param app: the app class to be registered 113 | :type app: object 114 | """ 115 | for pipeline_name in pipeline_names: 116 | cls.available_pipeline[pipeline_name] = app 117 | return app 118 | 119 | return decorator 120 | -------------------------------------------------------------------------------- /cars/pipelines/pipeline_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | this module contains the constants used for all pipelines 23 | """ 24 | 25 | # Sensor input 26 | 27 | INPUTS = "inputs" 28 | OUTPUT = "output" 29 | APPLICATIONS = "applications" 30 | ADVANCED = "advanced" 31 | ORCHESTRATOR = "orchestrator" 32 | -------------------------------------------------------------------------------- /cars/pipelines/pipeline_template.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | This module contains class pipeline template for 23 | templating the pipeline concept. 24 | """ 25 | 26 | from abc import ABCMeta, abstractmethod 27 | 28 | from json_checker import Checker, OptionalKey 29 | 30 | # CARS imports 31 | from cars.orchestrator import orchestrator 32 | from cars.pipelines import pipeline_constants 33 | 34 | # Disable pylint error: too few public method 35 | 36 | 37 | class PipelineTemplate(metaclass=ABCMeta): # pylint: disable=R0903 38 | """ 39 | Class for general specification of an pipeline 40 | """ 41 | 42 | def check_orchestrator(self, conf): 43 | """ 44 | Check the configuration of orchestrator 45 | 46 | :param conf: configuration of orchestrator 47 | :type conf: dict 48 | :return: overloaded orchestrator conf 49 | :rtype: dict 50 | """ 51 | 52 | with orchestrator.Orchestrator( 53 | orchestrator_conf=conf, out_dir=None, launch_worker=False 54 | ) as orchestrator_obj: 55 | conf = orchestrator_obj.get_conf() 56 | return conf 57 | 58 | def check_global_schema(self, conf): 59 | """ 60 | Check the given global configuration 61 | 62 | :param conf: configuration 63 | :type conf: dict 64 | """ 65 | 66 | # Validate inputs 67 | global_schema = { 68 | pipeline_constants.INPUTS: dict, 69 | pipeline_constants.OUTPUT: dict, 70 | OptionalKey(pipeline_constants.APPLICATIONS): dict, 71 | OptionalKey(pipeline_constants.ORCHESTRATOR): dict, 72 | OptionalKey(pipeline_constants.ADVANCED): dict, 73 | } 74 | 75 | checker_inputs = Checker(global_schema) 76 | checker_inputs.validate(conf) 77 | 78 | def merge_pipeline_conf(self, config1, config2): 79 | """ 80 | Merge two pipeline dict, generating a new configuration 81 | 82 | :param conf1: configuration 83 | :type conf1: dict 84 | :param conf2: configuration 85 | :type conf2: dict 86 | 87 | :return: merged conf 88 | :rtype: dict 89 | 90 | """ 91 | 92 | merged_dict = config1.copy() 93 | 94 | _merge_pipeline_conf_rec(merged_dict, config2) 95 | 96 | return merged_dict 97 | 98 | @abstractmethod 99 | def check_inputs(self, conf, config_json_dir=None): 100 | """ 101 | Check the inputs given 102 | 103 | :param conf: configuration of inputs 104 | :type conf: dict 105 | :param config_json_dir: directory of used json, if 106 | user filled paths with relative paths 107 | :type config_json_dir: str 108 | 109 | :return: overloader inputs 110 | :rtype: dict 111 | """ 112 | 113 | @abstractmethod 114 | def check_output(self, conf, pipeline): 115 | """ 116 | Check the output given 117 | 118 | :param conf: configuration of output 119 | :type conf: dict 120 | :param pipeline: name of corresponding pipeline 121 | :type pipeline_name: str 122 | 123 | :return overloader output 124 | :rtype : dict 125 | """ 126 | 127 | @abstractmethod 128 | def check_applications(self, conf): 129 | """ 130 | Check the given configuration for applications 131 | 132 | :param conf: configuration of applications 133 | :type conf: dict 134 | """ 135 | 136 | @abstractmethod 137 | def run(self): 138 | """ 139 | Run pipeline 140 | """ 141 | 142 | 143 | def _merge_pipeline_conf_rec(conf1, conf2): 144 | """ 145 | Merge secondary configuration on primary one 146 | 147 | :param conf1: configuration 148 | :type conf1: dict 149 | :param conf2: configuration 150 | :type conf2: dict 151 | 152 | :return: merged conf 153 | :rtype: dict 154 | 155 | """ 156 | 157 | for key in conf2.keys(): 158 | if key in conf1: 159 | if isinstance(conf1[key], dict) and isinstance(conf2[key], dict): 160 | _merge_pipeline_conf_rec(conf1[key], conf2[key]) 161 | else: 162 | conf1[key] = conf2[key] 163 | 164 | else: 165 | conf1[key] = conf2[key] 166 | -------------------------------------------------------------------------------- /cars/starter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | cars-starter: helper to create configuration file 23 | """ 24 | 25 | # pylint: disable=import-outside-toplevel 26 | # standard imports 27 | import argparse 28 | import json 29 | import os 30 | 31 | 32 | def inputfilename_to_sensor(inputfilename): 33 | """ 34 | Fill sensor dictionary according to an input filename 35 | """ 36 | sensor = {} 37 | 38 | absfilename = os.path.abspath(inputfilename) 39 | dirname = os.path.dirname(absfilename) 40 | basename = os.path.basename(absfilename) 41 | 42 | if basename.startswith("DIM") and basename.endswith("XML"): 43 | geomodel = os.path.join(dirname, "RPC" + basename[3:]) 44 | if os.path.exists(geomodel) is False: 45 | raise FileNotFoundError(geomodel + " does not exist") 46 | 47 | elif basename.endswith(".tif"): 48 | geomodel = os.path.splitext(absfilename)[0] + ".geom" 49 | if os.path.exists(geomodel) is False: 50 | raise FileNotFoundError(geomodel + " does not exist") 51 | else: 52 | raise ValueError(absfilename + " not supported") 53 | 54 | sensor["image"] = absfilename 55 | sensor["geomodel"] = geomodel 56 | 57 | return sensor 58 | 59 | 60 | def cars_starter(cli_params: dict = None, **kwargs) -> None: 61 | """ 62 | Main fonction. Expects a dictionary from the CLI (cli_params) 63 | or directly the input parameters. 64 | """ 65 | if cli_params and isinstance(cli_params, dict): 66 | config = cli_params 67 | else: 68 | params_name = set(kwargs.keys()) 69 | required_params = {"il", "out"} 70 | missing_params = required_params - params_name 71 | if len(missing_params) > 0: 72 | raise ValueError( 73 | "The following parameters are required: {}".format( 74 | ", ".join(list(missing_params)) 75 | ) 76 | ) 77 | config = kwargs 78 | 79 | cars_config = {"inputs": {"sensors": {}}, "output": {}} 80 | pipeline_name = "default" 81 | 82 | for idx, inputfilename in enumerate(config["il"]): 83 | cars_config["inputs"]["sensors"][str(idx)] = inputfilename_to_sensor( 84 | inputfilename 85 | ) 86 | 87 | # pairing with first image as reference 88 | pairing = list( 89 | zip( # noqa: B905 90 | ["0"] * (len(config["il"]) - 1), 91 | map(str, range(1, len(config["il"]))), 92 | ) 93 | ) 94 | 95 | cars_config["inputs"]["pairing"] = pairing 96 | cars_config["output"]["directory"] = config["out"] 97 | 98 | check = config["check"] if "check" in config.keys() else False 99 | full = config["full"] if "full" in config.keys() else False 100 | 101 | if check or full: 102 | # cars imports 103 | from cars.pipelines.pipeline import Pipeline 104 | 105 | used_pipeline = Pipeline(pipeline_name, cars_config, None) 106 | if full: 107 | cars_config = used_pipeline.used_conf 108 | 109 | print(json.dumps(cars_config, indent=2)) 110 | 111 | 112 | def cli(): 113 | """ 114 | Main cars-starter entrypoint (Command Line Interface) 115 | """ 116 | parser = argparse.ArgumentParser( 117 | "cars-starter", description="Helper to create configuration file" 118 | ) 119 | parser.add_argument( 120 | "-il", 121 | type=str, 122 | nargs="*", 123 | metavar="input.{tif,XML}", 124 | help="Input sensor list", 125 | required=True, 126 | ) 127 | 128 | parser.add_argument( 129 | "-out", 130 | type=str, 131 | metavar="out_dir", 132 | help="Output directory", 133 | required=True, 134 | ) 135 | 136 | parser.add_argument( 137 | "--full", action="store_true", help="Fill all default values" 138 | ) 139 | 140 | parser.add_argument("--check", action="store_true", help="Check inputs") 141 | 142 | args = parser.parse_args() 143 | cars_starter(vars(args)) 144 | 145 | 146 | if __name__ == "__main__": 147 | cli() 148 | -------------------------------------------------------------------------------- /meson.build: -------------------------------------------------------------------------------- 1 | project( 2 | 'cars', 3 | 'cpp', 4 | version: run_command( 5 | 'python', 6 | meson.source_root() / 'version.py', 7 | 'get-vcs', 8 | ).stdout().strip(), 9 | default_options: [ 10 | 'cpp_std=c++17', 11 | ], 12 | ) 13 | 14 | meson.add_dist_script('python', meson.source_root() / 'version.py', 'set-dist', meson.project_version()) 15 | 16 | py = import('python').find_installation(pure: false) 17 | 18 | pybind11_dep = dependency('pybind11') 19 | 20 | install_subdir( 21 | 'cars', 22 | install_dir: py.get_install_dir() / 'cars', 23 | strip_directory: true, 24 | ) 25 | 26 | subdir( 27 | 'cars/applications/dense_matching/cpp' 28 | ) 29 | 30 | subdir( 31 | 'cars/applications/dense_match_filling/cpp' 32 | ) 33 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # content of pytest.ini 2 | [pytest] 3 | addopts = -ra 4 | markers = 5 | unit_tests: Unit tests 6 | end2end_tests: End2end tests 7 | pbs_cluster_tests: PBS cluster unit tests 8 | slurm_cluster_tests: SLURM cluster unit tests 9 | notebook_tests: Notebook unit tests 10 | testpaths = tests 11 | norecursedirs = .git _build build tmp* venv* 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf8 3 | # 4 | # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). 5 | # 6 | # This file is part of CARS 7 | # (see https://github.com/CNES/cars). 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | """ 22 | CARS setup.py 23 | """ 24 | from setuptools import setup 25 | 26 | try: 27 | setup(use_scm_version={"fallback_version": "0.0.0"}) 28 | except Exception: 29 | print( 30 | "\n\nAn error occurred while building the project, " 31 | "please ensure you have the most updated version of pip, setuptools, " 32 | "setuptools_scm and wheel with:\n" 33 | " pip install -U pip setuptools setuptools_scm wheel\n\n" 34 | ) 35 | raise 36 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | # unique project identifier (required) 2 | sonar.projectKey=cars:cars 3 | 4 | # project metadata 5 | sonar.projectName=Cars 6 | 7 | # path to source directories (required) 8 | sonar.sources=cars/ 9 | 10 | # path to test source directories (optional) 11 | sonar.tests=tests/ 12 | 13 | # exclude files or directories 14 | #sonar.exclusions=src/ 15 | #sonar.test.exclusions=test.xml 16 | sonar.exclusions=cars/orchestrator/cluster/dask_config/**, cars/orchestrator/cluster/dask_config/**/**, cars/applications/dem_generation/bulldozer_config/**, cars/applications/dsm_filling/bulldozer_config/** 17 | 18 | # path to pylint analysis report (optional) 19 | # DEPRECATED not supported anymore in CNES sonarqube instance. 20 | #sonar.python.pylint.reportPath=pylint-report.txt 21 | 22 | # path to pytest report (optional) 23 | sonar.python.xunit.reportPath=pytest-report.xml 24 | 25 | # path to coverage report (optional) 26 | sonar.python.coverage.reportPaths=coverage.xml 27 | 28 | # deactivate duplicated lines in sonarqube in some chosen files: 29 | sonar.cpd.exclusions=cars/pipelines/**/*_pipeline.py, cars/applications/point_cloud_outlier_removal/*, cars/applications/dem_generation/*, cars/applications/dsm_filling/*, cars/applications/grid_generation/grid_correction.py, cars/orchestrator/log_wrapper.py 30 | 31 | # Deactivate complexity rule for pipelines 32 | sonar.issue.ignore.multicriteria=complexity1,complexity2, complexity3 33 | 34 | sonar.issue.ignore.multicriteria.complexity1.ruleKey=python:FunctionComplexity 35 | sonar.issue.ignore.multicriteria.complexity1.resourceKey=cars/pipelines/**/*_pipeline.py 36 | 37 | sonar.issue.ignore.multicriteria.complexity2.ruleKey=python:FunctionComplexity 38 | sonar.issue.ignore.multicriteria.complexity2.resourceKey=cars/applications/rasterization/simple_gaussian.py 39 | 40 | sonar.issue.ignore.multicriteria.complexity3.ruleKey=python:FunctionComplexity 41 | sonar.issue.ignore.multicriteria.complexity3.resourceKey=cars/applications/triangulation/line_of_sight_intersection.py 42 | -------------------------------------------------------------------------------- /version.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import sys 4 | 5 | 6 | def get_vcs(): 7 | subprocess.run(["python", "-m", "setuptools_scm"], check=True) 8 | 9 | 10 | def set_dist(version): 11 | meson_project_dist_root = os.getenv("MESON_PROJECT_DIST_ROOT") 12 | meson_rewrite = os.getenv("MESONREWRITE") 13 | 14 | if not meson_project_dist_root or not meson_rewrite: 15 | print( 16 | "Error: Required environment variables MESON_PROJECT_DIST_ROOT or MESONREWRITE are missing." 17 | ) 18 | sys.exit(1) 19 | 20 | print(meson_project_dist_root) 21 | rewrite_command = f"{meson_rewrite} --sourcedir {meson_project_dist_root} " 22 | rewrite_command += f"kwargs set project / version {version}" 23 | 24 | subprocess.run(rewrite_command.split(" ")) 25 | 26 | 27 | if __name__ == "__main__": 28 | if len(sys.argv) < 2: 29 | print("Usage: script.py [version]") 30 | sys.exit(1) 31 | 32 | command = sys.argv[1] 33 | 34 | if command == "get-vcs": 35 | get_vcs() 36 | elif command == "set-dist": 37 | if len(sys.argv) < 3: 38 | print("Error: Missing version argument for set-dist.") 39 | sys.exit(1) 40 | set_dist(sys.argv[2]) 41 | else: 42 | print("Error: Invalid command.") 43 | sys.exit(1) 44 | --------------------------------------------------------------------------------