├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.md │ ├── feature-request.md │ └── help-support.md └── workflows │ ├── ci_action.yml │ ├── ci_trigger.yml │ ├── docker_action.yml │ └── scheduled_caller.yml ├── .gitignore ├── .gitlab-ci.yml ├── .pre-commit-config.yaml ├── .readthedocs.yml ├── .zenodo.json ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── CREDITS.md ├── LICENSE ├── Makefile ├── README.md ├── docker ├── eolearn-examples.dockerfile └── eolearn.dockerfile ├── docs ├── Makefile ├── environment.yml └── source │ ├── _static │ └── style.css │ ├── _templates │ ├── layout.html │ ├── module.rst_t │ └── package.rst_t │ ├── conf.py │ ├── contribute.rst │ ├── custom_reference │ └── eolearn.rst │ ├── examples.rst │ ├── figures │ ├── eo-learn-illustration.png │ ├── eo-learn-logo-white.png │ ├── eo-learn-logo.png │ └── eo-learn-workflow.png │ ├── index.rst │ └── install.rst ├── eolearn ├── __init__.py ├── core │ ├── __init__.py │ ├── constants.py │ ├── core_tasks.py │ ├── eodata.py │ ├── eodata_io.py │ ├── eodata_merge.py │ ├── eoexecution.py │ ├── eonode.py │ ├── eotask.py │ ├── eoworkflow.py │ ├── eoworkflow_tasks.py │ ├── exceptions.py │ ├── extra │ │ ├── __init__.py │ │ └── ray.py │ ├── graph.py │ ├── types.py │ └── utils │ │ ├── __init__.py │ │ ├── common.py │ │ ├── fs.py │ │ ├── logging.py │ │ ├── parallelize.py │ │ ├── parsing.py │ │ ├── raster.py │ │ ├── testing.py │ │ └── types.py ├── coregistration │ ├── __init__.py │ └── coregistration.py ├── features │ ├── __init__.py │ ├── extra │ │ ├── __init__.py │ │ ├── clustering.py │ │ └── interpolation.py │ ├── feature_manipulation.py │ ├── ndi.py │ └── utils.py ├── geometry │ ├── __init__.py │ ├── morphology.py │ └── transformations.py ├── io │ ├── __init__.py │ ├── geometry_io.py │ ├── raster_io.py │ └── sentinelhub_process.py ├── mask │ ├── __init__.py │ ├── extra │ │ ├── __init__.py │ │ └── cloud_mask.py │ ├── masking.py │ └── snow_mask.py ├── ml_tools │ ├── __init__.py │ └── sampling.py ├── py.typed └── visualization │ ├── __init__.py │ ├── eoexecutor.py │ ├── eopatch.py │ ├── eoworkflow.py │ ├── report_templates │ └── report.html │ └── utils.py ├── example_data ├── REFERENCE_SCENES.npy ├── TestEOPatch │ ├── bbox.geojson │ ├── data │ │ ├── BANDS-S2-L1C.npy │ │ ├── CLP.npy │ │ ├── CLP_MULTI.npy │ │ ├── CLP_S2C.npy │ │ └── NDVI.npy │ ├── data_timeless │ │ ├── DEM.npy │ │ └── MAX_NDVI.npy │ ├── label │ │ ├── IS_CLOUDLESS.npy │ │ └── RANDOM_DIGIT.npy │ ├── label_timeless │ │ └── LULC_COUNTS.npy │ ├── mask │ │ ├── CLM.npy │ │ ├── CLM_INTERSSIM.npy │ │ ├── CLM_MULTI.npy │ │ ├── CLM_S2C.npy │ │ ├── IS_DATA.npy │ │ └── IS_VALID.npy │ ├── mask_timeless │ │ ├── LULC.npy │ │ ├── RANDOM_UINT8.npy │ │ └── VALID_COUNT.npy │ ├── meta_info │ │ ├── maxcc.json │ │ ├── service_type.json │ │ ├── size_x.json │ │ └── size_y.json │ ├── scalar │ │ └── CLOUD_COVERAGE.npy │ ├── scalar_timeless │ │ └── LULC_PERCENTAGE.npy │ ├── timestamps.json │ ├── vector │ │ └── CLM_VECTOR.gpkg │ └── vector_timeless │ │ └── LULC.gpkg ├── import-gpkg-test.gpkg ├── import-tiff-test1.tiff ├── import-tiff-test2.tiff └── svn_border.geojson ├── examples ├── README.md ├── core │ ├── .gitignore │ ├── CoreOverview.ipynb │ ├── TimeLapse.ipynb │ └── images │ │ └── eopatch.png ├── io │ └── SentinelHubIO.ipynb ├── land-cover-map │ ├── .gitignore │ ├── README.md │ ├── SI_LULC_pipeline.ipynb │ └── readme_figs │ │ ├── aoi_to_tiles.png │ │ ├── fraction_valid_pixels_per_frame_cleaned-eopatch-0.png │ │ ├── fraction_valid_pixels_per_frame_eopatch-0.png │ │ ├── hist_number_of_valid_observations_slovenia.png │ │ ├── hist_number_of_valid_observations_slovenia_s2c_vs_sh.png │ │ ├── number_of_valid_observations_eopatch_0.png │ │ ├── number_of_valid_observations_slovenia.png │ │ ├── number_of_valid_observations_slovenia_s2c.png │ │ ├── patch_0.png │ │ └── patch_31.png ├── mask │ └── ValidDataMask.ipynb ├── visualization │ └── EOPatchVisualization.ipynb └── water-monitor │ ├── WaterMonitorWorkflow.ipynb │ └── theewaterskloof_dam_nominal.wkt ├── install_all.py ├── pyproject.toml └── tests ├── __init__.py ├── core ├── __init__.py ├── conftest.py ├── stats │ └── test_save_stats.pkl ├── test_constants.py ├── test_core_tasks.py ├── test_eodata.py ├── test_eodata_io.py ├── test_eodata_merge.py ├── test_eoexecutor.py ├── test_eonode.py ├── test_eotask.py ├── test_eoworkflow.py ├── test_eoworkflow_tasks.py ├── test_extra │ ├── __init__.py │ └── test_ray.py ├── test_graph.py └── test_utils │ ├── test_common.py │ ├── test_fs.py │ ├── test_parallelize.py │ ├── test_parsing.py │ ├── test_raster.py │ └── test_testing.py ├── coregistration ├── conftest.py └── test_coregistration.py ├── features ├── conftest.py ├── extra │ ├── test_clustering.py │ └── test_interpolation.py ├── test_bands_extraction.py ├── test_feature_manipulation.py └── test_features_utils.py ├── geometry ├── conftest.py ├── test_morphology.py └── test_transformations.py ├── io ├── TestInputs │ ├── test_meteoblue_raster_input.bin │ └── test_meteoblue_vector_input.bin ├── conftest.py ├── test_geometry_io.py ├── test_raster_io.py └── test_sentinelhub_process.py ├── mask ├── conftest.py ├── extra │ └── test_cloud_mask.py ├── test_masking.py └── test_snow_mask.py ├── ml_tools ├── conftest.py └── test_sampling.py └── visualization ├── test_eoexecutor.py ├── test_eopatch.py └── test_eoworkflow.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | docs 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Bug report' 3 | about: Help us improve 4 | title: '[BUG]' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | 12 | A clear and concise description of what the bug is. 13 | 14 | NOTE: Remove any confidential information, such as Sentinel Hub credentials! 15 | 16 | **To Reproduce** 17 | 18 | Steps to reproduce the behavior: 19 | 20 | 1. 21 | 2. 22 | 3. 23 | 24 | **Expected behavior** 25 | 26 | A clear and concise description of what you expected to happen. 27 | 28 | **Environment** 29 | 30 | Details of the python environment 31 | 32 | **Stack trace or screenshots** 33 | 34 | If applicable, add screenshots to help explain your problem. 35 | 36 | **Desktop (please complete the following information):** 37 | 38 | - OS: [e.g. iOS] 39 | - Browser [e.g. chrome, safari] 40 | - Version [e.g. 22] 41 | 42 | **Additional context** 43 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Feature request' 3 | about: What feature is missing? 4 | title: '[FEAT]' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What is the problem? Please describe.** 11 | 12 | A clear and concise description of what the problem is. 13 | 14 | **Here's the solution** 15 | 16 | A clear and concise description of what you want to happen. 17 | 18 | **Alternatives** 19 | 20 | A clear and concise description of any alternative solutions or features you've considered. 21 | 22 | **Additional context** 23 | 24 | Add any other context or screenshots about the feature request here. 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/help-support.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Help/Support' 3 | about: 'Need help?' 4 | title: '[HELP]' 5 | labels: help wanted, question 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Resources** 11 | 12 | Before opening an issue for help/support, make sure you have consulted our extensive [documentation](https://eo-learn.readthedocs.io/en/latest/), our [existing issues](https://github.com/sentinel-hub/eo-learn/issues), and the Sentinel-Hub [forum](https://forum.sentinel-hub.com/). 13 | 14 | **Question** 15 | 16 | Formulate your question in a clear and structured way, providing context and links to the resources raising issues. 17 | 18 | NOTE: Remove any confidential information, such as Sentinel Hub credentials! 19 | 20 | **Additional context** 21 | 22 | Add any information that would help us supporting you. 23 | -------------------------------------------------------------------------------- /.github/workflows/ci_action.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - "master" 8 | - "develop" 9 | workflow_call: 10 | 11 | concurrency: 12 | # This will cancel outdated runs on the same pull-request, but not runs for other triggers 13 | group: ${{ github.head_ref || github.run_id }} 14 | cancel-in-progress: true 15 | 16 | env: 17 | # The only way to simulate if-else statement 18 | CHECKOUT_BRANCH: ${{ github.event_name == 'schedule' && 'develop' || github.ref }} 19 | 20 | jobs: 21 | check-pre-commit-hooks: 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Checkout branch 25 | uses: actions/checkout@v3 26 | with: 27 | ref: ${{ env.CHECKOUT_BRANCH }} 28 | 29 | - name: Setup Python 30 | uses: actions/setup-python@v4 31 | with: 32 | python-version: "3.8" 33 | 34 | - uses: pre-commit/action@v3.0.0 35 | with: 36 | extra_args: --all-files --verbose 37 | 38 | check-code-pylint-and-mypy: 39 | runs-on: ubuntu-latest 40 | steps: 41 | - name: Checkout branch 42 | uses: actions/checkout@v3 43 | with: 44 | ref: ${{ env.CHECKOUT_BRANCH }} 45 | 46 | - name: Setup Python 47 | uses: actions/setup-python@v4 48 | with: 49 | python-version: "3.8" 50 | cache: pip 51 | 52 | - name: Install packages 53 | run: pip install .[DEV] --upgrade --upgrade-strategy eager 54 | 55 | - name: Run pylint 56 | run: pylint eolearn 57 | 58 | - name: Run mypy 59 | if: success() || failure() 60 | run: mypy eolearn 61 | 62 | test-on-github: 63 | runs-on: ubuntu-latest 64 | strategy: 65 | matrix: 66 | python-version: 67 | - "3.9" 68 | - "3.10" 69 | - "3.11" 70 | include: 71 | # A flag marks whether full or partial tests should be run 72 | # We don't run integration tests on pull requests from outside repos, because they don't have secrets 73 | - python-version: "3.8" 74 | full_test_suite: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} 75 | steps: 76 | - name: Checkout branch 77 | uses: actions/checkout@v3 78 | with: 79 | ref: ${{ env.CHECKOUT_BRANCH }} 80 | 81 | - name: Setup Python 82 | uses: actions/setup-python@v4 83 | with: 84 | python-version: ${{ matrix.python-version }} 85 | cache: pip 86 | 87 | - name: Install packages 88 | run: | # must install async-timeout until ray fixes issue 89 | pip install -e .[DEV] --upgrade --upgrade-strategy eager 90 | pip install async-timeout 91 | 92 | - name: Run full tests and code coverage 93 | if: ${{ matrix.full_test_suite }} 94 | run: | 95 | sentinelhub.config \ 96 | --sh_client_id "${{ secrets.SH_CLIENT_ID }}" \ 97 | --sh_client_secret "${{ secrets.SH_CLIENT_SECRET }}" 98 | if [ ${{ github.event_name }} == 'push' ]; then 99 | pytest -m "not geopedia" --cov=eolearn --cov-report=term --cov-report=xml 100 | else 101 | pytest -m "not geopedia" 102 | fi 103 | 104 | - name: Run reduced tests 105 | if: ${{ !matrix.full_test_suite }} 106 | run: pytest -m "not sh_integration and not geopedia" 107 | 108 | # - name: Upload code coverage 109 | # if: ${{ matrix.full_test_suite && github.event_name == 'push' }} 110 | # uses: codecov/codecov-action@v2 111 | # with: 112 | # files: coverage.xml 113 | # fail_ci_if_error: true 114 | # verbose: false 115 | 116 | mirror-and-integration-test-on-gitlab: 117 | if: github.event_name == 'push' 118 | runs-on: ubuntu-latest 119 | steps: 120 | - uses: actions/checkout@v1 121 | - name: Mirror + trigger CI 122 | uses: SvanBoxel/gitlab-mirror-and-ci-action@master 123 | with: 124 | args: "https://hello.planet.com/code/eo/code/eo-learn" 125 | env: 126 | FOLLOW_TAGS: "true" 127 | GITLAB_HOSTNAME: "hello.planet.com/code" 128 | GITLAB_USERNAME: "github-action" 129 | GITLAB_PASSWORD: ${{ secrets.GITLAB_PASSWORD }} 130 | GITLAB_PROJECT_ID: "9715" 131 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 132 | -------------------------------------------------------------------------------- /.github/workflows/ci_trigger.yml: -------------------------------------------------------------------------------- 1 | name: trigger 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | 8 | jobs: 9 | trigger: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Trigger API 13 | run: > 14 | curl -X POST --fail \ 15 | -F token=${{ secrets.GITLAB_PIPELINE_TRIGGER_TOKEN }} \ 16 | -F ref=main \ 17 | -F variables[CUSTOM_RUN_TAG]=auto \ 18 | -F variables[LAYER_NAME]=dotai-eo \ 19 | https://hello.planet.com/code/api/v4/projects/9723/trigger/pipeline 20 | -------------------------------------------------------------------------------- /.github/workflows/docker_action.yml: -------------------------------------------------------------------------------- 1 | name: publish Docker images 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | build_and_push_to_registry: 9 | name: Push Docker image to Docker Hub 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check out the repo 13 | uses: actions/checkout@v2 14 | with: 15 | ref: master 16 | 17 | - name: Log in to Docker Hub 18 | uses: docker/login-action@v1 19 | with: 20 | username: ${{ secrets.DOCKERHUB_USERNAME }} 21 | password: ${{ secrets.DOCKERHUB_TOKEN }} 22 | 23 | - name: Build and push latest tag 24 | uses: docker/build-push-action@v2 25 | with: 26 | file: ./docker/eolearn.dockerfile 27 | push: true 28 | tags: sentinelhub/eolearn:latest 29 | 30 | - name: Build and push latest-examples tag 31 | uses: docker/build-push-action@v2 32 | with: 33 | file: ./docker/eolearn-examples.dockerfile 34 | push: true 35 | tags: sentinelhub/eolearn:latest-examples 36 | 37 | - name: Update Docker Hub description and readme 38 | uses: peter-evans/dockerhub-description@v2 39 | with: 40 | username: ${{ secrets.DOCKERHUB_USERNAME }} 41 | password: ${{ secrets.DOCKERHUB_PASSWORD }} 42 | repository: sentinelhub/eolearn 43 | short-description: "Official eo-learn Docker images with Jupyter notebook" 44 | readme-filepath: ./README.md 45 | -------------------------------------------------------------------------------- /.github/workflows/scheduled_caller.yml: -------------------------------------------------------------------------------- 1 | name: scheduled build caller 2 | 3 | on: 4 | schedule: 5 | # Schedule events are triggered by whoever last changed the cron schedule 6 | - cron: "0 0 * * *" 7 | 8 | jobs: 9 | call-workflow: 10 | uses: sentinel-hub/eo-learn/.github/workflows/ci_action.yml@develop 11 | secrets: inherit 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook, Jupyter Lab 71 | .ipynb_checkpoints 72 | .jupyter 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # SageMath parsed files 81 | *.sage.py 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | .venv 88 | venv/ 89 | ENV/ 90 | 91 | # pipenv 92 | Pipfile* 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | # Test output folder 108 | /TestOutputs 109 | 110 | # IDE config 111 | .idea/ 112 | .vscode/ 113 | 114 | # Files that Sphinx generates 115 | /docs/source/examples 116 | /docs/source/markdowns 117 | /docs/source/eotasks.rst 118 | /docs/source/reference/ 119 | 120 | # pytest 121 | .pytest_cache/ 122 | features/.pytest_cache 123 | 124 | # execution reports 125 | **execution-report** 126 | 127 | # MacOS cache 128 | .DS_Store 129 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | default: 2 | image: python:3.9 3 | tags: 4 | - sinergise-lju 5 | 6 | stages: 7 | - test 8 | 9 | run_sh_integration_tests: 10 | stage: test 11 | when: always 12 | before_script: 13 | - apt-get update 14 | - apt-get install -y build-essential libgdal-dev graphviz proj-bin gcc libproj-dev libspatialindex-dev 15 | script: 16 | - pip install .[DEV] 17 | - sentinelhub.config --sh_client_id "$SH_CLIENT_ID" --sh_client_secret "$SH_CLIENT_SECRET" > /dev/null # Gitlab can't mask SH_CLIENT_SECRET in logs 18 | - pytest -m sh_integration 19 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.6.0 4 | hooks: 5 | - id: end-of-file-fixer 6 | - id: requirements-txt-fixer 7 | - id: trailing-whitespace 8 | - id: debug-statements 9 | - id: check-json 10 | - id: check-toml 11 | - id: check-yaml 12 | - id: check-merge-conflict 13 | - id: debug-statements 14 | 15 | - repo: https://github.com/psf/black 16 | rev: 24.8.0 17 | hooks: 18 | - id: black 19 | language_version: python3 20 | 21 | - repo: https://github.com/charliermarsh/ruff-pre-commit 22 | rev: "v0.6.8" 23 | hooks: 24 | - id: ruff 25 | 26 | - repo: https://github.com/nbQA-dev/nbQA 27 | rev: 1.8.7 28 | hooks: 29 | - id: nbqa-black 30 | - id: nbqa-ruff 31 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | configuration: docs/source/conf.py 5 | 6 | build: 7 | os: "ubuntu-20.04" 8 | tools: 9 | python: "mambaforge-4.10" 10 | 11 | conda: 12 | environment: docs/environment.yml 13 | -------------------------------------------------------------------------------- /.zenodo.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "eo-learn", 3 | "description": "eo-learn makes extraction of valuable information from satellite imagery easy.\\nThe availability of open Earth observation (EO) data through the Copernicus and Landsat programs represents an unprecedented resource for many EO applications, ranging from ocean and land use and land cover monitoring, disaster control, emergency services and humanitarian relief. Given the large amount of high spatial resolution data at high revisit frequency, techniques able to automatically extract complex patterns in such spatio-temporaldata are needed.\\neo-learn is a collection of Python packages that have been developed to seamlessly access and process spatio-temporal image sequences acquired by any satellite fleet in a timely and automatic manner. eo-learn is easy to use, it's design modular, and encourages collaboration -- sharing and reusing of specific tasks in a typical EO-value-extraction workflows, such as cloud masking, image co-registration, feature extraction, classification, etc. Everyone is free to use any of the available tasks and is encouraged to improve the, develop new ones and share them with the rest of the community.", 4 | "keywords": [ 5 | "Earth Observation", 6 | "Python", 7 | "Workflows", 8 | "Remote sensing", 9 | "Data Processing", 10 | "Machine Learning" 11 | ], 12 | "creators": [ 13 | { 14 | "affiliation": "Sinergise", 15 | "name": "EO Research Team" 16 | } 17 | ], 18 | "contributors": [ 19 | { 20 | "name": "Matej Batič", 21 | "affiliation": "Sinergise", 22 | "type": "ProjectMember" 23 | }, 24 | { 25 | "name": "Sabina Dolenc", 26 | "affiliation": "Sinergise", 27 | "type": "ProjectMember" 28 | }, 29 | { 30 | "name": "Jan Geršak", 31 | "affiliation": "Sinergise", 32 | "type": "ProjectMember" 33 | }, 34 | { 35 | "name": "Chung-Xiang Hong", 36 | "affiliation": "Sentinel Hub", 37 | "type": "ProjectMember" 38 | }, 39 | { 40 | "name": "Domagoj Korais", 41 | "affiliation": "Sinergise", 42 | "type": "ProjectMember" 43 | }, 44 | { 45 | "name": "Matic Lubej", 46 | "affiliation": "Sinergise", 47 | "type": "ProjectMember" 48 | }, 49 | { 50 | "name": "Žiga Lukšič", 51 | "affiliation": "Sinergise", 52 | "type": "ProjectMember" 53 | }, 54 | { 55 | "name": "Grega Milčinski", 56 | "affiliation": "Sinergise", 57 | "type": "ProjectMember" 58 | }, 59 | { 60 | "name": "Nika Oman Kadunc", 61 | "affiliation": "Sinergise", 62 | "type": "ProjectMember" 63 | }, 64 | { 65 | "name": "Devis Peressutti", 66 | "affiliation": "Sinergise", 67 | "type": "ProjectMember" 68 | }, 69 | { 70 | "name": "Tomi Sljepčević", 71 | "affiliation": "Sinergise", 72 | "type": "ProjectMember" 73 | }, 74 | { 75 | "name": "Tamara Šuligoj", 76 | "affiliation": "Sinergise", 77 | "type": "ProjectMember" 78 | }, 79 | { 80 | "name": "Jovan Višnjić", 81 | "affiliation": "Sinergise", 82 | "type": "ProjectMember" 83 | }, 84 | { 85 | "name": "Anže Zupanc", 86 | "affiliation": "Sinergise", 87 | "type": "ProjectMember" 88 | }, 89 | { 90 | "name": "Matej Aleksandrov", 91 | "affiliation": "Sinergise", 92 | "type": "ProjectMember" 93 | }, 94 | { 95 | "name": "Andrej Burja", 96 | "affiliation": "Sinergise", 97 | "type": "ProjectMember" 98 | }, 99 | { 100 | "name": "Eva Erzin", 101 | "affiliation": "Sinergise", 102 | "type": "ProjectMember" 103 | }, 104 | { 105 | "name": "Jernej Puc", 106 | "affiliation": "Sinergise", 107 | "type": "ProjectMember" 108 | }, 109 | { 110 | "name": "Blaž Sovdat", 111 | "affiliation": "Sinergise", 112 | "type": "ProjectMember" 113 | }, 114 | { 115 | "name": "Lojze Žust", 116 | "affiliation": "Sinergise", 117 | "type": "ProjectMember" 118 | }, 119 | { 120 | "name": "Drew Bollinger", 121 | "affiliation": "DevelopmentSeed", 122 | "type": "Other" 123 | }, 124 | { 125 | "name": "Peter Fogh", 126 | "type": "Other" 127 | }, 128 | { 129 | "name": "Hugo Fournier", 130 | "affiliation": "Magellium", 131 | "type": "Other" 132 | }, 133 | { 134 | "name": "Ben Huff", 135 | "type": "Other" 136 | }, 137 | { 138 | "name": "Filip Koprivec", 139 | "affiliation": "Jožef Stefan Institute", 140 | "type": "Other" 141 | }, 142 | { 143 | "name": "Colin Moldenhauer", 144 | "affiliation": "Technical University of Munich", 145 | "type": "Other" 146 | }, 147 | { 148 | "name": "William Ouellette", 149 | "affiliation": "TomTom", 150 | "type": "Other" 151 | }, 152 | { 153 | "name": "Radoslav Pitoňák", 154 | "type": "Other" 155 | }, 156 | { 157 | "name": "Johannes Schmid", 158 | "affiliation": "GeoVille", 159 | "type": "Other" 160 | }, 161 | { 162 | "name": "Nour Soltani", 163 | "type": "Other" 164 | }, 165 | { 166 | "name": "Beno Šircelj", 167 | "affiliation": "Jožef Stefan Institute", 168 | "type": "Other" 169 | }, 170 | { 171 | "name": "Andrew Tedstone", 172 | "type": "Other" 173 | }, 174 | { 175 | "name": "Raaj Tilak Sarma", 176 | "type": "Other" 177 | }, 178 | { 179 | "name": "Zhuangfang Yi 依庄防", 180 | "type": "Other" 181 | }, 182 | { 183 | "name": "Patrick Zippenfenig", 184 | "affiliation": "meteoblue", 185 | "type": "Other" 186 | }, 187 | { 188 | "name": "fred-sch", 189 | "type": "Other" 190 | }, 191 | { 192 | "name": "Gnilliw", 193 | "type": "Other" 194 | }, 195 | { 196 | "name": "theirix", 197 | "type": "Other" 198 | } 199 | ], 200 | "license": { 201 | "id": "MIT" 202 | }, 203 | "grants": [ 204 | { 205 | "id": "776115" 206 | }, 207 | { 208 | "id": "101004112" 209 | }, 210 | { 211 | "id": "101059548" 212 | }, 213 | { 214 | "id": "101086461" 215 | } 216 | ] 217 | } 218 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we, as contributors and maintainers, pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at eoresearch@sinergise.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 44 | 45 | [homepage]: https://www.contributor-covenant.org 46 | 47 | For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq 48 | -------------------------------------------------------------------------------- /CREDITS.md: -------------------------------------------------------------------------------- 1 | # Credits 2 | 3 | ------- 4 | 5 | For a full list of contributors you can also see GitHub [contributors](https://github.com/sentinel-hub/eo-learn/graphs/contributors) 6 | page or mine the [commit history](https://github.com/sentinel-hub/eo-learn/commits/master). 7 | 8 | 9 | ## core eo-learn team 10 | 11 | * Matej Batič (Sinergise) 12 | * Sabina Dolenc (Sinergise) 13 | * Jan Geršak (Sinergise) 14 | * Chung-Xiang Hong (Sentinel Hub) 15 | * Domagoj Korais (Sinergise) 16 | * Matic Lubej (Sinergise) 17 | * Žiga Lukšič (Sinergise) 18 | * Grega Milčinski (Sinergise) 19 | * Nika Oman Kadunc (Sinergise) 20 | * Devis Peressutti (Sinergise) 21 | * Tomi Sljepčević (Sinergise) 22 | * Tamara Šuligoj (Sinergise) 23 | * Jovan Višnjić (Sinergise) 24 | * Anže Zupanc (Sinergise) 25 | 26 | 27 | ## Past members of core eo-learn team 28 | 29 | * Matej Aleksandrov (Sinergise) 30 | * Andrej Burja (Sinergise) 31 | * Eva Erzin (Sinergise) 32 | * Jernej Puc (Sinergise) 33 | * Blaž Sovdat (Sinergise) 34 | * Lojze Žust (Sinergise) 35 | 36 | 37 | ## Other contributors 38 | 39 | * Drew Bollinger (DevelopmentSeed) 40 | * Michael Engel (Technical University of Munich) 41 | * Peter Fogh 42 | * Hugo Fournier (Magellium) 43 | * Ben Huff 44 | * Filip Koprivec (Jožef Stefan Institute) 45 | * Colin Moldenhauer (Technical University of Munich) 46 | * William Ouellette (TomTom) 47 | * Radoslav Pitoňák 48 | * Johannes Schmid (GeoVille) 49 | * Nour Soltani 50 | * Beno Šircelj (Jožef Stefan Institute) 51 | * Andrew Tedstone 52 | * Raaj Tilak Sarma 53 | * Zhuangfang Yi 依庄防 54 | * Patrick Zippenfenig (meteoblue) 55 | * fred-sch 56 | * Gnilliw 57 | * theirix 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017-2022 Matej Aleksandrov, Matej Batič, Grega Milčinski, Domagoj Korais, Miha Kadunc (Sinergise) 4 | Copyright (c) 2017-2022 Matic Lubej, Žiga Lukšič, Devis Peressutti, Tomislav Slijepčević, Nejc Vesel (Sinergise) 5 | Copyright (c) 2017-2022 Jovan Višnjić, Anže Zupanc (Sinergise) 6 | Copyright (c) 2019-2020 Jernej Puc, Lojze Žust (Sinergise) 7 | Copyright (c) 2017-2019 Blaž Sovdat, Andrej Burja, Eva Erzin (Sinergise) 8 | Copyright (c) 2018-2019 Hugo Fournier (Magellium) 9 | Copyright (c) 2018-2020 Filip Koprivec, Beno Šircelj (Jožef Stefan Institute) 10 | Copyright (c) 2018-2020 William Ouellette 11 | Copyright (c) 2018-2019 Johannes Schmid (GeoVille) 12 | Copyright (c) 2019 Drew Bollinger (DevelopmentSeed) 13 | 14 | Permission is hereby granted, free of charge, to any person obtaining a copy 15 | of this software and associated documentation files (the "Software"), to deal 16 | in the Software without restriction, including without limitation the rights 17 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 18 | copies of the Software, and to permit persons to whom the Software is 19 | furnished to do so, subject to the following conditions: 20 | 21 | The above copyright notice and this permission notice shall be included in all 22 | copies or substantial portions of the Software. 23 | 24 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 25 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 26 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 27 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 28 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 29 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | SOFTWARE. 31 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for creating a new release of the package and uploading it to PyPI 2 | 3 | PYTHON = python3 4 | 5 | .PHONY: $(PACKAGES:test) 6 | 7 | help: 8 | @echo "Use 'make test-upload' to upload the package to testPyPi" 9 | @echo "Use 'make upload' to upload the package to PyPi" 10 | 11 | 12 | upload: 13 | rm -r dist | true 14 | python -m build --sdist --wheel 15 | twine upload --skip-existing dist/* 16 | 17 | # For testing: 18 | test-upload: 19 | rm -r dist | true 20 | python -m build --sdist --wheel 21 | twine upload --repository testpypi --skip-existing dist/* 22 | -------------------------------------------------------------------------------- /docker/eolearn-examples.dockerfile: -------------------------------------------------------------------------------- 1 | FROM sentinelhub/eolearn:latest 2 | 3 | LABEL description=\ 4 | "An official eo-learn docker image with a full eo-learn installation, Jupyter notebook, all \ 5 | example notebooks, and some additional dependencies required to run those notebooks." 6 | 7 | RUN apt-get update && apt-get install -y \ 8 | ffmpeg \ 9 | && apt-get clean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* 10 | 11 | RUN pip3 install --no-cache-dir \ 12 | ffmpeg-python \ 13 | ipyleaflet 14 | 15 | COPY ./examples ./examples 16 | COPY ./example_data ./example_data 17 | -------------------------------------------------------------------------------- /docker/eolearn.dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster 2 | 3 | LABEL maintainer="Sinergise EO research team " 4 | LABEL description="An official eo-learn docker image with a full eo-learn installation and Jupyter notebook." 5 | 6 | RUN apt-get update && apt-get install -y \ 7 | gcc \ 8 | libgdal-dev \ 9 | graphviz \ 10 | proj-bin \ 11 | libproj-dev \ 12 | libspatialindex-dev \ 13 | && apt-get clean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* 14 | 15 | ENV CPLUS_INCLUDE_PATH=/usr/include/gdal 16 | ENV C_INCLUDE_PATH=/usr/include/gdal 17 | 18 | RUN pip3 install --no-cache-dir pip --upgrade 19 | RUN pip3 install --no-cache-dir shapely --no-binary :all: 20 | 21 | WORKDIR /tmp 22 | 23 | COPY eolearn eolearn 24 | COPY pyproject.toml README.md LICENSE ./ 25 | 26 | RUN pip3 install --no-cache-dir .[FULL] 27 | 28 | RUN pip3 install --no-cache-dir \ 29 | . \ 30 | rtree \ 31 | jupyter 32 | 33 | RUN rm -r ./* 34 | 35 | ENV TINI_VERSION=v0.19.0 36 | ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini 37 | RUN chmod +x /tini 38 | ENTRYPOINT ["/tini", "--"] 39 | 40 | WORKDIR /home/eolearner 41 | 42 | EXPOSE 8888 43 | CMD ["/usr/local/bin/jupyter", "notebook", "--no-browser", "--port=8888", "--ip=0.0.0.0", \ 44 | "--NotebookApp.token=''", "--allow-root"] 45 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = eo-learn 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | rm -rf ./build/ 21 | $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 22 | # rm -r source/examples source/markdowns source/reference source/eotasks.rst 23 | -------------------------------------------------------------------------------- /docs/environment.yml: -------------------------------------------------------------------------------- 1 | name: eo-learn-docs 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - rasterio 6 | - pip 7 | - python=3.10 8 | - pip: 9 | - Sphinx==7.1.2 10 | - sphinx_rtd_theme==1.3.0 11 | - nbsphinx 12 | - jupyter 13 | - sphinx_mdinclude==0.5.4 14 | 15 | - ./../.[FULL] 16 | -------------------------------------------------------------------------------- /docs/source/_static/style.css: -------------------------------------------------------------------------------- 1 | .wy-nav-content { 2 | max-width: 1200px !important; 3 | } 4 | -------------------------------------------------------------------------------- /docs/source/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | {% block extrahead %} 3 | 4 | {% endblock %} 5 | -------------------------------------------------------------------------------- /docs/source/_templates/module.rst_t: -------------------------------------------------------------------------------- 1 | {%- if show_headings %} 2 | {{- basename | e | heading }} 3 | 4 | {% endif -%} 5 | .. automodule:: {{ qualname }} 6 | {%- for option in automodule_options %} 7 | :{{ option }}: 8 | {%- endfor %} 9 | -------------------------------------------------------------------------------- /docs/source/_templates/package.rst_t: -------------------------------------------------------------------------------- 1 | {%- macro automodule(modname, options) -%} 2 | .. automodule:: {{ modname }} 3 | {%- for option in options %} 4 | :{{ option }}: 5 | {%- endfor %} 6 | {%- endmacro %} 7 | 8 | {%- macro toctree(docnames) -%} 9 | .. toctree:: 10 | :maxdepth: {{ maxdepth }} 11 | {% for docname in docnames %} 12 | {{ docname }} 13 | {%- endfor %} 14 | {%- endmacro %} 15 | 16 | {%- if is_namespace %} 17 | {{- pkgname | e | heading }} 18 | {% else %} 19 | {{- pkgname | e | heading }} 20 | {% endif %} 21 | 22 | {%- if is_namespace %} 23 | .. py:module:: {{ pkgname }} 24 | {% endif %} 25 | 26 | {%- if modulefirst and not is_namespace %} 27 | {{ automodule(pkgname, automodule_options) }} 28 | {% endif %} 29 | 30 | {%- if subpackages %} 31 | 32 | {{ toctree(subpackages) }} 33 | {% endif %} 34 | 35 | {%- if submodules %} 36 | 37 | {% if separatemodules %} 38 | {{ toctree(submodules) }} 39 | {% else %} 40 | {%- for submodule in submodules %} 41 | {% if show_headings %} 42 | {{- [submodule, "module"] | join(" ") | e | heading(2) }} 43 | {% endif %} 44 | {{ automodule(submodule, automodule_options) }} 45 | {% endfor %} 46 | {%- endif %} 47 | {%- endif %} 48 | 49 | {%- if not modulefirst and not is_namespace %} 50 | 51 | {{ automodule(pkgname, automodule_options) }} 52 | {% endif %} 53 | -------------------------------------------------------------------------------- /docs/source/contribute.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: markdowns/CONTRIBUTING.md 2 | -------------------------------------------------------------------------------- /docs/source/custom_reference/eolearn.rst: -------------------------------------------------------------------------------- 1 | Package content 2 | =============== 3 | 4 | Subpackages: 5 | 6 | .. toctree:: 7 | 8 | eolearn.core 9 | eolearn.coregistration 10 | eolearn.features 11 | eolearn.geometry 12 | eolearn.io 13 | eolearn.mask 14 | eolearn.ml_tools 15 | eolearn.visualization 16 | -------------------------------------------------------------------------------- /docs/source/examples.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | 5 | Land Use and Land Cover 6 | ----------------------- 7 | 8 | .. toctree:: 9 | 10 | examples/land-cover-map/SI_LULC_pipeline.ipynb 11 | 12 | 13 | Water Monitor 14 | ------------- 15 | 16 | .. toctree:: 17 | 18 | examples/water-monitor/WaterMonitorWorkflow.ipynb 19 | 20 | 21 | IO Examples 22 | ----------- 23 | 24 | .. toctree:: 25 | 26 | examples/io/SentinelHubIO.ipynb 27 | 28 | 29 | Data masks 30 | ---------- 31 | 32 | .. toctree:: 33 | 34 | examples/mask/ValidDataMask.ipynb 35 | 36 | 37 | Visualizations 38 | -------------- 39 | 40 | .. toctree:: 41 | 42 | examples/visualization/EOPatchVisualization.ipynb 43 | 44 | 45 | Timelapse 46 | --------- 47 | 48 | .. toctree:: 49 | 50 | examples/core/TimeLapse.ipynb 51 | -------------------------------------------------------------------------------- /docs/source/figures/eo-learn-illustration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/docs/source/figures/eo-learn-illustration.png -------------------------------------------------------------------------------- /docs/source/figures/eo-learn-logo-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/docs/source/figures/eo-learn-logo-white.png -------------------------------------------------------------------------------- /docs/source/figures/eo-learn-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/docs/source/figures/eo-learn-logo.png -------------------------------------------------------------------------------- /docs/source/figures/eo-learn-workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/docs/source/figures/eo-learn-workflow.png -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: markdowns/INTRO.md 2 | 3 | .. toctree:: 4 | :maxdepth: 1 5 | :hidden: 6 | :caption: Contents: 7 | 8 | self 9 | install 10 | examples/core/CoreOverview.ipynb 11 | eotasks 12 | examples 13 | reference/eolearn 14 | contribute 15 | -------------------------------------------------------------------------------- /docs/source/install.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: markdowns/INSTALL.md 2 | -------------------------------------------------------------------------------- /eolearn/__init__.py: -------------------------------------------------------------------------------- 1 | """Main module of the `eolearn` package.""" 2 | 3 | __version__ = "1.5.7" 4 | 5 | import importlib.util 6 | import warnings 7 | 8 | SUBPACKAGES = ["core", "coregistration", "features", "geometry", "io", "mask", "ml_tools", "visualization"] 9 | deprecated_installs = [ 10 | subpackage for subpackage in SUBPACKAGES if importlib.util.find_spec(f"deprecated_eolearn_{subpackage}") is not None 11 | ] 12 | if deprecated_installs: 13 | warnings.warn( 14 | f"You are currently using an outdated installation of `eo-learn` for submodules {deprecated_installs}. You can" 15 | " find instructions on how to install `eo-learn` correctly at" 16 | " https://github.com/sentinel-hub/eo-learn/issues/733." 17 | ) 18 | -------------------------------------------------------------------------------- /eolearn/core/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The following objects and functions are the core of eo-learn package 3 | """ 4 | 5 | from .constants import FeatureType, OverwritePermission 6 | from .core_tasks import ( 7 | AddFeatureTask, 8 | CopyTask, 9 | CreateEOPatchTask, 10 | DeepCopyTask, 11 | DuplicateFeatureTask, 12 | ExplodeBandsTask, 13 | ExtractBandsTask, 14 | InitializeFeatureTask, 15 | LoadTask, 16 | MapFeatureTask, 17 | MergeEOPatchesTask, 18 | MergeFeatureTask, 19 | MoveFeatureTask, 20 | RemoveFeatureTask, 21 | RenameFeatureTask, 22 | SaveTask, 23 | TemporalSubsetTask, 24 | ZipFeatureTask, 25 | ) 26 | from .eodata import EOPatch 27 | from .eodata_merge import merge_eopatches 28 | from .eoexecution import EOExecutor 29 | from .eonode import EONode, linearly_connect_tasks 30 | from .eotask import EOTask 31 | from .eoworkflow import EOWorkflow, WorkflowResults 32 | from .eoworkflow_tasks import OutputTask 33 | from .utils.common import deep_eq 34 | from .utils.fs import get_filesystem, load_s3_filesystem, pickle_fs, unpickle_fs 35 | from .utils.parallelize import execute_with_mp_lock, join_futures, join_futures_iter, parallelize 36 | from .utils.parsing import FeatureParser 37 | -------------------------------------------------------------------------------- /eolearn/core/eonode.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements the EONode class, which specifies the local dependencies of an EOWorkflow 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import datetime as dt 13 | from dataclasses import dataclass, field 14 | from typing import Sequence, cast 15 | 16 | from .eotask import EOTask 17 | from .utils.common import generate_uid 18 | 19 | 20 | @dataclass(frozen=True) 21 | class EONode: 22 | """Class representing a node in EOWorkflow graph. 23 | 24 | The object id is kept to help with serialization issues. Tasks created in different sessions have a small chance 25 | of having an id clash. For this reason all tasks of a workflow should be created in the same session. 26 | 27 | :param task: An instance of `EOTask` that is carried out at the node when executed 28 | :param inputs: A sequence of `EONode` instances whose results this node takes as input 29 | :param name: Custom name of the node 30 | """ 31 | 32 | task: EOTask 33 | inputs: Sequence[EONode] = field(default_factory=tuple) 34 | name: str | None = field(default=None) 35 | uid: str = field(init=False, repr=False) 36 | 37 | def __post_init__(self) -> None: 38 | """Additionally verifies the parameters and adds a unique id to the node.""" 39 | if not isinstance(self.task, EOTask): 40 | raise ValueError(f"Value of `task` should be an instance of {EOTask.__name__}, got {self.task}.") 41 | 42 | if not isinstance(self.inputs, Sequence): 43 | raise ValueError(f"Value of `inputs` should be a sequence (`list`, `tuple`, ...), got {self.inputs}.") 44 | for input_node in self.inputs: 45 | if not isinstance(input_node, EONode): 46 | raise ValueError(f"Values in `inputs` should be instances of {EONode.__name__}, got {input_node}.") 47 | super().__setattr__("inputs", tuple(self.inputs)) 48 | 49 | if self.name is None: 50 | super().__setattr__("name", self.task.__class__.__name__) 51 | 52 | super().__setattr__("uid", generate_uid(self.task.__class__.__name__)) 53 | 54 | def __hash__(self) -> int: 55 | return self.uid.__hash__() 56 | 57 | def get_name(self, suffix_number: int = 0) -> str: 58 | """Provides node name according to the class of the contained task and a given number.""" 59 | if suffix_number: 60 | return f"{self.name}_{suffix_number}" 61 | return cast(str, self.name) 62 | 63 | def get_dependencies(self, *, _memo: dict[EONode, set[EONode]] | None = None) -> set[EONode]: 64 | """Returns a set of nodes that this node depends on. Set includes the node itself.""" 65 | _memo = _memo if _memo is not None else {} 66 | if self not in _memo: 67 | result = {self}.union(*(input_node.get_dependencies(_memo=_memo) for input_node in self.inputs)) 68 | _memo[self] = result 69 | 70 | return _memo[self] 71 | 72 | 73 | def linearly_connect_tasks(*tasks: EOTask | tuple[EOTask, str]) -> list[EONode]: 74 | """Creates a list of linearly linked nodes, suitable to construct an EOWorkflow. 75 | 76 | Nodes depend on each other in such a way, that the node containing the task at index `i` is the input node for the 77 | node at index `i+1`. Nodes are returned in the order of execution, so the task at index `j` is contained in the node 78 | at index `j`, making it easier to construct execution arguments. 79 | 80 | :param tasks: A sequence containing tasks and/or (task, name) pairs 81 | """ 82 | nodes = [] 83 | endpoint: Sequence[EONode] = tuple() 84 | for task_spec in tasks: 85 | if isinstance(task_spec, EOTask): 86 | node = EONode(task_spec, inputs=endpoint) 87 | else: 88 | task, name = task_spec 89 | node = EONode(task, inputs=endpoint, name=name) 90 | nodes.append(node) 91 | endpoint = [node] 92 | 93 | return nodes 94 | 95 | 96 | @dataclass(frozen=True) 97 | class NodeStats: 98 | """An object containing statistical info about a node execution.""" 99 | 100 | node_uid: str 101 | node_name: str 102 | start_time: dt.datetime 103 | end_time: dt.datetime 104 | exception_info: ExceptionInfo | None = None 105 | 106 | 107 | @dataclass(frozen=True) 108 | class ExceptionInfo: 109 | """Contains information on exceptions that occur when executing a node.""" 110 | 111 | exception: BaseException 112 | traceback: str 113 | origin: str 114 | -------------------------------------------------------------------------------- /eolearn/core/eotask.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements the core class hierarchy for implementing EO tasks. An EO task is any class the inherits 3 | from the abstract EOTask class. Each EO task has to implement the execute method; invoking __call__ on a EO task 4 | instance invokes the execute method. EO tasks are meant primarily to operate on EO patches (i.e. instances of EOPatch). 5 | 6 | EO task classes are generally lightweight (i.e. not too complicated), short, and do one thing well. For example, an 7 | EO task might take as input an EOPatch containing cloud mask and return as a result the cloud coverage for that mask. 8 | 9 | Copyright (c) 2017- Sinergise and contributors 10 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 11 | 12 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 13 | """ 14 | 15 | from __future__ import annotations 16 | 17 | import inspect 18 | import logging 19 | from abc import ABCMeta, abstractmethod 20 | from dataclasses import dataclass 21 | from typing import Any, Callable, Iterable, TypeVar 22 | 23 | from typing_extensions import deprecated 24 | 25 | from .constants import FeatureType 26 | from .eodata import EOPatch 27 | from .exceptions import EODeprecationWarning 28 | from .types import EllipsisType, Feature, FeaturesSpecification, SingleFeatureSpec 29 | from .utils.parsing import FeatureParser, parse_feature, parse_features, parse_renamed_feature, parse_renamed_features 30 | 31 | LOGGER = logging.getLogger(__name__) 32 | 33 | Self = TypeVar("Self") 34 | 35 | PARSE_RENAMED_DEPRECATE_MSG = ( 36 | "The method will no longer be a method of `EOTask`, but can be imported as a function from" 37 | " `eolearn.core.utils.parsing`." 38 | ) 39 | 40 | 41 | class EOTask(metaclass=ABCMeta): 42 | """Base class for EOTask.""" 43 | 44 | parse_renamed_feature = staticmethod( 45 | deprecated(PARSE_RENAMED_DEPRECATE_MSG, category=EODeprecationWarning)(parse_renamed_feature) 46 | ) 47 | parse_renamed_features = staticmethod( 48 | deprecated(PARSE_RENAMED_DEPRECATE_MSG, category=EODeprecationWarning)(parse_renamed_features) 49 | ) 50 | 51 | def __new__(cls: type[Self], *args: Any, **kwargs: Any) -> Self: 52 | """Stores initialization parameters and the order to the instance attribute `init_args`.""" 53 | self = super().__new__(cls) # type: ignore[misc] 54 | 55 | init_args: dict[str, object] = {} 56 | for arg, value in zip(inspect.getfullargspec(self.__init__).args[1 : len(args) + 1], args): 57 | init_args[arg] = repr(value) 58 | for arg in inspect.getfullargspec(self.__init__).args[len(args) + 1 :]: 59 | if arg in kwargs: 60 | init_args[arg] = repr(kwargs[arg]) 61 | 62 | self._private_task_config = _PrivateTaskConfig(init_args=init_args) 63 | 64 | return self 65 | 66 | @property 67 | def private_task_config(self) -> _PrivateTaskConfig: 68 | """Keeps track of the arguments for which the task was initialized for better logging. 69 | 70 | :return: The initial configuration arguments of the task 71 | """ 72 | return self._private_task_config # type: ignore[attr-defined] 73 | 74 | def __call__(self, *args: Any, **kwargs: Any) -> Any: 75 | """Syntactic sugar for task execution""" 76 | # The type cannot be more precise unless we know the type of `execute`. Possible improvement with generics + 77 | # the use of ParamSpec. 78 | return self.execute(*args, **kwargs) 79 | 80 | @abstractmethod 81 | def execute(self, *eopatches, **kwargs): # type: ignore[no-untyped-def] # must be ignored so subclasses can change 82 | """Override to specify action performed by task.""" 83 | 84 | @staticmethod 85 | def parse_feature( 86 | feature: SingleFeatureSpec, 87 | eopatch: EOPatch | None = None, 88 | allowed_feature_types: EllipsisType | Iterable[FeatureType] | Callable[[FeatureType], bool] = ..., 89 | ) -> Feature: 90 | """See `eolearn.core.utils.parse_feature`.""" 91 | return parse_feature(feature, eopatch, allowed_feature_types) 92 | 93 | @staticmethod 94 | def parse_features( 95 | features: FeaturesSpecification, 96 | eopatch: EOPatch | None = None, 97 | allowed_feature_types: EllipsisType | Iterable[FeatureType] | Callable[[FeatureType], bool] = ..., 98 | ) -> list[Feature]: 99 | """See `eolearn.core.utils.parse_features`.""" 100 | return parse_features(features, eopatch, allowed_feature_types) 101 | 102 | @staticmethod 103 | def get_feature_parser( 104 | features: FeaturesSpecification, 105 | allowed_feature_types: EllipsisType | Iterable[FeatureType] | Callable[[FeatureType], bool] = ..., 106 | ) -> FeatureParser: 107 | """See :class:`FeatureParser`.""" 108 | return FeatureParser(features, allowed_feature_types=allowed_feature_types) 109 | 110 | 111 | @dataclass(frozen=True) 112 | class _PrivateTaskConfig: 113 | """A container for configuration parameters about an EOTask itself. 114 | 115 | :param init_args: A dictionary of parameters and values used for EOTask initialization 116 | """ 117 | 118 | init_args: dict[str, object] 119 | -------------------------------------------------------------------------------- /eolearn/core/eoworkflow_tasks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module implementing tasks that have a special effect in `EOWorkflow` 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | from .eodata import EOPatch 13 | from .eotask import EOTask 14 | from .types import FeaturesSpecification 15 | from .utils.common import generate_uid 16 | 17 | 18 | class InputTask(EOTask): 19 | """Introduces data into an EOWorkflow, where the data can be specified at initialization or at execution.""" 20 | 21 | def __init__(self, value: object | None = None): 22 | """ 23 | :param value: Default value that the task should provide as a result. Can be overridden in execution arguments 24 | """ 25 | self.value = value 26 | 27 | def execute(self, *, value: object | None = None) -> object: 28 | """ 29 | :param value: A value that the task should provide as its result. If not set uses the value from initialization 30 | :return: Directly returns `value` 31 | """ 32 | return value or self.value 33 | 34 | 35 | class OutputTask(EOTask): 36 | """Stores data as an output of `EOWorkflow` results.""" 37 | 38 | def __init__(self, name: str | None = None, features: FeaturesSpecification = ...): 39 | """ 40 | :param name: A name under which the data will be saved in `WorkflowResults`, auto-generated if `None` 41 | :param features: A collection of features to be kept if the data is an `EOPatch` 42 | """ 43 | self._name = name or generate_uid("output") 44 | self.features = features 45 | 46 | @property 47 | def name(self) -> str: 48 | """Provides a name under which data will be saved in `WorkflowResults`. 49 | 50 | :return: A name 51 | """ 52 | return self._name 53 | 54 | def execute(self, data: object) -> object: 55 | """ 56 | :param data: input data 57 | :return: Same data, to be stored in results. For `EOPatch` returns shallow copy containing `features` and 58 | possibly BBox and timestamps (see `copy` method of `EOPatch`). 59 | """ 60 | if isinstance(data, EOPatch): 61 | return data.copy(features=self.features, copy_timestamps=True) 62 | return data 63 | -------------------------------------------------------------------------------- /eolearn/core/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implementation of custom eo-learn exceptions and warnings 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import warnings 13 | 14 | 15 | class EODeprecationWarning(DeprecationWarning): 16 | """A custom deprecation warning for eo-learn package.""" 17 | 18 | 19 | class EOUserWarning(UserWarning): 20 | """A custom user warning for eo-learn package.""" 21 | 22 | 23 | class EORuntimeWarning(RuntimeWarning): 24 | """A custom runtime warning for eo-learn package.""" 25 | 26 | 27 | class TemporalDimensionWarning(RuntimeWarning): 28 | """A custom runtime warning for cases where EOPatches are temporally ill defined.""" 29 | 30 | 31 | warnings.simplefilter("default", EODeprecationWarning) 32 | warnings.simplefilter("default", EOUserWarning) 33 | warnings.simplefilter("always", EORuntimeWarning) 34 | -------------------------------------------------------------------------------- /eolearn/core/extra/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A subfolder containing integrations with other packages 3 | """ 4 | -------------------------------------------------------------------------------- /eolearn/core/types.py: -------------------------------------------------------------------------------- 1 | """ 2 | Types and type aliases used throughout the code. 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import sys 13 | 14 | # pylint: disable=unused-import 15 | from typing import Dict, Iterable, Sequence, Tuple, Union 16 | 17 | from .constants import FeatureType 18 | 19 | if sys.version_info >= (3, 10): 20 | from types import EllipsisType # pylint: disable=ungrouped-imports 21 | from typing import TypeAlias 22 | else: 23 | import builtins # noqa: F401, RUF100 24 | 25 | from typing_extensions import TypeAlias 26 | 27 | EllipsisType: TypeAlias = "builtins.ellipsis" 28 | 29 | 30 | # DEVELOPER NOTE: the #: comments are applied as docstrings 31 | Feature: TypeAlias = Tuple[FeatureType, str] 32 | 33 | SingleFeatureSpec: TypeAlias = Union[Feature, Tuple[FeatureType, str, str]] 34 | 35 | SequenceFeatureSpec: TypeAlias = Sequence[Union[SingleFeatureSpec, FeatureType, Tuple[FeatureType, EllipsisType]]] 36 | DictFeatureSpec: TypeAlias = Dict[FeatureType, Union[EllipsisType, Iterable[Union[str, Tuple[str, str]]]]] 37 | MultiFeatureSpec: TypeAlias = Union[ 38 | EllipsisType, FeatureType, Tuple[FeatureType, EllipsisType], SequenceFeatureSpec, DictFeatureSpec 39 | ] 40 | 41 | #: Specification of a single or multiple features. See :class:`FeatureParser`. 42 | FeaturesSpecification: TypeAlias = Union[SingleFeatureSpec, MultiFeatureSpec] 43 | -------------------------------------------------------------------------------- /eolearn/core/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A subfolder containing utilities 3 | """ 4 | -------------------------------------------------------------------------------- /eolearn/core/utils/common.py: -------------------------------------------------------------------------------- 1 | """ 2 | The utilities module is a collection of classes and functions used across the eolearn package, such as checking whether 3 | two objects are deeply equal, padding of an image, etc. 4 | 5 | Copyright (c) 2017- Sinergise and contributors 6 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 7 | 8 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 9 | """ 10 | 11 | from __future__ import annotations 12 | 13 | import uuid 14 | from typing import Callable, Mapping, Sequence, cast 15 | 16 | import geopandas as gpd 17 | import numpy as np 18 | from geopandas.testing import assert_geodataframe_equal 19 | 20 | 21 | def deep_eq(fst_obj: object, snd_obj: object) -> bool: 22 | """Compares whether fst_obj and snd_obj are deeply equal. 23 | 24 | In case when both fst_obj and snd_obj are of type np.ndarray or either np.memmap, they are compared using 25 | np.array_equal(fst_obj, snd_obj). Otherwise, when they are lists or tuples, they are compared for length and then 26 | deep_eq is applied component-wise. When they are dict, they are compared for key set equality, and then deep_eq is 27 | applied value-wise. For all other data types that are not list, tuple, dict, or np.ndarray, the method falls back 28 | to the __eq__ method. 29 | 30 | Because np.ndarray is not a hashable object, it is impossible to form a set of numpy arrays, hence deep_eq works 31 | correctly. 32 | 33 | :param fst_obj: First object compared 34 | :param snd_obj: Second object compared 35 | :return: `True` if objects are deeply equal, `False` otherwise 36 | """ 37 | # pylint: disable=too-many-return-statements 38 | if not isinstance(fst_obj, type(snd_obj)): 39 | return False 40 | 41 | if isinstance(fst_obj, np.ndarray): 42 | snd_obj = cast(np.ndarray, snd_obj) 43 | if fst_obj.dtype != snd_obj.dtype: 44 | return False 45 | fst_nan_mask = np.isnan(fst_obj) 46 | snd_nan_mask = np.isnan(snd_obj) 47 | return np.array_equal(fst_obj[~fst_nan_mask], snd_obj[~snd_nan_mask]) and np.array_equal( 48 | fst_nan_mask, snd_nan_mask 49 | ) 50 | 51 | if isinstance(fst_obj, gpd.GeoDataFrame): 52 | try: 53 | # We allow differences in index types and in dtypes of columns 54 | assert_geodataframe_equal(fst_obj, snd_obj, check_index_type=False, check_dtype=False) 55 | return True 56 | except AssertionError: 57 | return False 58 | 59 | if isinstance(fst_obj, (tuple, list)): 60 | snd_obj = cast(Sequence, snd_obj) 61 | 62 | return len(fst_obj) == len(snd_obj) and all(map(deep_eq, fst_obj, snd_obj)) 63 | 64 | if isinstance(fst_obj, (dict, Mapping)): 65 | snd_obj = cast(dict, snd_obj) 66 | 67 | if fst_obj.keys() != snd_obj.keys(): 68 | return False 69 | 70 | return all(deep_eq(fst_obj[key], snd_obj[key]) for key in fst_obj) 71 | 72 | return fst_obj == snd_obj 73 | 74 | 75 | def generate_uid(prefix: str) -> str: 76 | """Generates a (sufficiently) unique ID starting with the `prefix`. 77 | 78 | The ID is composed of the prefix, a hexadecimal string obtained from the current time and a random hexadecimal 79 | string. This makes the uid sufficiently unique. 80 | """ 81 | time_uid = uuid.uuid1(node=0).hex[:-12] 82 | random_uid = uuid.uuid4().hex[:12] 83 | return f"{prefix}-{time_uid}-{random_uid}" 84 | 85 | 86 | def is_discrete_type(number_type: np.dtype | type) -> bool: 87 | """Checks if a given `numpy` type is a discrete numerical type.""" 88 | return np.issubdtype(number_type, np.integer) or np.issubdtype(number_type, bool) 89 | 90 | 91 | def _apply_to_spatial_axes( 92 | function: Callable[[np.ndarray], np.ndarray], data: np.ndarray, spatial_axes: tuple[int, int] 93 | ) -> np.ndarray: 94 | """Helper function for applying a 2D -> 2D function to a higher dimension array 95 | 96 | Recursively slices data into smaller-dimensional ones, until only the spatial axes remain. The indices of spatial 97 | axes have to be adjusted if the recursion-axis is smaller than either one, e.g. spatial axes (1, 2) become (0, 1) 98 | after splitting the 3D data along axis 0 into 2D arrays. 99 | 100 | After achieving 2D data slices the mapping function is applied. The data is then reconstructed into original form. 101 | """ 102 | 103 | ax1, ax2 = spatial_axes 104 | if ax1 >= ax2: 105 | raise ValueError( 106 | f"For parameter `spatial_axes` the second axis must be greater than first, got {spatial_axes}." 107 | ) 108 | 109 | if ax2 >= data.ndim: 110 | raise ValueError( 111 | f"Values in `spatial_axes` must be smaller than `data.ndim`, got {spatial_axes} for data of dimension" 112 | f" {data.ndim}." 113 | ) 114 | 115 | if data.ndim <= 2: 116 | return function(data) 117 | 118 | axis = next(i for i in range(data.ndim) if i not in spatial_axes) 119 | data = np.moveaxis(data, axis, 0) 120 | 121 | ax1, ax2 = (ax if axis > ax else ax - 1 for ax in spatial_axes) 122 | 123 | mapped_slices = [_apply_to_spatial_axes(function, data_slice, (ax1, ax2)) for data_slice in data] 124 | return np.moveaxis(np.stack(mapped_slices), 0, axis) 125 | -------------------------------------------------------------------------------- /eolearn/core/utils/logging.py: -------------------------------------------------------------------------------- 1 | """ 2 | The utilities module is a collection of classes and functions used across the eolearn package, such as checking whether 3 | two objects are deeply equal, padding of an image, etc. 4 | 5 | Copyright (c) 2017- Sinergise and contributors 6 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 7 | 8 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 9 | """ 10 | 11 | from __future__ import annotations 12 | 13 | import logging 14 | from logging import Filter, LogRecord 15 | from typing import Any 16 | 17 | LOGGER = logging.getLogger(__name__) 18 | 19 | 20 | class LogFileFilter(Filter): 21 | """Filters log messages passed to log file.""" 22 | 23 | def __init__(self, thread_name: str | None, *args: Any, **kwargs: Any): 24 | """ 25 | :param thread_name: Name of the thread by which to filter logs. By default, it won't filter by any name. 26 | """ 27 | self.thread_name = thread_name 28 | super().__init__(*args, **kwargs) 29 | 30 | def filter(self, record: LogRecord) -> bool: 31 | """Shows everything from the thread that it was initialized in.""" 32 | return record.threadName == self.thread_name 33 | -------------------------------------------------------------------------------- /eolearn/core/utils/raster.py: -------------------------------------------------------------------------------- 1 | """ 2 | Useful utilities for working with raster data, typically `numpy` arrays. 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | from typing import Literal 13 | 14 | import numpy as np 15 | 16 | 17 | def fast_nanpercentile(data: np.ndarray, percentile: float, *, method: str = "linear") -> np.ndarray: 18 | """This is an alternative implementation of `numpy.nanpercentile`. For cases where the size of the first dimension 19 | is relatively small compared to the size of the entire array it works much faster than the original. 20 | 21 | The algorithm divides pixel data over the first axis into groups by how many NaN values they have. In each group 22 | NaN values are removed and `numpy.percentile` function is applied. If any series contains only NaN values also any 23 | percentile of that series will be NaN. 24 | 25 | This function differs from `numpy` implementations only in the following: 26 | 27 | - In case the size of the first dimension of `data` is `0` this method will still return an output array with 28 | all NaN values. This matches with `numpy.nanpercentile` while `numpy.percentile` raises an error. 29 | - The output dtype of this method will be always the same as the input dtype while `numpy` implementations 30 | in many cases use `float64` as the output dtype. 31 | 32 | :param data: An array for which percentiles will be calculated along the first axis. 33 | :param percentile: A percentile to compute, which must be between `0` and `100` inclusive. 34 | :param method: A method for estimating the percentile. This parameter is propagated to `numpy.percentile`. 35 | :return: An array of percentiles and a shape equal to the shape of `data` array without the first dimension. 36 | """ 37 | method_kwargs = {"method" if np.__version__ >= "1.22.0" else "interpolation": method} 38 | 39 | combined_data = np.zeros(data.shape[1:], dtype=data.dtype) 40 | 41 | no_data_counts = np.count_nonzero(np.isnan(data), axis=0) 42 | for no_data_num in np.unique(no_data_counts): 43 | mask = no_data_counts == no_data_num 44 | 45 | chunk = data[..., mask] 46 | time_size, sample_size = chunk.shape 47 | 48 | if time_size == no_data_num: 49 | result = np.full(sample_size, np.nan, dtype=chunk.dtype) 50 | else: 51 | chunk = chunk.flatten(order="F") 52 | chunk = chunk[~np.isnan(chunk)] 53 | chunk = chunk.reshape((time_size - no_data_num, sample_size), order="F") 54 | 55 | result = np.percentile(chunk, q=percentile, axis=0, **method_kwargs) # type: ignore[call-overload] 56 | 57 | combined_data[mask] = result 58 | 59 | return combined_data 60 | 61 | 62 | def constant_pad( # noqa: C901 63 | array: np.ndarray, 64 | multiple_of: tuple[int, int], 65 | up_down_rule: Literal["even", "up", "down"] = "even", 66 | left_right_rule: Literal["even", "left", "right"] = "even", 67 | pad_value: float = 0, 68 | ) -> np.ndarray: 69 | """Function pads an image of shape (rows, columns, channels) with zeros. 70 | 71 | It pads an image so that the shape becomes (rows + padded_rows, columns + padded_columns, channels), where 72 | padded_rows = (int(rows/multiple_of[0]) + 1) * multiple_of[0] - rows 73 | 74 | Same rule is applied to columns. 75 | 76 | :param array: Array with shape `(rows, columns, ...)` to be padded. 77 | :param multiple_of: make array' rows and columns multiple of this tuple 78 | :param up_down_rule: Add padded rows evenly to the top/bottom of the image, or up (top) / down (bottom) only 79 | :param left_right_rule: Add padded columns evenly to the left/right of the image, or left / right only 80 | :param pad_value: Value to be assigned to padded rows and columns 81 | """ 82 | rows, columns = array.shape[:2] 83 | row_padding, col_padding = 0, 0 84 | 85 | if rows % multiple_of[0]: 86 | row_padding = (int(rows / multiple_of[0]) + 1) * multiple_of[0] - rows 87 | 88 | if columns % multiple_of[1]: 89 | col_padding = (int(columns / multiple_of[1]) + 1) * multiple_of[1] - columns 90 | 91 | row_padding_up, row_padding_down, col_padding_left, col_padding_right = 0, 0, 0, 0 92 | 93 | if row_padding > 0: 94 | if up_down_rule == "up": 95 | row_padding_up = row_padding 96 | elif up_down_rule == "down": 97 | row_padding_down = row_padding 98 | elif up_down_rule == "even": 99 | row_padding_up = int(row_padding / 2) 100 | row_padding_down = row_padding_up + (row_padding % 2) 101 | else: 102 | raise ValueError("Padding rule for rows not supported. Choose between even, down or up!") 103 | 104 | if col_padding > 0: 105 | if left_right_rule == "left": 106 | col_padding_left = col_padding 107 | elif left_right_rule == "right": 108 | col_padding_right = col_padding 109 | elif left_right_rule == "even": 110 | col_padding_left = int(col_padding / 2) 111 | col_padding_right = col_padding_left + (col_padding % 2) 112 | else: 113 | raise ValueError("Padding rule for columns not supported. Choose between even, left or right!") 114 | 115 | return np.pad( 116 | array, 117 | ((row_padding_up, row_padding_down), (col_padding_left, col_padding_right)), 118 | "constant", 119 | constant_values=((pad_value, pad_value), (pad_value, pad_value)), 120 | ) 121 | -------------------------------------------------------------------------------- /eolearn/core/utils/testing.py: -------------------------------------------------------------------------------- 1 | """ 2 | The eodata module provides core objects for handling remote sensing multi-temporal data (such as satellite imagery). 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import datetime as dt 13 | import string 14 | from dataclasses import dataclass, field 15 | from typing import Any 16 | 17 | import geopandas as gpd 18 | import numpy as np 19 | import pandas as pd 20 | from geopandas.testing import assert_geodataframe_equal 21 | from numpy.testing import assert_array_equal 22 | 23 | from sentinelhub import CRS, BBox 24 | 25 | from ..constants import FeatureType 26 | from ..eodata import EOPatch 27 | from ..types import FeaturesSpecification 28 | from ..utils.parsing import FeatureParser 29 | 30 | DEFAULT_BBOX = BBox((0, 0, 100, 100), crs=CRS("EPSG:32633")) 31 | 32 | 33 | @dataclass 34 | class PatchGeneratorConfig: 35 | """Dataclass containing a more complex setup of the PatchGenerator class.""" 36 | 37 | num_timestamps: int = 5 38 | timestamps_range: tuple[dt.datetime, dt.datetime] = (dt.datetime(2019, 1, 1), dt.datetime(2019, 12, 31)) 39 | timestamps: list[dt.datetime] = field(init=False, repr=False) 40 | 41 | max_integer_value: int = 256 42 | raster_shape: tuple[int, int] = (98, 151) 43 | depth_range: tuple[int, int] = (1, 3) 44 | 45 | def __post_init__(self) -> None: 46 | self.timestamps = list(pd.date_range(*self.timestamps_range, periods=self.num_timestamps).to_pydatetime()) 47 | 48 | 49 | def generate_eopatch( 50 | features: FeaturesSpecification | None = None, 51 | bbox: BBox = DEFAULT_BBOX, 52 | timestamps: list[dt.datetime] | None = None, 53 | seed: int = 42, 54 | config: PatchGeneratorConfig | None = None, 55 | ) -> EOPatch: 56 | """A class for generating EOPatches with dummy data.""" 57 | config = config if config is not None else PatchGeneratorConfig() 58 | 59 | parsed_features = FeatureParser( 60 | features or [], lambda feature_type: feature_type.is_array() or feature_type == FeatureType.META_INFO 61 | ).get_features() 62 | 63 | rng = np.random.default_rng(seed) 64 | timestamps = timestamps if timestamps is not None else config.timestamps 65 | patch = EOPatch(bbox=bbox, timestamps=timestamps) 66 | 67 | # fill eopatch with random data 68 | # note: the patch generation functionality could be extended by generating extra random features 69 | for ftype, fname in parsed_features: 70 | if ftype == FeatureType.META_INFO: 71 | patch[(ftype, fname)] = "".join(rng.choice(list(string.ascii_letters), 20)) 72 | else: 73 | shape = _get_feature_shape(rng, ftype, timestamps, config) 74 | patch[(ftype, fname)] = _generate_feature_data(rng, ftype, shape, config) 75 | 76 | return patch 77 | 78 | 79 | def _generate_feature_data( 80 | rng: np.random.Generator, ftype: FeatureType, shape: tuple[int, ...], config: PatchGeneratorConfig 81 | ) -> np.ndarray: 82 | if ftype.is_discrete(): 83 | return rng.integers(config.max_integer_value, size=shape) 84 | return rng.normal(size=shape) 85 | 86 | 87 | def _get_feature_shape( 88 | rng: np.random.Generator, ftype: FeatureType, timestamps: list[dt.datetime], config: PatchGeneratorConfig 89 | ) -> tuple[int, ...]: 90 | time, height, width, depth = len(timestamps), *config.raster_shape, rng.integers(*config.depth_range) 91 | 92 | if ftype.is_spatial() and not ftype.is_vector(): 93 | return (time, height, width, depth) if ftype.is_temporal() else (height, width, depth) 94 | return (time, depth) if ftype.is_temporal() else (depth,) 95 | 96 | 97 | def assert_feature_data_equal(tested_feature: Any, expected_feature: Any) -> None: 98 | """Asserts that the data of two features is equal. Cases are specialized for common data found in EOPatches.""" 99 | if isinstance(tested_feature, np.ndarray) and isinstance(expected_feature, np.ndarray): 100 | assert_array_equal(tested_feature, expected_feature) 101 | elif isinstance(tested_feature, gpd.GeoDataFrame) and isinstance(expected_feature, gpd.GeoDataFrame): 102 | assert CRS(tested_feature.crs) == CRS(expected_feature.crs) 103 | assert_geodataframe_equal( 104 | tested_feature, expected_feature, check_crs=False, check_index_type=False, check_dtype=False 105 | ) 106 | else: 107 | assert tested_feature == expected_feature 108 | -------------------------------------------------------------------------------- /eolearn/core/utils/types.py: -------------------------------------------------------------------------------- 1 | """Deprecated module for types, moved to `eolearn.core.types`.""" 2 | 3 | from __future__ import annotations 4 | 5 | from warnings import warn 6 | 7 | from ..exceptions import EODeprecationWarning 8 | 9 | # pylint: disable-next=wildcard-import,unused-wildcard-import 10 | from ..types import * # noqa: F403 11 | 12 | warn( 13 | "The module `eolearn.core.utils.types` is deprecated, use `eolearn.core.types` instead.", 14 | category=EODeprecationWarning, 15 | stacklevel=2, 16 | ) 17 | -------------------------------------------------------------------------------- /eolearn/coregistration/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A collection of tools and EOTasks for image co-registration 3 | """ 4 | 5 | from .coregistration import ECCRegistrationTask, get_gradient 6 | -------------------------------------------------------------------------------- /eolearn/features/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A collection of EOTasks for feature manipulation 3 | """ 4 | 5 | from .feature_manipulation import FilterTimeSeriesTask, SimpleFilterTask 6 | from .ndi import NormalizedDifferenceIndexTask 7 | -------------------------------------------------------------------------------- /eolearn/features/extra/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A collection of EOTasks with non-standard dependencies. Use the extra `[EXTRA]` to install dependencies or 3 | install the module-specific dependencies by hand. 4 | """ 5 | -------------------------------------------------------------------------------- /eolearn/features/extra/clustering.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for computing clusters in EOPatch 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | from typing import Callable, Literal 13 | 14 | import numpy as np 15 | from sklearn.cluster import AgglomerativeClustering 16 | from sklearn.feature_extraction.image import grid_to_graph 17 | 18 | from eolearn.core import EOPatch, EOTask, FeatureType 19 | from eolearn.core.types import Feature 20 | 21 | 22 | class ClusteringTask(EOTask): 23 | """ 24 | Tasks computes clusters on selected features using `sklearn.cluster.AgglomerativeClustering`. 25 | 26 | The algorithm produces a timeless data feature where each cell has a natural number which corresponds to specific 27 | group. The cells marked with -1 are not marking clusters. They are either being excluded by a mask or later removed 28 | by depending on the 'remove_small' threshold. 29 | """ 30 | 31 | def __init__( 32 | self, 33 | features: Feature, 34 | new_feature_name: str, 35 | distance_threshold: float | None = None, 36 | n_clusters: int | None = None, 37 | affinity: Literal["euclidean", "l1", "l2", "manhattan", "cosine"] = "cosine", 38 | linkage: Literal["ward", "complete", "average", "single"] = "single", 39 | remove_small: int = 0, 40 | connectivity: None | np.ndarray | Callable = None, 41 | mask_name: str | None = None, 42 | ): 43 | """Class constructor 44 | 45 | :param features: A collection of features used for clustering. The features need to be of type DATA_TIMELESS 46 | :param new_feature_name: Name of feature that is the result of clustering 47 | :param distance_threshold: The linkage distance threshold above which, clusters will not be merged. If non None, 48 | n_clusters must be None nd compute_full_tree must be True 49 | :param n_clusters: The number of clusters found by the algorithm. If distance_threshold=None, it will be equal 50 | to the given n_clusters 51 | :param affinity: Metric used to compute the linkage. Can be “euclidean”, “l1”, “l2”, “manhattan”, “cosine”. 52 | :param linkage: Which linkage criterion to use. The linkage criterion determines which distance to use between 53 | sets of observation. The algorithm will merge the pairs of cluster that minimize this criterion. 54 | - ward minimizes the variance of the clusters being merged. 55 | - average uses the average of the distances of each observation of the two sets. 56 | - complete or maximum linkage uses the maximum distances between all observations of the two sets. 57 | - single uses the minimum of the distances between all observations of the two sets. 58 | :param remove_small: If greater than 0, removes all clusters that have fewer points as "remove_small" 59 | :param connectivity: Connectivity matrix. Defines for each sample the neighboring samples following a given 60 | structure of the data. This can be a connectivity matrix itself or a callable that transforms the data into 61 | a connectivity matrix, such as derived from neighbors_graph. If set to None it uses the graph that has 62 | adjacent pixels connected. 63 | :param mask_name: An optional mask feature used for exclusion of the area from clustering 64 | """ 65 | self.features_parser = self.get_feature_parser(features, allowed_feature_types=[FeatureType.DATA_TIMELESS]) 66 | self.distance_threshold = distance_threshold 67 | self.affinity = affinity 68 | self.linkage = linkage 69 | self.new_feature_name = new_feature_name 70 | self.n_clusters = n_clusters 71 | self.compute_full_tree: Literal["auto"] | bool = "auto" if distance_threshold is None else True 72 | self.remove_small = remove_small 73 | self.connectivity = connectivity 74 | self.mask_name = mask_name 75 | 76 | def execute(self, eopatch: EOPatch) -> EOPatch: 77 | """ 78 | :param eopatch: Input EOPatch 79 | :return: Transformed EOPatch 80 | """ 81 | relevant_features = self.features_parser.get_features(eopatch) 82 | data = np.concatenate([eopatch[feature] for feature in relevant_features], axis=2) 83 | 84 | # Reshapes the data, because AgglomerativeClustering method only takes one dimensional arrays of vectors 85 | height, width, num_channels = data.shape 86 | data = np.reshape(data, (-1, num_channels)) 87 | 88 | graph_args = {"n_x": height, "n_y": width} 89 | 90 | # All connections to masked pixels are removed 91 | if self.mask_name is not None: 92 | mask = eopatch.mask_timeless[self.mask_name].squeeze(axis=-1) 93 | graph_args["mask"] = mask 94 | data = data[np.ravel(mask) != 0] 95 | 96 | # If connectivity is not set, it uses pixel-to-pixel connections 97 | if not self.connectivity: 98 | self.connectivity = grid_to_graph(**graph_args) 99 | 100 | model = AgglomerativeClustering( 101 | distance_threshold=self.distance_threshold, 102 | metric=self.affinity, 103 | linkage=self.linkage, 104 | connectivity=self.connectivity, 105 | n_clusters=self.n_clusters, 106 | compute_full_tree=self.compute_full_tree, 107 | ) 108 | 109 | model.fit(data) 110 | result = model.labels_ 111 | if self.remove_small > 0: 112 | for label, count in zip(*np.unique(result, return_counts=True)): 113 | if count < self.remove_small: 114 | result[result == label] = -1 115 | 116 | # Transforms data back to original shape and setting all masked regions to -1 117 | if self.mask_name is not None: 118 | unmasked_result = np.full(height * width, -1) 119 | unmasked_result[np.ravel(mask) != 0] = result 120 | result = unmasked_result 121 | 122 | eopatch[FeatureType.DATA_TIMELESS, self.new_feature_name] = np.reshape(result, (height, width, 1)) 123 | 124 | return eopatch 125 | -------------------------------------------------------------------------------- /eolearn/features/ndi.py: -------------------------------------------------------------------------------- 1 | """ 2 | A collection of bands extraction EOTasks 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import numpy as np 13 | 14 | from eolearn.core import MapFeatureTask 15 | from eolearn.core.types import Feature 16 | 17 | 18 | class NormalizedDifferenceIndexTask(MapFeatureTask): 19 | """The task calculates a Normalized Difference Index (NDI) between two bands A and B as: 20 | 21 | :math:`NDI = \\dfrac{A-B+c}{A+B+c}`, 22 | 23 | where c is provided as the *acorvi_constant* argument. For the reasoning behind using the acorvi_constant in the 24 | equation, check the article `Using NDVI with atmospherically corrected data 25 | `_. 26 | """ 27 | 28 | def __init__( 29 | self, 30 | input_feature: Feature, 31 | output_feature: Feature, 32 | bands: tuple[int, int], 33 | acorvi_constant: float = 0, 34 | undefined_value: float = np.nan, 35 | ): 36 | """ 37 | :param input_feature: A source feature from which to take the bands. 38 | :param output_feature: An output feature to which to write the NDI. 39 | :param bands: A list of bands from which to calculate the NDI. 40 | :param acorvi_constant: A constant to be used in the NDI calculation. It is set to 0 by default. 41 | :param undefined_value: A value to override any calculation result that is not a finite value (e.g.: inf, nan). 42 | """ 43 | super().__init__(input_feature, output_feature) 44 | 45 | if not isinstance(bands, (list, tuple)) or len(bands) != 2 or not all(isinstance(x, int) for x in bands): 46 | raise ValueError("bands argument should be a list or tuple of two integers!") 47 | 48 | self.band_a, self.band_b = bands 49 | self.undefined_value = undefined_value 50 | self.acorvi_constant = acorvi_constant 51 | 52 | def map_method(self, feature: np.ndarray) -> np.ndarray: 53 | """ 54 | :param feature: An eopatch on which to calculate the NDI. 55 | """ 56 | band_a, band_b = feature[..., self.band_a], feature[..., self.band_b] 57 | 58 | with np.errstate(divide="ignore", invalid="ignore"): 59 | ndi = (band_a - band_b + self.acorvi_constant) / (band_a + band_b + self.acorvi_constant) 60 | 61 | ndi[~np.isfinite(ndi)] = self.undefined_value 62 | 63 | return ndi[..., np.newaxis] 64 | -------------------------------------------------------------------------------- /eolearn/geometry/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Subpackage containing EOTasks for geometrical transformations 3 | """ 4 | 5 | from .morphology import ErosionTask, MorphologicalFilterTask, MorphologicalOperations, MorphologicalStructFactory 6 | from .transformations import RasterToVectorTask, VectorToRasterTask 7 | -------------------------------------------------------------------------------- /eolearn/geometry/morphology.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module containing tasks for morphological operations 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import itertools as it 13 | from enum import Enum 14 | from functools import partial 15 | 16 | import cv2 17 | import numpy as np 18 | 19 | from eolearn.core import EOPatch, EOTask, MapFeatureTask 20 | from eolearn.core.types import FeaturesSpecification, SingleFeatureSpec 21 | from eolearn.core.utils.parsing import parse_renamed_feature 22 | 23 | 24 | class ErosionTask(EOTask): 25 | """ 26 | The task performs an erosion to the provided mask 27 | 28 | :param mask_feature: The mask which is to be eroded 29 | :param disk_radius: Radius of the erosion disk (in pixels). Default is set to `1` 30 | :param erode_labels: List of labels to erode. If `None`, all unique labels are eroded. Default is `None` 31 | :param no_data_label: Value used to replace eroded pixels. Default is set to `0` 32 | """ 33 | 34 | def __init__( 35 | self, 36 | mask_feature: SingleFeatureSpec, 37 | disk_radius: int = 1, 38 | erode_labels: list[int] | None = None, 39 | no_data_label: int = 0, 40 | ): 41 | if not isinstance(disk_radius, int) or disk_radius is None or disk_radius < 1: 42 | raise ValueError("Disk radius should be an integer larger than 0!") 43 | 44 | parsed_mask_feature = parse_renamed_feature(mask_feature, allowed_feature_types=lambda fty: fty.is_array()) 45 | 46 | self.mask_type, self.mask_name, self.new_mask_name = parsed_mask_feature 47 | self.disk = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (disk_radius, disk_radius)) 48 | self.erode_labels = erode_labels 49 | self.no_data_label = no_data_label 50 | 51 | def execute(self, eopatch: EOPatch) -> EOPatch: 52 | feature_array = eopatch[(self.mask_type, self.mask_name)].squeeze(axis=-1).copy() 53 | 54 | all_labels = np.unique(feature_array) 55 | erode_labels = self.erode_labels if self.erode_labels else all_labels 56 | 57 | erode_labels = set(erode_labels) - {self.no_data_label} 58 | other_labels = set(all_labels) - set(erode_labels) - {self.no_data_label} 59 | 60 | eroded_masks = [cv2.erode((feature_array == label).astype(np.uint8), self.disk) for label in erode_labels] 61 | other_masks = [feature_array == label for label in other_labels] 62 | 63 | merged_mask = np.logical_or.reduce(eroded_masks + other_masks, axis=0) 64 | 65 | feature_array[~merged_mask] = self.no_data_label 66 | eopatch[(self.mask_type, self.new_mask_name)] = np.expand_dims(feature_array, axis=-1) 67 | 68 | return eopatch 69 | 70 | 71 | class MorphologicalOperations(Enum): 72 | """Enum class of morphological operations""" 73 | 74 | OPENING = "opening" 75 | CLOSING = "closing" 76 | DILATION = "dilation" 77 | EROSION = "erosion" 78 | 79 | @classmethod 80 | def get_operation(cls, morph_type: MorphologicalOperations) -> int: 81 | """Maps morphological operation type to function 82 | 83 | :param morph_type: Morphological operation type 84 | """ 85 | return { 86 | cls.OPENING: cv2.MORPH_OPEN, 87 | cls.CLOSING: cv2.MORPH_CLOSE, 88 | cls.DILATION: cv2.MORPH_DILATE, 89 | cls.EROSION: cv2.MORPH_ERODE, 90 | }[morph_type] 91 | 92 | 93 | class MorphologicalStructFactory: 94 | """ 95 | Factory methods for generating morphological structuring elements 96 | """ 97 | 98 | @staticmethod 99 | def get_disk(radius: int) -> np.ndarray: 100 | """ 101 | :param radius: Radius of disk 102 | :return: The structuring element where elements of the neighborhood are 1 and 0 otherwise. 103 | """ 104 | return cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (radius, radius)) 105 | 106 | @staticmethod 107 | def get_rectangle(width: int, height: int) -> np.ndarray: 108 | """ 109 | :param width: Width of rectangle 110 | :param height: Height of rectangle 111 | :return: A structuring element consisting only of ones, i.e. every pixel belongs to the neighborhood. 112 | """ 113 | return cv2.getStructuringElement(cv2.MORPH_RECT, (height, width)) 114 | 115 | @staticmethod 116 | def get_square(width: int) -> np.ndarray: 117 | """ 118 | :param width: Size of square 119 | :return: A structuring element consisting only of ones, i.e. every pixel belongs to the neighborhood. 120 | """ 121 | return cv2.getStructuringElement(cv2.MORPH_RECT, (width, width)) 122 | 123 | 124 | class MorphologicalFilterTask(MapFeatureTask): 125 | """Performs morphological operations on masks.""" 126 | 127 | def __init__( 128 | self, 129 | input_features: FeaturesSpecification, 130 | output_features: FeaturesSpecification | None = None, 131 | *, 132 | morph_operation: MorphologicalOperations, 133 | struct_elem: np.ndarray | None = None, 134 | ): 135 | """ 136 | :param input_features: Input features to be processed. 137 | :param output_features: Outputs of input features. If not provided the `input_features` are overwritten. 138 | :param morph_operation: A morphological operation. 139 | :param struct_elem: A structuring element to be used with the morphological operation. Usually it is generated 140 | with a factory method from MorphologicalStructElements 141 | """ 142 | if output_features is None: 143 | output_features = input_features 144 | super().__init__(input_features, output_features) 145 | 146 | self.morph_operation = MorphologicalOperations.get_operation(morph_operation) 147 | self.struct_elem = struct_elem 148 | 149 | def map_method(self, feature: np.ndarray) -> np.ndarray: 150 | """Applies the morphological operation to a raster feature.""" 151 | feature = feature.copy() 152 | is_bool = feature.dtype == bool 153 | if is_bool: 154 | feature = feature.astype(np.uint8) 155 | 156 | morph_func = partial(cv2.morphologyEx, kernel=self.struct_elem, op=self.morph_operation) 157 | if feature.ndim == 3: 158 | for channel in range(feature.shape[2]): 159 | feature[..., channel] = morph_func(feature[..., channel]) 160 | elif feature.ndim == 4: 161 | for time, channel in it.product(range(feature.shape[0]), range(feature.shape[3])): 162 | feature[time, ..., channel] = morph_func(feature[time, ..., channel]) 163 | else: 164 | raise ValueError(f"Invalid number of dimensions: {feature.ndim}") 165 | 166 | return feature.astype(bool) if is_bool else feature 167 | -------------------------------------------------------------------------------- /eolearn/io/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A collection of input and output EOTasks 3 | """ 4 | 5 | from .geometry_io import VectorImportTask 6 | from .raster_io import ExportToTiffTask, ImportFromTiffTask 7 | from .sentinelhub_process import ( 8 | SentinelHubDemTask, 9 | SentinelHubEvalscriptTask, 10 | SentinelHubInputTask, 11 | get_available_timestamps, 12 | ) 13 | -------------------------------------------------------------------------------- /eolearn/mask/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Public classes and functions of mask subpackage 3 | """ 4 | 5 | from .masking import JoinMasksTask, MaskFeatureTask 6 | from .snow_mask import SnowMaskTask 7 | -------------------------------------------------------------------------------- /eolearn/mask/extra/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A collection of EOTasks with non-standard dependencies. Use the extra `[EXTRA]` to install dependencies or 3 | install the module-specific dependencies by hand. 4 | """ 5 | -------------------------------------------------------------------------------- /eolearn/mask/extra/cloud_mask.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for cloud masking 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import logging 13 | 14 | import numpy as np 15 | from s2cloudless import S2PixelCloudDetector 16 | 17 | from eolearn.core import EOPatch, EOTask 18 | from eolearn.core.types import Feature 19 | 20 | LOGGER = logging.getLogger(__name__) 21 | 22 | 23 | class CloudMaskTask(EOTask): 24 | """Cloud masking with the s2cloudless model. Outputs a cloud mask and optionally the cloud probabilities.""" 25 | 26 | def __init__( 27 | self, 28 | data_feature: Feature, 29 | valid_data_feature: Feature, 30 | output_mask_feature: Feature, 31 | output_proba_feature: Feature | None = None, 32 | all_bands: bool = True, 33 | threshold: float = 0.4, 34 | average_over: int | None = 4, 35 | dilation_size: int | None = 2, 36 | ): 37 | """ 38 | :param data_feature: A data feature which stores raw Sentinel-2 reflectance bands. 39 | :param valid_data_feature: A mask feature which indicates whether data is valid. 40 | :param output_mask_feature: The output feature containing cloud masks. 41 | :param output_proba_feature: The output feature containing cloud probabilities. By default this is not saved. 42 | :param all_bands: Flag which indicates whether images will consist of all 13 Sentinel-2 L1C bands or only 43 | the required 10. 44 | :param threshold: Cloud probability threshold for the classifier. 45 | :param average_over: Size of the pixel neighbourhood used in the averaging post-processing step. Set to `None` 46 | to skip this post-processing step. 47 | :param dilation_size: Size of the dilation post-processing step. Set to `None` to skip this post-processing 48 | step. 49 | """ 50 | self.data_feature = self.parse_feature(data_feature) 51 | self.data_indices = (0, 1, 3, 4, 7, 8, 9, 10, 11, 12) if all_bands else tuple(range(10)) 52 | self.valid_data_feature = self.parse_feature(valid_data_feature) 53 | 54 | self.output_mask_feature = self.parse_feature(output_mask_feature) 55 | self.output_proba_feature = None 56 | if output_proba_feature is not None: 57 | self.output_proba_feature = self.parse_feature(output_proba_feature) 58 | 59 | self.threshold = threshold 60 | 61 | self.classifier = S2PixelCloudDetector( 62 | threshold=threshold, average_over=average_over, dilation_size=dilation_size, all_bands=all_bands 63 | ) 64 | 65 | def execute(self, eopatch: EOPatch) -> EOPatch: 66 | """Add selected features (cloud probabilities and masks) to an EOPatch instance. 67 | 68 | :param eopatch: Input `EOPatch` instance 69 | :return: `EOPatch` with additional features 70 | """ 71 | data = eopatch[self.data_feature].astype(np.float32) 72 | valid_data = eopatch[self.valid_data_feature].astype(bool) 73 | 74 | patch_bbox = eopatch.bbox 75 | if patch_bbox is None: 76 | raise ValueError("Cannot run cloud masking on an EOPatch without a BBox.") 77 | 78 | cloud_proba = self.classifier.get_cloud_probability_maps(data) 79 | cloud_mask = self.classifier.get_mask_from_prob(cloud_proba, threshold=self.threshold) 80 | 81 | eopatch[self.output_mask_feature] = (cloud_mask[..., np.newaxis] * valid_data).astype(bool) 82 | if self.output_proba_feature is not None: 83 | eopatch[self.output_proba_feature] = (cloud_proba[..., np.newaxis] * valid_data).astype(np.float32) 84 | 85 | return eopatch 86 | -------------------------------------------------------------------------------- /eolearn/mask/masking.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for creating mask features 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | from typing import Callable, Iterable, Literal 13 | 14 | import numpy as np 15 | 16 | from eolearn.core import EOPatch, EOTask, FeatureType, ZipFeatureTask 17 | from eolearn.core.types import Feature, FeaturesSpecification, SingleFeatureSpec 18 | from eolearn.core.utils.parsing import parse_renamed_feature 19 | 20 | 21 | class JoinMasksTask(ZipFeatureTask): 22 | """Joins together masks with the provided logical operation.""" 23 | 24 | def __init__( 25 | self, 26 | input_features: FeaturesSpecification, 27 | output_feature: Feature, 28 | join_operation: Literal["and", "or", "xor"] | Callable = "and", 29 | ): 30 | """ 31 | :param input_features: Mask features to be joined together. 32 | :param output_feature: Feature to which to save the joined mask. 33 | :param join_operation: How to join masks. Supports `'and'`, `'or'`, `'xor'`, or a `Callable` object. 34 | """ 35 | self.join_method: Callable[[np.ndarray, np.ndarray], np.ndarray] 36 | if isinstance(join_operation, str): 37 | methods: dict[str, Callable[[np.ndarray, np.ndarray], np.ndarray]] = { 38 | "and": np.logical_and, 39 | "or": np.logical_or, 40 | "xor": np.logical_xor, 41 | } 42 | if join_operation not in methods: 43 | raise ValueError( 44 | f"Join operation {join_operation} is not a viable choice. For operations other than {list(methods)}" 45 | "the user must provide a `Callable` object." 46 | ) 47 | self.join_method = methods[join_operation] 48 | else: 49 | self.join_method = join_operation 50 | 51 | super().__init__(input_features, output_feature) 52 | 53 | def zip_method(self, *masks: np.ndarray) -> np.ndarray: 54 | """Joins masks using the provided operation""" 55 | final_mask, *masks_to_join = masks 56 | for mask in masks_to_join: 57 | final_mask = self.join_method(final_mask, mask) 58 | return final_mask 59 | 60 | 61 | class MaskFeatureTask(EOTask): 62 | """Masks out values of a feature using defined values of a given mask feature. 63 | 64 | As an example, it can be used to mask the data feature using values from the Sen2cor Scene Classification 65 | Layer (SCL). 66 | 67 | Contributor: Johannes Schmid, GeoVille Information Systems GmbH, 2018 68 | """ 69 | 70 | def __init__( 71 | self, 72 | feature: SingleFeatureSpec, 73 | mask_feature: Feature, 74 | mask_values: Iterable[int], 75 | no_data_value: float = np.nan, 76 | ): 77 | """ 78 | :param feature: A feature to be masked with optional new feature name 79 | :param mask_feature: Masking feature. Values of this mask will be used to mask values of `feature` 80 | :param mask_values: List of values of `mask_feature` to be used for masking `feature` 81 | :param no_data_value: Value that replaces masked values in `feature`. Default is `NaN` 82 | :return: The same `eopatch` instance with a masked array 83 | """ 84 | self.renamed_feature = parse_renamed_feature(feature) 85 | self.mask_feature = self.parse_feature(mask_feature) 86 | self.mask_values = mask_values 87 | self.no_data_value = no_data_value 88 | 89 | if not isinstance(self.mask_values, list): 90 | raise ValueError("Incorrect format or values of argument 'mask_values'") 91 | 92 | def execute(self, eopatch: EOPatch) -> EOPatch: 93 | """Mask values of `feature` according to the `mask_values` in `mask_feature` 94 | 95 | :param eopatch: `eopatch` to be processed 96 | :return: Same `eopatch` instance with masked `feature` 97 | """ 98 | feature_type, feature_name, new_feature_name = self.renamed_feature 99 | mask_feature_type, mask_feature_name = self.mask_feature 100 | 101 | data = np.copy(eopatch[feature_type][feature_name]) 102 | mask = eopatch[mask_feature_type][mask_feature_name] 103 | 104 | for value in self.mask_values: 105 | data = apply_mask(data, mask, value, self.no_data_value, feature_type, mask_feature_type) 106 | 107 | eopatch[feature_type][new_feature_name] = data 108 | return eopatch 109 | 110 | 111 | def apply_mask( 112 | data: np.ndarray, 113 | mask: np.ndarray, 114 | old_value: float, 115 | new_value: float, 116 | data_type: FeatureType, 117 | mask_type: FeatureType, 118 | ) -> np.ndarray: 119 | """A general masking function 120 | 121 | :param data: A data feature 122 | :param mask: A mask feature 123 | :param old_value: An old value in data that will be replaced 124 | :param new_value: A new value that will replace the old value in data 125 | :param data_type: A data feature type 126 | :param mask_type: A mask feature type 127 | """ 128 | if not (data_type.is_spatial() and mask_type.is_spatial()): 129 | raise ValueError("Masking with non-spatial data types is not yet supported") 130 | 131 | if data_type.is_timeless() and mask_type.is_temporal(): 132 | raise ValueError("Cannot mask timeless data feature with time dependent mask feature") 133 | 134 | if data.shape[-3:-1] != mask.shape[-3:-1]: 135 | raise ValueError("Data feature and mask feature have different spatial dimensions") 136 | if mask_type.is_temporal() and data.shape[0] != mask.shape[0]: 137 | raise ValueError("Data feature and mask feature have different temporal dimensions") 138 | 139 | if mask.shape[-1] == data.shape[-1]: 140 | data[..., mask == old_value] = new_value 141 | elif mask.shape[-1] == 1: 142 | data[..., mask[..., 0] == old_value, :] = new_value 143 | else: 144 | raise ValueError( 145 | f"Mask feature has {mask.shape[-1]} number of bands while data feature has {data.shape[-1]} number of bands" 146 | ) 147 | return data 148 | -------------------------------------------------------------------------------- /eolearn/mask/snow_mask.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for snow masking 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import logging 13 | 14 | import cv2 15 | import numpy as np 16 | 17 | from eolearn.core import EOPatch, EOTask, FeatureType 18 | from eolearn.core.types import Feature 19 | 20 | LOGGER = logging.getLogger(__name__) 21 | 22 | 23 | class SnowMaskTask(EOTask): 24 | """The task calculates the snow mask using the given thresholds. 25 | 26 | The default values were optimised based on the Sentinel-2 L1C processing level. Values might not be optimal for L2A 27 | processing level 28 | """ 29 | 30 | NDVI_THRESHOLD = 0.1 31 | 32 | def __init__( 33 | self, 34 | data_feature: Feature, 35 | band_indices: list[int], 36 | ndsi_threshold: float = 0.4, 37 | brightness_threshold: float = 0.3, 38 | dilation_size: int = 0, 39 | undefined_value: int = 0, 40 | mask_name: str = "SNOW_MASK", 41 | ): 42 | """ 43 | :param data_feature: EOPatch feature represented by a tuple in the form of `(FeatureType, 'feature_name')` 44 | containing the bands 2, 3, 7, 11, i.e. (FeatureType.DATA, 'BANDS') 45 | :param band_indices: A list containing the indices at which the required bands can be found in the data_feature. 46 | The required bands are B03, B04, B08 and B11 and the indices should be provided in this order. If the 47 | 'BANDS' array contains all 13 L1C bands, then `band_indices=[2, 3, 7, 11]`. If the 'BANDS' are the 12 bands 48 | with L2A values, then `band_indices=[2, 3, 7, 10]` 49 | :param ndsi_threshold: Minimum value of the NDSI required to classify the pixel as snow 50 | :param brightness_threshold: Minimum value of the red band for a pixel to be classified as bright 51 | """ 52 | self.bands_feature = self.parse_feature(data_feature, allowed_feature_types={FeatureType.DATA}) 53 | self.band_indices = band_indices 54 | self.ndsi_threshold = ndsi_threshold 55 | self.brightness_threshold = brightness_threshold 56 | self.disk_size = 2 * dilation_size + 1 57 | self.undefined_value = undefined_value 58 | self.mask_feature = (FeatureType.MASK, mask_name) 59 | 60 | def _apply_dilation(self, snow_masks: np.ndarray) -> np.ndarray: 61 | """Apply binary dilation for each mask in the series""" 62 | if self.disk_size > 0: 63 | disk = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (self.disk_size, self.disk_size)) 64 | dilated_masks = np.array([cv2.dilate(mask.astype(np.uint8), disk) for mask in snow_masks]) 65 | snow_masks = dilated_masks.reshape(snow_masks.shape) # edge case where data is empty 66 | return snow_masks.astype(bool) 67 | 68 | def execute(self, eopatch: EOPatch) -> EOPatch: 69 | bands = eopatch[self.bands_feature][..., self.band_indices] 70 | with np.errstate(divide="ignore", invalid="ignore"): 71 | # (B03 - B11) / (B03 + B11) 72 | ndsi = (bands[..., 0] - bands[..., 3]) / (bands[..., 0] + bands[..., 3]) 73 | # (B08 - B04) / (B08 + B04) 74 | ndvi = (bands[..., 2] - bands[..., 1]) / (bands[..., 2] + bands[..., 1]) 75 | 76 | ndsi_invalid, ndvi_invalid = ~np.isfinite(ndsi), ~np.isfinite(ndvi) 77 | ndsi[ndsi_invalid] = self.undefined_value 78 | ndvi[ndvi_invalid] = self.undefined_value 79 | 80 | ndi_criterion = (ndsi >= self.ndsi_threshold) | (np.abs(ndvi - self.NDVI_THRESHOLD) < self.NDVI_THRESHOLD / 2) 81 | brightnes_criterion = bands[..., 0] >= self.brightness_threshold 82 | snow_mask = np.where(ndi_criterion & brightnes_criterion, 1, 0) 83 | 84 | snow_mask = self._apply_dilation(snow_mask) 85 | 86 | snow_mask[ndsi_invalid | ndvi_invalid] = self.undefined_value 87 | 88 | eopatch[self.mask_feature] = snow_mask[..., np.newaxis].astype(bool) 89 | return eopatch 90 | -------------------------------------------------------------------------------- /eolearn/ml_tools/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Public classes and functions of ml_tools subpackage 3 | """ 4 | 5 | from .sampling import BlockSamplingTask, FractionSamplingTask, GridSamplingTask, sample_by_values 6 | -------------------------------------------------------------------------------- /eolearn/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/eolearn/py.typed -------------------------------------------------------------------------------- /eolearn/visualization/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | A collection of visualization utilities 3 | """ 4 | 5 | from .eopatch import PlotBackend, PlotConfig 6 | -------------------------------------------------------------------------------- /eolearn/visualization/eoworkflow.py: -------------------------------------------------------------------------------- 1 | """ 2 | Visualization of EOWorkflow 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | from typing import Sequence 13 | 14 | from graphviz import Digraph 15 | 16 | from eolearn.core import EONode 17 | 18 | 19 | class EOWorkflowVisualization: 20 | """Class handling EOWorkflow visualization""" 21 | 22 | def __init__(self, nodes: Sequence[EONode]): 23 | """ 24 | :param nodes: A sequence of topologically ordered workflow nodes 25 | """ 26 | self.nodes = nodes 27 | 28 | def dependency_graph(self, filename: str | None = None) -> Digraph: 29 | """Visualize the computational graph. 30 | 31 | :param filename: Filename of the output image together with file extension. Supported formats: `png`, `jpg`, 32 | `pdf`, ... . Check `graphviz` Python package for more options 33 | :return: The DOT representation of the computational graph, with some more formatting 34 | """ 35 | dot = self.get_dot() 36 | dot.attr(rankdir="LR") # Show graph from left to right 37 | 38 | if filename is not None: 39 | file_name, file_format = filename.rsplit(".", 1) 40 | 41 | dot.render(filename=file_name, format=file_format, cleanup=True) 42 | 43 | return dot 44 | 45 | def get_dot(self) -> Digraph: 46 | """Generates the DOT description of the underlying computational graph. 47 | 48 | :return: The DOT representation of the computational graph 49 | """ 50 | dot = Digraph(format="png") 51 | 52 | node_uid_to_dot_name = self._get_node_uid_to_dot_name_mapping(self.nodes) 53 | 54 | for node in self.nodes: 55 | for input_node in node.inputs: 56 | dot.edge(node_uid_to_dot_name[input_node.uid], node_uid_to_dot_name[node.uid]) 57 | return dot 58 | 59 | @staticmethod 60 | def _get_node_uid_to_dot_name_mapping(nodes: Sequence[EONode]) -> dict[str, str]: 61 | """Creates mapping between EONode classes and names used in DOT graph. To do that, it has to collect nodes with 62 | the same name and assign them different indices.""" 63 | dot_name_to_nodes: dict[str, list[EONode]] = {} 64 | for node in nodes: 65 | dot_name_to_nodes[node.get_name()] = dot_name_to_nodes.get(node.get_name(), []) 66 | dot_name_to_nodes[node.get_name()].append(node) 67 | 68 | node_to_dot_name = {} 69 | for same_name_nodes in dot_name_to_nodes.values(): 70 | if len(same_name_nodes) == 1: 71 | node = same_name_nodes[0] 72 | node_to_dot_name[node.uid] = node.get_name() 73 | else: 74 | for idx, node in enumerate(same_name_nodes): 75 | node_to_dot_name[node.uid] = node.get_name(idx + 1) 76 | 77 | return node_to_dot_name 78 | -------------------------------------------------------------------------------- /eolearn/visualization/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | The module provides some utility functions for plotting 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import itertools 13 | 14 | import matplotlib.pyplot as plt 15 | import numpy as np 16 | from matplotlib.colors import Colormap 17 | 18 | 19 | def plot_confusion_matrix( 20 | confusion_matrix: np.ndarray, 21 | classes: list[str], 22 | normalize: bool = True, 23 | title: str = "Confusion matrix", 24 | cmap: str | Colormap | None = plt.cm.Blues, 25 | xlabel: str = "Predicted label", 26 | ylabel: str = "True label", 27 | ) -> None: 28 | """Make a single confusion matrix plot.""" 29 | if normalize: 30 | normalisation_factor = confusion_matrix.sum(axis=1)[:, np.newaxis] + np.finfo(float).eps 31 | confusion_matrix = confusion_matrix.astype(float) / normalisation_factor 32 | 33 | plt.imshow(confusion_matrix, interpolation="nearest", cmap=cmap, vmin=0, vmax=1) 34 | plt.title(title, fontsize=20) 35 | tick_marks = np.arange(len(classes)) 36 | plt.xticks(tick_marks, classes, rotation=90, fontsize=20) 37 | plt.yticks(tick_marks, classes, fontsize=20) 38 | 39 | fmt = ".2f" if normalize else "d" 40 | threshold = confusion_matrix.max() / 2.0 41 | for i, j in itertools.product(range(confusion_matrix.shape[0]), range(confusion_matrix.shape[1])): 42 | plt.text( 43 | j, 44 | i, 45 | format(confusion_matrix[i, j], fmt), 46 | horizontalalignment="center", 47 | fontsize=12, 48 | color="white" if confusion_matrix[i, j] > threshold else "black", 49 | ) 50 | 51 | plt.tight_layout() 52 | plt.ylabel(ylabel, fontsize=20) 53 | plt.xlabel(xlabel, fontsize=20) 54 | -------------------------------------------------------------------------------- /example_data/REFERENCE_SCENES.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/REFERENCE_SCENES.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/bbox.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "crs": { 3 | "type": "name", 4 | "properties": { 5 | "name": "urn:ogc:def:crs:EPSG::32633" 6 | } 7 | }, 8 | "type": "Polygon", 9 | "coordinates": [ 10 | [ 11 | [ 12 | 465181.0522318204, 13 | 5079244.8912012065 14 | ], 15 | [ 16 | 465181.0522318204, 17 | 5080254.63349641 18 | ], 19 | [ 20 | 466180.53145382757, 21 | 5080254.63349641 22 | ], 23 | [ 24 | 466180.53145382757, 25 | 5079244.8912012065 26 | ], 27 | [ 28 | 465181.0522318204, 29 | 5079244.8912012065 30 | ] 31 | ] 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /example_data/TestEOPatch/data/BANDS-S2-L1C.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/data/BANDS-S2-L1C.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/data/CLP.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/data/CLP.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/data/CLP_MULTI.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/data/CLP_MULTI.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/data/CLP_S2C.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/data/CLP_S2C.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/data/NDVI.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/data/NDVI.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/data_timeless/DEM.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/data_timeless/DEM.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/data_timeless/MAX_NDVI.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/data_timeless/MAX_NDVI.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/label/IS_CLOUDLESS.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/label/IS_CLOUDLESS.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/label/RANDOM_DIGIT.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/label/RANDOM_DIGIT.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/label_timeless/LULC_COUNTS.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/label_timeless/LULC_COUNTS.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/mask/CLM.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/mask/CLM.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/mask/CLM_INTERSSIM.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/mask/CLM_INTERSSIM.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/mask/CLM_MULTI.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/mask/CLM_MULTI.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/mask/CLM_S2C.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/mask/CLM_S2C.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/mask/IS_DATA.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/mask/IS_DATA.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/mask/IS_VALID.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/mask/IS_VALID.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/mask_timeless/LULC.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/mask_timeless/LULC.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/mask_timeless/RANDOM_UINT8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/mask_timeless/RANDOM_UINT8.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/mask_timeless/VALID_COUNT.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/mask_timeless/VALID_COUNT.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/meta_info/maxcc.json: -------------------------------------------------------------------------------- 1 | 0.8 2 | -------------------------------------------------------------------------------- /example_data/TestEOPatch/meta_info/service_type.json: -------------------------------------------------------------------------------- 1 | "wcs" 2 | -------------------------------------------------------------------------------- /example_data/TestEOPatch/meta_info/size_x.json: -------------------------------------------------------------------------------- 1 | "10m" 2 | -------------------------------------------------------------------------------- /example_data/TestEOPatch/meta_info/size_y.json: -------------------------------------------------------------------------------- 1 | "10m" 2 | -------------------------------------------------------------------------------- /example_data/TestEOPatch/scalar/CLOUD_COVERAGE.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/scalar/CLOUD_COVERAGE.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/scalar_timeless/LULC_PERCENTAGE.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/scalar_timeless/LULC_PERCENTAGE.npy -------------------------------------------------------------------------------- /example_data/TestEOPatch/timestamps.json: -------------------------------------------------------------------------------- 1 | [ 2 | "2015-07-11T10:00:08", 3 | "2015-07-31T10:00:09", 4 | "2015-08-20T10:07:28", 5 | "2015-08-30T10:05:47", 6 | "2015-09-09T10:00:17", 7 | "2015-09-19T10:05:43", 8 | "2015-09-29T10:06:33", 9 | "2015-12-08T10:04:09", 10 | "2015-12-08T10:11:25", 11 | "2015-12-18T10:12:15", 12 | "2015-12-28T10:14:55", 13 | "2016-01-07T10:12:43", 14 | "2016-01-17T10:10:30", 15 | "2016-02-06T10:02:03", 16 | "2016-03-17T10:06:59", 17 | "2016-03-27T10:00:12", 18 | "2016-04-26T10:01:28", 19 | "2016-05-06T10:05:27", 20 | "2016-05-16T10:06:47", 21 | "2016-05-26T10:06:11", 22 | "2016-06-05T10:06:50", 23 | "2016-06-15T10:06:08", 24 | "2016-06-25T10:06:17", 25 | "2016-07-25T10:06:02", 26 | "2016-08-04T10:06:13", 27 | "2016-08-14T10:06:04", 28 | "2016-08-24T10:06:07", 29 | "2016-09-13T10:05:04", 30 | "2016-09-23T10:06:25", 31 | "2016-10-23T10:00:47", 32 | "2016-12-12T10:04:09", 33 | "2016-12-22T10:06:06", 34 | "2017-01-01T10:04:07", 35 | "2017-01-11T10:03:51", 36 | "2017-02-20T10:06:35", 37 | "2017-03-02T10:00:20", 38 | "2017-03-12T10:07:06", 39 | "2017-04-01T10:00:22", 40 | "2017-04-11T10:00:25", 41 | "2017-04-21T10:05:41", 42 | "2017-05-01T10:00:29", 43 | "2017-05-21T10:00:29", 44 | "2017-05-31T10:05:36", 45 | "2017-06-10T10:00:27", 46 | "2017-06-20T10:04:53", 47 | "2017-07-05T10:00:26", 48 | "2017-07-10T10:05:40", 49 | "2017-07-15T10:00:26", 50 | "2017-07-20T10:00:27", 51 | "2017-07-25T10:05:36", 52 | "2017-07-30T10:05:35", 53 | "2017-08-04T10:06:08", 54 | "2017-08-09T10:00:28", 55 | "2017-08-24T10:00:22", 56 | "2017-08-29T10:00:26", 57 | "2017-09-08T10:06:55", 58 | "2017-09-18T10:00:23", 59 | "2017-09-23T10:05:02", 60 | "2017-09-28T10:06:17", 61 | "2017-10-08T10:03:22", 62 | "2017-10-13T10:00:12", 63 | "2017-10-18T10:02:00", 64 | "2017-11-12T10:02:29", 65 | "2017-11-17T10:03:38", 66 | "2017-11-27T10:03:39", 67 | "2017-12-07T10:07:25", 68 | "2017-12-17T10:05:40", 69 | "2017-12-22T10:04:15" 70 | ] 71 | -------------------------------------------------------------------------------- /example_data/TestEOPatch/vector/CLM_VECTOR.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/vector/CLM_VECTOR.gpkg -------------------------------------------------------------------------------- /example_data/TestEOPatch/vector_timeless/LULC.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/TestEOPatch/vector_timeless/LULC.gpkg -------------------------------------------------------------------------------- /example_data/import-gpkg-test.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/import-gpkg-test.gpkg -------------------------------------------------------------------------------- /example_data/import-tiff-test1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/import-tiff-test1.tiff -------------------------------------------------------------------------------- /example_data/import-tiff-test2.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/example_data/import-tiff-test2.tiff -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # eo-learn examples 2 | 3 | This folder contains a small example of Earth observation workflows that extract valuable information from satellite imagery. The examples, covering much larger scope, are available in [`eo-learn-examples`](https://github.com/sentinel-hub/eo-learn-examples) repository. 4 | 5 | ##### Requirements 6 | 7 | In order to run the example you'll need a Sentinel Hub account. You can get a trial version [here](https://www.sentinel-hub.com). 8 | 9 | Once you have the account set up, login to [Sentinel Hub Configurator](https://apps.sentinel-hub.com/configurator/). By default you will already have the default confoguration with an **instance ID** (alpha-numeric code of length 36). For these examples it is recommended that you create a new configuration (`"Add new configuration"`) and set the configuration to be based on **Python scripts template**. Such configuration will already contain all layers used in these examples. Otherwise you will have to define the layers for your configuration yourself. 10 | 11 | After you have decided which configuration to use, you have two options You can either put configuration's **instance ID** into `sentinelhub` package's configuration file following the [configuration instructions](http://sentinelhub-py.readthedocs.io/en/latest/configure.html) or you can write it down in the example notebooks. 12 | -------------------------------------------------------------------------------- /examples/core/.gitignore: -------------------------------------------------------------------------------- 1 | outputs* 2 | -------------------------------------------------------------------------------- /examples/core/images/eopatch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/core/images/eopatch.png -------------------------------------------------------------------------------- /examples/land-cover-map/.gitignore: -------------------------------------------------------------------------------- 1 | bbox_splitter.pickle 2 | data 3 | figs 4 | tile-def 5 | eopatches 6 | eopatches_sampled 7 | results 8 | predicted_tiff 9 | -------------------------------------------------------------------------------- /examples/land-cover-map/README.md: -------------------------------------------------------------------------------- 1 | # Land Use and Land Cover (LULC) classification example 2 | 3 | This example showcases how it's possible to perform an automatic LULC classification 4 | on a country scale (Slovenia taken as an example) using the `eo-learn` package. The 5 | classification is performed on a time-series of all Sentinel-2 scenes from 2017. 6 | 7 | The example is separated into multiple steps, where different aspects of the workflow are 8 | explained. 9 | 10 | ##### Requirements 11 | 12 | In order to run the example you'll need a Sentinel Hub account. You can get a trial version [here](https://www.sentinel-hub.com) and a free R&D account [here](https://earth.esa.int/aos/OSEO) 13 | 14 | Once you have the account set up, login to [Sentinel Hub Configurator](https://apps.sentinel-hub.com/configurator/). By default you will already have the default confoguration with an **instance ID** (alpha-numeric code of length 36). For this tutorial it is recommended that you create a new configuration (`"Add new configuration"`) and set the configuration to be based on **Python scripts template**. Such configuration will already contain all layers used in these examples. Otherwise you will have to define the layers for your configuration yourself. 15 | 16 | After you have decided which configuration to use, you have two options You can either put configuration's **instance ID** into `sentinelhub` package's configuration file following the [configuration instructions](http://sentinelhub-py.readthedocs.io/en/latest/configure.html) or you can write it down in the example notebooks. 17 | 18 | **Note: The full end-to-end LULC workflow is not yet shown in the example so make sure to 19 | check regularly for the updates.** 20 | 21 | --- 22 | 23 | ## 1. Split area of interest into tiles 24 | 25 | The area of interest (AOI) at a country level is too large and needs to be tiled into smaller 26 | pieces in order to be able to fit the one-year-long time series into memory. 27 | Each smaller piece is called an `EOPatch` in the `eo-learn` package. In order to create an 28 | `EOPatch` we simply need the bounding box in a given coordinate reference system. We'll 29 | use `BBOXSplitter` from [`sentinelhub`](https://github.com/sentinel-hub/sentinelhub-py) python package. 30 | 31 | **Notebook: 1_split-AOI.ipynb** 32 | 33 | - Inputs: 34 | - a geo-json file defining the AOI. In our case, a buffered version of Slovenia. (`reference-data/slovenia-def/svn_buffered.geojson`) 35 | - Outputs: 36 | - pickled `BBoxSplitter` with 293 tiles (seeds for 293 `EOPatch`es) 37 | - geopandas dataframe of tiles 38 | 39 | ![SLO-tiles](./readme_figs/aoi_to_tiles.png) 40 | 41 | **Figure:** AOI (Slovenia) tiled into smaller rectangular tiles. 42 | 43 | ## 2. Create EOPatches 44 | 45 | Now is time to create an `EOPatch` for each out of 293 tiles of the AOI. The `EOPatch` is created by filling it with Sentinel-2 data using Sentinel Hub services. We will add the following data to each `EOPatch`: 46 | 47 | - L1C RGB (bands B04, B03, and B02) 48 | - cloud probability and cloud mask from SentinelHub's `s2cloudless` cloud detector 49 | - in order to perform the cloud detection the 10 L1C bands needed by the `s2cloudless` cloud detector are going to be downloaded (and removed before saving) 50 | 51 | ### 2.1 Determine the number of valid observations 52 | 53 | We can count how many times in a time series a pixel is valid or not from the or SentinelHub's cloud mask. 54 | 55 | **Notebook: 2_eopatch-L1C.ipynb** 56 | 57 | - Inputs: 58 | - pickled `BBoxSplitter` with 293 tiles (seeds for 293 `EOPatch`es) 59 | - Outputs: 60 | - `EOPatch`(es) with the following content: 61 | - cloud probability map per frame 62 | - cloud mask per frame 63 | - `IS_DATA` mask 64 | - `VALID_DATA` mask 65 | - map of number of valid frames per pixel 66 | - fraction of valid pixels per frame 67 | - geo-referenced tiff files with number of valid observations 68 | - Tasks (this example shows how to): 69 | - create an EOPatch and fill it with data (Sentinel-2 L1C) using SentinelHub services 70 | - run SentinelHub's cloud detector (`s2cloudless`) 71 | - remove features from an EOPatch 72 | - validate pixels using custom (user-specified) predicate 73 | - count number of valid observations per pixel 74 | - export a feature to geo-referenced tiff file 75 | - add custom (user-defined) feature to EOPatch 76 | - remove frames from an EOPatch using custom custom (user-specified) predicate 77 | - save EOPatch to disk 78 | - load EOPatch from disk 79 | 80 | If we take a look into the first `EOPatch` this is what we'll find: 81 | 82 | A. Visualise a couple of frames 83 | 84 | ![eopatch-0-frame-0](./readme_figs/patch_0.png) 85 | 86 | **Figure:** Frame with index 0 for an `EOPatch`. 87 | 88 | ![eopatch-0-frame-31](./readme_figs/patch_31.png) 89 | 90 | **Figure:** Frame with index 31 for the same `EOPatch`. 91 | 92 | B. Number of valid observations per EOPatch and entire AOI 93 | 94 | ![eopatch-valid](./readme_figs/number_of_valid_observations_eopatch_0.png) 95 | 96 | **Figure:** Number of valid observations for an `EOPatch` with index 0. 97 | 98 | ![slovenia-valid](./readme_figs/number_of_valid_observations_slovenia.png) 99 | 100 | **Figure:** Number of valid Sentinel-2 observations for Slovenia in 2017. This image can be created by merging all geo-referenced tiffs into a single one using `gdal_merge.py`. 101 | 102 | ![slovenia-valid-hist](./readme_figs/hist_number_of_valid_observations_slovenia.png) 103 | 104 | **Figure:** Distribution of number of valid Senintel-2 observations for Slovenia in 2017. 105 | 106 | ![slovenia-fraction-valid-before](./readme_figs/fraction_valid_pixels_per_frame_eopatch-0.png) 107 | 108 | **Figure:** Fraction of valid pixels per frame for an EOPatch with index 0. 109 | 110 | ![slovenia-fraction-valid-before](./readme_figs/fraction_valid_pixels_per_frame_cleaned-eopatch-0.png) 111 | 112 | **Figure:** Fraction of valid pixels per frame for an EOPatch with index 0 with frames with this fraction below 60% removed. 113 | 114 | **Notebook: 2_eopatch-L2A.ipynb** 115 | 116 | This notebook does almost the same thing as the notebook `2_eopatch-L1C.ipynb`. The main difference that here the input collection is Sentinel-2 L2A (bottom of atmosphere or atmosphericaly corrected refelectances) produced with Sen2Cor. The cloud and cloud shadow masking is based on Sen2Cor's scene classification. 117 | 118 | ![slovenia-valid-s2c](./readme_figs/number_of_valid_observations_slovenia_s2c.png) 119 | 120 | **Figure:** Number of valid Sentinel-2 observations for Slovenia in 2017 as determined from Sen2Cor's scene classification. 121 | 122 | ![slovenia-valid-hist-comp](./readme_figs/hist_number_of_valid_observations_slovenia_s2c_vs_sh.png) 123 | 124 | **Figure:** Distributions of number of valid Senintel-2 observations for Slovenia in 2017 as determined using Sen2Cor's scene classification in red and Sentinel Hub's cloud detector `s2cloudless` in blue. 125 | -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/aoi_to_tiles.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/aoi_to_tiles.png -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/fraction_valid_pixels_per_frame_cleaned-eopatch-0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/fraction_valid_pixels_per_frame_cleaned-eopatch-0.png -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/fraction_valid_pixels_per_frame_eopatch-0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/fraction_valid_pixels_per_frame_eopatch-0.png -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/hist_number_of_valid_observations_slovenia.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/hist_number_of_valid_observations_slovenia.png -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/hist_number_of_valid_observations_slovenia_s2c_vs_sh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/hist_number_of_valid_observations_slovenia_s2c_vs_sh.png -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/number_of_valid_observations_eopatch_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/number_of_valid_observations_eopatch_0.png -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/number_of_valid_observations_slovenia.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/number_of_valid_observations_slovenia.png -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/number_of_valid_observations_slovenia_s2c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/number_of_valid_observations_slovenia_s2c.png -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/patch_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/patch_0.png -------------------------------------------------------------------------------- /examples/land-cover-map/readme_figs/patch_31.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/examples/land-cover-map/readme_figs/patch_31.png -------------------------------------------------------------------------------- /install_all.py: -------------------------------------------------------------------------------- 1 | """ 2 | A script for installing all subpackages at once 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import subprocess 13 | import sys 14 | import warnings 15 | 16 | if __name__ == "__main__": 17 | subprocess.check_call([sys.executable, "-m", "pip", "install", *sys.argv[1:], "."]) 18 | warnings.warn( 19 | "Installing via `install_all.py` is no longer necessary and has been deprecated. Use `pip install" 20 | " eo-learn` instead." 21 | ) 22 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Required for Ray tests 2 | -------------------------------------------------------------------------------- /tests/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Required for Ray tests 2 | -------------------------------------------------------------------------------- /tests/core/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with global fixtures 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | 6 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import os 13 | 14 | import pytest 15 | 16 | from eolearn.core import EOPatch 17 | 18 | 19 | @pytest.fixture(scope="session", name="test_eopatch_path") 20 | def test_eopatch_path_fixture() -> str: 21 | return os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "example_data", "TestEOPatch") 22 | 23 | 24 | @pytest.fixture(name="test_eopatch") 25 | def test_eopatch_fixture(test_eopatch_path) -> EOPatch: 26 | return EOPatch.load(test_eopatch_path) 27 | -------------------------------------------------------------------------------- /tests/core/stats/test_save_stats.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/tests/core/stats/test_save_stats.pkl -------------------------------------------------------------------------------- /tests/core/test_constants.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import warnings 11 | 12 | import pytest 13 | 14 | from eolearn.core.constants import FeatureType 15 | from eolearn.core.exceptions import EODeprecationWarning 16 | 17 | # ruff: noqa:B018 18 | 19 | with warnings.catch_warnings(): 20 | warnings.simplefilter("ignore") 21 | TEST_CASES = [ 22 | (FeatureType.TIMESTAMP, FeatureType.TIMESTAMPS), 23 | (FeatureType["TIMESTAMP"], FeatureType["TIMESTAMPS"]), 24 | (FeatureType("timestamp"), FeatureType("timestamps")), 25 | ] 26 | 27 | 28 | @pytest.mark.parametrize( 29 | ("old_ftype", "new_ftype"), 30 | TEST_CASES, 31 | ids=["attribute access", "name access", "value access"], 32 | ) 33 | def test_timestamp_featuretype(old_ftype, new_ftype) -> None: 34 | assert old_ftype is new_ftype 35 | 36 | 37 | def test_timestamps_bbox_deprecation() -> None: 38 | with warnings.catch_warnings(): # make warnings errors 39 | warnings.simplefilter("error") 40 | 41 | FeatureType.DATA 42 | FeatureType["MASK"] 43 | FeatureType("label") 44 | FeatureType(FeatureType.META_INFO) 45 | 46 | with pytest.warns(EODeprecationWarning): 47 | FeatureType.TIMESTAMPS 48 | with pytest.warns(EODeprecationWarning): 49 | FeatureType["BBOX"] 50 | with pytest.warns(EODeprecationWarning): 51 | FeatureType("bbox") 52 | -------------------------------------------------------------------------------- /tests/core/test_eonode.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | from eolearn.core import EONode, EOTask, OutputTask, linearly_connect_tasks 11 | 12 | 13 | class InputTask(EOTask): 14 | def execute(self, *, val=None): 15 | return val 16 | 17 | 18 | class DivideTask(EOTask): 19 | def execute(self, x, y, *, z=0): 20 | return x / y + z 21 | 22 | 23 | class Inc(EOTask): 24 | def execute(self, x, *, d=1): 25 | return x + d 26 | 27 | 28 | def test_nodes_different_uids(): 29 | uids = set() 30 | task = Inc() 31 | for _ in range(5000): 32 | node = EONode(task) 33 | uids.add(node.uid) 34 | 35 | assert len(uids) == 5000, "Different nodes should have different uids." 36 | 37 | 38 | def test_hashing(): 39 | """This tests that nodes are hashable. If this test is slow then hashing of large workflows is slow. 40 | Probably due to structural hashing (should be avoided). 41 | """ 42 | task1 = Inc() 43 | task2 = DivideTask() 44 | 45 | _ = {EONode(task1): "Can be hashed!"} 46 | 47 | many_nodes = {} 48 | for _ in range(5000): 49 | many_nodes[EONode(task1)] = "We should all be different!" 50 | assert len(many_nodes) == 5000, "Hash clashes happen." 51 | 52 | branch_1, branch_2 = EONode(task1), EONode(task1) 53 | for _ in range(500): 54 | branch_1 = EONode(task2, inputs=(branch_1, branch_2)) 55 | branch_2 = EONode(task2, inputs=(branch_2, EONode(task1))) 56 | 57 | branch_1.__hash__() 58 | branch_2.__hash__() 59 | 60 | 61 | def test_get_dependencies(): 62 | input_node1 = EONode(InputTask()) 63 | input_node2 = EONode(InputTask(), name="some name") 64 | divide_node1 = EONode(DivideTask(), inputs=(input_node1, input_node2), name="some name") 65 | divide_node2 = EONode(DivideTask(), inputs=(divide_node1, input_node2), name="some name") 66 | output_node = EONode(OutputTask(name="output"), inputs=[divide_node2]) 67 | all_nodes = {input_node1, input_node2, divide_node1, divide_node2, output_node} 68 | 69 | assert len(output_node.get_dependencies()) == len(all_nodes), "Wrong number of nodes returned" 70 | 71 | assert all_nodes == set(output_node.get_dependencies()) 72 | 73 | 74 | def test_linearly_connect_tasks(): 75 | tasks = [InputTask(), Inc(), (Inc(), "special inc"), Inc(), OutputTask(name="out")] 76 | nodes = linearly_connect_tasks(*tasks) 77 | 78 | assert all(isinstance(node, EONode) for node in nodes), "Function does not return EONodes" 79 | 80 | assert len(tasks) == len(nodes), "Function returns incorrect number of nodes" 81 | 82 | pure_tasks = tasks[:] 83 | pure_tasks[2] = pure_tasks[2][0] 84 | assert all(task == node.task for task, node in zip(pure_tasks, nodes)), "Nodes do not contain correct tasks" 85 | 86 | for i, node in enumerate(nodes): 87 | previous_endpoint = () if i == 0 else (nodes[i - 1],) 88 | assert node.inputs == previous_endpoint, "Nodes are not linked linearly" 89 | 90 | assert nodes[2].name == "special inc", "Names are not handled correctly" 91 | -------------------------------------------------------------------------------- /tests/core/test_eotask.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | from eolearn.core import EOTask 11 | 12 | 13 | class PlusOneTask(EOTask): 14 | @staticmethod 15 | def execute(x): 16 | return x + 1 17 | 18 | 19 | class PlusConstSquaredTask(EOTask): 20 | def __init__(self, const): 21 | self.const = const 22 | 23 | def execute(self, x): 24 | return (x + self.const) ** 2 25 | 26 | 27 | class SelfRecursiveTask(EOTask): 28 | def __init__(self, x, *args, **kwargs): 29 | self.recursive = self 30 | self.arg_x = x 31 | self.args = args 32 | self.kwargs = kwargs 33 | 34 | def execute(self, _): 35 | return self.arg_x 36 | 37 | 38 | def test_call_equals_execute(): 39 | task = PlusOneTask() 40 | assert task(1) == task.execute(1), "t(x) should given the same result as t.execute(x)" 41 | task = PlusConstSquaredTask(20) 42 | assert task(14) == task.execute(14), "t(x) should given the same result as t.execute(x)" 43 | -------------------------------------------------------------------------------- /tests/core/test_eoworkflow_tasks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test for module eoworkflow_tasks 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | from eolearn.core import EONode, EOTask, EOWorkflow, FeatureType, LoadTask, OutputTask 13 | from eolearn.core.eoworkflow_tasks import InputTask 14 | 15 | 16 | class DummyTask(EOTask): 17 | def execute(self, *eopatches): 18 | return eopatches[0] 19 | 20 | 21 | def test_input_task(): 22 | """Test basic functionalities of InputTask""" 23 | task = InputTask(value=31) 24 | assert task.execute() == 31 25 | assert task.execute(value=42) == 42 26 | 27 | task = InputTask() 28 | assert task.execute() is None 29 | assert task.execute(value=42) == 42 30 | 31 | 32 | def test_output_task(test_eopatch): 33 | """Tests basic functionalities of OutputTask""" 34 | task = OutputTask(name="my-task", features=[(FeatureType.DATA, "NDVI")]) 35 | 36 | assert task.name == "my-task" 37 | 38 | new_eopatch = task.execute(test_eopatch) 39 | assert id(new_eopatch) != id(test_eopatch) 40 | 41 | assert len(new_eopatch.get_features()) == 1 42 | assert new_eopatch.bbox == test_eopatch.bbox 43 | 44 | 45 | def test_output_task_in_workflow(test_eopatch_path, test_eopatch): 46 | load = EONode(LoadTask(test_eopatch_path)) 47 | output = EONode(OutputTask(name="result-name"), inputs=[load]) 48 | 49 | workflow = EOWorkflow([load, output, EONode(DummyTask(), inputs=[load])]) 50 | 51 | results = workflow.execute() 52 | 53 | assert len(results.outputs) == 1 54 | assert results.outputs["result-name"] == test_eopatch 55 | -------------------------------------------------------------------------------- /tests/core/test_extra/__init__.py: -------------------------------------------------------------------------------- 1 | # Required for Ray tests 2 | -------------------------------------------------------------------------------- /tests/core/test_graph.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import functools 11 | 12 | import pytest 13 | from hypothesis import given 14 | from hypothesis import strategies as st 15 | 16 | from eolearn.core.graph import CyclicDependencyError, DirectedGraph 17 | 18 | 19 | @pytest.fixture(name="test_graph") 20 | def test_graph_fixture(): 21 | return DirectedGraph({1: [2, 3], 2: [4], 3: [4]}) 22 | 23 | 24 | def test_is_edge(): 25 | graph = DirectedGraph({1: [2]}) 26 | assert graph.is_edge(1, 2) 27 | assert not graph.is_edge(2, 1) 28 | 29 | 30 | def test_add_edge(): 31 | graph = DirectedGraph() 32 | assert not graph.is_edge(1, 2) 33 | assert graph.get_outdegree(1) == 0 34 | 35 | graph.add_edge(1, 2) 36 | 37 | assert graph.get_outdegree(1) == 1 38 | assert graph.get_indegree(2) == 1 39 | assert graph.is_edge(1, 2) 40 | 41 | graph.add_edge(1, 2) 42 | 43 | assert graph.get_indegree(1) == 0 44 | assert graph.get_outdegree(1) == 2 45 | assert graph.get_indegree(2) == 2 46 | 47 | 48 | def test_del_edge(): 49 | graph = DirectedGraph({1: [2]}) 50 | assert graph.is_edge(1, 2) 51 | graph.del_edge(1, 2) 52 | assert not graph.is_edge(1, 2) 53 | 54 | graph = DirectedGraph({1: [2, 2]}) 55 | assert graph.is_edge(1, 2) 56 | graph.del_edge(1, 2) 57 | assert graph.is_edge(1, 2) 58 | graph.del_edge(1, 2) 59 | assert not graph.is_edge(1, 2) 60 | 61 | 62 | def test_neigbors(test_graph): 63 | assert test_graph.get_neighbors(1) == [2, 3] 64 | assert test_graph.get_neighbors(2) == [4] 65 | assert test_graph.get_neighbors(3) == [4] 66 | assert test_graph.get_neighbors(4) == [] 67 | 68 | 69 | def test_get_indegree(test_graph): 70 | assert test_graph.get_indegree(1) == 0 71 | assert test_graph.get_indegree(2) == 1 72 | assert test_graph.get_indegree(3) == 1 73 | assert test_graph.get_indegree(4) == 2 74 | 75 | 76 | def test_get_outdegree(test_graph): 77 | assert test_graph.get_outdegree(1) == 2 78 | assert test_graph.get_outdegree(2) == 1 79 | assert test_graph.get_outdegree(3) == 1 80 | assert test_graph.get_outdegree(4) == 0 81 | 82 | 83 | def test_vertices(test_graph): 84 | assert test_graph.get_vertices() == {1, 2, 3, 4} 85 | 86 | graph = DirectedGraph() 87 | graph.add_edge(1, 2) 88 | graph.add_edge(2, 3) 89 | graph.add_edge(3, 4) 90 | assert graph.get_vertices() == {1, 2, 3, 4} 91 | 92 | 93 | def test_add_vertex(): 94 | graph = DirectedGraph({1: [2, 3], 2: [4], 3: [4]}) 95 | graph.add_vertex(5) 96 | assert 5 in graph 97 | assert graph.get_indegree(5) == 0 98 | assert graph.get_outdegree(5) == 0 99 | assert graph.get_neighbors(5) == [] 100 | 101 | 102 | def test_del_vertex(): 103 | graph = DirectedGraph({1: [2, 3], 2: [4], 3: [4]}) 104 | 105 | assert graph.get_outdegree(1) == 2 106 | assert graph.get_indegree(4) == 2 107 | assert len(graph) == 4 108 | assert 2 in graph 109 | 110 | graph.del_vertex(2) 111 | 112 | assert graph.get_outdegree(1) == 1 113 | assert graph.get_indegree(4) == 1 114 | assert len(graph) == 3 115 | assert 2 not in graph 116 | 117 | 118 | @given( 119 | st.lists( 120 | st.tuples(st.integers(min_value=0, max_value=10), st.integers(min_value=0, max_value=10)).filter( 121 | lambda p: p[0] != p[1] 122 | ), 123 | min_size=1, 124 | max_size=110, 125 | ) 126 | ) 127 | def test_resolve_dependencies(edges): 128 | graph = DirectedGraph.from_edges(edges) 129 | 130 | if DirectedGraph._is_cyclic(graph): # noqa: SLF001 131 | with pytest.raises(CyclicDependencyError): 132 | graph.topologically_ordered_vertices() 133 | else: 134 | vertex_position = {vertex: i for i, vertex in enumerate(graph.topologically_ordered_vertices())} 135 | assert functools.reduce(lambda p, q: p and q, [vertex_position[u] < vertex_position[v] for u, v in edges]) 136 | -------------------------------------------------------------------------------- /tests/core/test_utils/test_common.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import dataclasses 11 | import warnings 12 | from functools import partial 13 | from typing import Callable 14 | 15 | import numpy as np 16 | import pytest 17 | from numpy.testing import assert_array_equal 18 | 19 | from eolearn.core.utils.common import _apply_to_spatial_axes, is_discrete_type 20 | 21 | with warnings.catch_warnings(): 22 | warnings.simplefilter("ignore", DeprecationWarning) 23 | DTYPE_TEST_CASES = [ 24 | (int, True), 25 | (bool, True), 26 | (float, False), 27 | (str, False), 28 | (bytes, False), 29 | (complex, False), 30 | (np.number, False), 31 | (np.byte, True), 32 | (np.bool_, True), 33 | (np.integer, True), 34 | (np.dtype("uint16"), True), 35 | (np.int8, True), 36 | (np.longlong, True), 37 | (np.int64, True), 38 | (np.double, False), 39 | (np.float64, False), 40 | (np.datetime64, False), 41 | (np.object_, False), 42 | (np.generic, False), 43 | ] 44 | 45 | 46 | @pytest.mark.parametrize(("number_type", "is_discrete"), DTYPE_TEST_CASES) 47 | def test_is_discrete_type(number_type, is_discrete): 48 | """Checks the given type and its numpy dtype against the expected answer.""" 49 | assert is_discrete_type(number_type) is is_discrete 50 | 51 | with warnings.catch_warnings(): 52 | warnings.simplefilter("ignore", DeprecationWarning) 53 | numpy_dtype = np.dtype(number_type) 54 | 55 | assert is_discrete_type(numpy_dtype) is is_discrete 56 | 57 | 58 | @dataclasses.dataclass 59 | class ApplyToAxesTestCase: 60 | function: Callable[[np.ndarray], np.ndarray] 61 | data: np.ndarray 62 | spatial_axes: tuple[int, int] 63 | expected: np.ndarray | None = None 64 | 65 | 66 | APPLY_TO_TEST_CASES = [ 67 | ApplyToAxesTestCase( 68 | function=partial(np.resize, new_shape=(3, 2)), 69 | data=np.ones((2, 3)), 70 | spatial_axes=(0, 1), 71 | expected=np.ones((3, 2)), 72 | ), 73 | ApplyToAxesTestCase( 74 | function=partial(np.resize, new_shape=(4, 3)), 75 | data=np.ones((2, 3, 2, 3, 1)), 76 | spatial_axes=(1, 2), 77 | expected=np.ones((2, 4, 3, 3, 1)), 78 | ), 79 | ApplyToAxesTestCase( 80 | function=partial(np.resize, new_shape=(5, 6)), 81 | data=np.ones((2, 3, 4)), 82 | spatial_axes=(0, 2), 83 | expected=np.ones((5, 3, 6)), 84 | ), 85 | ApplyToAxesTestCase( 86 | function=partial(np.resize, new_shape=(2, 4)), 87 | data=np.ones((2, 3, 4, 1)), 88 | spatial_axes=(2, 3), 89 | expected=np.ones((2, 3, 2, 4)), 90 | ), 91 | ApplyToAxesTestCase( 92 | function=partial(np.resize, new_shape=(2, 2)), 93 | data=np.arange(2 * 3 * 4).reshape((2, 3, 4, 1)), 94 | spatial_axes=(1, 2), 95 | expected=np.array([[[[0], [1]], [[2], [3]]], [[[12], [13]], [[14], [15]]]]), 96 | ), 97 | ApplyToAxesTestCase( 98 | function=partial(np.flip, axis=0), 99 | data=np.arange(2 * 3 * 4).reshape((2, 3, 4)), 100 | spatial_axes=(1, 2), 101 | expected=np.array( 102 | [[[8, 9, 10, 11], [4, 5, 6, 7], [0, 1, 2, 3]], [[20, 21, 22, 23], [16, 17, 18, 19], [12, 13, 14, 15]]] 103 | ), 104 | ), 105 | ApplyToAxesTestCase( 106 | function=lambda x: x + 1, 107 | data=np.arange(24).reshape((2, 3, 4, 1)), 108 | spatial_axes=(1, 2), 109 | expected=np.arange(1, 25).reshape((2, 3, 4, 1)), 110 | ), 111 | ] 112 | 113 | 114 | @pytest.mark.parametrize("test_case", APPLY_TO_TEST_CASES) 115 | def test_apply_to_spatial_axes(test_case: ApplyToAxesTestCase) -> None: 116 | image = _apply_to_spatial_axes(test_case.function, test_case.data, test_case.spatial_axes) 117 | assert_array_equal(image, test_case.expected) 118 | 119 | 120 | APPLY_TO_FAIL_TEST_CASES = [ 121 | ApplyToAxesTestCase( 122 | function=partial(np.resize, new_shape=(2, 2)), 123 | data=np.zeros(shape=0), 124 | spatial_axes=(1, 2), 125 | ), 126 | ApplyToAxesTestCase( 127 | function=partial(np.resize, new_shape=(2, 4)), 128 | data=np.ones((2, 3, 4, 1)), 129 | spatial_axes=(3, 2), 130 | ), 131 | ApplyToAxesTestCase( 132 | function=partial(np.flip, axis=0), 133 | data=np.arange(2 * 3 * 4).reshape((2, 3, 4, 1)), 134 | spatial_axes=(2, 2), 135 | ), 136 | ApplyToAxesTestCase( 137 | function=lambda x: x + 1, 138 | data=np.arange(24).reshape((2, 3, 4, 1)), 139 | spatial_axes=(1,), 140 | ), 141 | ApplyToAxesTestCase( 142 | function=lambda x: x + 1, 143 | data=np.arange(24).reshape((2, 3, 4, 1)), 144 | spatial_axes=(1, 2, 3), 145 | ), 146 | ] 147 | 148 | 149 | @pytest.mark.parametrize("test_case", APPLY_TO_FAIL_TEST_CASES) 150 | def test_apply_to_spatial_axes_fails(test_case: ApplyToAxesTestCase) -> None: 151 | with pytest.raises(ValueError): 152 | _apply_to_spatial_axes(test_case.function, test_case.data, test_case.spatial_axes) 153 | -------------------------------------------------------------------------------- /tests/core/test_utils/test_parallelize.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import itertools as it 11 | from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor 12 | 13 | import pytest 14 | 15 | from eolearn.core.utils.parallelize import ( 16 | _decide_processing_type, 17 | _ProcessingType, 18 | execute_with_mp_lock, 19 | join_futures, 20 | join_futures_iter, 21 | parallelize, 22 | submit_and_monitor_execution, 23 | ) 24 | 25 | 26 | @pytest.mark.parametrize( 27 | ("workers", "multiprocess", "expected_type"), 28 | [ 29 | (1, False, _ProcessingType.SINGLE_PROCESS), 30 | (1, True, _ProcessingType.SINGLE_PROCESS), 31 | (3, False, _ProcessingType.MULTITHREADING), 32 | (2, True, _ProcessingType.MULTIPROCESSING), 33 | ], 34 | ) 35 | def test_decide_processing_type(workers, multiprocess, expected_type): 36 | processing_type = _decide_processing_type(workers=workers, multiprocess=multiprocess) 37 | assert processing_type is expected_type 38 | 39 | 40 | def test_execute_with_mp_lock(): 41 | """For now just a basic dummy test.""" 42 | result = execute_with_mp_lock(sorted, range(10), key=lambda value: -value) 43 | assert result == list(range(9, -1, -1)) 44 | 45 | 46 | @pytest.mark.parametrize( 47 | ("workers", "multiprocess"), 48 | [ 49 | (1, True), 50 | (3, False), 51 | (2, True), 52 | ], 53 | ) 54 | def test_parallelize(workers, multiprocess): 55 | results = parallelize(max, range(10), it.repeat(5), workers=workers, multiprocess=multiprocess, desc="Test") 56 | assert results == [5] * 5 + list(range(5, 10)) 57 | 58 | 59 | @pytest.mark.parametrize("executor_class", [ThreadPoolExecutor, ProcessPoolExecutor]) 60 | def test_submit_and_monitor_execution(executor_class): 61 | with executor_class(max_workers=2) as executor: 62 | results = submit_and_monitor_execution(executor, max, range(10), it.repeat(5), disable=True) 63 | 64 | assert results == [5] * 5 + list(range(5, 10)) 65 | 66 | 67 | def plus_one(value): 68 | return value + 1 69 | 70 | 71 | @pytest.mark.parametrize("executor_class", [ThreadPoolExecutor, ProcessPoolExecutor]) 72 | def test_join_futures(executor_class): 73 | with executor_class() as executor: 74 | futures = [executor.submit(plus_one, value) for value in range(5)] 75 | results = join_futures(futures) 76 | 77 | assert results == list(range(1, 6)) 78 | assert futures == [] 79 | 80 | 81 | @pytest.mark.parametrize("executor_class", [ThreadPoolExecutor, ProcessPoolExecutor]) 82 | def test_join_futures_iter(executor_class): 83 | with executor_class() as executor: 84 | futures = [executor.submit(plus_one, value) for value in range(5)] 85 | results = [] 86 | for value in join_futures_iter(futures): 87 | assert futures == [] 88 | results.append(value) 89 | 90 | assert sorted(results) == [(num, num + 1) for num in range(5)] 91 | -------------------------------------------------------------------------------- /tests/core/test_utils/test_raster.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | from typing import Literal 11 | 12 | import numpy as np 13 | import pytest 14 | from numpy.testing import assert_array_equal 15 | 16 | from eolearn.core.utils.raster import constant_pad, fast_nanpercentile 17 | 18 | # ruff: noqa: NPY002 19 | 20 | 21 | @pytest.mark.parametrize("size", [0, 5]) 22 | @pytest.mark.parametrize("percentile", [0, 1.5, 50, 80.99, 100]) 23 | @pytest.mark.parametrize("nan_ratio", [0, 0.05, 0.1, 0.5, 0.9, 1]) 24 | @pytest.mark.parametrize("dtype", [np.float64, np.float32, np.int16]) 25 | @pytest.mark.parametrize("method", ["linear", "nearest"]) 26 | @pytest.mark.filterwarnings("ignore::RuntimeWarning") 27 | def test_fast_nanpercentile(size: int, percentile: float, nan_ratio: float, dtype: type, method: str): 28 | data_shape = (size, 3, 2, 4) 29 | data = np.random.rand(*data_shape) 30 | data[data < nan_ratio] = np.nan 31 | 32 | if np.issubdtype(dtype, np.integer): 33 | data *= 1000 34 | data = data.astype(dtype) 35 | 36 | method_kwargs = {"method" if np.__version__ >= "1.22.0" else "interpolation": method} 37 | expected_result = np.nanpercentile(data, q=percentile, axis=0, **method_kwargs).astype(data.dtype) 38 | 39 | result = fast_nanpercentile(data, percentile, method=method) 40 | 41 | assert_array_equal(result, expected_result) 42 | 43 | 44 | @pytest.mark.parametrize( 45 | argnames="array, multiple_of, up_down_rule, left_right_rule, pad_value, expected_result", 46 | argvalues=[ 47 | (np.arange(2).reshape((1, 2)), (3, 3), "even", "right", 5, np.array([[5, 5, 5], [0, 1, 5], [5, 5, 5]])), 48 | (np.arange(2).reshape((1, 2)), (3, 3), "up", "even", 5, np.array([[5, 5, 5], [5, 5, 5], [0, 1, 5]])), 49 | (np.arange(4).reshape((2, 2)), (3, 3), "down", "left", 7, np.array([[7, 0, 1], [7, 2, 3], [7, 7, 7]])), 50 | (np.arange(20).reshape((4, 5)), (3, 3), "down", "left", 3, None), 51 | (np.arange(60).reshape((6, 10)), (11, 11), "even", "even", 3, None), 52 | (np.ones((167, 210)), (256, 256), "even", "even", 3, None), 53 | (np.arange(6).reshape((2, 3)), (2, 2), "down", "even", 9, np.array([[0, 1, 2, 9], [3, 4, 5, 9]])), 54 | ( 55 | np.arange(6).reshape((3, 2)), 56 | (4, 4), 57 | "down", 58 | "even", 59 | 9, 60 | np.array([[9, 0, 1, 9], [9, 2, 3, 9], [9, 4, 5, 9], [9, 9, 9, 9]]), 61 | ), 62 | ], 63 | ) 64 | def test_constant_pad( 65 | array: np.ndarray, 66 | multiple_of: tuple[int, int], 67 | up_down_rule: Literal["even", "up", "down"], 68 | left_right_rule: Literal["even", "left", "right"], 69 | pad_value: float, 70 | expected_result: np.ndarray | None, 71 | ): 72 | """Checks that the function pads correctly and minimally. In larger cases only the shapes are checked.""" 73 | padded = constant_pad(array, multiple_of, up_down_rule, left_right_rule, pad_value) 74 | if expected_result is not None: 75 | assert_array_equal(padded, expected_result) 76 | 77 | # correct amount of padding is present 78 | assert np.sum(padded == pad_value) - np.sum(array == pad_value) == np.prod(padded.shape) - np.prod(array.shape) 79 | for dim in (0, 1): 80 | assert (padded.shape[dim] - array.shape[dim]) // multiple_of[dim] == 0 # least amount of padding 81 | assert padded.shape[dim] % multiple_of[dim] == 0 # is divisible 82 | -------------------------------------------------------------------------------- /tests/core/test_utils/test_testing.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import datetime as dt 4 | from dataclasses import dataclass 5 | from typing import Any 6 | 7 | import numpy as np 8 | import pytest 9 | 10 | from sentinelhub import CRS, BBox 11 | from sentinelhub.testing_utils import assert_statistics_match 12 | 13 | from eolearn.core.constants import FeatureType 14 | from eolearn.core.types import Feature, FeaturesSpecification 15 | from eolearn.core.utils.parsing import parse_features 16 | from eolearn.core.utils.testing import PatchGeneratorConfig, generate_eopatch 17 | 18 | 19 | def test_generate_eopatch_set_bbox_timestamps() -> None: 20 | bbox = BBox((0, 0, 10, 10), crs=CRS("EPSG:32633")) 21 | timestamps = [dt.datetime(2019, 1, 1)] 22 | patch = generate_eopatch((FeatureType.DATA, "bands"), bbox=bbox, timestamps=timestamps) 23 | 24 | assert patch.bbox == bbox 25 | assert patch.timestamps == timestamps 26 | 27 | assert patch[(FeatureType.DATA, "bands")].shape[0] == len(timestamps) 28 | 29 | 30 | @pytest.mark.parametrize( 31 | "config", 32 | [ 33 | {"num_timestamps": 3, "max_integer_value": 1, "raster_shape": (1, 1), "depth_range": (1, 2)}, 34 | {"num_timestamps": 7, "max_integer_value": 333, "raster_shape": (3, 27), "depth_range": (5, 15)}, 35 | ], 36 | ) 37 | def test_generate_eopatch_config(config: dict[str, Any]) -> None: 38 | mask_feature = (FeatureType.MASK, "mask") 39 | 40 | patch = generate_eopatch(mask_feature, config=PatchGeneratorConfig(**config)) 41 | 42 | time, raster_y, raster_x, depth = patch[mask_feature].shape 43 | assert time == config["num_timestamps"] 44 | assert (raster_y, raster_x) == config["raster_shape"] 45 | assert config["depth_range"][0] <= depth <= config["depth_range"][1] 46 | 47 | assert np.max(patch[mask_feature]) < config["max_integer_value"] 48 | 49 | 50 | @pytest.mark.parametrize("seed", [0, 1, 42, 100]) 51 | @pytest.mark.parametrize( 52 | "features", 53 | [ 54 | {}, 55 | {FeatureType.DATA: ["bands, CLP"]}, 56 | {FeatureType.DATA: ["bands, CLP"], FeatureType.MASK_TIMELESS: "LULC"}, 57 | { 58 | FeatureType.DATA: "data", 59 | FeatureType.MASK: "mask", 60 | FeatureType.SCALAR: "scalar", 61 | FeatureType.LABEL: "label", 62 | FeatureType.DATA_TIMELESS: "data_timeless", 63 | FeatureType.MASK_TIMELESS: "mask_timeless", 64 | FeatureType.SCALAR_TIMELESS: "scalar_timeless", 65 | FeatureType.LABEL_TIMELESS: "label_timeless", 66 | FeatureType.META_INFO: "meta_info", 67 | }, 68 | ], 69 | ) 70 | def test_generate_eopatch_seed(seed: int, features: FeaturesSpecification) -> None: 71 | patch1 = generate_eopatch(features, seed=seed) 72 | patch2 = generate_eopatch(features, seed=seed) 73 | assert patch1 == patch2 74 | 75 | 76 | @dataclass 77 | class GenerateTestCase: 78 | data_features: FeaturesSpecification 79 | seed: int 80 | expected_statistics: dict[Feature, dict[str, Any]] 81 | 82 | 83 | @pytest.mark.parametrize( 84 | "test_case", 85 | [ 86 | GenerateTestCase( 87 | seed=3, 88 | data_features=(FeatureType.MASK, "CLM"), 89 | expected_statistics={ 90 | (FeatureType.MASK, "CLM"): { 91 | "exp_shape": (5, 98, 151, 2), 92 | "exp_min": 0, 93 | "exp_max": 255, 94 | "exp_mean": 127.389, 95 | } 96 | }, 97 | ), 98 | GenerateTestCase( 99 | seed=1, 100 | data_features={FeatureType.DATA: ["data"], FeatureType.MASK_TIMELESS: ["LULC", "IS_VALUE"]}, 101 | expected_statistics={ 102 | (FeatureType.DATA, "data"): { 103 | "exp_shape": (5, 98, 151, 1), 104 | "exp_min": -4.030404, 105 | "exp_max": 4.406353, 106 | "exp_mean": -0.005156786, 107 | }, 108 | (FeatureType.MASK_TIMELESS, "LULC"): { 109 | "exp_shape": (98, 151, 2), 110 | "exp_min": 0, 111 | "exp_max": 255, 112 | "exp_mean": 127.0385, 113 | }, 114 | (FeatureType.MASK_TIMELESS, "IS_VALUE"): { 115 | "exp_shape": (98, 151, 2), 116 | "exp_min": 0, 117 | "exp_max": 255, 118 | "exp_mean": 127.3140, 119 | }, 120 | }, 121 | ), 122 | GenerateTestCase( 123 | seed=100, 124 | data_features=[(FeatureType.SCALAR, "scalar"), (FeatureType.SCALAR_TIMELESS, "scalar_timeless")], 125 | expected_statistics={ 126 | (FeatureType.SCALAR, "scalar"): { 127 | "exp_shape": (5, 2), 128 | "exp_min": -0.9613, 129 | "exp_max": 2.24297, 130 | "exp_mean": 0.7223, 131 | }, 132 | (FeatureType.SCALAR_TIMELESS, "scalar_timeless"): { 133 | "exp_shape": (2,), 134 | "exp_min": -0.611493, 135 | "exp_max": 0.0472111, 136 | "exp_mean": -0.28214, 137 | }, 138 | }, 139 | ), 140 | ], 141 | ) 142 | def test_generate_eopatch_data(test_case: GenerateTestCase) -> None: 143 | patch = generate_eopatch(test_case.data_features, seed=test_case.seed) 144 | for feature in parse_features(test_case.data_features): 145 | assert_statistics_match(patch[feature], **test_case.expected_statistics[feature], rel_delta=0.0001) 146 | 147 | 148 | @pytest.mark.parametrize( 149 | "feature", 150 | [ 151 | (FeatureType.VECTOR, "vector"), 152 | (FeatureType.VECTOR_TIMELESS, "vector_timeless"), 153 | {FeatureType.VECTOR_TIMELESS: ["vector_timeless"], FeatureType.META_INFO: ["test_meta"]}, 154 | ], 155 | ) 156 | def test_generate_eopatch_fails(feature: FeaturesSpecification) -> None: 157 | with pytest.raises(ValueError): 158 | generate_eopatch(feature) 159 | 160 | 161 | def test_generate_meta_data() -> None: 162 | patch = generate_eopatch((FeatureType.META_INFO, "test_meta")) 163 | assert isinstance(patch.meta_info["test_meta"], str) 164 | -------------------------------------------------------------------------------- /tests/coregistration/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with global fixtures 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import os 13 | 14 | import pytest 15 | 16 | from eolearn.core import EOPatch 17 | 18 | pytest.register_assert_rewrite("sentinelhub.testing_utils") # makes asserts in helper functions work with pytest 19 | 20 | EXAMPLE_DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "example_data") 21 | EXAMPLE_EOPATCH_PATH = os.path.join(EXAMPLE_DATA_PATH, "TestEOPatch") 22 | 23 | 24 | @pytest.fixture(name="example_eopatch") 25 | def example_eopatch_fixture(): 26 | return EOPatch.load(EXAMPLE_EOPATCH_PATH, lazy_loading=True) 27 | -------------------------------------------------------------------------------- /tests/coregistration/test_coregistration.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import logging 11 | 12 | import cv2 13 | import numpy as np 14 | import pytest 15 | 16 | from eolearn.core import FeatureType 17 | from eolearn.core.exceptions import EORuntimeWarning 18 | from eolearn.coregistration import ECCRegistrationTask 19 | 20 | logging.basicConfig(level=logging.DEBUG) 21 | 22 | 23 | def test_registration(example_eopatch): 24 | apply_to_features = [(FeatureType.DATA, "NDVI"), (FeatureType.DATA, "BANDS-S2-L1C"), (FeatureType.MASK, "CLM")] 25 | 26 | reg = ECCRegistrationTask( 27 | registration_feature=(FeatureType.DATA, "NDVI"), 28 | reference_feature=(FeatureType.DATA_TIMELESS, "MAX_NDVI"), 29 | channel=0, 30 | valid_mask_feature=None, 31 | apply_to_features=apply_to_features, 32 | interpolation_mode=cv2.INTER_LINEAR, 33 | warp_mode=cv2.MOTION_TRANSLATION, 34 | max_iter=100, 35 | gauss_kernel_size=1, 36 | border_mode=cv2.BORDER_REPLICATE, 37 | border_value=0, 38 | num_threads=-1, 39 | max_translation=5.0, 40 | ) 41 | with pytest.warns(EORuntimeWarning): 42 | reopatch = reg(example_eopatch) 43 | 44 | assert example_eopatch.data["BANDS-S2-L1C"].shape == reopatch.data["BANDS-S2-L1C"].shape 45 | assert example_eopatch.data["NDVI"].shape == reopatch.data["NDVI"].shape 46 | assert example_eopatch.mask["CLM"].shape == reopatch.mask["CLM"].shape 47 | assert not np.allclose( 48 | example_eopatch.data["BANDS-S2-L1C"], reopatch.data["BANDS-S2-L1C"] 49 | ), "Registration did not warp .data['bands']" 50 | assert not np.allclose( 51 | example_eopatch.data["NDVI"], reopatch.data["NDVI"] 52 | ), "Registration did not warp .data['ndvi']" 53 | assert not np.allclose(example_eopatch.mask["CLM"], reopatch.mask["CLM"]), "Registration did not warp .mask['cm']" 54 | assert "warp_matrices" in reopatch.meta_info 55 | 56 | for warp_matrix in reopatch.meta_info["warp_matrices"].values(): 57 | assert np.linalg.norm(np.array(warp_matrix)[:, 2]) <= 5.0, "Estimated translation is larger than max value" 58 | -------------------------------------------------------------------------------- /tests/features/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with global fixtures 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | 6 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import os 13 | 14 | import numpy as np 15 | import pytest 16 | 17 | from eolearn.core import EOPatch 18 | 19 | pytest.register_assert_rewrite("sentinelhub.testing_utils") # makes asserts in helper functions work with pytest 20 | 21 | 22 | @pytest.fixture(scope="session", name="example_data_path") 23 | def example_data_path_fixture() -> str: 24 | return os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "example_data") 25 | 26 | 27 | @pytest.fixture(name="example_eopatch") 28 | def example_eopatch_fixture(): 29 | path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "example_data", "TestEOPatch") 30 | return EOPatch.load(path, lazy_loading=True) 31 | 32 | 33 | @pytest.fixture(name="small_ndvi_eopatch") 34 | def small_ndvi_eopatch_fixture(example_eopatch: EOPatch): 35 | ndvi = example_eopatch.data["NDVI"][:, :20, :20] 36 | ndvi[np.isnan(ndvi)] = 0 37 | example_eopatch.data["NDVI"] = ndvi 38 | return example_eopatch.temporal_subset(range(10)) 39 | -------------------------------------------------------------------------------- /tests/features/extra/test_clustering.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import logging 11 | import sys 12 | 13 | import numpy as np 14 | import pytest 15 | 16 | from eolearn.core import FeatureType 17 | from eolearn.features.extra.clustering import ClusteringTask 18 | 19 | logging.basicConfig(level=logging.DEBUG) 20 | 21 | 22 | def test_clustering(example_eopatch): 23 | test_features = {FeatureType.DATA_TIMELESS: ["DEM", "MAX_NDVI"]} 24 | mask = np.zeros_like(example_eopatch.mask_timeless["LULC"], dtype=np.uint8) 25 | mask[:90, :95] = 1 26 | example_eopatch.mask_timeless["mask"] = mask 27 | 28 | ClusteringTask( 29 | features=test_features, 30 | new_feature_name="clusters_small", 31 | n_clusters=100, 32 | affinity="cosine", 33 | linkage="single", 34 | remove_small=3, 35 | ).execute(example_eopatch) 36 | 37 | ClusteringTask( 38 | features=test_features, 39 | new_feature_name="clusters_mask", 40 | distance_threshold=0.00000001, 41 | affinity="cosine", 42 | linkage="average", 43 | mask_name="mask", 44 | remove_small=10, 45 | ).execute(example_eopatch) 46 | 47 | clusters = example_eopatch.data_timeless["clusters_small"].squeeze(axis=-1) 48 | 49 | assert len(np.unique(clusters)) == 22, "Wrong number of clusters." 50 | assert np.median(clusters) == 2 51 | 52 | assert np.mean(clusters) == pytest.approx(2.19109 if sys.version_info < (3, 9) else 2.201188) 53 | 54 | clusters = example_eopatch.data_timeless["clusters_mask"].squeeze(axis=-1) 55 | 56 | assert len(np.unique(clusters)) == 8, "Wrong number of clusters." 57 | assert np.median(clusters) == 0 58 | assert np.mean(clusters) == pytest.approx(-0.0550495) 59 | assert np.all(clusters[90:, 95:] == -1), "Wrong area" 60 | -------------------------------------------------------------------------------- /tests/features/test_bands_extraction.py: -------------------------------------------------------------------------------- 1 | """ 2 | A collection of bands extraction EOTasks 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import numpy as np 13 | import pytest 14 | 15 | from sentinelhub import CRS, BBox 16 | 17 | from eolearn.core import EOPatch, FeatureType 18 | from eolearn.features import NormalizedDifferenceIndexTask 19 | 20 | INPUT_FEATURE = (FeatureType.DATA, "TEST") 21 | 22 | 23 | @pytest.mark.parametrize("bad_input", [(1, 2, 3), "test", 0.5]) 24 | def test_bad_input(bad_input): 25 | with pytest.raises(ValueError): 26 | NormalizedDifferenceIndexTask(INPUT_FEATURE, (FeatureType.DATA, "NDI"), bands=bad_input) 27 | 28 | 29 | def test_ndi(): 30 | eopatch = EOPatch(bbox=BBox((0, 0, 1, 1), CRS(3857)), timestamps=["1234-05-06"] * 4) 31 | eopatch[INPUT_FEATURE] = np.zeros((4, 3, 3, 9)) 32 | 33 | band_a, band_b = 4.123, 3.321 34 | eopatch[INPUT_FEATURE][..., 0] = band_a 35 | eopatch[INPUT_FEATURE][..., 1] = band_b 36 | eopatch = NormalizedDifferenceIndexTask(INPUT_FEATURE, (FeatureType.DATA, "NDI"), bands=[0, 1]).execute(eopatch) 37 | assert (eopatch.data["NDI"] == ((band_a - band_b) / (band_a + band_b))).all() 38 | 39 | eopatch[INPUT_FEATURE][..., 5] = np.nan 40 | eopatch[INPUT_FEATURE][..., 7] = np.inf 41 | eopatch = NormalizedDifferenceIndexTask(INPUT_FEATURE, (FeatureType.DATA, "NAN_INF_INPUT"), bands=[5, 7]).execute( 42 | eopatch 43 | ) 44 | assert np.isnan(eopatch.data["NAN_INF_INPUT"]).all() 45 | 46 | eopatch[INPUT_FEATURE][..., 1] = 1 47 | eopatch[INPUT_FEATURE][..., 3] = -1 48 | eopatch = NormalizedDifferenceIndexTask(INPUT_FEATURE, (FeatureType.DATA, "DIV_ZERO_NAN"), bands=[1, 3]).execute( 49 | eopatch 50 | ) 51 | assert np.isnan(eopatch.data["DIV_ZERO_NAN"]).all() 52 | 53 | eopatch[INPUT_FEATURE][..., 1] = 0 54 | eopatch[INPUT_FEATURE][..., 3] = 0 55 | eopatch = NormalizedDifferenceIndexTask( 56 | INPUT_FEATURE, (FeatureType.DATA, "DIV_INVALID"), bands=[1, 3], undefined_value=123 57 | ).execute(eopatch) 58 | assert (eopatch.data["DIV_INVALID"] == 123).all() 59 | -------------------------------------------------------------------------------- /tests/features/test_feature_manipulation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import datetime 11 | 12 | import numpy as np 13 | import pytest 14 | 15 | from sentinelhub import CRS, BBox 16 | 17 | from eolearn.core import EOPatch, FeatureType 18 | from eolearn.core.types import Feature 19 | from eolearn.features import FilterTimeSeriesTask, SimpleFilterTask 20 | from eolearn.features.feature_manipulation import SpatialResizeTask 21 | from eolearn.features.utils import ResizeParam 22 | 23 | DUMMY_BBOX = BBox((0, 0, 1, 1), CRS(3857)) 24 | # ruff: noqa: NPY002 25 | 26 | 27 | @pytest.mark.parametrize("feature", [(FeatureType.DATA, "BANDS-S2-L1C"), (FeatureType.LABEL, "IS_CLOUDLESS")]) 28 | def test_simple_filter_task_filter_all(example_eopatch: EOPatch, feature): 29 | filter_all_task = SimpleFilterTask(feature, filter_func=lambda _: False) 30 | filtered_eopatch = filter_all_task.execute(example_eopatch) 31 | 32 | assert filtered_eopatch is not example_eopatch 33 | assert filtered_eopatch.data["CLP"].shape == (0, 101, 100, 1) 34 | assert filtered_eopatch.scalar["CLOUD_COVERAGE"].shape == (0, 1) 35 | assert len(filtered_eopatch.vector["CLM_VECTOR"]) == 0 36 | assert np.array_equal(filtered_eopatch.mask_timeless["LULC"], example_eopatch.mask_timeless["LULC"]) 37 | assert filtered_eopatch.timestamps == [] 38 | 39 | 40 | @pytest.mark.parametrize("feature", [(FeatureType.MASK, "CLM"), (FeatureType.SCALAR, "CLOUD_COVERAGE")]) 41 | def test_simple_filter_task_filter_nothing(example_eopatch: EOPatch, feature): 42 | filter_all_task = SimpleFilterTask(feature, filter_func=lambda _: True) 43 | filtered_eopatch = filter_all_task.execute(example_eopatch) 44 | 45 | assert filtered_eopatch is not example_eopatch 46 | assert filtered_eopatch == example_eopatch 47 | 48 | 49 | @pytest.mark.parametrize( 50 | "invalid_feature", 51 | [(FeatureType.VECTOR, "foo"), (FeatureType.VECTOR_TIMELESS, "bar"), (FeatureType.MASK_TIMELESS, "foobar")], 52 | ) 53 | def test_simple_filter_invalid_feature(invalid_feature: Feature): 54 | with pytest.raises(ValueError): 55 | SimpleFilterTask(invalid_feature, filter_func=lambda _: True) 56 | 57 | 58 | def test_content_after_time_filter(): 59 | timestamps = [ 60 | datetime.datetime(2017, 1, 1, 10, 4, 7), 61 | datetime.datetime(2017, 1, 4, 10, 14, 5), 62 | datetime.datetime(2017, 1, 11, 10, 3, 51), 63 | datetime.datetime(2017, 1, 14, 10, 13, 46), 64 | datetime.datetime(2017, 1, 24, 10, 14, 7), 65 | datetime.datetime(2017, 2, 10, 10, 1, 32), 66 | datetime.datetime(2017, 2, 20, 10, 6, 35), 67 | datetime.datetime(2017, 3, 2, 10, 0, 20), 68 | datetime.datetime(2017, 3, 12, 10, 7, 6), 69 | datetime.datetime(2017, 3, 15, 10, 12, 14), 70 | ] 71 | data = np.random.rand(10, 100, 100, 3) 72 | 73 | new_start, new_end = 4, -3 74 | 75 | eop = EOPatch(bbox=DUMMY_BBOX, timestamps=timestamps, data={"data": data}) 76 | 77 | filter_task = FilterTimeSeriesTask(start_date=timestamps[new_start], end_date=timestamps[new_end]) 78 | filtered_eop = filter_task.execute(eop) 79 | 80 | assert filtered_eop is not eop 81 | assert filtered_eop.timestamps == timestamps[new_start : new_end + 1] 82 | assert np.array_equal(filtered_eop.data["data"], data[new_start : new_end + 1, ...]) 83 | 84 | 85 | @pytest.mark.parametrize( 86 | ("resize_type", "height_param", "width_param", "features_call", "features_check", "outputs"), 87 | [ 88 | (ResizeParam.NEW_SIZE, 50, 70, ("data", "CLP"), ("data", "CLP"), (68, 50, 70, 1)), 89 | (ResizeParam.NEW_SIZE, 50, 70, ("data", "CLP"), ("mask", "CLM"), (68, 101, 100, 1)), 90 | (ResizeParam.NEW_SIZE, 50, 70, ..., ("data", "CLP"), (68, 50, 70, 1)), 91 | (ResizeParam.NEW_SIZE, 50, 70, ..., ("mask", "CLM"), (68, 50, 70, 1)), 92 | (ResizeParam.NEW_SIZE, 50, 70, ("data", "CLP", "CLP_small"), ("data", "CLP_small"), (68, 50, 70, 1)), 93 | (ResizeParam.NEW_SIZE, 50, 70, ("data", "CLP", "CLP_small"), ("data", "CLP"), (68, 101, 100, 1)), 94 | (ResizeParam.SCALE_FACTORS, 2, 2, ("data", "CLP"), ("data", "CLP"), (68, 202, 200, 1)), 95 | (ResizeParam.SCALE_FACTORS, 0.5, 2, ("data", "CLP"), ("data", "CLP"), (68, 50, 200, 1)), 96 | (ResizeParam.SCALE_FACTORS, 0.1, 0.1, ("data", "CLP"), ("data", "CLP"), (68, 10, 10, 1)), 97 | (ResizeParam.RESOLUTION, 5, 5, ("data", "CLP"), ("data", "CLP"), (68, 202, 200, 1)), 98 | (ResizeParam.RESOLUTION, 20, 20, ("data", "CLP"), ("data", "CLP"), (68, 50, 50, 1)), 99 | (ResizeParam.RESOLUTION, 5, 20, ("data", "CLP"), ("data", "CLP"), (68, 202, 50, 1)), 100 | ], 101 | ) 102 | @pytest.mark.filterwarnings("ignore::RuntimeWarning") 103 | def test_spatial_resize_task( 104 | example_eopatch, resize_type, height_param, width_param, features_call, features_check, outputs 105 | ): 106 | # Warnings occur due to lossy casting in the downsampling procedure 107 | 108 | resize = SpatialResizeTask( 109 | resize_type=resize_type, height_param=height_param, width_param=width_param, features=features_call 110 | ) 111 | assert resize(example_eopatch)[features_check].shape == outputs 112 | 113 | 114 | def test_spatial_resize_task_exception(): 115 | with pytest.raises(ValueError): 116 | SpatialResizeTask(features=("mask", "CLM"), resize_type="blabla", height_param=20, width_param=20) 117 | -------------------------------------------------------------------------------- /tests/features/test_features_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import numpy as np 4 | import pytest 5 | 6 | from eolearn.features.utils import ResizeLib, ResizeMethod, spatially_resize_image 7 | 8 | 9 | @pytest.mark.parametrize("method", ResizeMethod) 10 | @pytest.mark.parametrize("library", ResizeLib) 11 | @pytest.mark.parametrize("dtype", [np.float32, np.int32, np.uint8, bool]) 12 | @pytest.mark.parametrize("new_size", [(50, 50), (35, 39), (271, 271)]) 13 | def test_spatially_resize_image_new_size( 14 | method: ResizeMethod, library: ResizeLib, dtype: np.dtype | type, new_size: tuple[int, int] 15 | ): 16 | """Test that all methods and backends are able to downscale and upscale images of various dtypes.""" 17 | if library is ResizeLib.CV2: # noqa: SIM102 18 | if np.issubdtype(dtype, np.integer) and method is ResizeMethod.CUBIC or dtype is bool: 19 | return 20 | 21 | old_shape = (111, 111) 22 | data_2d = np.arange(np.prod(old_shape)).astype(dtype).reshape(old_shape) 23 | result = spatially_resize_image(data_2d, new_size, resize_method=method, resize_library=library) 24 | assert result.shape == new_size 25 | assert result.dtype == dtype 26 | 27 | old_shape = (111, 111, 3) 28 | data_3d = np.arange(np.prod(old_shape)).astype(dtype).reshape(old_shape) 29 | result = spatially_resize_image(data_3d, new_size, resize_method=method, resize_library=library) 30 | assert result.shape == (*new_size, 3) 31 | assert result.dtype == dtype 32 | 33 | old_shape = (5, 111, 111, 3) 34 | data_4d = np.arange(np.prod(old_shape)).astype(dtype).reshape(old_shape) 35 | result = spatially_resize_image(data_4d, new_size, resize_method=method, resize_library=library) 36 | assert result.shape == (5, *new_size, 3) 37 | assert result.dtype == dtype 38 | 39 | old_shape = (2, 1, 111, 111, 3) 40 | data_5d = np.arange(np.prod(old_shape)).astype(dtype).reshape(old_shape) 41 | result = spatially_resize_image( 42 | data_5d, new_size, resize_method=method, spatial_axes=(2, 3), resize_library=library 43 | ) 44 | assert result.shape == (2, 1, *new_size, 3) 45 | assert result.dtype == dtype 46 | 47 | 48 | @pytest.mark.parametrize("method", ResizeMethod) 49 | @pytest.mark.parametrize("library", ResizeLib) 50 | @pytest.mark.parametrize("scale_factors", [(2, 2), (0.25, 0.25)]) 51 | def test_spatially_resize_image_scale_factors( 52 | method: ResizeMethod, library: ResizeLib, scale_factors: tuple[float, float] 53 | ): 54 | height, width = 120, 120 55 | old_shape = (height, width, 3) 56 | data_3d = np.arange(np.prod(old_shape)).astype(np.float32).reshape(old_shape) 57 | 58 | result = spatially_resize_image(data_3d, scale_factors=scale_factors, resize_method=method, resize_library=library) 59 | 60 | assert result.shape == (height * scale_factors[0], width * scale_factors[1], 3) 61 | 62 | 63 | @pytest.mark.parametrize("library", [ResizeLib.PIL]) 64 | @pytest.mark.parametrize( 65 | "dtype", 66 | [ 67 | bool, 68 | np.int8, 69 | np.uint8, 70 | np.int16, 71 | np.uint16, 72 | np.int32, 73 | np.uint32, 74 | np.int64, 75 | np.uint64, 76 | int, 77 | np.float16, 78 | np.float32, 79 | np.float64, 80 | float, 81 | ], 82 | ) 83 | @pytest.mark.filterwarnings("ignore::RuntimeWarning") 84 | def test_spatially_resize_image_dtype(library: ResizeLib, dtype: np.dtype | type): 85 | # Warnings occur due to lossy casting in the downsampling procedure 86 | old_shape = (111, 111) 87 | data_2d = np.arange(np.prod(old_shape)).astype(dtype).reshape(old_shape) 88 | result = spatially_resize_image(data_2d, (50, 50), resize_library=library) 89 | assert result.dtype == dtype 90 | 91 | 92 | @pytest.fixture(name="test_image", scope="module") 93 | def test_image_fixture(): 94 | """Image with a distinct value in each of the quadrants. In the bottom right quadrant there is a special layer.""" 95 | example = np.zeros((3, 100, 100, 4)) 96 | example[:, 50:, :50, :] = 1 97 | example[:, :50, 50:, :] = 2 98 | example[:, 50:, 50:, :] = 3 99 | example[0, 50:, 50:, 0] = 10 100 | example[0, 50:, 50:, 1] = 20 101 | example[0, 50:, 50:, 2] = 30 102 | example[0, 50:, 50:, 3] = 40 103 | return example.astype(np.float32) 104 | 105 | 106 | @pytest.mark.parametrize("method", ResizeMethod) 107 | @pytest.mark.parametrize("library", ResizeLib) 108 | @pytest.mark.parametrize("new_size", [(50, 50), (217, 271)]) 109 | def test_spatially_resize_image_correctness( 110 | method: ResizeMethod, library: ResizeLib, new_size: tuple[int, int], test_image: np.ndarray 111 | ): 112 | """Test that resizing is correct on a very basic example. It tests midpoints of the 4 quadrants.""" 113 | height, width = new_size 114 | x1, x2 = width // 4, 3 * width // 4 115 | y1, y2 = height // 4, 3 * height // 4 116 | 117 | result = spatially_resize_image(test_image, new_size, resize_method=method, resize_library=library) 118 | assert result.shape == (3, *new_size, 4) 119 | assert result[1, x1, y1, :] == pytest.approx([0, 0, 0, 0]) 120 | assert result[2, x2, y1, :] == pytest.approx([1, 1, 1, 1]) 121 | assert result[1, x1, y2, :] == pytest.approx([2, 2, 2, 2]) 122 | assert result[0, x2, y2, :] == pytest.approx([10, 20, 30, 40]), "The first temporal layer of image is incorrect." 123 | assert result[1, x2, y2, :] == pytest.approx([3, 3, 3, 3]) 124 | -------------------------------------------------------------------------------- /tests/geometry/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with global fixtures 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import os 13 | 14 | import pytest 15 | 16 | from eolearn.core import EOPatch 17 | 18 | EXAMPLE_DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "example_data") 19 | TEST_EOPATCH_PATH = os.path.join(EXAMPLE_DATA_PATH, "TestEOPatch") 20 | 21 | 22 | @pytest.fixture(name="test_eopatch") 23 | def test_eopatch_fixture(): 24 | return EOPatch.load(TEST_EOPATCH_PATH, lazy_loading=True) 25 | -------------------------------------------------------------------------------- /tests/geometry/test_morphology.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import numpy as np 11 | import pytest 12 | from numpy.testing import assert_array_equal 13 | 14 | from eolearn.core import EOPatch, FeatureType 15 | from eolearn.core.utils.testing import PatchGeneratorConfig, generate_eopatch 16 | from eolearn.geometry import ErosionTask, MorphologicalFilterTask, MorphologicalOperations, MorphologicalStructFactory 17 | 18 | CLASSES = (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) 19 | MASK_FEATURE = FeatureType.MASK, "mask" 20 | MASK_TIMELESS_FEATURE = FeatureType.MASK_TIMELESS, "timeless_mask" 21 | # ruff: noqa: NPY002 22 | 23 | 24 | @pytest.fixture(name="patch") 25 | def patch_fixture() -> EOPatch: 26 | config = PatchGeneratorConfig(max_integer_value=10, raster_shape=(50, 100), depth_range=(3, 4)) 27 | patch = generate_eopatch([MASK_FEATURE, MASK_TIMELESS_FEATURE], config=config) 28 | patch[MASK_FEATURE] = patch[MASK_FEATURE].astype(np.uint8) 29 | patch[MASK_TIMELESS_FEATURE] = patch[MASK_TIMELESS_FEATURE] < 1 30 | patch[MASK_TIMELESS_FEATURE][10:20, 20:32] = 0 31 | patch[MASK_TIMELESS_FEATURE][30:, 50:] = 1 32 | 33 | return patch 34 | 35 | 36 | @pytest.mark.parametrize("invalid_input", [None, 0, "a"]) 37 | def test_erosion_value_error(invalid_input): 38 | with pytest.raises(ValueError): 39 | ErosionTask((FeatureType.MASK_TIMELESS, "LULC", "TEST"), disk_radius=invalid_input) 40 | 41 | 42 | def test_erosion_full(test_eopatch): 43 | erosion_task = ErosionTask((FeatureType.MASK_TIMELESS, "LULC", "LULC_ERODED"), disk_radius=3) 44 | eopatch = erosion_task.execute(test_eopatch) 45 | 46 | mask_after = eopatch.mask_timeless["LULC_ERODED"] 47 | 48 | assert_array_equal(np.unique(mask_after, return_counts=True)[1], [1942, 6950, 1069, 87, 52]) 49 | 50 | 51 | def test_erosion_partial(test_eopatch): 52 | # skip forest and artificial surface 53 | specific_labels = [0, 1, 3, 4] 54 | erosion_task = ErosionTask( 55 | mask_feature=(FeatureType.MASK_TIMELESS, "LULC", "LULC_ERODED"), disk_radius=3, erode_labels=specific_labels 56 | ) 57 | eopatch = erosion_task.execute(test_eopatch) 58 | 59 | mask_after = eopatch.mask_timeless["LULC_ERODED"] 60 | 61 | assert_array_equal(np.unique(mask_after, return_counts=True)[1], [1145, 7601, 1069, 87, 198]) 62 | 63 | 64 | @pytest.mark.parametrize( 65 | ("morph_operation", "struct_element", "mask_counts", "mask_timeless_counts"), 66 | [ 67 | ( 68 | MorphologicalOperations.DILATION, 69 | None, 70 | [6, 34, 172, 768, 2491, 7405, 19212, 44912], 71 | [4882, 10118], 72 | ), 73 | ( 74 | MorphologicalOperations.EROSION, 75 | MorphologicalStructFactory.get_disk(4), 76 | [54555, 15639, 3859, 770, 153, 19, 5], 77 | [12391, 2609], 78 | ), 79 | ( 80 | MorphologicalOperations.OPENING, 81 | MorphologicalStructFactory.get_disk(3), 82 | [8850, 13652, 16866, 14632, 11121, 6315, 2670, 761, 133], 83 | [11981, 3019], 84 | ), 85 | (MorphologicalOperations.CLOSING, MorphologicalStructFactory.get_disk(11), [770, 74230], [661, 14339]), 86 | ( 87 | MorphologicalOperations.OPENING, 88 | MorphologicalStructFactory.get_rectangle(3, 3), 89 | [15026, 23899, 20363, 9961, 4328, 1128, 280, 15], 90 | [12000, 3000], 91 | ), 92 | ( 93 | MorphologicalOperations.DILATION, 94 | MorphologicalStructFactory.get_rectangle(5, 6), 95 | [2, 19, 198, 3929, 70852], 96 | [803, 14197], 97 | ), 98 | ], 99 | ) 100 | def test_morphological_filter(patch, morph_operation, struct_element, mask_counts, mask_timeless_counts): 101 | task = MorphologicalFilterTask( 102 | [MASK_FEATURE, MASK_TIMELESS_FEATURE], morph_operation=morph_operation, struct_elem=struct_element 103 | ) 104 | task.execute(patch) 105 | 106 | assert patch[MASK_FEATURE].shape == (5, 50, 100, 3) 107 | assert patch[MASK_TIMELESS_FEATURE].shape == (50, 100, 3) 108 | assert patch[MASK_TIMELESS_FEATURE].dtype == bool 109 | assert_array_equal(np.unique(patch[MASK_FEATURE], return_counts=True)[1], mask_counts) 110 | assert_array_equal(np.unique(patch[MASK_TIMELESS_FEATURE], return_counts=True)[1], mask_timeless_counts) 111 | -------------------------------------------------------------------------------- /tests/io/TestInputs/test_meteoblue_raster_input.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/tests/io/TestInputs/test_meteoblue_raster_input.bin -------------------------------------------------------------------------------- /tests/io/TestInputs/test_meteoblue_vector_input.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sentinel-hub/eo-learn/df8bbe80a0a0dbd9326c05b2c2d94ff41b152e3d/tests/io/TestInputs/test_meteoblue_vector_input.bin -------------------------------------------------------------------------------- /tests/io/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with global fixtures 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import os 13 | 14 | import boto3 15 | import pytest 16 | from botocore.exceptions import ClientError, NoCredentialsError 17 | 18 | pytest.register_assert_rewrite("sentinelhub.testing_utils") # makes asserts in helper functions work with pytest 19 | 20 | from sentinelhub import SHConfig # noqa: E402 21 | 22 | from eolearn.core import EOPatch # noqa: E402 23 | 24 | EXAMPLE_DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "example_data") 25 | TEST_EOPATCH_PATH = os.path.join(EXAMPLE_DATA_PATH, "TestEOPatch") 26 | 27 | 28 | @pytest.fixture(name="test_eopatch") 29 | def test_eopatch_fixture(): 30 | return EOPatch.load(TEST_EOPATCH_PATH, lazy_loading=True) 31 | 32 | 33 | @pytest.fixture(name="example_data_path") 34 | def example_data_path_fixture(): 35 | return EXAMPLE_DATA_PATH 36 | 37 | 38 | @pytest.fixture(name="gpkg_file") 39 | def local_gpkg_example_file_fixture(): 40 | """A pytest fixture to retrieve a gpkg example file""" 41 | return os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../example_data/import-gpkg-test.gpkg") 42 | 43 | 44 | @pytest.fixture(name="s3_gpkg_file") 45 | def s3_gpkg_example_file_fixture(): 46 | """A pytest fixture to retrieve a gpkg example file""" 47 | config = SHConfig() 48 | aws_config = { 49 | "region_name": "eu-central-1", 50 | } 51 | if config.aws_access_key_id and config.aws_secret_access_key: 52 | aws_config["aws_access_key_id"] = config.aws_access_key_id 53 | aws_config["aws_secret_access_key"] = config.aws_secret_access_key 54 | 55 | client = boto3.client("s3", **aws_config) 56 | 57 | try: 58 | client.head_bucket(Bucket="eolearn-io") 59 | except (ClientError, NoCredentialsError): 60 | return pytest.skip(reason="No access to the bucket.") 61 | 62 | return "s3://eolearn-io/import-gpkg-test.gpkg" 63 | 64 | 65 | @pytest.fixture(name="geodb_client") 66 | def geodb_client_fixture(): 67 | """A geoDB client object""" 68 | geodb = pytest.importorskip("xcube_geodb.core.geodb") 69 | 70 | client_id = os.getenv("GEODB_AUTH_CLIENT_ID") 71 | client_secret = os.getenv("GEODB_AUTH_CLIENT_SECRET") 72 | 73 | if not (client_id or client_secret): 74 | raise ValueError("Could not initiate geoDB client, GEODB_AUTH_CLIENT_ID and GEODB_AUTH_CLIENT_SECRET missing!") 75 | 76 | return geodb.GeoDBClient(client_id=client_id, client_secret=client_secret) 77 | -------------------------------------------------------------------------------- /tests/io/test_geometry_io.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import pytest 11 | 12 | from sentinelhub import CRS, BBox 13 | 14 | from eolearn.core import FeatureType 15 | from eolearn.io import VectorImportTask 16 | 17 | 18 | @pytest.mark.parametrize( 19 | argnames="reproject, clip, n_features, bbox, crs", 20 | ids=["simple", "bbox", "bbox_full", "bbox_smaller"], 21 | argvalues=[ 22 | (False, False, 193, None, None), 23 | (False, False, 193, BBox([857000, 6521500, 861000, 6525500], CRS("epsg:2154")), None), 24 | (True, True, 193, BBox([657089, 5071037, 661093, 5075039], CRS.UTM_31N), CRS.UTM_31N), 25 | (True, True, 125, BBox([657690, 5071637, 660493, 5074440], CRS.UTM_31N), CRS.UTM_31N), 26 | ], 27 | ) 28 | class TestVectorImportTask: 29 | """Class for testing vector imports from local file, s3 bucket object and layer from Geopedia""" 30 | 31 | def test_import_local_file(self, gpkg_file, reproject, clip, n_features, bbox, crs): 32 | self._test_import(bbox, clip, crs, gpkg_file, n_features, reproject) 33 | 34 | def test_import_s3_file(self, s3_gpkg_file, reproject, clip, n_features, bbox, crs): 35 | self._test_import(bbox, clip, crs, s3_gpkg_file, n_features, reproject) 36 | 37 | @staticmethod 38 | def _test_import(bbox, clip, crs, gpkg_example, n_features, reproject): 39 | feature = FeatureType.VECTOR_TIMELESS, "lpis_iacs" 40 | import_task = VectorImportTask(feature=feature, path=gpkg_example, reproject=reproject, clip=clip) 41 | eop = import_task.execute(bbox=bbox) 42 | assert len(eop[feature]) == n_features, "Wrong number of features!" 43 | to_crs = crs or import_task.dataset_crs 44 | assert eop[feature].crs == to_crs.pyproj_crs() 45 | 46 | 47 | def test_clipping_wrong_crs(gpkg_file): 48 | """Test for trying to clip using different CRS than the data is in""" 49 | feature = FeatureType.VECTOR_TIMELESS, "lpis_iacs" 50 | import_task = VectorImportTask(feature=feature, path=gpkg_file, reproject=False, clip=True) 51 | with pytest.raises(ValueError): 52 | import_task.execute(bbox=BBox([657690, 5071637, 660493, 5074440], CRS.UTM_31N)) 53 | -------------------------------------------------------------------------------- /tests/mask/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with global fixtures 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import os 13 | 14 | import pytest 15 | 16 | from eolearn.core import EOPatch 17 | 18 | pytest.register_assert_rewrite("sentinelhub.testing_utils") 19 | 20 | EXAMPLE_DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "example_data") 21 | TEST_EOPATCH_PATH = os.path.join(EXAMPLE_DATA_PATH, "TestEOPatch") 22 | 23 | 24 | @pytest.fixture(name="test_eopatch") 25 | def test_eopatch_fixture(): 26 | return EOPatch.load(TEST_EOPATCH_PATH, lazy_loading=True) 27 | -------------------------------------------------------------------------------- /tests/mask/extra/test_cloud_mask.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | from numpy.testing import assert_array_equal 11 | 12 | from eolearn.core import FeatureType 13 | from eolearn.mask.extra.cloud_mask import CloudMaskTask 14 | 15 | 16 | def test_cloud_detection(test_eopatch): 17 | add_tcm = CloudMaskTask( 18 | data_feature=(FeatureType.DATA, "BANDS-S2-L1C"), 19 | valid_data_feature=(FeatureType.MASK, "IS_DATA"), 20 | output_mask_feature=(FeatureType.MASK, "CLM_TEST"), 21 | output_proba_feature=(FeatureType.DATA, "CLP_TEST"), 22 | threshold=0.4, 23 | average_over=4, 24 | dilation_size=2, 25 | ) 26 | eop_clm = add_tcm(test_eopatch) 27 | 28 | assert_array_equal(eop_clm.mask["CLM_TEST"], test_eopatch.mask["CLM_S2C"]) 29 | assert_array_equal(eop_clm.data["CLP_TEST"], test_eopatch.data["CLP_S2C"]) 30 | -------------------------------------------------------------------------------- /tests/mask/test_masking.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import numpy as np 11 | import pytest 12 | 13 | from sentinelhub import CRS, BBox 14 | 15 | from eolearn.core import EOPatch, FeatureType 16 | from eolearn.mask import JoinMasksTask, MaskFeatureTask 17 | 18 | BANDS_FEATURE = FeatureType.DATA, "BANDS-S2-L1C" 19 | NDVI_FEATURE = FeatureType.DATA, "NDVI" 20 | CLOUD_MASK_FEATURE = FeatureType.MASK, "CLM" 21 | LULC_FEATURE = FeatureType.MASK_TIMELESS, "LULC" 22 | 23 | 24 | def test_bands_with_clm(test_eopatch): 25 | ftype, old_name, new_name = FeatureType.DATA, "BANDS-S2-L1C", "BANDS-S2-L1C_MASKED" 26 | 27 | mask_task = MaskFeatureTask([ftype, old_name, new_name], CLOUD_MASK_FEATURE, mask_values=[True], no_data_value=-1) 28 | eop = mask_task(test_eopatch) 29 | 30 | masked_count = np.count_nonzero(eop[ftype, new_name] == -1) 31 | clm_count = np.count_nonzero(eop[CLOUD_MASK_FEATURE]) 32 | bands_num = eop[BANDS_FEATURE].shape[-1] 33 | assert masked_count == clm_count * bands_num 34 | 35 | 36 | def test_ndvi_with_clm(test_eopatch): 37 | ftype, old_name, new_name = FeatureType.DATA, "NDVI", "NDVI_MASKED" 38 | 39 | mask_task = MaskFeatureTask([ftype, old_name, new_name], CLOUD_MASK_FEATURE, mask_values=[True]) 40 | eop = mask_task(test_eopatch) 41 | 42 | masked_count = np.count_nonzero(np.isnan(eop[ftype, new_name])) 43 | clm_count = np.count_nonzero(eop[CLOUD_MASK_FEATURE]) 44 | assert masked_count == clm_count 45 | 46 | 47 | def test_clm_with_lulc(test_eopatch): 48 | ftype, old_name, new_name = FeatureType.MASK, "CLM", "CLM_MASKED" 49 | 50 | mask_task = MaskFeatureTask([ftype, old_name, new_name], LULC_FEATURE, mask_values=[2], no_data_value=255) 51 | eop = mask_task(test_eopatch) 52 | 53 | masked_count = np.count_nonzero(eop[ftype, new_name] == 255) 54 | lulc_count = np.count_nonzero(eop[LULC_FEATURE] == 2) 55 | bands_num = eop[CLOUD_MASK_FEATURE].shape[-1] 56 | time_num = eop[CLOUD_MASK_FEATURE].shape[0] 57 | assert masked_count == lulc_count * time_num * bands_num 58 | 59 | 60 | def test_lulc_with_lulc(test_eopatch): 61 | ftype, old_name, new_name = FeatureType.MASK_TIMELESS, "LULC", "LULC_MASKED" 62 | 63 | mask_task = MaskFeatureTask([ftype, old_name, new_name], LULC_FEATURE, mask_values=[1], no_data_value=100) 64 | eop = mask_task(test_eopatch) 65 | 66 | masked_count = np.count_nonzero(eop[ftype, new_name] == 100) 67 | lulc_count = np.count_nonzero(eop[LULC_FEATURE] == 1) 68 | assert masked_count == lulc_count 69 | 70 | 71 | def test_wrong_arguments(): 72 | with pytest.raises(ValueError): 73 | MaskFeatureTask(BANDS_FEATURE, CLOUD_MASK_FEATURE, mask_values=10) 74 | 75 | 76 | def test_join_masks(): 77 | eopatch = EOPatch(bbox=BBox((0, 0, 1, 1), CRS(3857))) 78 | 79 | mask1 = (FeatureType.MASK_TIMELESS, "Mask1") 80 | mask_data1 = np.zeros((10, 10, 1), dtype=np.uint8) 81 | mask_data1[2:5, 2:5] = 1 82 | eopatch[mask1] = mask_data1 83 | 84 | mask2 = (FeatureType.MASK_TIMELESS, "Mask2") 85 | mask_data2 = np.zeros((10, 10, 1), dtype=np.uint8) 86 | mask_data2[0:3, 7:8] = 1 87 | eopatch[mask2] = mask_data2 88 | 89 | mask3 = (FeatureType.MASK_TIMELESS, "Mask3") 90 | mask_data3 = np.zeros((10, 10, 1), dtype=np.uint8) 91 | mask_data3[1:1] = 1 92 | eopatch[mask3] = mask_data3 93 | 94 | input_features = [mask1, mask2, mask3] 95 | output_feature = (FeatureType.MASK_TIMELESS, "Output") 96 | 97 | task1 = JoinMasksTask(input_features, output_feature) 98 | expected_result = mask_data1 & mask_data2 & mask_data3 99 | task1(eopatch) 100 | assert np.array_equal(eopatch[output_feature], expected_result) 101 | 102 | task2 = JoinMasksTask(input_features, output_feature, "or") 103 | expected_result = mask_data1 | mask_data2 | mask_data3 104 | task2(eopatch) 105 | assert np.array_equal(eopatch[output_feature], expected_result) 106 | 107 | task3 = JoinMasksTask(input_features, output_feature, lambda x, y: x + y) 108 | expected_result = mask_data1 + mask_data2 + mask_data3 109 | task3(eopatch) 110 | assert np.array_equal(eopatch[output_feature], expected_result) 111 | -------------------------------------------------------------------------------- /tests/mask/test_snow_mask.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import numpy as np 11 | import pytest 12 | 13 | from eolearn.core import EOPatch, FeatureType 14 | from eolearn.mask import SnowMaskTask 15 | 16 | 17 | @pytest.mark.parametrize( 18 | ("task", "result"), 19 | [ 20 | (SnowMaskTask((FeatureType.DATA, "BANDS-S2-L1C"), [2, 3, 7, 11], mask_name="TEST_SNOW_MASK"), (50468, 1405)), 21 | ], 22 | ) 23 | def test_snow_coverage(task, result, test_eopatch): 24 | resulting_eopatch = task(test_eopatch) 25 | output = resulting_eopatch[task.mask_feature] 26 | 27 | assert output.ndim == 4 28 | assert output.shape[:-1] == test_eopatch.data["BANDS-S2-L1C"].shape[:-1] 29 | assert output.shape[-1] == 1 30 | 31 | assert output.dtype == bool 32 | 33 | snow_pixels = np.sum(output, axis=(1, 2, 3)) 34 | assert np.sum(snow_pixels) == result[0], "Sum of snowy pixels does not match" 35 | assert snow_pixels[-4] == result[1], "Snowy pixels on specified frame do not match" 36 | 37 | 38 | def test_snow_empty_eopatch(test_eopatch): 39 | _, h, w, c = test_eopatch.data["BANDS-S2-L1C"].shape 40 | empty_eopatch = EOPatch(bbox=test_eopatch.bbox, timestamps=[]) 41 | empty_eopatch.data["BANDS-S2-L1C"] = np.array([], dtype=np.float32).reshape((0, h, w, c)) 42 | 43 | task = SnowMaskTask((FeatureType.DATA, "BANDS-S2-L1C"), [2, 3, 7, 11], mask_name="TEST_SNOW_MASK") 44 | resulting_eopatch = task(empty_eopatch) # checks if the task runs without errors 45 | assert resulting_eopatch.mask["TEST_SNOW_MASK"].shape == (0, h, w, 1) 46 | -------------------------------------------------------------------------------- /tests/ml_tools/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with global fixtures 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import os 13 | 14 | import pytest 15 | 16 | from eolearn.core import EOPatch 17 | 18 | EXAMPLE_DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "example_data") 19 | TEST_EOPATCH_PATH = os.path.join(EXAMPLE_DATA_PATH, "TestEOPatch") 20 | 21 | 22 | @pytest.fixture(name="test_eopatch") 23 | def test_eopatch_fixture() -> EOPatch: 24 | return EOPatch.load(TEST_EOPATCH_PATH, lazy_loading=True) 25 | -------------------------------------------------------------------------------- /tests/visualization/test_eoexecutor.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import logging 11 | import os 12 | import tempfile 13 | 14 | import pytest 15 | 16 | from eolearn.core import EOExecutor, EONode, EOTask, EOWorkflow 17 | 18 | logging.basicConfig(level=logging.DEBUG) 19 | 20 | 21 | class ExampleTask(EOTask): 22 | def execute(self, *_, **kwargs): 23 | my_logger = logging.getLogger(__file__) 24 | my_logger.info("Info statement of Example task with kwargs: %s", kwargs) 25 | my_logger.warning("Warning statement of Example task with kwargs: %s", kwargs) 26 | my_logger.debug("Debug statement of Example task with kwargs: %s", kwargs) 27 | 28 | if "arg1" in kwargs and kwargs["arg1"] is None: 29 | raise RuntimeError(f"Oh no, i spilled my kwargs all over the floor! {kwargs}!") 30 | 31 | 32 | NODE = EONode(ExampleTask()) 33 | WORKFLOW = EOWorkflow([NODE, EONode(task=ExampleTask(), inputs=[NODE, NODE])]) 34 | EXECUTION_KWARGS = [ 35 | {NODE: {"arg1": 1}}, 36 | {}, 37 | {NODE: {"arg1": 3, "arg3": 10}}, 38 | {NODE: {"arg1": None}}, 39 | {NODE: {"arg1": None, "arg3": 10}}, 40 | ] 41 | 42 | 43 | @pytest.mark.parametrize("save_logs", [True, False]) 44 | @pytest.mark.parametrize("include_logs", [True, False]) 45 | def test_report_creation(save_logs, include_logs): 46 | with tempfile.TemporaryDirectory() as tmp_dir_name: 47 | executor = EOExecutor( 48 | WORKFLOW, 49 | EXECUTION_KWARGS, 50 | logs_folder=tmp_dir_name, 51 | save_logs=save_logs, 52 | execution_names=["ex 1", 2, 0.4, None, "beep"], 53 | ) 54 | executor.run(workers=10) 55 | executor.make_report(include_logs=include_logs) 56 | 57 | assert os.path.exists(executor.get_report_path()), "Execution report was not created" 58 | -------------------------------------------------------------------------------- /tests/visualization/test_eopatch.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for `EOPatch` visualizations 3 | 4 | Copyright (c) 2017- Sinergise and contributors 5 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 6 | 7 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 8 | """ 9 | 10 | from __future__ import annotations 11 | 12 | import os 13 | 14 | import numpy as np 15 | import pytest 16 | from matplotlib.pyplot import Axes 17 | 18 | from eolearn.core import EOPatch, FeatureType 19 | from eolearn.visualization import PlotConfig 20 | 21 | 22 | @pytest.fixture(name="eopatch", scope="module") 23 | def eopatch_fixture(): 24 | path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "example_data", "TestEOPatch") 25 | return EOPatch.load(path) 26 | 27 | 28 | @pytest.mark.parametrize( 29 | ("feature", "params"), 30 | [ 31 | ((FeatureType.DATA, "BANDS-S2-L1C"), {"rgb": [3, 2, 1]}), 32 | ((FeatureType.DATA, "BANDS-S2-L1C"), {"times": [7, 14, 67], "channels": slice(4, 8)}), 33 | ((FeatureType.MASK, "CLM"), {}), 34 | ((FeatureType.MASK, "IS_VALID"), {}), 35 | ((FeatureType.SCALAR, "CLOUD_COVERAGE"), {"times": [5, 8, 13, 21]}), 36 | ((FeatureType.SCALAR, "CLOUD_COVERAGE"), {"times": slice(20, 25)}), 37 | ((FeatureType.LABEL, "IS_CLOUDLESS"), {"channels": [0]}), 38 | ((FeatureType.LABEL, "RANDOM_DIGIT"), {}), 39 | ((FeatureType.VECTOR, "CLM_VECTOR"), {}), 40 | ((FeatureType.VECTOR, "CLM_VECTOR"), {"channels": slice(10, 20)}), 41 | ((FeatureType.VECTOR, "CLM_VECTOR"), {"channels": [0, 5, 8]}), 42 | ((FeatureType.DATA_TIMELESS, "DEM"), {}), 43 | ((FeatureType.MASK_TIMELESS, "RANDOM_UINT8"), {}), 44 | ((FeatureType.SCALAR_TIMELESS, "LULC_PERCENTAGE"), {}), 45 | ((FeatureType.LABEL_TIMELESS, "LULC_COUNTS"), {}), 46 | ((FeatureType.LABEL_TIMELESS, "LULC_COUNTS"), {"channels": [0]}), 47 | ((FeatureType.VECTOR_TIMELESS, "LULC"), {}), 48 | ], 49 | ) 50 | @pytest.mark.sh_integration 51 | def test_eopatch_plot(eopatch: EOPatch, feature, params): 52 | """A simple test of EOPatch plotting for different features.""" 53 | # We reduce width and height otherwise running matplotlib.pyplot.subplots in combination with pytest would 54 | # kill the Python kernel. 55 | config = PlotConfig(subplot_width=1, subplot_height=1) 56 | 57 | axes = eopatch.plot(feature, config=config, **params) 58 | axes = axes.flatten() 59 | 60 | assert isinstance(axes, np.ndarray) 61 | for item in axes: 62 | assert isinstance(item, Axes) 63 | -------------------------------------------------------------------------------- /tests/visualization/test_eoworkflow.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) 2017- Sinergise and contributors 3 | For the full list of contributors, see the CREDITS file in the root directory of this source tree. 4 | 5 | This source code is licensed under the MIT license, see the LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import pytest 11 | from graphviz import Digraph 12 | 13 | from eolearn.core import EOTask, EOWorkflow, linearly_connect_tasks 14 | 15 | 16 | class FooTask(EOTask): 17 | def execute(self, *eopatch): 18 | return eopatch 19 | 20 | 21 | class BarTask(EOTask): 22 | def execute(self, eopatch): 23 | return eopatch 24 | 25 | 26 | @pytest.fixture(name="workflow") 27 | def linear_workflow_fixture(): 28 | nodes = linearly_connect_tasks(FooTask(), FooTask(), BarTask()) 29 | return EOWorkflow(nodes) 30 | 31 | 32 | def test_graph_nodes_and_edges(workflow): 33 | dot = workflow.get_dot() 34 | assert isinstance(dot, Digraph) 35 | dot_repr = str(dot).strip("\n") 36 | assert dot_repr == "digraph {\n\tFooTask_1 -> FooTask_2\n\tFooTask_2 -> BarTask\n}" 37 | 38 | digraph = workflow.dependency_graph() 39 | assert isinstance(digraph, Digraph) 40 | digraph_repr = str(digraph).strip("\n") 41 | assert digraph_repr == "digraph {\n\tFooTask_1 -> FooTask_2\n\tFooTask_2 -> BarTask\n\trankdir=LR\n}" 42 | --------------------------------------------------------------------------------