├── .github ├── dependabot.yml └── workflows │ ├── nightly-build.yaml │ ├── publish-book.yaml │ ├── trigger-book-build.yaml │ ├── trigger-delete-preview.yaml │ ├── trigger-link-check.yaml │ ├── trigger-preview.yaml │ └── trigger-replace-links.yaml ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── CITATION.cff ├── LICENSE ├── Makefile ├── README.md ├── _config.yml ├── _gallery_info.yml ├── _static ├── custom.css └── footer-logo-nsf.png ├── _templates └── footer-extra.html ├── _toc.yml ├── environment.yml └── notebooks ├── courses ├── microwave-remote-sensing.ipynb ├── microwave-remote-sensing.yml └── microwave-remote-sensing │ ├── 01_in_class_exercise.ipynb │ ├── 02_in_class_exercise.ipynb │ ├── 03_in_class_exercise.ipynb │ ├── 04_in_class_exercise.ipynb │ ├── 05_in_class_exercise.ipynb │ ├── 06_in_class_exercise.ipynb │ ├── 07_in_class_exercise.ipynb │ ├── 08_in_class_exercise.ipynb │ └── 09_in_class_exercise.ipynb ├── how-to-cite.md ├── images ├── ProjectPythia_Logo_Final-01-Blue.svg ├── cmaps │ └── 06_color_mapping.json ├── icons │ └── favicon.ico ├── logos │ ├── NSF-NCAR_Lockup-UCAR-Dark_102523.svg │ ├── UAlbany-A2-logo-purple-gold.svg │ ├── Unidata_logo_horizontal_1200x300.svg │ ├── pythia_logo-white-notext.svg │ ├── pythia_logo-white-rtext.svg │ └── tuw-geo_eodc_logo_horizontal.png ├── ridgecrest.gif ├── side_looking_image_distortions.png ├── speckle_effect.png └── tuw-geo_eodc_logo_vertical.png ├── references.bib ├── references.ipynb ├── templates ├── classification.ipynb ├── classification.yml └── prereqs-templates.ipynb └── tutorials ├── floodmapping.ipynb ├── floodmapping.yml └── prereqs-tutorials.ipynb /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # - package-ecosystem: pip 4 | # directory: "/" 5 | # schedule: 6 | # interval: daily 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | # Check for updates once a week 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/workflows/nightly-build.yaml: -------------------------------------------------------------------------------- 1 | name: nightly-build 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: "0 0 * * *" # Daily “At 00:00” 7 | 8 | jobs: 9 | build: 10 | if: ${{ github.repository_owner == 'ProjectPythia' }} 11 | uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main 12 | with: 13 | build_command: jupyter-book build . ; jupyter-book build . 14 | environment_name: eo-datascience-cookbook-dev 15 | 16 | link-check: 17 | if: ${{ github.repository_owner == 'ProjectPythia' }} 18 | uses: ProjectPythia/cookbook-actions/.github/workflows/link-checker.yaml@main 19 | -------------------------------------------------------------------------------- /.github/workflows/publish-book.yaml: -------------------------------------------------------------------------------- 1 | name: publish-book 2 | 3 | on: 4 | # Trigger the workflow on push to main branch 5 | push: 6 | branches: 7 | - main 8 | - dev 9 | workflow_dispatch: 10 | 11 | jobs: 12 | build: 13 | uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main 14 | with: 15 | build_command: jupyter-book build . ; jupyter-book build . 16 | environment_name: eo-datascience-cookbook-dev 17 | 18 | deploy: 19 | needs: build 20 | uses: ProjectPythia/cookbook-actions/.github/workflows/deploy-book.yaml@main 21 | -------------------------------------------------------------------------------- /.github/workflows/trigger-book-build.yaml: -------------------------------------------------------------------------------- 1 | name: trigger-book-build 2 | on: 3 | pull_request: 4 | 5 | jobs: 6 | build: 7 | uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main 8 | with: 9 | environment_name: eo-datascience-cookbook-dev 10 | artifact_name: book-zip-${{ github.event.number }} 11 | # Other input options are possible, see ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml 12 | -------------------------------------------------------------------------------- /.github/workflows/trigger-delete-preview.yaml: -------------------------------------------------------------------------------- 1 | name: trigger-delete-preview 2 | 3 | on: 4 | pull_request_target: 5 | types: closed 6 | 7 | jobs: 8 | delete: 9 | uses: ProjectPythia/cookbook-actions/.github/workflows/delete-preview.yaml@main 10 | -------------------------------------------------------------------------------- /.github/workflows/trigger-link-check.yaml: -------------------------------------------------------------------------------- 1 | name: trigger-link-check 2 | on: 3 | pull_request: 4 | 5 | jobs: 6 | link-check: 7 | uses: ProjectPythia/cookbook-actions/.github/workflows/link-checker.yaml@main 8 | -------------------------------------------------------------------------------- /.github/workflows/trigger-preview.yaml: -------------------------------------------------------------------------------- 1 | name: trigger-preview 2 | on: 3 | workflow_run: 4 | workflows: 5 | - trigger-book-build 6 | types: 7 | - requested 8 | - completed 9 | 10 | jobs: 11 | find-pull-request: 12 | uses: ProjectPythia/cookbook-actions/.github/workflows/find-pull-request.yaml@main 13 | deploy-preview: 14 | needs: find-pull-request 15 | if: github.event.workflow_run.conclusion == 'success' 16 | uses: ProjectPythia/cookbook-actions/.github/workflows/deploy-book.yaml@main 17 | with: 18 | artifact_name: book-zip-${{ needs.find-pull-request.outputs.number }} 19 | destination_dir: _preview/${{ needs.find-pull-request.outputs.number }} # deploy to subdirectory labeled with PR number 20 | is_preview: "true" 21 | 22 | preview-comment: 23 | needs: find-pull-request 24 | uses: ProjectPythia/cookbook-actions/.github/workflows/preview-comment.yaml@main 25 | with: 26 | pull_request_number: ${{ needs.find-pull-request.outputs.number }} 27 | sha: ${{ needs.find-pull-request.outputs.sha }} 28 | -------------------------------------------------------------------------------- /.github/workflows/trigger-replace-links.yaml: -------------------------------------------------------------------------------- 1 | name: trigger-replace-links 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | contents: write 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Find and Replace Repository Name 15 | uses: jacobtomlinson/gha-find-replace@v3 16 | with: 17 | find: "ProjectPythia/cookbook-template" 18 | replace: "${{ github.repository_owner }}/${{ github.event.repository.name }}" 19 | regex: false 20 | exclude: ".github/workflows/trigger-replace-links.yaml" 21 | 22 | - name: Find and Replace Repository ID 23 | uses: jacobtomlinson/gha-find-replace@v3 24 | with: 25 | find: "475509405" 26 | replace: "${{ github.repository_id}}" 27 | regex: false 28 | exclude: ".github/workflows/trigger-replace-links.yaml" 29 | 30 | - name: Push changes 31 | uses: stefanzweifel/git-auto-commit-action@v5 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Default JupyterBook build output dir 2 | _build/ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | notebooks/_build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | pip-wheel-metadata/ 28 | share/python-wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .nox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | *.py,cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | db.sqlite3-journal 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 99 | __pypackages__/ 100 | 101 | # Celery stuff 102 | celerybeat-schedule 103 | celerybeat.pid 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # Environments 109 | .env 110 | .venv 111 | env/ 112 | venv/ 113 | ENV/ 114 | env.bak/ 115 | venv.bak/ 116 | 117 | # Spyder project settings 118 | .spyderproject 119 | .spyproject 120 | 121 | # Rope project settings 122 | .ropeproject 123 | 124 | # mkdocs documentation 125 | /site 126 | 127 | # mypy 128 | .mypy_cache/ 129 | .dmypy.json 130 | dmypy.json 131 | 132 | # Pyre type checker 133 | .pyre/ 134 | 135 | # Ephemeral .nfs files 136 | .nfs* 137 | 138 | # Data 139 | data/ 140 | **/data/ -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party = 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.5.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-docstring-first 8 | - id: check-json 9 | - id: check-yaml 10 | - id: double-quote-string-fixer 11 | 12 | - repo: https://github.com/psf/black 13 | rev: 23.12.1 14 | hooks: 15 | - id: black 16 | 17 | - repo: https://github.com/keewis/blackdoc 18 | rev: v0.3.9 19 | hooks: 20 | - id: blackdoc 21 | 22 | - repo: https://github.com/PyCQA/flake8 23 | rev: 7.0.0 24 | hooks: 25 | - id: flake8 26 | 27 | - repo: https://github.com/asottile/seed-isort-config 28 | rev: v2.2.0 29 | hooks: 30 | - id: seed-isort-config 31 | 32 | - repo: https://github.com/PyCQA/isort 33 | rev: 5.13.2 34 | hooks: 35 | - id: isort 36 | 37 | - repo: https://github.com/pre-commit/mirrors-prettier 38 | rev: v3.1.0 39 | hooks: 40 | - id: prettier 41 | additional_dependencies: [prettier@v2.7.1] 42 | 43 | - repo: https://github.com/nbQA-dev/nbQA 44 | rev: 1.7.1 45 | hooks: 46 | - id: nbqa-black 47 | additional_dependencies: [black] 48 | - id: nbqa-pyupgrade 49 | additional_dependencies: [pyupgrade] 50 | exclude: foundations/quickstart.ipynb 51 | - id: nbqa-isort 52 | additional_dependencies: [isort] 53 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this cookbook, please cite it as below." 3 | authors: 4 | # add additional entries for each author -- see https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md 5 | - family-names: Wagner 6 | given-names: Wolfgang 7 | orcid: https://orcid.org/0000-0001-7704-6857 8 | website: https://www.tuwien.at/mg/dekanat/mitarbeiter-innen 9 | affiliation: Technische Universität Wien, Vienna, Austria, EODC Earth Observation Data Centre for Water Resources Monitoring, Austria 10 | - family-names: Schobben 11 | given-names: Martin 12 | orcid: https://orcid.org/0000-0001-8560-0037 13 | website: https://github.com/martinschobben 14 | affiliation: Technische Universität Wien, Vienna, Austria 15 | - family-names: Pikall 16 | given-names: Nikolas 17 | website: https://github.com/npikall 18 | affiliation: Technische Universität Wien, Vienna, Austria 19 | - family-names: Wagner 20 | given-names: Joseph 21 | affiliation: Technische Universität Wien, Vienna, Austria 22 | - family-names: Festa 23 | given-names: Davide 24 | affiliation: Technische Universität Wien, Vienna, Austria 25 | - family-names: Reuß 26 | given-names: Felix David 27 | affiliation: Technische Universität Wien, Vienna, Austria 28 | - family-names: Jovic 29 | given-names: Luka 30 | affiliation: Technische Universität Wien, Vienna, Austria 31 | - name: "Earth Observation Data Science contributors" # use the 'name' field to acknowledge organizations 32 | website: "https://github.com/TUW-GEO/eo-datascience-cookbook/graphs/contributors" 33 | title: "Earth Observation Data Science Cookbook" 34 | abstract: "Earth Observation Data Science Cookbook provides training material \ 35 | centered around Earth Observation data while honoring the Pangeo Philosophy. \ 36 | The examples used in the notebooks represent some of the main research lines \ 37 | of the Remote Sensing Unit at the Department of Geodesy and Geoinformation at \ 38 | the TU Wien (Austria). In addition, the content familiarizes the reader with \ 39 | the data available at the EODC (Earth Observation Data Centre For Water \ 40 | Resources Monitoring) as well as the computational resources to process 41 | large amounts of data." 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .ONESHELL: 2 | SHELL = /bin/bash 3 | .PHONY: help clean environment kernel teardown post-render dev 4 | 5 | YML = $(wildcard notebooks/**/*.yml) 6 | REQ := $(basename $(notdir $(YML))) 7 | 8 | CONDA_ENV != conda info --base 9 | CONDA_ACTIVATE := source $(CONDA_ENV)/etc/profile.d/conda.sh ; \ 10 | conda activate ; conda activate 11 | CONDA_ENV_DIR := $(foreach i,$(REQ),$(CONDA_ENV)/envs/$(i)) 12 | KERNEL_DIR != $(CONDA_ACTIVATE) eo-datascience-cookbook-dev; jupyter --data-dir 13 | KERNEL_DIR := $(foreach i,$(REQ),$(KERNEL_DIR)/kernels/$(i)) 14 | 15 | help: 16 | @echo "Makefile for setting up environment, kernel, and rendering book" 17 | @echo "" 18 | @echo "Usage:" 19 | @echo " make environment - Create Conda environments" 20 | @echo " make kernel - Create Conda environments and Jupyter kernels" 21 | @echo " " 22 | @echo " make teardown - Remove Conda environments and Jupyter kernels" 23 | @echo " make clean - Removes ipynb_checkpoints and quarto \ 24 | temporary files" 25 | @echo " make help - Display this help message" 26 | 27 | $(CONDA_ENV)/envs/eo-datascience-cookbook-dev: 28 | - conda update -n base -c conda-forge conda -y 29 | conda env create --file environment.yml 30 | 31 | $(CONDA_ENV_DIR): 32 | $(foreach f, $(YML), conda env create --file $(f); ) 33 | 34 | environment: $(CONDA_ENV_DIR) 35 | @echo -e "conda environments are ready." 36 | 37 | $(KERNEL_DIR): 38 | $(foreach f, $(REQ), \ 39 | $(CONDA_ACTIVATE) $(f); \ 40 | python -m ipykernel install --user --name $(f) --display-name $(f); \ 41 | conda deactivate; ) 42 | 43 | kernel: $(CONDA_ENV)/envs/eo-datascience-cookbook-dev $(CONDA_ENV_DIR) $(KERNEL_DIR) 44 | @echo -e "jupyter kernels are ready." 45 | 46 | clean: 47 | rm --force --recursive **/.ipynb_checkpoints **/**/.ipynb_checkpoints \ 48 | **/**/**/.ipynb_checkpoints ./pytest_cache **/.jupyter_cache \ 49 | **/**/.jupyter_cache **/**/**/.jupyter_cache 50 | 51 | teardown: 52 | conda remove -n eo-datascience-cookbook-dev --all -y 53 | $(foreach f, $(REQ), \ 54 | $(CONDA_ACTIVATE) $(f); \ 55 | jupyter kernelspec uninstall -y $(f); \ 56 | conda deactivate; \ 57 | conda remove -n $(f) --all -y ; \ 58 | conda deactivate; ) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | tuw-geo-logo 2 | 3 | # Earth Observation Data Science Cookbook 4 | 5 | [![nightly-build](https://github.com/ProjectPythia/eo-datascience-cookbook/actions/workflows/nightly-build.yaml/badge.svg)](https://github.com/ProjectPythia/eo-datascience-cookbook/actions/workflows/nightly-build.yaml) 6 | [![Binder](https://binder.projectpythia.org/badge_logo.svg)](https://binder.projectpythia.org/v2/gh/ProjectPythia/eo-datascience-cookbook/main?labpath=notebooks) 7 | [![DOI](https://zenodo.org/badge/830421828.svg)](https://zenodo.org/badge/latestdoi/830421828) 8 | 9 | This Project Pythia Cookbook covers a range of Earth observation examples employing 10 | the Pangeo philosophy. The examples represent the main research lines and BSc/MSc 11 | courses at the Department of Geodesy and Geoinformation at the TU Wien (Austria). 12 | The department has strong ties with the EODC (Earth Observation Data Centre For 13 | Water Resources Monitoring), which hosts e.g., analysis-ready Sentinel-1 14 | (imaging radar mission) data, and has the computational resources to process 15 | large data volumes. 16 | 17 | ## Motivation 18 | 19 | The motivation behind this book is to provide examples of Pangeo-based workflows 20 | applied to realistic examples in Earth observation data science. Creating an 21 | effective learning environment for Earth observation students is a challenging 22 | task due to the rapidly growing volume of remotely sensed, climate, and other 23 | Earth observation data, along with the evolving demands from the tech industry. 24 | Today's Earth observation students are increasingly becoming a blend of traditional 25 | Earth system scientists and "big data scientists", with expertise spanning computer 26 | architectures, programming paradigms, statistics, and machine learning for 27 | predictive modeling. As a result, it is essential to equip educators with the 28 | proper tools for instruction, including training materials, access to data, and 29 | the necessary skills to support scalable and reproducible research. 30 | 31 | ## Authors 32 | 33 | [Wolfgang Wagner](https://github.com/wagner-wolfgang), [Martin Schobben](https://github.com/martinschobben), 34 | [Nikolas Pikall](https://github.com/npikall), [Joseph Wagner](https://github.com/wagnerjoseph), [Davide Festa](https://github.com/maybedave), 35 | [Felix David Reuß](https://github.com/FelixReuss), [Luka Jovic](https://github.com/lukojovic) 36 | 37 | ### Contributors 38 | 39 | 40 | 41 | 42 | 43 | ## Structure 44 | 45 | This book comprises examples of data science concerning Earth Observation (EO) data, 46 | including course material on remote sensing and data products produced by the TU 47 | Wien. It also serves to showcase the data and services offered by the EODC, including 48 | a [STAC](https://docs.eodc.eu/services/stac.html) catalogue and a 49 | [Dask Gateway](https://docs.eodc.eu/services/dask.html) for distributed data processing. 50 | 51 | ### Courses 52 | 53 | This section offers an overview of notebooks, which are used in **courses** from 54 | the Department of Geodesy and Geoinformation at TU Wien. 55 | 56 | ### Templates 57 | 58 | This section provides a collection of general **examples** of earth observation 59 | related tasks and workflows, which are not directly related to a specific course 60 | or product. 61 | 62 | ### Tutorials 63 | 64 | In this section you will find a collection of lessons, which explain certain 65 | **products** or methods that have been developed at the Department of Geodesy and 66 | Geoinformation at TU Wien. 67 | 68 | ## Running the Notebooks 69 | 70 | You can either run the notebook using [Binder](https://binder.projectpythia.org/v2/gh/ProjectPythia/eo-datascience-cookbook/main?labpath=notebooks) 71 | or on your local machine. 72 | 73 | ### Running on Binder 74 | 75 | The simplest way to interact with a Jupyter Notebook is through 76 | [Binder](https://binder.projectpythia.org/v2/gh/ProjectPythia/eo-datascience-cookbook/main?labpath=notebooks), which enables the execution of a 77 | [Jupyter Book](https://jupyterbook.org) in the cloud. The details of how this works are not 78 | important for now. All you need to know is how to launch a Pythia 79 | Cookbooks chapter via Binder. Simply navigate your mouse to 80 | the top right corner of the book chapter you are viewing and click 81 | on the rocket ship icon, (see figure below), and be sure to select 82 | “launch Binder”. After a moment you should be presented with a 83 | notebook that you can interact with. I.e. you'll be able to execute 84 | and even change the example programs. You'll see that the code cells 85 | have no output at first, until you execute them by pressing 86 | {kbd}`Shift`\+{kbd}`Enter`. Complete details on how to interact with 87 | a live Jupyter notebook are described in [Getting Started with 88 | Jupyter](https://foundations.projectpythia.org/foundations/getting-started-jupyter.html). 89 | 90 | ### Running on Your Own Machine 91 | 92 | If you are interested in running this material locally on your computer, you will 93 | need to follow this workflow: 94 | 95 | 1. Clone the `https://github.com/ProjectPythia/eo-datascience-cookbook` repository: 96 | 97 | ```bash 98 | git clone https://github.com/TUW-GEO/eo-datascience-cookbook 99 | ``` 100 | 101 | 1. Move into the `eo-datascience-cookbook` directory 102 | ```bash 103 | cd eo-datascience-cookbook 104 | ``` 105 | 1. Create and activate your conda environment from the ``environment.yml`` file 106 | ```bash 107 | conda env create -f environment.yml 108 | conda activate eo-datascience-cookbook-dev 109 | ``` 110 | 1. Move into the `notebooks` directory and start up Jupyterlab 111 | ```bash 112 | cd notebooks/ 113 | jupyter lab 114 | ``` 115 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | # Book settings 2 | # Learn more at https://jupyterbook.org/customize/config.html 3 | 4 | title: Earth Observation Data Science Cookbook 5 | author: the Project Pythia Community 6 | logo: notebooks/images/logos/pythia_logo-white-rtext.svg 7 | copyright: "2025" 8 | 9 | bibtex_bibfiles: 10 | - notebooks/references.bib 11 | 12 | execute: 13 | # To execute notebooks via a Binder instead, replace 'cache' with 'binder' 14 | execute_notebooks: cache 15 | timeout: 600 16 | allow_errors: False # cells with expected failures must set the `raises-exception` cell tag 17 | 18 | # Add a few extensions to help with parsing content 19 | parse: 20 | myst_enable_extensions: # default extensions to enable in the myst parser. See https://myst-parser.readthedocs.io/en/latest/using/syntax-optional.html 21 | - amsmath 22 | - colon_fence 23 | - deflist 24 | - dollarmath 25 | - html_admonition 26 | - html_image 27 | - replacements 28 | - smartquotes 29 | - substitution 30 | 31 | sphinx: 32 | config: 33 | linkcheck_ignore: [ 34 | "https://doi.org/*", 35 | "https://zenodo.org/badge/*", 36 | "https://services.eodc.eu/browser/*", 37 | "https://binder.eo-datascience-cookbook.org/*", 38 | "https://www.mdpi.com/*", 39 | ] # don't run link checker on DOI links since they are immutable 40 | nb_execution_raise_on_error: true # raise exception in build if there are notebook errors (this flag is ignored if building on binder) 41 | html_favicon: notebooks/images/icons/favicon.ico 42 | html_last_updated_fmt: "%-d %B %Y" 43 | html_theme: sphinx_pythia_theme 44 | html_permalinks_icon: '' 45 | html_theme_options: 46 | home_page_in_toc: true 47 | repository_url: https://github.com/ProjectPythia/eo-datascience-cookbook # Online location of your book 48 | repository_branch: main # Which branch of the repository should be used when creating links (optional) 49 | use_issues_button: true 50 | use_repository_button: true 51 | use_edit_page_button: true 52 | use_fullscreen_button: true 53 | analytics: 54 | google_analytics_id: G-T52X8HNYE8 55 | github_url: https://github.com/ProjectPythia 56 | icon_links: 57 | - name: YouTube 58 | url: https://www.youtube.com/channel/UCoZPBqJal5uKpO8ZiwzavCw 59 | icon: fab fa-youtube-square 60 | type: fontawesome 61 | launch_buttons: 62 | binderhub_url: https://binder.projectpythia.org 63 | notebook_interface: jupyterlab 64 | logo: 65 | link: https://projectpythia.org 66 | navbar_start: 67 | - navbar-logo 68 | navbar_end: 69 | - navbar-icon-links 70 | navbar_links: 71 | - name: Home 72 | url: https://projectpythia.org 73 | - name: Foundations 74 | url: https://foundations.projectpythia.org 75 | - name: Cookbooks 76 | url: https://cookbooks.projectpythia.org 77 | - name: Resources 78 | url: https://projectpythia.org/resource-gallery.html 79 | - name: Community 80 | url: https://projectpythia.org/index.html#join-us 81 | footer_logos: 82 | NCAR: notebooks/images/logos/NSF-NCAR_Lockup-UCAR-Dark_102523.svg 83 | Unidata: notebooks/images/logos/Unidata_logo_horizontal_1200x300.svg 84 | UAlbany: notebooks/images/logos/UAlbany-A2-logo-purple-gold.svg 85 | footer_start: 86 | - footer-logos 87 | - footer-info 88 | - footer-extra 89 | -------------------------------------------------------------------------------- /_gallery_info.yml: -------------------------------------------------------------------------------- 1 | thumbnail: notebooks/images/logos/tuw-geo_eodc_logo_horizontal.png 2 | tags: 3 | domains: 4 | - remote-sensing 5 | - microwave-remote-sensing 6 | - earth-observation 7 | - sentinel-1 8 | - stac 9 | packages: 10 | - xarray 11 | - holoviews 12 | - pystac-client 13 | - odc-stac 14 | - rioxarray 15 | -------------------------------------------------------------------------------- /_static/custom.css: -------------------------------------------------------------------------------- 1 | .bd-main .bd-content .bd-article-container { 2 | max-width: 100%; /* default is 60em */ 3 | } 4 | .bd-page-width { 5 | max-width: 100%; /* default is 88rem */ 6 | } 7 | -------------------------------------------------------------------------------- /_static/footer-logo-nsf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProjectPythia/eo-datascience-cookbook/4f2eb4968e10868a7b99e152739ec001ace1b3d5/_static/footer-logo-nsf.png -------------------------------------------------------------------------------- /_templates/footer-extra.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | Theme by Project Pythia. All code 5 | in Pythia Cookbooks is licensed under 6 | Apache 2.0. All 7 | other non-code content is licensed under 8 | 9 | Creative Commons BY 4.0 (CC BY 4.0). 11 |
12 |

13 |
14 |

15 | 19 | This material is based upon work supported by the National Science 20 | Foundation under Grant Nos. 2026863 and 2026899. Any opinions, findings, 21 | and conclusions or recommendations expressed in this material are those 22 | of the author(s) and do not necessarily reflect the views of the 23 | National Science Foundation. 24 |

25 |
26 |
27 |
28 | -------------------------------------------------------------------------------- /_toc.yml: -------------------------------------------------------------------------------- 1 | format: jb-book 2 | parts: 3 | - caption: Preamble 4 | chapters: 5 | - file: notebooks/how-to-cite 6 | - caption: Courses 7 | chapters: 8 | - file: notebooks/courses/microwave-remote-sensing 9 | sections: 10 | - file: notebooks/courses/microwave-remote-sensing/01_in_class_exercise 11 | - file: notebooks/courses/microwave-remote-sensing/02_in_class_exercise 12 | - file: notebooks/courses/microwave-remote-sensing/03_in_class_exercise 13 | - file: notebooks/courses/microwave-remote-sensing/04_in_class_exercise 14 | - file: notebooks/courses/microwave-remote-sensing/05_in_class_exercise 15 | - file: notebooks/courses/microwave-remote-sensing/06_in_class_exercise 16 | - file: notebooks/courses/microwave-remote-sensing/07_in_class_exercise 17 | - file: notebooks/courses/microwave-remote-sensing/08_in_class_exercise 18 | - file: notebooks/courses/microwave-remote-sensing/09_in_class_exercise 19 | - caption: Templates 20 | chapters: 21 | - file: notebooks/templates/prereqs-templates 22 | sections: 23 | - file: notebooks/templates/classification 24 | - caption: Tutorials 25 | chapters: 26 | - file: notebooks/tutorials/prereqs-tutorials 27 | sections: 28 | - file: notebooks/tutorials/floodmapping 29 | - caption: References 30 | chapters: 31 | - file: notebooks/references 32 | root: README 33 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: eo-datascience-cookbook-dev 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - aiohttp 6 | - bokeh 7 | - cartopy 8 | - cmcrameri 9 | - dask==2025.2.0 10 | - datashader 11 | - folium 12 | - geopandas 13 | - geoviews 14 | - graphviz 15 | - holoviews 16 | - huggingface_hub 17 | - hvplot 18 | - intake 19 | - intake-xarray 20 | - jupyter 21 | - jupyter-book 22 | - jupyter-cache 23 | - jupyter_bokeh 24 | - mamba 25 | - matplotlib 26 | - nbformat 27 | - nbstripout 28 | - nodejs 29 | - numpy 30 | - odc-stac 31 | - openssl 32 | - pip 33 | - pre-commit 34 | - pyproj 35 | - pystac-client 36 | - pytest 37 | - python=3.12 38 | - rasterio 39 | - requests 40 | - rioxarray 41 | - scikit-image 42 | - scikit-learn 43 | - scipy 44 | - seaborn 45 | - snaphu 46 | - sphinx-pythia-theme 47 | - stackstac 48 | - xarray 49 | - zarr=2.18.4 50 | -------------------------------------------------------------------------------- /notebooks/courses/microwave-remote-sensing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Microwave Remote Sensing\n", 9 | "\n", 10 | "\n", 11 | "This course at the TU Wien teaches students to read, visualize and analyze\n", 12 | "Synthetic Aperture Radar (SAR) data. This will aid interpretation of SAR data\n", 13 | "based upon a physical understanding of sensing principles and the interaction of\n", 14 | "microwaves with natural objects.\n", 15 | "\n", 16 | "| Concepts | Importance | Notes |\n", 17 | "|---|---|---|\n", 18 | "| [Intro to xarray](https://foundations.projectpythia.org/core/xarray/xarray-intro.html) | Necessary | |\n", 19 | "| [Dask Arrays](https://foundations.projectpythia.org/core/xarray/dask-arrays-xarray.html)| Necessary| |\n", 20 | "| [Intake](https://projectpythia.org/intake-cookbook/README.html)|Helpful| |\n", 21 | "| [Matplotlib](https://foundations.projectpythia.org/core/matplotlib.html)|Helpful|Ploting in Python|\n", 22 | "| [Documentation hvPlot](https://hvplot.holoviz.org/)|Helpful|Interactive plotting|\n", 23 | "| [Documentation odc-stac](https://odc-stac.readthedocs.io/en/latest/)|Helpful|Data access|\n", 24 | "\n", 25 | "- **Time to learn**: 90 min\n", 26 | "\n", 27 | ":::{note}\n", 28 | "These notebooks contain interactive elements. The full interactive elements can\n", 29 | "only be viewed on Binder by clicking on the Binder badge or 🚀 button.\n", 30 | ":::\n" 31 | ] 32 | } 33 | ], 34 | "metadata": { 35 | "kernelspec": { 36 | "display_name": "Python 3 (ipykernel)", 37 | "language": "python", 38 | "name": "python3", 39 | "path": "/opt/hostedtoolcache/Python/3.11.11/x64/share/jupyter/kernels/python3" 40 | }, 41 | "language_info": { 42 | "codemirror_mode": { 43 | "name": "ipython", 44 | "version": 3 45 | }, 46 | "file_extension": ".py", 47 | "mimetype": "text/x-python", 48 | "name": "python", 49 | "nbconvert_exporter": "python", 50 | "pygments_lexer": "ipython3", 51 | "version": "3.11.11" 52 | } 53 | }, 54 | "nbformat": 4, 55 | "nbformat_minor": 5 56 | } 57 | -------------------------------------------------------------------------------- /notebooks/courses/microwave-remote-sensing.yml: -------------------------------------------------------------------------------- 1 | name: microwave-remote-sensing 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - aiohttp 6 | - bokeh 7 | - cmcrameri 8 | - dask 9 | - datashader 10 | - folium 11 | - graphviz 12 | - holoviews 13 | - huggingface_hub 14 | - hvplot 15 | - intake 16 | - intake-xarray 17 | - jupyter 18 | - jupyter_bokeh 19 | - mamba 20 | - matplotlib 21 | - nodejs 22 | - numpy 23 | - odc-stac 24 | - pyproj 25 | - pystac-client 26 | - python=3.11 27 | - requests 28 | - rioxarray 29 | - scikit-image 30 | - scipy 31 | - seaborn 32 | - snaphu 33 | - xarray 34 | - zarr=2.18.4 -------------------------------------------------------------------------------- /notebooks/courses/microwave-remote-sensing/01_in_class_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Discover and Read SAR Data\n", 9 | "\n", 10 | "\n", 11 | "\n", 12 | "This notebook demonstrates how to access radar data in a SpatioTemporal Asset Catalog (STAC) Catalogue using the `pystac` library. In this example, we use Sentinel-1 data from the EODC (earth observation data and high performance computing service provider based in Vienna) STAC catalog. In the further process, we will learn how to query a STAC catalog, select specific items, and display the metadata and the actual image.\n", 13 | "\n" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "id": "1", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import folium\n", 24 | "import pystac_client\n", 25 | "from odc import stac as odc_stac" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "2", 31 | "metadata": {}, 32 | "source": [ 33 | "## Data Discovery\n" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": null, 39 | "id": "3", 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "eodc_catalog = pystac_client.Client.open(\"https://stac.eodc.eu/api/v1\")\n", 44 | "\n", 45 | "eodc_catalog" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "id": "4", 51 | "metadata": {}, 52 | "source": [ 53 | "The URL `https://stac.eodc.eu/api/v1`, served over Hypertext Transfer Protocol (HTTP), is a STAC-compliant API endpoint (specific URL address where an API service is available) that leads to the EODC Catalogue. Besides EODC's, other catalogues can be found on [STAC Index](https://stacindex.org/catalogs), such as United States Geological Survey (USGS) Landsat imagery, Sentinel Hub, Copernicus Data Space Ecosystem, and so on. Briefly spoken, STAC can be used to search, discover, and access metadata of these datasets with the same code. The EODC Catalogue can be accessed on the web via this [link](https://services.eodc.eu/browser/#/?.language=en) as well.\n", 54 | "\n", 55 | "Each STAC catalog, composed by different providers, has many collections. To get all collections of a catalog, we can print all of them and their ids, which are used to fetch them from the catalog.\n" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "id": "5", 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "collections = eodc_catalog.get_collections()\n", 66 | "\n", 67 | "# length of string of collection.id, for pretty print\n", 68 | "max_length = max(len(collection.id) for collection in collections)\n", 69 | "\n", 70 | "for collection in eodc_catalog.get_collections():\n", 71 | " print(f\"{collection.id.ljust(max_length)}: {collection.title}\")" 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "id": "6", 77 | "metadata": {}, 78 | "source": [ 79 | "To get a specific collection from the catalog, we can use the `client.get_collection()` method and provide the collection name. We can then display its description, id, temporal and spatial extent, license, etc. In this notebook, we will work with the Sentinel-1 sigma naught 20m collection.\n" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "id": "7", 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "colllection_id = \"SENTINEL1_SIG0_20M\"\n", 90 | "\n", 91 | "collection = eodc_catalog.get_collection(colllection_id)\n", 92 | "collection" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "id": "8", 98 | "metadata": {}, 99 | "source": [ 100 | "Each collection has multiple items. An item is one spatio-temporal instance in the collection, for instance a satellite image. If items are needed for a specific timeframe or for a specific region of interest, we can define this as a query.\n" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "id": "9", 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "time_range = \"2022-10-01/2022-10-07\" # a closed range\n", 111 | "# time_range = \"2022-01\" # whole month, same can be done for a year and a day\n", 112 | "# time_range = \"2022-01-01/..\" # up to the current date, an open range\n", 113 | "# time_range = \"2022-01-01T05:34:46\" # a specific time instance" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "id": "10", 119 | "metadata": {}, 120 | "source": [ 121 | "A spatial region of interest can be defined in different ways. One option is to define a simple bounding box:\n" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "id": "11", 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [ 131 | "latmin, latmax = 46.3, 49.3 # South to North\n", 132 | "lonmin, lonmax = 13.8, 17.8 # West to East\n", 133 | "\n", 134 | "bounding_box = [lonmin, latmin, lonmax, latmax]" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "id": "12", 140 | "metadata": {}, 141 | "source": [ 142 | "If the region of interest is not rectangular, we can also define a polygon:\n" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "id": "13", 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "# GEOJSON can be created on geojson.io\n", 153 | "\n", 154 | "# This specific area of interest is a rectangle, but since it is\n", 155 | "# a closed polygon it seems like it has five nodes\n", 156 | "\n", 157 | "area_of_interest = {\n", 158 | " \"coordinates\": [\n", 159 | " [\n", 160 | " [17.710928010825853, 49.257630084442496],\n", 161 | " [13.881798300915221, 49.257630084442496],\n", 162 | " [13.881798300915221, 46.34747715326259],\n", 163 | " [17.710928010825853, 46.34747715326259],\n", 164 | " [17.710928010825853, 49.257630084442496],\n", 165 | " ]\n", 166 | " ],\n", 167 | " \"type\": \"Polygon\",\n", 168 | "}" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "id": "14", 174 | "metadata": {}, 175 | "source": [ 176 | "Using our previously loaded STAC catalog, we can now search for items fullfilling our query. In this example we are using the bounding box. If we want to use an area of interest specified in the geojson format - one has to use the intersects parameter as documented in the comment below.\n" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": null, 182 | "id": "15", 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "search = eodc_catalog.search(\n", 187 | " collections=colllection_id, # can also be a list of several collections\n", 188 | " bbox=bounding_box, # search by bounding box\n", 189 | " # intersects=area_of_interest, # GeoJSON search\n", 190 | " datetime=time_range,\n", 191 | " # max_items=1 # number of max items to load\n", 192 | ")\n", 193 | "\n", 194 | "# If we comment everything besides colllection_id, we will load whole\n", 195 | "# collection for available region and time_range\n", 196 | "\n", 197 | "items_eodc = search.item_collection()\n", 198 | "print(f\"On EODC we found {len(items_eodc)} items for the given search query\")" 199 | ] 200 | }, 201 | { 202 | "cell_type": "markdown", 203 | "id": "16", 204 | "metadata": {}, 205 | "source": [ 206 | "Now, we can fetch a single item, in this case a Sentinel-1 image, from the query results. A good practice is to always check what metadata the data provider has stored on the item level. This can be done by looking into the item properties.\n" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": null, 212 | "id": "17", 213 | "metadata": {}, 214 | "outputs": [], 215 | "source": [ 216 | "item = items_eodc[0]\n", 217 | "item.properties" 218 | ] 219 | }, 220 | { 221 | "cell_type": "markdown", 222 | "id": "18", 223 | "metadata": {}, 224 | "source": [ 225 | "For now, let's display only the vertical-vertical (VV) polarized band of the item and some information about the data.\n" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": null, 231 | "id": "19", 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [ 235 | "item.assets[\"VV\"].extra_fields.get(\"raster:bands\")[0]" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "id": "20", 241 | "metadata": {}, 242 | "source": [ 243 | "In the EODC STAC catalogue an item can conveniently be displayed using its thumbnail.\n" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": null, 249 | "id": "21", 250 | "metadata": {}, 251 | "outputs": [], 252 | "source": [ 253 | "item.assets[\"thumbnail\"].href" 254 | ] 255 | }, 256 | { 257 | "cell_type": "markdown", 258 | "id": "22", 259 | "metadata": {}, 260 | "source": [ 261 | "Now we will plot the data on a map using the thumbnail and the python package [folium](https://python-visualization.github.io/folium/latest/user_guide.html). This is an easy way to quickly check how the data found by a search query looks on a map.\n" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": null, 267 | "id": "23", 268 | "metadata": {}, 269 | "outputs": [], 270 | "source": [ 271 | "map = folium.Map(\n", 272 | " location=[(latmin + latmax) / 2, (lonmin + lonmax) / 2],\n", 273 | " zoom_start=7,\n", 274 | " zoom_control=False,\n", 275 | " scrollWheelZoom=False,\n", 276 | " dragging=False,\n", 277 | ")\n", 278 | "\n", 279 | "folium.GeoJson(area_of_interest, name=\"Area of Interest\").add_to(map)\n", 280 | "\n", 281 | "for item in items_eodc:\n", 282 | " # url leading to display of an item, can also be used as hyperlink\n", 283 | " image_url = item.assets[\"thumbnail\"].href\n", 284 | " bounds = item.bbox\n", 285 | " folium.raster_layers.ImageOverlay(\n", 286 | " image=image_url,\n", 287 | " bounds=[[bounds[1], bounds[0]], [bounds[3], bounds[2]]],\n", 288 | " ).add_to(map)\n", 289 | "\n", 290 | "folium.LayerControl().add_to(map)\n", 291 | "\n", 292 | "map" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "id": "24", 298 | "metadata": {}, 299 | "source": [ 300 | "*Figure 1: Map of study area. Blue rectangle is the area covered by the discovered data.*\n", 301 | "\n", 302 | "## Data Reading\n", 303 | "\n", 304 | "STAC can also be a useful tool for the discovery of data, however it only loads metadata. This saves memory, but if one would like to do further analysis, the data has to be loaded into memory or downloaded on disk.\n", 305 | "\n", 306 | "In the following, we will demonstrate this with the library `odc-stac`. Here we can define what data will loaded as `bands`; in this case VV sigma naught. Moreover we can resample the data by providing any coordinate reference system (CRS) and resolution as well as a method for resampling of continuos data (e.g. bilinear resampling). In the example below we use the EQUI7 Grid of Europe and a 20 meter sampling. This is the native format of sigma naught stored at EODC, so there will be no actual resampling. Note, also, that resampling is not advisable for this data, as it is provided on a logarithmic scale. More about this in the notebook \"Backscattering Coefficients\".\n", 307 | "\n", 308 | "*The chunks argument is an advancement method for performing parallel computations on the data. We will not cover this in further detail.*\n" 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": null, 314 | "id": "25", 315 | "metadata": {}, 316 | "outputs": [], 317 | "source": [ 318 | "bands = \"VV\" # Vertical-vertical polarized\n", 319 | "crs = \"EPSG:27704\" # Coordinate Reference System: EQUI7 Grid of Europe\n", 320 | "res = 20 # 20 meter\n", 321 | "chunks = {\"time\": 1, \"latitude\": 1000, \"longitude\": 1000}\n", 322 | "sig0_dc = odc_stac.load(\n", 323 | " items_eodc,\n", 324 | " bands=bands,\n", 325 | " crs=crs,\n", 326 | " resolution=res,\n", 327 | " bbox=bounding_box,\n", 328 | " chunks=chunks,\n", 329 | " resampling=\"bilinear\",\n", 330 | ")" 331 | ] 332 | }, 333 | { 334 | "cell_type": "markdown", 335 | "id": "26", 336 | "metadata": {}, 337 | "source": [ 338 | "Let's have a look at the VV polarized band of the dataset.\n" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": null, 344 | "id": "27", 345 | "metadata": {}, 346 | "outputs": [], 347 | "source": [ 348 | "sig0_dc.VV" 349 | ] 350 | }, 351 | { 352 | "cell_type": "markdown", 353 | "id": "28", 354 | "metadata": {}, 355 | "source": [ 356 | "As we can see, the data is stored as a `xarray` DataArray. Xarray is a convenient package for multidimensional labeled arrays, like temperature, humidity, pressure, different bands of satellite imagery, and so on. [The link](https://docs.xarray.dev/en/stable/index.html) provides detailed documentation. In a later notebook we will explore some more of the functionality of `xarray`. As we can see in the coordinates, the data here consists of 21 time steps.\n", 357 | "\n", 358 | "In general, data from STAC is \"lazily\" loaded, which means that the structure of the DataArray is constructed, but the data is not loaded yet. It is loaded only at instance when it is needed, for example, for plotting, computations, and so on.\n", 359 | "\n", 360 | "Since the DataArray has currently a size of almost 18 GiB, we will subset it to the region of Vienna.\n" 361 | ] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "execution_count": null, 366 | "id": "29", 367 | "metadata": {}, 368 | "outputs": [], 369 | "source": [ 370 | "# Create a bounding box covering the region of Vienna\n", 371 | "latmin_smaller, latmax_smaller = 48, 48.4\n", 372 | "lonmin_smaller, lonmax_smaller = 16, 16.5\n", 373 | "\n", 374 | "smaller_bounding_box = [\n", 375 | " [latmin_smaller, lonmin_smaller],\n", 376 | " [latmax_smaller, lonmax_smaller],\n", 377 | "]\n", 378 | "\n", 379 | "map = folium.Map(\n", 380 | " location=[\n", 381 | " (latmin_smaller + latmax_smaller) / 2,\n", 382 | " (lonmin_smaller + lonmax_smaller) / 2,\n", 383 | " ],\n", 384 | " zoom_start=8,\n", 385 | " zoom_control=False,\n", 386 | " scrollWheelZoom=False,\n", 387 | " dragging=False,\n", 388 | ")\n", 389 | "\n", 390 | "folium.GeoJson(area_of_interest, name=\"Area of Interest\").add_to(map)\n", 391 | "\n", 392 | "folium.Rectangle(\n", 393 | " bounds=smaller_bounding_box,\n", 394 | " color=\"red\",\n", 395 | ").add_to(map)\n", 396 | "\n", 397 | "for item in items_eodc:\n", 398 | " image_url = item.assets[\"thumbnail\"].href\n", 399 | " bounds = item.bbox\n", 400 | " folium.raster_layers.ImageOverlay(\n", 401 | " image=image_url,\n", 402 | " bounds=[[bounds[1], bounds[0]], [bounds[3], bounds[2]]],\n", 403 | " ).add_to(map)\n", 404 | "\n", 405 | "folium.LayerControl().add_to(map)\n", 406 | "\n", 407 | "map" 408 | ] 409 | }, 410 | { 411 | "cell_type": "markdown", 412 | "id": "30", 413 | "metadata": {}, 414 | "source": [ 415 | "*Figure 2: Map of study area. Blue rectangle is the area covered by the discovered data. Red rectangle covers the selected data.*\n", 416 | "\n", 417 | "Create a new dataset with the smaller bounding box covering the region of Vienna. We will leave out the arguments for resampling and directly use the native format as defined in the metadata.\n" 418 | ] 419 | }, 420 | { 421 | "cell_type": "code", 422 | "execution_count": null, 423 | "id": "31", 424 | "metadata": {}, 425 | "outputs": [], 426 | "source": [ 427 | "sig0_dc = odc_stac.load(\n", 428 | " items_eodc,\n", 429 | " bands=bands,\n", 430 | " bbox=[lonmin_smaller, latmin_smaller, lonmax_smaller, latmax_smaller],\n", 431 | " chunks=chunks,\n", 432 | ")" 433 | ] 434 | }, 435 | { 436 | "cell_type": "markdown", 437 | "id": "32", 438 | "metadata": {}, 439 | "source": [ 440 | "Due to the way the data is acquired and stored, some items include \"no data\" areas. In our case, no data has the value -9999, but this can vary from data provider to data provider. This information can usually be found in the metadata. Furthermore, to save memory, data is often stored as integer (e.g. 25) and not in float (e.g. 2.5) format. For this reason, the backscatter values are often multiplied by a scale factor, in this case factor 10.\n", 441 | "\n", 442 | "As Sentinel-1 satellites overpasses Austria every few days, only some time steps of the dataset will have physical data. As a final step, we will now decode the data and create a plot of two consecutive Sentinel-1 acquisitions of Vienna.\n" 443 | ] 444 | }, 445 | { 446 | "cell_type": "code", 447 | "execution_count": null, 448 | "id": "33", 449 | "metadata": {}, 450 | "outputs": [], 451 | "source": [ 452 | "# Retrieve the scale factor and NoData value from the metadata. raster:bands is\n", 453 | "# a STAC raster extension\n", 454 | "scale = item.assets[\"VV\"].extra_fields.get(\"raster:bands\")[0][\"scale\"]\n", 455 | "nodata = item.assets[\"VV\"].extra_fields.get(\"raster:bands\")[0][\"nodata\"]\n", 456 | "\n", 457 | "# Decode data with the NoData value and the scale factor\n", 458 | "sig0_dc = sig0_dc.where(sig0_dc != nodata) / scale\n", 459 | "\n", 460 | "# We should remove unnecessary dates when there was no data\n", 461 | "# (no satellite overpass)\n", 462 | "sig0_dc = sig0_dc.dropna(dim=\"time\")" 463 | ] 464 | }, 465 | { 466 | "cell_type": "code", 467 | "execution_count": null, 468 | "id": "34", 469 | "metadata": {}, 470 | "outputs": [], 471 | "source": [ 472 | "sig0_dc.VV.plot(col=\"time\", robust=True, cmap=\"Greys_r\", aspect=1, size=10)" 473 | ] 474 | }, 475 | { 476 | "cell_type": "markdown", 477 | "id": "35", 478 | "metadata": {}, 479 | "source": [ 480 | "*Figure 3: Sentinel-1 microwave backscatter image for two timeslices.*" 481 | ] 482 | } 483 | ], 484 | "metadata": { 485 | "kernelspec": { 486 | "display_name": "Python 3 (ipykernel)", 487 | "language": "python", 488 | "name": "python3" 489 | }, 490 | "language_info": { 491 | "codemirror_mode": { 492 | "name": "ipython", 493 | "version": 3 494 | }, 495 | "file_extension": ".py", 496 | "mimetype": "text/x-python", 497 | "name": "python", 498 | "nbconvert_exporter": "python", 499 | "pygments_lexer": "ipython3", 500 | "version": "3.11.11" 501 | } 502 | }, 503 | "nbformat": 4, 504 | "nbformat_minor": 5 505 | } 506 | -------------------------------------------------------------------------------- /notebooks/courses/microwave-remote-sensing/02_in_class_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Unit Conversion\n", 9 | "\n", 10 | "\n", 11 | "In this notebook, we are going to have a look at the conversion of units. Sentinel-1 data, and most other SAR data, is usually provided in decibels (dB). In this notebook, we will discover the advantages of displaying SAR data in decibels and why we need to convert the data to a linear scale in order to make meaningful calculations. Let's start with importing some libraries.\n", 12 | "\n", 13 | "$$\n", 14 | "\\text{logarithmic} \\longleftrightarrow \\text{linear}\n", 15 | "$$\n", 16 | "$$\n", 17 | "[\\text{dB}] \\longleftrightarrow [\\text{m}^2 \\cdot \\text{m}^{-2}]\n", 18 | "$$\n", 19 | "\n" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "id": "1", 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "import matplotlib.pyplot as plt\n", 30 | "import numpy as np\n", 31 | "import odc.stac\n", 32 | "import pystac_client\n", 33 | "import rioxarray # noqa: F401\n", 34 | "import xarray as xr" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "id": "2", 40 | "metadata": {}, 41 | "source": [ 42 | "## Exploring the Data\n", 43 | "\n", 44 | "Let's start by loading some sample data, in order to demonstrate why this conversion is important.\n", 45 | "Here we will have a look at some SAR data from the Sentinel-1. The data is provided in decibels (dB).\n", 46 | "In the following example, we will:\n", 47 | "\n", 48 | "- load data from Sentinel-1\n", 49 | "- visualize the data in logarithmic scale\n", 50 | "- compare the data with linear scale\n", 51 | "\n", 52 | "## Search for some Data\n", 53 | "\n", 54 | "Now, we start by loading data from Sentinel-1 from the EODC STAC Catalogue. We do this in the same way as in the previous notebook \"Discover and Read SAR Data\".\n" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "id": "3", 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "latmin, latmax = 48, 48.5\n", 65 | "lonmin, lonmax = 16, 17\n", 66 | "bounds = (lonmin, latmin, lonmax, latmax)\n", 67 | "\n", 68 | "time_range = \"2022-07-01/2022-07-31\"\n", 69 | "\n", 70 | "items = (\n", 71 | " pystac_client.Client.open(\"https://stac.eodc.eu/api/v1\")\n", 72 | " .search(\n", 73 | " bbox=bounds,\n", 74 | " collections=[\"SENTINEL1_SIG0_20M\"],\n", 75 | " datetime=time_range,\n", 76 | " limit=100,\n", 77 | " )\n", 78 | " .item_collection()\n", 79 | ")\n", 80 | "\n", 81 | "print(len(items), \"scenes found\")" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "id": "4", 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "bands = \"VV\"\n", 92 | "crs = \"EPSG:27704\" # Coordinate Reference System: EQUI7 Grid of Europe\n", 93 | "res = 20 # 20 meter\n", 94 | "\n", 95 | "sig0_dc = odc.stac.stac_load(\n", 96 | " items,\n", 97 | " bands=bands,\n", 98 | " bbox=bounds,\n", 99 | " chunks={\"time\": 5, \"x\": 600, \"y\": 600},\n", 100 | ")\n", 101 | "\n", 102 | "nodata = items[0].assets[\"VV\"].extra_fields[\"raster:bands\"][0][\"nodata\"]\n", 103 | "scale = items[0].assets[\"VV\"].extra_fields[\"raster:bands\"][0][\"scale\"]\n", 104 | "\n", 105 | "sig0_dc = (sig0_dc.where(sig0_dc != nodata) / scale).VV\n", 106 | "sig0_dc" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "id": "5", 112 | "metadata": {}, 113 | "source": [ 114 | "## Comparison of the Data in dB and Linear Scale\n", 115 | "\n", 116 | "In the next two cells we will select a subset of the data. This is done to reduce the amount of data we are working with. The precise workflow is not important for now, since the theory is explained after the cells. They are just here to show the data we are working with.\n" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "id": "6", 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "subset = sig0_dc.sel(time=slice(\"2022-07-01\", \"2022-07-07\"))\n", 127 | "subset = subset.dropna(\"time\", how=\"all\")" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "id": "7", 133 | "metadata": {}, 134 | "source": [ 135 | "Now plot the data.\n" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "id": "8", 142 | "metadata": { 143 | "jupyter": { 144 | "source_hidden": true 145 | } 146 | }, 147 | "outputs": [], 148 | "source": [ 149 | "aoi = subset.isel(time=0, x=slice(0, 500), y=slice(0, 500))\n", 150 | "aoi_lin = 10 ** (aoi / 10)\n", 151 | "\n", 152 | "fig, ax = plt.subplots(2, 3, figsize=(14, 8))\n", 153 | "# upper left\n", 154 | "ax_ul = ax[0, 0]\n", 155 | "aoi.plot.imshow(robust=True, ax=ax_ul, cmap=\"Greys_r\")\n", 156 | "ax_ul.set_title(r\"$\\sigma^0$ [$dB$] (robust plot)\")\n", 157 | "ax_ul.axes.set_aspect(\"equal\")\n", 158 | "\n", 159 | "# upper middle\n", 160 | "ax_um = ax[0, 1]\n", 161 | "aoi.plot.imshow(robust=False, ax=ax_um, cmap=\"Greys_r\")\n", 162 | "ax_um.set_title(r\"$\\sigma^0$ [$dB$] (not robust plot)\")\n", 163 | "ax_um.axes.set_aspect(\"equal\")\n", 164 | "\n", 165 | "# upper right\n", 166 | "ax_ur = ax[0, 2]\n", 167 | "aoi.plot.hist(bins=50, ax=ax_ur, edgecolor=\"black\")\n", 168 | "ax_ur.set_xlabel(r\"$\\sigma^0$ [$dB$]\")\n", 169 | "ax_ur.set_title(r\"$\\sigma^0$ [$dB$] distribution\")\n", 170 | "ax_ur.set_ylabel(\"n (number of pixels)\")\n", 171 | "\n", 172 | "# lower left\n", 173 | "ax_ll = ax[1, 0]\n", 174 | "aoi_lin.plot.imshow(robust=True, ax=ax_ll, cmap=\"Greys_r\")\n", 175 | "ax_ll.set_title(r\"$\\sigma^0$ [$m^2 \\cdot m^{-2}$] (robust plot)\")\n", 176 | "ax_ll.axes.set_aspect(\"equal\")\n", 177 | "\n", 178 | "# lower middle\n", 179 | "ax_lm = ax[1, 1]\n", 180 | "aoi_lin.plot.imshow(robust=False, ax=ax_lm, cmap=\"Greys_r\")\n", 181 | "ax_lm.set_title(r\"$\\sigma^0$ [$m^2 \\cdot m^{-2}$] (not robust plot)\")\n", 182 | "ax_lm.axes.set_aspect(\"equal\")\n", 183 | "\n", 184 | "# lower right\n", 185 | "ax_lr = ax[1, 2]\n", 186 | "aoi_lin.plot.hist(bins=50, ax=ax_lr, edgecolor=\"black\")\n", 187 | "ax_lr.set_xlabel(r\"$\\sigma^0$ [$m^2 \\cdot m^{-2}$]\")\n", 188 | "ax_lr.set_ylabel(\"n (number of pixels)\")\n", 189 | "ax_lr.set_title(r\"$\\sigma^0$ [$m^2 \\cdot m^{-2}$] distribution\")\n", 190 | "\n", 191 | "title = (\n", 192 | " r\"Sentinel-1 backscatter $\\sigma^0$ comparison\"\n", 193 | " + r\" in linear and logarithmic domain\"\n", 194 | ")\n", 195 | "fig.suptitle(title, horizontalalignment=\"center\")\n", 196 | "plt.tight_layout()" 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "id": "9", 202 | "metadata": {}, 203 | "source": [ 204 | "*Figure 1: Visually comparing $\\sigma^0$ on a logarithmic and linear scale (left column). In addition, the benefit of using the robust plotting method is shown (middle column). The robust argument uses the 2^nd^ and 98^th^ percentiles of the data to compute the color limits to eliminate washing out the plot due to data outliers.*\n", 205 | "\n", 206 | "In the plot above you can see the difference between the two scales. The values in dB are more evenly distributed and are therefore easier to plot. The values in linear scale are more spread out and are therefore harder to interpret.\n", 207 | "This is why we use the dB scale for plotting/visualization.\n", 208 | "\n", 209 | "While the logarithmic scale facilitates visual interpretation, it has implications for mathematical operations. In the following, we'll have a closer look at this. But first, let's see how we can convert between the linear and the logarithmic domains.\n", 210 | "\n", 211 | "## Conversion Formulas\n", 212 | "\n", 213 | "The decibel (dB) is a logarithmic unit used to express the ratio of two values of a physical quantity, often power or intensity. In the case of SAR data, the backscatter coefficient is often expressed in dB to facilitate visualization.\n", 214 | "\n", 215 | "In order to convert the data from dB to linear scale, we use the following formula.\n", 216 | "Let $D$ be the original value (dB) and $I$ the converted value ($m^2m^{-2}$). The conversion of units can be expressed as:\n", 217 | "$$\n", 218 | "D = 10 \\cdot \\log_{10} (I) = 10 \\cdot \\log_{10} (e) \\cdot \\ln (I)\\longrightarrow [dB]\n", 219 | "$$\n", 220 | "Similarly, the conversion back to the original unit can be expressed as:\n", 221 | "$$\n", 222 | "I = e^{\\frac{D}{10\\cdot \\log_{10}(e)}} = 10^{\\frac{D}{10}} \\longrightarrow [m^2m^{-2}]\n", 223 | "$$\n", 224 | "You can find these formulas in the script for `Microwave Remote Sensing` on ``page 136 (equation 6.40)``.\n", 225 | "\n", 226 | "Now let's implement the conversion in Python.\n" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": null, 232 | "id": "10", 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [ 236 | "def lin2db(val: float | int) -> float:\n", 237 | " \"\"\"\n", 238 | " Converts value from linear to dB units.\n", 239 | "\n", 240 | " :param val: Value in linear units.\n", 241 | " :type val: float|int\n", 242 | " :return: Value in dB.\n", 243 | " :rtype: float\n", 244 | " \"\"\"\n", 245 | " return 10 * np.log10(val)\n", 246 | "\n", 247 | "\n", 248 | "def db2lin(val: float | int) -> float:\n", 249 | " \"\"\"\n", 250 | " Converts value from dB to linear units.\n", 251 | "\n", 252 | " :param val: Value in dB.\n", 253 | " :type val: float|int\n", 254 | " :return: Value in linear units.\n", 255 | " :rtype: float\n", 256 | " \"\"\"\n", 257 | " return 10 ** (val / 10)" 258 | ] 259 | }, 260 | { 261 | "cell_type": "markdown", 262 | "id": "11", 263 | "metadata": {}, 264 | "source": [ 265 | "When performing mathematical operations with SAR data it is important to be aware, that adding values in the logarithmic scale doesn't work in the same way as adding regular (linear) values. This is because in the logarithmic scale, each unit step represents an equal multiplication. This means that an addition of two values in the logarithmic scale equals a multiplication of the values in the linear scale. Vice versa, a subtraction in a logarithmic scale equals a division in a linear scale. Let's have a look at an example, where we add two values, once without the conversion to linear scale and once with the conversion to linear scale.\n" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": null, 271 | "id": "12", 272 | "metadata": {}, 273 | "outputs": [], 274 | "source": [ 275 | "# Logarithmic addition\n", 276 | "# Values in linear and decibel units\n", 277 | "val1_db, val2_db = 10, 12\n", 278 | "\n", 279 | "# Logarithmic addition\n", 280 | "sum_db = val1_db + val2_db\n", 281 | "print(\"Logarithmic Addition:\")\n", 282 | "print(f\"Logarithmic values: \\t{val1_db: <5}, {val2_db: <5} [dB]\")\n", 283 | "print(f\"Logarithmic sum: \\t{val1_db} + {val2_db} = {sum_db: <5} [dB]\")\n", 284 | "\n", 285 | "# Linear addition\n", 286 | "val1_lin, val2_lin = db2lin(val1_db), db2lin(val2_db)\n", 287 | "sum_lin = val1_lin + val2_lin\n", 288 | "print(\"\\nLinear Addition:\")\n", 289 | "print(\n", 290 | " f\"\"\"Linear values: \\t\\t{val1_lin: <5}, {val2_lin: <5.2f} [lin]\n", 291 | " \\t\\t\\t(converted from dB)\"\"\"\n", 292 | ")\n", 293 | "print(f\"Linear sum: \\t\\t{val1_lin} + {val2_lin: .2f} = {sum_lin: .2f} [lin]\")\n", 294 | "print(f\"\\t\\t\\t= {lin2db(sum_lin): .2f} [dB]\")" 295 | ] 296 | }, 297 | { 298 | "cell_type": "markdown", 299 | "id": "13", 300 | "metadata": {}, 301 | "source": [ 302 | "As you can see, the values in dB and in linear scale differ quite a bit. In the example above, the values differ by a factor of around 6 when looked at in linear scale.\n", 303 | "\n", 304 | "Now that we have some data, we will have a look at some practical examples where we will convert the data to linear scale.\n", 305 | "When we try to calculate the average $\\sigma^0$ value across the scene, we need to do this by converting the data to linear scale first and then calculating the average and converting it back to dB.\n", 306 | "\n", 307 | "## Creating a Monthly Mosaic\n", 308 | "\n", 309 | "So in the beginning we have lazily loaded data for an area across a whole year. We therefore have around 700 images. We will now essentially compress the data of each month into one timestamp. This is done by using the ``resampling`` method together with an operation method like ``mean`` that includes summation. Since the data is in dB we need to convert it to linear scale first, then we can resample the data and convert it back to dB.\n" 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": null, 315 | "id": "14", 316 | "metadata": {}, 317 | "outputs": [], 318 | "source": [ 319 | "# Convert to linear scale and calculate monthly means\n", 320 | "# Conversion by calculating with the xarray Object\n", 321 | "sig0_lin = 10 ** (sig0_dc / 10)\n", 322 | "\n", 323 | "# Resample to monthly means. Time accepts intervals identical to the pandas\n", 324 | "# resample function. 'D' for days, 'W' for weeks, 'ME' for months.\n", 325 | "sig0_lin_monthly = sig0_lin.resample(time=\"1ME\").mean()\n", 326 | "\n", 327 | "# Convert back to dB scale\n", 328 | "# Conversion by applying a function\n", 329 | "sig0_monthly = lin2db(sig0_lin_monthly)\n", 330 | "sig0_monthly" 331 | ] 332 | }, 333 | { 334 | "cell_type": "markdown", 335 | "id": "15", 336 | "metadata": {}, 337 | "source": [ 338 | "The dataset has now only 12 timestamps, one for each month. Next, we want to calculate the average $\\sigma^0$ value across the scene for one month. We will do this again by converting the data to linear scale first and then calculating the average and converting it back to dB.\n" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": null, 344 | "id": "16", 345 | "metadata": {}, 346 | "outputs": [], 347 | "source": [ 348 | "# Lets take a data array with db values\n", 349 | "db_array = sig0_monthly.sel(time=\"2022-07-30\", method=\"nearest\")\n", 350 | "\n", 351 | "# Compute the linear values\n", 352 | "lin_array = db2lin(db_array)" 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": null, 358 | "id": "17", 359 | "metadata": {}, 360 | "outputs": [], 361 | "source": [ 362 | "# Compute the average backscatter value in linear units across the whole scene\n", 363 | "lin_mean = lin_array.mean()\n", 364 | "print(f\"Average backscatter value in linear units: {lin_mean.values: .3f}\")\n", 365 | "db_from_lin_mean = lin2db(lin_mean)\n", 366 | "print(f\"That value in dB: {db_from_lin_mean.values: .3f}\\n\")\n", 367 | "\n", 368 | "# Compute the average backscatter value in dB across the whole scene\n", 369 | "db_mean = db_array.mean()\n", 370 | "print(f\"Average backscatter value in dB: {db_mean.values: .3f}\")" 371 | ] 372 | }, 373 | { 374 | "cell_type": "markdown", 375 | "id": "18", 376 | "metadata": {}, 377 | "source": [ 378 | "As you can see in the example, the mean values across the scene are different in dB and linear scale. Therefore, it is important to be aware in which scale the data is stored to perform the correct type of mathematical operation or always convert the data to linear scale before doing any calculations.\n", 379 | "\n", 380 | "## Save Mean Mosaic as Tif File\n", 381 | "\n", 382 | "Often we want to store the output of a computation permanently on a file system. The most common file format for this is a GeoTIFF (TIF file with additional information on the georeference). The following cell indicates how this can be easily done with Xarray. When we want to store the data as a GeoTIFF we need to make sure to provide a spatial reference to geolocate the data. The best way to check whether the Xarray has a coordinate reference system (CRS) is by using the rioxarray `rio.crs` accessor. More about this in notebook \"Datacubes\".\n" 383 | ] 384 | }, 385 | { 386 | "cell_type": "code", 387 | "execution_count": null, 388 | "id": "19", 389 | "metadata": {}, 390 | "outputs": [], 391 | "source": [ 392 | "# Select some data which we want to save\n", 393 | "data_2_save = sig0_monthly.sel(time=\"2022-07-30\", method=\"nearest\")\n", 394 | "data_2_save.rio.crs" 395 | ] 396 | }, 397 | { 398 | "cell_type": "markdown", 399 | "id": "20", 400 | "metadata": {}, 401 | "source": [ 402 | "In this case, the spatial reference is the EPSG Code `EPSG:27704`, which is the `Equi7Grid Europe`.\n", 403 | "As the output data array already has a spatial reference we now save it as a raster file\n" 404 | ] 405 | }, 406 | { 407 | "cell_type": "code", 408 | "execution_count": null, 409 | "id": "21", 410 | "metadata": {}, 411 | "outputs": [], 412 | "source": [ 413 | "# Save the data\n", 414 | "data_2_save.rio.to_raster(\n", 415 | " \"sig0_mean_mosaic_july.tif\", tiled=True, driver=\"GTiff\", compress=\"LZW\"\n", 416 | ")" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": null, 422 | "id": "22", 423 | "metadata": {}, 424 | "outputs": [], 425 | "source": [ 426 | "# Load the data again (for demonstration purposes)\n", 427 | "loaded_data = xr.open_dataset(\"sig0_mean_mosaic_july.tif\", engine=\"rasterio\")\n", 428 | "loaded_data" 429 | ] 430 | } 431 | ], 432 | "metadata": { 433 | "kernelspec": { 434 | "display_name": "Python 3 (ipykernel)", 435 | "language": "python", 436 | "name": "python3" 437 | }, 438 | "language_info": { 439 | "codemirror_mode": { 440 | "name": "ipython", 441 | "version": 3 442 | }, 443 | "file_extension": ".py", 444 | "mimetype": "text/x-python", 445 | "name": "python", 446 | "nbconvert_exporter": "python", 447 | "pygments_lexer": "ipython3", 448 | "version": "3.11.11" 449 | } 450 | }, 451 | "nbformat": 4, 452 | "nbformat_minor": 5 453 | } 454 | -------------------------------------------------------------------------------- /notebooks/courses/microwave-remote-sensing/03_in_class_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Backscattering Coefficients\n", 9 | "\n", 10 | "\n", 11 | "In this notebook, we will introduce some of the steps involved in the processing of Sentinel-1 Level1 Ground Range Detected (`GRD`) data to $\\sigma^0$ (`sig0`) and $\\gamma^0$ (`gmr`). Moreover, the notebook illustrates the importance and impact of geometric and radiometric terrain correction. As the processing of SAR data is a very time and hardware-intense task, we won't perform the actual processing in this notebook. Instead, data at different processing steps is illustrated to highlight the impact of the processing steps.\n", 12 | "\n" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "id": "1", 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import hvplot.xarray # noqa: F401\n", 23 | "import intake\n", 24 | "import matplotlib.pyplot as plt # noqa: F401\n", 25 | "import numpy as np\n", 26 | "import rioxarray # noqa: F401\n", 27 | "import xarray as xr" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "id": "2", 33 | "metadata": {}, 34 | "source": [ 35 | "## Loading Backscatter Data\n", 36 | "\n", 37 | "We first load our data from the following [intake](https://intake.readthedocs.io/en/latest/) catalog. Intake is somewhat similar to STAC in that it makes it easy to discover and load data. More importantly, this package allows us to hide some of the complexities involved with getting the data in the right format, which are not of concern in this notebook.\n" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "id": "3", 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "url = \"https://huggingface.co/datasets/martinschobben/microwave-remote-sensing/resolve/main/microwave-remote-sensing.yml\"\n", 48 | "cat = intake.open_catalog(url)\n", 49 | "gtc_dc = cat.gtc.read().compute()\n", 50 | "gtc_dc" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "id": "4", 56 | "metadata": {}, 57 | "source": [ 58 | "## Geometric Terrain Correction\n", 59 | "\n", 60 | "Level 1 SAR data, such as the “GRD” product only takes into account the ellipsoidal model of the earth (i.e. slant-range distortion; script Chapter 4), without the description of the relief. This makes mountains appear to lean towards the radar system as visible in the below plot of the GRD scene. This latter distortion originates from the side-looking geometry of the SAR system. Due to this, the radar pulse reaches mountain slope facing the sensor before it reaches the base. Consequently, these slopes appear compressed and leaning toward the sensor. During processing to a level 1C product, these slant range distortions are partly corrected using a terrain correction algorithm and a Digital Elevation Model (DEM). The most common algorithm for this is the Range Doppler Terrain Correction.\n", 61 | "\n", 62 | "![](https://www.esri.com/arcgis-blog/wp-content/uploads/2024/05/emenezes-esriBlog-GP-tool-fig_Apply-Geometric-Terrain-Correction-detailed.png)\n", 63 | "\n", 64 | "*Figure 1: Geometric terrain correction. The lower bar shows the GRD without geometric terrain correction in slant geometry. In areas where the ground is elevated, the time of the signal to travel to the earth’s surface and back to the sensor is distorted, causing geometric shifts (foreshortening, lengthening, etc). Using a DEM and the Range Doppler Terrain Correction, the distortions are corrected and the image is orthorectified. (Source: ESRI)*\n", 65 | "\n", 66 | "Let's visualize this geometric terrain correction (GTC) with some actual data using the `xarray` method `hvplot` of the `gtc_dc` object.\n" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "id": "5", 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "gtc_dc.hvplot.image(\n", 77 | " x=\"x\",\n", 78 | " y=\"y\",\n", 79 | " robust=True,\n", 80 | " data_aspect=1,\n", 81 | " cmap=\"Greys_r\",\n", 82 | " groupby=\"band\",\n", 83 | " rasterize=True,\n", 84 | ").opts(frame_height=600, framewise=False, aspect=\"equal\")" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "id": "6", 90 | "metadata": {}, 91 | "source": [ 92 | "*Figure 2: The ground range detected values and geometrically terrain corrected values can be selected on the right-hand side of the graphic.*\n", 93 | "\n", 94 | "The geometrically terrain corrected values from the `gtc_dc` object (Figure 1) can be approximated to a certain extent, as we have sufficiently detailed information of topography in this area. This corrects for at least one typically occurring distortion in mountainous regions: \"foreshortening\".\n", 95 | "\n", 96 | "![](../../images/side_looking_image_distortions.png)\n", 97 | "\n", 98 | "*Figure 3: Side Looking radar distortions (script Chapter 4).*\n", 99 | "\n", 100 | "Foreshortening can be spotted by eye, as it often has a radiometric consequence, where unusually bright areas fringe mountain ridges; a phenomenon called \"highlighting\". This geometric artifact occurs due to the compression of the distance in the image of slopes facing the radar system and the consequentially higher density of scatterers per unit length. Now let's zoom in on an example from the same datacube and display the original and corrected values side-by-side.\n" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "id": "7", 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "for_dc = gtc_dc.sel(x=slice(9.651, 9.706), y=slice(47.134, 47.079)).band_data\n", 111 | "\n", 112 | "fig, ax = plt.subplots(1, 2, figsize=(20, 8))\n", 113 | "\n", 114 | "bbox = dict(boxstyle=\"round\", fc=\"0.8\")\n", 115 | "\n", 116 | "\n", 117 | "ax[1].annotate(\n", 118 | " \"foreshortening/layover\",\n", 119 | " xy=(9.674, 47.092),\n", 120 | " xytext=(0.574, 0.192),\n", 121 | " textcoords=\"subfigure fraction\",\n", 122 | " bbox=bbox,\n", 123 | " arrowprops=dict(facecolor=\"red\", shrink=0.05),\n", 124 | ")\n", 125 | "ax[1].annotate(\n", 126 | " \"radar shadows\",\n", 127 | " xy=(9.68, 47.119),\n", 128 | " xytext=(0.6, 0.625),\n", 129 | " textcoords=\"subfigure fraction\",\n", 130 | " bbox=bbox,\n", 131 | " arrowprops=dict(facecolor=\"red\", shrink=0.05),\n", 132 | ")\n", 133 | "\n", 134 | "ax[0].axes.set_aspect(\"equal\")\n", 135 | "ax[1].axes.set_aspect(\"equal\")\n", 136 | "\n", 137 | "for_dc.sel(band=\"grd\").plot(ax=ax[0], robust=True, cmap=\"Greys_r\")\n", 138 | "for_dc.sel(band=\"sig0_gtc\").plot(ax=ax[1], robust=True, cmap=\"Greys_r\")" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "id": "8", 144 | "metadata": {}, 145 | "source": [ 146 | "*Figure 4: Close-up inspection of geometric distortions in side-looking radar*\n", 147 | "\n", 148 | "As we can see, not all the geometric distortions can be corrected by the algorithm. Some of the pixels at the mountain ranges appear stretched, as in these areas not enough valid measurements are available. Moreover, we can see dark areas which are indicating radar shadows. These are image areas that could not be captured by the radar sensor and have values close to the noise floor of the Sensor (minimum detectable signal strength) ~ -28dB. It is important to note, that radar shadows are not the same for every image, as they depend on the acquisition geometry, in particular, the incidence angle and the flight direction of the satellite.\n", 149 | "\n", 150 | "## Backscattering Coefficients\n", 151 | "\n", 152 | "In this chapter, we will look at some of the different backscatter coefficients in more detail ($\\sigma^0_E$ or $\\gamma^0_E$), where both coefficients are geometrically terrain corrected. The difference is the plane of the reference area, which is the ground area as a tangent on an ellipsoidal Earth model for $\\sigma^0_E$ and perpendicular to the line of sight for $\\gamma^0_E$ (Figure 5). For this, we load a new datacube which includes $\\sigma^0_E$ and the Incidence Angle for each pixel. We visualize the cube with the same method as before.\n" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "id": "9", 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "coef_dc = cat.coef.read().compute()\n", 163 | "coef_dc.hvplot.image(\n", 164 | " x=\"x\",\n", 165 | " y=\"y\",\n", 166 | " robust=True,\n", 167 | " data_aspect=1,\n", 168 | " cmap=\"Greys_r\",\n", 169 | " groupby=\"band\",\n", 170 | " rasterize=True,\n", 171 | ").opts(frame_height=600, framewise=False, aspect=\"equal\")" 172 | ] 173 | }, 174 | { 175 | "cell_type": "markdown", 176 | "id": "10", 177 | "metadata": {}, 178 | "source": [ 179 | "*Figure 5: The $\\sigma^0_E$ and the incidence angle can be selected on the right-hand side of the graphic.*\n", 180 | "\n", 181 | "In Figure 5 we can see the incidence angle image of our scene. We can see, that it depicts the differences between near to far range, but not the actual terrain as it refers to the ellipsoid. The slight patterns of the terrain that are visible are originating from the geometric terrain correction. We will use this information now to convert our ($\\sigma^0_E$ to $\\gamma^0_E$) with the following equation (equation 6.20 in the script):\n", 182 | "\n", 183 | "$$ \\gamma^0_E = \\sigma^0_E / \\cos(\\theta_i) $$\n", 184 | "\n", 185 | "We can perform this transformation with basic `numpy` operations on the `xarray` datacube.\n" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "id": "11", 192 | "metadata": {}, 193 | "outputs": [], 194 | "source": [ 195 | "# linear scale\n", 196 | "sig0_db = coef_dc.sel(band=\"sig0_gtc\") / 10\n", 197 | "sig0_lin = 10 ** (coef_dc.sel(band=\"sig0_gtc\") / 10)\n", 198 | "# conversion to gamma\n", 199 | "gam0_lin = sig0_lin / np.cos(np.radians(coef_dc.sel(band=\"incidence_angle\")))\n", 200 | "# dB scale\n", 201 | "gam0_db = 10 * np.log(gam0_lin)\n", 202 | "# add to existing cube\n", 203 | "coef_dc = xr.concat(\n", 204 | " [coef_dc.sel(band=\"sig0_gtc\"), gam0_db.expand_dims(band=[\"gam0_gtc\"])], dim=\"band\"\n", 205 | ")\n", 206 | "\n", 207 | "coef_dc.hvplot.image(\n", 208 | " x=\"x\",\n", 209 | " y=\"y\",\n", 210 | " robust=False,\n", 211 | " data_aspect=1,\n", 212 | " cmap=\"Greys_r\",\n", 213 | " groupby=\"band\",\n", 214 | " rasterize=True,\n", 215 | ").opts(frame_height=600, framewise=False, aspect=\"equal\")" 216 | ] 217 | }, 218 | { 219 | "cell_type": "markdown", 220 | "id": "12", 221 | "metadata": {}, 222 | "source": [ 223 | "*Figure 6: $\\sigma^0_E$, and $\\gamma^0_E$ can be selected on the right-hand side of the graphic.*\n", 224 | "\n", 225 | "Comparing $\\sigma^0_E$ and $\\gamma^0_E$ in the figure, we can see that both look identical except for the range. This is because the only difference between $\\sigma^0_E$ and $\\gamma^0_E$ is the change of the reference area. While $\\sigma^0_E$ is defined to be ground range, $\\gamma^0_E$ is defined to be in the plane perpendicular to the line of sight from the sensor. This way, $\\gamma^0_E$ mitigates the impact of the incidence angle. However, $\\gamma^0_E$ is still based on the ellipsoid and does not account for the impact of the terrain on the radiometry.\n", 226 | "\n", 227 | "# Radiometric Terrain Correction\n", 228 | "\n", 229 | "So far, we corrected geometric distortions and compared the impact of the choice of the reference area. However, we still haven't corrected the backscatter intensity of pixels which are distorted by the terrain. In this last step, we will show that we can also correct radiometric artifacts to a certain degree. For this, we will load radiometrically terrain corrected (`rtc`) $\\gamma^0_T$ and plot it along the other coefficients.\n" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "id": "13", 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "rtc_dc = cat.rtc.read().compute()\n", 240 | "\n", 241 | "# add to existing cube\n", 242 | "rtc_dc = xr.concat([coef_dc, rtc_dc], dim=\"band\")\n", 243 | "\n", 244 | "rtc_dc.hvplot.image(\n", 245 | " x=\"x\",\n", 246 | " y=\"y\",\n", 247 | " robust=True,\n", 248 | " data_aspect=1,\n", 249 | " cmap=\"Greys_r\",\n", 250 | " groupby=\"band\",\n", 251 | " rasterize=True,\n", 252 | ").opts(frame_height=600, framewise=False, aspect=\"equal\")" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "id": "14", 258 | "metadata": {}, 259 | "source": [ 260 | "*Figure 7: $\\sigma^0_E$, $\\gamma^0_E$, and $\\gamma^0_T$ can be selected on the right-hand side of the graphic.*\n", 261 | "\n", 262 | "When comparing $\\gamma^0_E$ and $\\gamma^0_T$ in the plot we can clearly see the impact of the radiometric correction in the mountainous areas. This correction is necessary, because for slopes facing towards the sensor, a larger ground area contributes to the backscatter value of a slant range resolution cell, than for slopes lying in the opposite direction. This results in significant brightness changes, where foreshortening areas appear brighter and lengthening areas darker. $\\gamma^0_T$ adjusts the backscatter to represent what it would be if the terrain was flat, thus reducing these effects. This significantly reduces the impact of the terrain on the backscatter values, allowing for more accurate comparisons across different terrain types and locations. The correction is done by using a DEM to determine the local illuminated area at each radar position. The above illustrated approach is also referred to as terrain flattening because in the resulting image, mountains appear flat. As $\\gamma^0_T$ is corrected for geometric and radiometric distortions, it is also referred to as Normalized Radar Backscatter (NRB) and is the current standard for Analysis-Ready-Backscatter (ARD)." 263 | ] 264 | } 265 | ], 266 | "metadata": { 267 | "kernelspec": { 268 | "display_name": "Python 3 (ipykernel)", 269 | "language": "python", 270 | "name": "python3" 271 | }, 272 | "language_info": { 273 | "codemirror_mode": { 274 | "name": "ipython", 275 | "version": 3 276 | }, 277 | "file_extension": ".py", 278 | "mimetype": "text/x-python", 279 | "name": "python", 280 | "nbconvert_exporter": "python", 281 | "pygments_lexer": "ipython3", 282 | "version": "3.11.11" 283 | } 284 | }, 285 | "nbformat": 4, 286 | "nbformat_minor": 5 287 | } 288 | -------------------------------------------------------------------------------- /notebooks/courses/microwave-remote-sensing/04_in_class_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Datacubes\n", 9 | "\n", 10 | "\n", 11 | "\n", 12 | "In this notebook we discuss how we can easily compare images of two or more different time slices, satellites or other earth observation products. We limit our selves to products on a regular grid with an associated coordinate reference system (CRS), known as a raster. This means that each cell of the raster contains an attribute value and location coordinates. The process of combining such rasters to form datacubes is called raster stacking. We can have datacubes in many forms, such as the spatiotemporal datacube:\n", 13 | "\n", 14 | "$$Z = f(x,y,t) \\quad \\text{,}$$\n", 15 | "\n", 16 | "or when dealing with electromagnetic spectrum, the spectral wavelengths may form an additional dimension of a cube:\n", 17 | "\n", 18 | "$$Z = f(x,y,t, \\lambda ) \\quad \\text{.} $$\n", 19 | "\n", 20 | "We also have already encountered the case where $Z$ consists of multiple variables, such as seen in the `xarray` dataset.\n", 21 | "\n", 22 | "$${Z_1,Z_2,...,Z_3} = f(x,y,t) $$\n", 23 | "\n", 24 | "To perform raster stacking, we generally follow a certain routine (see also Figure 1).\n", 25 | "\n", 26 | "1. Collect data (GeoTIFF, NetCDF, Zarr)\n", 27 | "2. Select an area of interest\n", 28 | "3. Reproject all rasters to the same projection, resolution, and region\n", 29 | "4. Stack the individual rasters\n", 30 | "\n", 31 | "To get the same projection, resolution, and region we have to resample one (or more) products. The desired projection, resolution, and region can be adopted from one of the original rasters or it can be a completely new projection of the data.\n", 32 | "\n", 33 | "![](https://eox.at/images/eodcaas-mosaic-data-cube-kopp.png)\n", 34 | "\n", 35 | "*Figure 1: Stacking of arrays to form datacubes (source: https://eox.at)*.\n", 36 | "\n", 37 | "In this notebook we will study two different SAR products. SAR data from the Advanced Land Observing Satellite (Alos-2), which is a Japanese platform with an L-band sensor from the Japan Aerospace Exploration Agency (JAXA), and C-band data from the Copernicus Sentinel-1 mission. It is our goal to compare C- with L-band, so we need to somehow stack these arrays.\n", 38 | "\n" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "id": "1", 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "from functools import partial\n", 49 | "from pathlib import Path\n", 50 | "\n", 51 | "import folium\n", 52 | "import hvplot.xarray # noqa: F401\n", 53 | "import numpy as np\n", 54 | "import pandas as pd\n", 55 | "import rioxarray # noqa: F401\n", 56 | "import xarray as xr\n", 57 | "from huggingface_hub import snapshot_download\n", 58 | "from rasterio.enums import Resampling\n", 59 | "from shapely import affinity\n", 60 | "from shapely.geometry import box, mapping" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "2", 66 | "metadata": {}, 67 | "source": [ 68 | "## Download Data\n", 69 | "\n", 70 | "For this exercise we will download a set of GeoTIFF files for both Sentinel-1 and Alos-2, where each image has it's own timestamp for the acquisition.\n" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "id": "3", 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "data_path = snapshot_download(\n", 81 | " repo_id=\"martinschobben/microwave-remote-sensing\",\n", 82 | " repo_type=\"dataset\",\n", 83 | " allow_patterns=\"*.tif\",\n", 84 | ")\n", 85 | "data_path = Path(data_path)" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "id": "4", 91 | "metadata": {}, 92 | "source": [ 93 | "## Loading Data\n", 94 | "\n", 95 | "Before loading the data into memory we will first look at the area covered by the Sentinel-1 dataset on a map. This way we can select a region of interest for our hypothetical study. We will extract and transform the bounds of the data to longitude and latitude.\n" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "id": "5", 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "bbox = xr.open_mfdataset(\n", 106 | " (data_path / \"sentinel-1\").glob(\"**/*.tif\"),\n", 107 | " engine=\"rasterio\",\n", 108 | " combine=\"nested\",\n", 109 | " concat_dim=\"band\",\n", 110 | ").rio.transform_bounds(\"EPSG:4326\")\n", 111 | "\n", 112 | "bbox = box(*bbox)\n", 113 | "\n", 114 | "map = folium.Map(\n", 115 | " max_bounds=True,\n", 116 | " location=[bbox.centroid.y, bbox.centroid.x],\n", 117 | " scrollWheelZoom=False,\n", 118 | ")\n", 119 | "\n", 120 | "# bounds of image\n", 121 | "folium.GeoJson(mapping(bbox), name=\"Area of Interest\", color=\"red\").add_to(map)\n", 122 | "\n", 123 | "# minimum longitude, minimum latitude, maximum longitude, maximum latitude\n", 124 | "area_of_interest = box(10.3, 45.5, 10.6, 45.6)\n", 125 | "\n", 126 | "folium.GeoJson(mapping(area_of_interest), name=\"Area of Interest\").add_to(map)\n", 127 | "\n", 128 | "map" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "id": "6", 134 | "metadata": {}, 135 | "source": [ 136 | "*Figure 2: Map of study area. Red rectangle is the area covered by the Sentinel-1 raster. Blue rectangle is the area of interest.*\n", 137 | "\n", 138 | "On the map we have drawn rectangles of the area covered by the images and of our selected study area. To prevent loading too much data we will now only load the data as defined by the blue rectangle on the `folium` map.\n", 139 | "\n", 140 | "The Sentinel-1 data is now stored on disk as separate two-dimensional GeoTIFF files with a certain timestamp. The following `s1_preprocess` function allows to load all files in one go as a spatiotemporal datacube. Basically, the preprocessing function helps reading the timestamp from the file and adds this as a new dimension to the array. The latter allows a concatenation procedure where all files are joined along the new time dimension. In addition by providing `area_of_interest.bounds` to the parameter `bbox` we will only load the data of the previously defined area of interest.\n" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "id": "7", 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "def s1_preprocess(x, bbox, scale):\n", 151 | " \"\"\"\n", 152 | " Preprocess file.\n", 153 | "\n", 154 | " Parameters\n", 155 | " ----------\n", 156 | " x : xarray.Dataset\n", 157 | " bbox: tuple\n", 158 | " minimum longitude minimum latitude maximum longitude maximum latitude\n", 159 | " scale: float\n", 160 | " scaling factor\n", 161 | " Returns\n", 162 | " -------\n", 163 | " xarray.Dataset\n", 164 | " \"\"\"\n", 165 | "\n", 166 | " path = Path(x.encoding[\"source\"])\n", 167 | " filename = path.name\n", 168 | " x = x.rio.clip_box(*bbox, crs=\"EPSG:4326\")\n", 169 | "\n", 170 | " date_str = filename.split(\"_\")[0][1:]\n", 171 | " time_str = filename.split(\"_\")[1][:6]\n", 172 | " datetime_str = date_str + time_str\n", 173 | " date = pd.to_datetime(datetime_str, format=\"%Y%m%d%H%M%S\")\n", 174 | " x = x.expand_dims(dim={\"time\": [date]})\n", 175 | "\n", 176 | " x = (\n", 177 | " x.rename({\"band_data\": \"s1_\" + path.parent.stem})\n", 178 | " .squeeze(\"band\")\n", 179 | " .drop_vars(\"band\")\n", 180 | " )\n", 181 | "\n", 182 | " return x * scale" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "id": "8", 188 | "metadata": {}, 189 | "source": [ 190 | "We load the data again with `open_mfdataset` and by providing the preprocess function, including the bounds of the area of interest and the scaling factor, as follows:\n" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "id": "9", 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "partial_ = partial(s1_preprocess, bbox=area_of_interest.bounds, scale=0.01)\n", 201 | "\n", 202 | "s1_ds = xr.open_mfdataset(\n", 203 | " (data_path / \"sentinel-1\").glob(\"**/*.tif\"),\n", 204 | " engine=\"rasterio\",\n", 205 | " combine=\"nested\",\n", 206 | " chunks=-1,\n", 207 | " preprocess=partial_,\n", 208 | ")" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "id": "10", 214 | "metadata": {}, 215 | "source": [ 216 | "## Unlocking Geospatial Information\n", 217 | "\n", 218 | "To enable further stacking of ALOS-2 and Sentinel-1 data we need to know some more information about the raster. Hence we define the following function `print_raster` to get the projection (CRS), resolution, and region (bounds). The function leverages the functionality of `rioxarray`; a package for rasters.\n" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": null, 224 | "id": "11", 225 | "metadata": {}, 226 | "outputs": [], 227 | "source": [ 228 | "def print_raster(raster, name):\n", 229 | " \"\"\"\n", 230 | " Print Raster Metadata\n", 231 | "\n", 232 | " Parameters\n", 233 | " ----------\n", 234 | " raster: xarray.DataArray|xarray.DataSet\n", 235 | " raster to process\n", 236 | " y: string\n", 237 | " name of product\n", 238 | " \"\"\"\n", 239 | "\n", 240 | " print(\n", 241 | " f\"{name} Raster: \\n----------------\\n\"\n", 242 | " f\"resolution: {raster.rio.resolution()} {raster.rio.crs.units_factor}\\n\" # noqa\n", 243 | " f\"bounds: {raster.rio.bounds()}\\n\"\n", 244 | " f\"CRS: {raster.rio.crs}\\n\"\n", 245 | " )\n", 246 | "\n", 247 | "\n", 248 | "print_raster(s1_ds, \"Sentinel-1\")" 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "id": "12", 254 | "metadata": {}, 255 | "source": [ 256 | "The CRS \"EPSG 27704\" is part of the EQUI7Grid. This grid provides equal-area tiles, meaning each tile represents the same area, which helps reducing distorsions. This feature is important for remote sensing as it reduces the so-called oversampling due to geometric distortions when projecting on a sphere. This particular projection is developed by TUWien.\n", 257 | "\n", 258 | "Now we will proceed with loading the ALOS-2 L-band data in much the same fashion as for Sentinel-1. Again timeslices are stored separately as individual GeoTIFFS and they need to be concatenated along the time dimension. We use a slightly different preprocessing function `alos_preprocess` for this purpose. The most notable difference of this function is the inclusion of a scaling factor for the 16-bit digital numbers (DN):\n", 259 | "\n", 260 | "$$\\gamma^0_T = 10 * log_{10}(\\text{DN}^2) - 83.0 \\,dB$$\n", 261 | "\n", 262 | "to correctly convert the integers to $\\gamma^0_T$ in the dB range.\n" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": null, 268 | "id": "13", 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [ 272 | "def alos_preprocess(x, bbox):\n", 273 | " \"\"\"\n", 274 | " Preprocess file.\n", 275 | "\n", 276 | " Parameters\n", 277 | " ----------\n", 278 | " x : xarray.Dataset\n", 279 | " bbox: tuple\n", 280 | " minimum longitude minimum latitude maximum longitude maximum latitude\n", 281 | " Returns\n", 282 | " -------\n", 283 | " xarray.Dataset\n", 284 | " \"\"\"\n", 285 | "\n", 286 | " path = Path(x.encoding[\"source\"])\n", 287 | " filename = path.name\n", 288 | " x = x.rio.clip_box(*bbox, crs=\"EPSG:4326\")\n", 289 | "\n", 290 | " date_str = filename.split(\"_\")[0][15:22]\n", 291 | " date = pd.to_datetime(date_str, format=\"%y%m%d\")\n", 292 | " x = x.expand_dims(dim={\"time\": [date]})\n", 293 | "\n", 294 | " x = (\n", 295 | " x.rename({\"band_data\": \"alos_\" + path.parent.stem})\n", 296 | " .squeeze(\"band\")\n", 297 | " .drop_vars(\"band\")\n", 298 | " )\n", 299 | "\n", 300 | " # conversion to dB scale of alos\n", 301 | " return 10 * np.log10(x**2) - 83.0" 302 | ] 303 | }, 304 | { 305 | "cell_type": "markdown", 306 | "id": "14", 307 | "metadata": {}, 308 | "source": [ 309 | "Now we load the data with the `open_mfdataset` function of `xarray` and we provide the preprocessing function (see above), which includes the selection of the bounds of an area of interest and the extraction of time stamps from the file name.\n" 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": null, 315 | "id": "15", 316 | "metadata": {}, 317 | "outputs": [], 318 | "source": [ 319 | "area_of_interest = affinity.scale(area_of_interest, xfact=1.7, yfact=1.7)\n", 320 | "partial_ = partial(alos_preprocess, bbox=area_of_interest.bounds)\n", 321 | "\n", 322 | "alos_ds = xr.open_mfdataset(\n", 323 | " (data_path / \"alos-2\").glob(\"**/*.tif\"),\n", 324 | " engine=\"rasterio\",\n", 325 | " combine=\"nested\",\n", 326 | " chunks=-1,\n", 327 | " preprocess=partial_,\n", 328 | ")" 329 | ] 330 | }, 331 | { 332 | "cell_type": "markdown", 333 | "id": "16", 334 | "metadata": {}, 335 | "source": [ 336 | "Also, for this dataset we will look at the metadata in order to compare it with Sentinel-1.\n" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": null, 342 | "id": "17", 343 | "metadata": {}, 344 | "outputs": [], 345 | "source": [ 346 | "print_raster(alos_ds, \"ALOS-2\")" 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "id": "18", 352 | "metadata": {}, 353 | "source": [ 354 | "## Reprojecting\n", 355 | "\n", 356 | "The ALOS-2 is projected on an UTM grid. We would therefore like to reproject this data to match the projection of Sentinel-1. Furthermore, we will upsample the data to match the Sentinel-1 sampling. The `rioxarray` package has a very convenient method that can do this all in one go:`reproject_match`. For continuous data it is best to use a bilinear resampling strategy. As always you have to consider again that we deal with values in the dB range, so we need to convert to the linear scale before bilinear resampling.\n" 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": null, 362 | "id": "19", 363 | "metadata": {}, 364 | "outputs": [], 365 | "source": [ 366 | "alos_ds_lin = 10 ** (alos_ds / 10)\n", 367 | "alos_ds_lin = alos_ds_lin.rio.reproject_match(\n", 368 | " s1_ds,\n", 369 | " resampling=Resampling.bilinear,\n", 370 | ")\n", 371 | "alos_ds = 10 * np.log10(alos_ds_lin)" 372 | ] 373 | }, 374 | { 375 | "cell_type": "markdown", 376 | "id": "20", 377 | "metadata": {}, 378 | "source": [ 379 | "We will overwrite the coordinate values of ALOS-2 with those of Sentinel-1. If we would not do this last step, small errors in how the numbers are stored would prevent stacking of the rasters.\n" 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": null, 385 | "id": "21", 386 | "metadata": {}, 387 | "outputs": [], 388 | "source": [ 389 | "alos_ds = alos_ds.assign_coords(\n", 390 | " {\n", 391 | " \"x\": s1_ds.x.data,\n", 392 | " \"y\": s1_ds.y.data,\n", 393 | " }\n", 394 | ")" 395 | ] 396 | }, 397 | { 398 | "cell_type": "markdown", 399 | "id": "22", 400 | "metadata": {}, 401 | "source": [ 402 | "Lastly, we will turn the `xarray.DataSet` to an `xarray.DataArray` where a new dimension will constitute the sensor for measurement (satellite + polarization).\n" 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": null, 408 | "id": "23", 409 | "metadata": {}, 410 | "outputs": [], 411 | "source": [ 412 | "s1_da = s1_ds.to_array(dim=\"sensor\")\n", 413 | "alos_da = alos_ds.to_array(dim=\"sensor\")\n", 414 | "s1_da" 415 | ] 416 | }, 417 | { 418 | "cell_type": "markdown", 419 | "id": "24", 420 | "metadata": {}, 421 | "source": [ 422 | "## Stacking of Multiple Arrays\n", 423 | "\n", 424 | "Now we are finally ready to stack Sentinel-1 C-band and ALOS-2 L-band arrays with the function `concat` of `xarray`. Now we can use the newly defined `\"sensor\"` dimension to concatenate the two arrays.\n" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": null, 430 | "id": "25", 431 | "metadata": {}, 432 | "outputs": [], 433 | "source": [ 434 | "fused_da = xr.concat([s1_da, alos_da], dim=\"sensor\").rename(\"gam0\")\n", 435 | "fused_da" 436 | ] 437 | }, 438 | { 439 | "cell_type": "markdown", 440 | "id": "26", 441 | "metadata": {}, 442 | "source": [ 443 | "The measurements for both satellites don't occur at the same time. Hence the cube is now padded with 2-D arrays entirely filled with NaN (Not A Number) for some time slices. As we have learned in notebook 2 we can use the `resample` method to make temporally coherent timeslices for each month. To deal with the dB scale backscatter values as well as the low number of observations per month we use a median of the samples. As taking the median only sorts the samples according to the sample quantiles we do not have to convert the observations to the linear scale.\n" 444 | ] 445 | }, 446 | { 447 | "cell_type": "code", 448 | "execution_count": null, 449 | "id": "27", 450 | "metadata": {}, 451 | "outputs": [], 452 | "source": [ 453 | "fused_da = fused_da.resample(time=\"ME\", skipna=True).median().compute()" 454 | ] 455 | }, 456 | { 457 | "cell_type": "markdown", 458 | "id": "28", 459 | "metadata": {}, 460 | "source": [ 461 | "We can plot each of the variables: \"ALOS-2\" and \"Sentinel-1\" to check our results.\n" 462 | ] 463 | }, 464 | { 465 | "cell_type": "code", 466 | "execution_count": null, 467 | "id": "29", 468 | "metadata": {}, 469 | "outputs": [], 470 | "source": [ 471 | "fused_da.hvplot.image(robust=True, data_aspect=1, cmap=\"Greys_r\", rasterize=True).opts(\n", 472 | " frame_height=600, aspect=\"equal\"\n", 473 | ")" 474 | ] 475 | }, 476 | { 477 | "cell_type": "markdown", 478 | "id": "30", 479 | "metadata": {}, 480 | "source": [ 481 | "*Figure 3: Stacked array with ALOS-2 L-band and Sentinel-1 C-band $\\gamma^0_T (dB)$.*" 482 | ] 483 | } 484 | ], 485 | "metadata": { 486 | "kernelspec": { 487 | "display_name": "Python 3 (ipykernel)", 488 | "language": "python", 489 | "name": "python3" 490 | }, 491 | "language_info": { 492 | "codemirror_mode": { 493 | "name": "ipython", 494 | "version": 3 495 | }, 496 | "file_extension": ".py", 497 | "mimetype": "text/x-python", 498 | "name": "python", 499 | "nbconvert_exporter": "python", 500 | "pygments_lexer": "ipython3", 501 | "version": "3.11.11" 502 | } 503 | }, 504 | "nbformat": 4, 505 | "nbformat_minor": 5 506 | } 507 | -------------------------------------------------------------------------------- /notebooks/courses/microwave-remote-sensing/05_in_class_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Wavelength and Polarization\n", 9 | "\n", 10 | "\n", 11 | "\n", 12 | "In this notebook, we aim to demonstrate how C-band (4–8 GHz, wavelengths of approximately 3.75–7.5 cm) and L-band (1–2 GHz, wavelengths of approximately 15–30 cm) radio frequencies differ for different land covers and times of the year. In addition, we'll look at co- and cross-polarized backscattering:\n", 13 | "\n", 14 | "+ Sentinel-1 (C-band)\n", 15 | " + VV\n", 16 | " + VH\n", 17 | "+ Alos-2 (L-band):\n", 18 | " + HH\n", 19 | " + HV\n", 20 | "\n" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "id": "1", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "import holoviews as hv\n", 31 | "import hvplot.xarray # noqa: F401\n", 32 | "import intake\n", 33 | "import matplotlib.pyplot as plt\n", 34 | "import numpy as np" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "id": "2", 40 | "metadata": {}, 41 | "source": [ 42 | "## Data Loading\n", 43 | "\n", 44 | "We load the data again with the help of `intake`.\n" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "id": "3", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "url = \"https://huggingface.co/datasets/martinschobben/microwave-remote-sensing/resolve/main/microwave-remote-sensing.yml\"\n", 55 | "cat = intake.open_catalog(url)\n", 56 | "fused_ds = cat.fused.read()\n", 57 | "fused_ds" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "id": "4", 63 | "metadata": {}, 64 | "source": [ 65 | "The loaded data contains the Leaf Area Index (LAI), which is used as an estimate of foliage cover of forest canopies. So high LAI is interpreted as forested area, whereas low values account for less vegetated areas (shrubs, grass-land, and crops).\n", 66 | "\n", 67 | "First we'll have a look at the mean and standard deviation of LAI over all timeslices. This can be achieved by using the `mean` and `std` methods of the `xarray` object and by supplying a dimension over which these aggregating operations will be applied. We use the dimension \"time\", thereby flattening the cube to a 2-D array with dimensions x and y.\n" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "id": "5", 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "fig, ax = plt.subplots(1, 2, figsize=(15, 6))\n", 78 | "\n", 79 | "LAI_dc = fused_ds.LAI\n", 80 | "LAI_mean = LAI_dc.mean(\"time\")\n", 81 | "LAI_std = LAI_dc.std(\"time\")\n", 82 | "\n", 83 | "LAI_mean.plot(ax=ax[0], vmin=0, vmax=6).axes.set_aspect(\"equal\")\n", 84 | "LAI_std.plot(ax=ax[1], vmin=0, vmax=3).axes.set_aspect(\"equal\")\n", 85 | "plt.tight_layout()" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "id": "6", 91 | "metadata": {}, 92 | "source": [ 93 | "*Figure 1: Map of mean LAI (left) and the associated standard deviation (right) for each pixel over time around Lake Garda.*\n", 94 | "\n", 95 | "It appears that the northern parts of our study area contain more and variable amounts of green elements per unit area. This might indicate a more complete coverage of foliage and thus forest.\n", 96 | "\n", 97 | "## Timeseries\n", 98 | "\n", 99 | "Now that we have detected possible forested areas, let's delve a bit deeper into the data. Remember that we deal with a spatiotemporal datacube. This gives us the possibility to study changes for each time increment. Hence we can show what happens to LAI for areas marked with generally low values as well as high values. We can achieve this by filtering the datacube with the `where` method for areas marked with low and high mean LAI values. In turn we will aggregate the remaining datacube over the spatial dimensions (\"x\" and \"y\") to get a mean values for each time increment.\n" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "id": "7", 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "fig, ax = plt.subplots(1, 2, figsize=(15, 4))\n", 110 | "\n", 111 | "LAI_low = LAI_dc.where(LAI_mean < 4)\n", 112 | "LAI_high = LAI_dc.where(LAI_mean > 4)\n", 113 | "\n", 114 | "LAI_low.mean([\"x\", \"y\"]).plot.scatter(x=\"time\", ax=ax[0], ylim=(0, 6))\n", 115 | "LAI_high.mean([\"x\", \"y\"]).plot.scatter(x=\"time\", ax=ax[1], ylim=(0, 6))\n", 116 | "ax[0].set_title(\"Low Mean LAI ($\\\\bar{LAI} < 4$)\")\n", 117 | "ax[1].set_title(\"High Mean LAI ($\\\\bar{LAI} > 4$)\")\n", 118 | "plt.tight_layout()" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "id": "8", 124 | "metadata": {}, 125 | "source": [ 126 | "*Figure 2: Timeseries of mean LAI per timeslice for areas with low (left) and high (right) mean LAI of Figure1.*\n", 127 | "\n", 128 | "Now we can see that areas with high mean LAI values (Figure 1) show a drop-off to values as low as those for areas with low mean LAI during the autumn months (Figure 2 ; right panel). Hence we might deduce that we deal with deciduous forest that becomes less green during autumn, as can be expected for the study area.\n", 129 | "\n", 130 | "Remember that longer wavelengths like L-bands are more likely to penetrate through a forest canopy and would interact more readily with larger object like tree trunks and the forest floor. In turn, C-band microwaves are more likely to interact with sparse and shrub vegetation. The polarization of the emitted and received microwaves is on the other hand dependent on the type of backscattering with co-polarization (HH and VV) happening more frequently with direct backscatter or double bounce scattering. Whereas volume scattering occurs when the radar signal is subject to multiple reflections within 3-dimensional matter, as the orientation of the main scatterers is random, the polarization of the backscattered signal is also random. Volume scattering can therefore cause an increase of cross-polarized intensity.\n", 131 | "\n", 132 | "Let's put this to the test by checking the microwave backscatter signatures over forested and sparsely vegetated areas as well as water bodies (Lake Garda). Let's first look at the different sensor readings for the beginning of summer and autumn.\n" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "id": "9", 139 | "metadata": {}, 140 | "outputs": [], 141 | "source": [ 142 | "hv.output(widget_location=\"bottom\")\n", 143 | "\n", 144 | "t1 = (\n", 145 | " fused_ds.gam0.isel(time=2)\n", 146 | " .hvplot.image(\n", 147 | " robust=True, data_aspect=1, cmap=\"Greys_r\", rasterize=True, clim=(-25, 0)\n", 148 | " )\n", 149 | " .opts(frame_height=400, aspect=\"equal\")\n", 150 | ")\n", 151 | "\n", 152 | "t2 = (\n", 153 | " fused_ds.gam0.isel(time=-1)\n", 154 | " .hvplot.image(\n", 155 | " robust=True, data_aspect=1, cmap=\"Greys_r\", rasterize=True, clim=(-25, 0)\n", 156 | " )\n", 157 | " .opts(frame_height=400, aspect=\"equal\")\n", 158 | ")\n", 159 | "\n", 160 | "t1 + t2" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "id": "10", 166 | "metadata": {}, 167 | "source": [ 168 | "*Figure 3: Maps of Sentinel-1 and Alos-2 $\\gamma^0_T \\,[dB]$ for the beginning of summer (left) and autumn (right).*\n", 169 | "\n", 170 | "The most notable difference is the lower energy received for cross-polarized than for co-polarized microwaves for both Sentinel-1 and Alos-2. The latter differences are independent of the time of year. However, one can also note small changes in the received energy for the same satellite dependent on the time of year. To get a better feel for these changes over time we generate the following interactive plot. On the following plot one can select areas of a certain mean LAI (by clicking on the map) and see the associated timeseries of $\\gamma^0_T$ for each of the sensors.\n" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": null, 176 | "id": "11", 177 | "metadata": {}, 178 | "outputs": [], 179 | "source": [ 180 | "LAI_image = LAI_mean.hvplot.image(rasterize=True, cmap=\"viridis\", clim=(0, 6)).opts(\n", 181 | " title=\"Mean LAI (Selectable)\", frame_height=400, aspect=\"equal\"\n", 182 | ")\n", 183 | "\n", 184 | "\n", 185 | "def get_timeseries(x, y):\n", 186 | " \"\"\"\n", 187 | " Callback Function Holoviews\n", 188 | "\n", 189 | " Parameters\n", 190 | " ----------\n", 191 | " x: float\n", 192 | " numeric value for x selected on LAI map\n", 193 | " y: float\n", 194 | " numeric value for y selected on LAI map\n", 195 | " \"\"\"\n", 196 | "\n", 197 | " lai_value = LAI_mean.sel(x=x, y=y, method=\"nearest\").values\n", 198 | "\n", 199 | " if np.isnan(lai_value):\n", 200 | " select = fused_ds.where(LAI_mean.isnull())\n", 201 | " label = \"Water\"\n", 202 | " else:\n", 203 | " mask = np.isclose(LAI_mean, lai_value, atol=0.05)\n", 204 | " select = fused_ds.where(mask)\n", 205 | " label = \"Mean LAI: \" + str(np.round(lai_value, 1))\n", 206 | "\n", 207 | " time_series = (\n", 208 | " select.gam0.to_dataset(\"sensor\")\n", 209 | " .median([\"x\", \"y\"], skipna=True)\n", 210 | " .hvplot.scatter(ylim=(-30, 5))\n", 211 | " .opts(title=label, frame_height=400)\n", 212 | " )\n", 213 | "\n", 214 | " return time_series\n", 215 | "\n", 216 | "\n", 217 | "point_stream = hv.streams.SingleTap(source=LAI_image)\n", 218 | "time_series = hv.DynamicMap(get_timeseries, streams=[point_stream])\n", 219 | "LAI_image + time_series" 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "id": "12", 225 | "metadata": {}, 226 | "source": [ 227 | "*Figure 4: Map of MEAN LAI around Lake Garda. The pixel values can be seen by hovering your mouse over the pixels. Clicking on the pixel will generate the timeseries for the associated mean LAI on the right hand-side. (Right) Timeseries of for Sentinel-1 and Alos-2 $\\gamma^0_T [dB]$.*\n", 228 | "\n", 229 | "Can you see some patterns when analyzing the different wavelengths and polarizations?\n", 230 | "\n", 231 | "Remember again that we deal with a logarithmic scale. A measurement of 10 dB is 10 times brighter than the intensity measured at 0 dB, and 100 times brighter at 20 dB. The most notable difference is that the offset between cross- and co-polarised signals becomes larger at low LAI and lower at higher LAI. This might indicate the effect of volume scattering in forested areas where co- and cross-polarization render backscattering values more equal. You will study the differences among cross- and co-polarized backscattering in more detail in the homework exercise." 232 | ] 233 | } 234 | ], 235 | "metadata": { 236 | "kernelspec": { 237 | "display_name": "Python 3 (ipykernel)", 238 | "language": "python", 239 | "name": "python3" 240 | }, 241 | "language_info": { 242 | "codemirror_mode": { 243 | "name": "ipython", 244 | "version": 3 245 | }, 246 | "file_extension": ".py", 247 | "mimetype": "text/x-python", 248 | "name": "python", 249 | "nbconvert_exporter": "python", 250 | "pygments_lexer": "ipython3", 251 | "version": "3.11.11" 252 | } 253 | }, 254 | "nbformat": 4, 255 | "nbformat_minor": 5 256 | } 257 | -------------------------------------------------------------------------------- /notebooks/courses/microwave-remote-sensing/06_in_class_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Dielectric Properties\n", 9 | "\n", 10 | "\n", 11 | "\n", 12 | "In this notebook, we will investigate the varying backscatter values associated with different land surfaces like water bodies, forests, grasslands and urban areas. We will use backscatter data from the Sentinel-1 satellite and we will utilize the CORINE Land Cover dataset to classify and extrapolate these surfaces, enabling us to analyze how different land cover types influence backscatter responses.\n", 13 | "\n" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "id": "1", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import json\n", 24 | "\n", 25 | "import holoviews as hv\n", 26 | "import intake\n", 27 | "import matplotlib.patches as mpatches\n", 28 | "import matplotlib.pyplot as plt\n", 29 | "import numpy as np\n", 30 | "import rioxarray # noqa: F401\n", 31 | "import xarray as xr\n", 32 | "from holoviews.streams import RangeXY\n", 33 | "from matplotlib.colors import BoundaryNorm, ListedColormap\n", 34 | "\n", 35 | "hv.extension(\"bokeh\")" 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "id": "2", 41 | "metadata": {}, 42 | "source": [ 43 | "## Load Sentinel-1 Data\n", 44 | "\n", 45 | "For our analysis we are using sigma naught backscatering data from Sentinel-1. The images we are analyzing cover the region south of Vienna and west of Lake Neusiedl. We load the data and and apply again a preprocessing function. Here we extract the scaling factor and the date the image was taken from the metadata. We will focus our attention to a smaller area containing a part of the Lake Neusiedl Lake and its surrounding land. The obtained`xarray` dataset and is then converted to an array, because we only have one variable, the VV backscatter values.\n" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "id": "3", 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "url = \"https://huggingface.co/datasets/martinschobben/microwave-remote-sensing/resolve/main/microwave-remote-sensing.yml\"\n", 56 | "cat = intake.open_catalog(url)\n", 57 | "sig0_da = cat.neusiedler.read().sig0.compute()" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "id": "4", 63 | "metadata": {}, 64 | "source": [ 65 | "Let's have a look at the data by plotting the first timeslice.\n" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": null, 71 | "id": "5", 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "sig0_da.isel(time=0).plot(robust=True, cmap=\"Greys_r\").axes.set_aspect(\"equal\")" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "id": "6", 81 | "metadata": {}, 82 | "source": [ 83 | "## Load CORINE Landcover Data\n", 84 | "\n", 85 | "We will load the CORINE Land Cover, which is a pan-European land cover and land use inventory with 44 thematic classes. The resolution of this classification is 100 by 100m and the file was created in 2018\n", 86 | "([CORINE Land Cover](https://land.copernicus.eu/en/products/corine-land-cover)).\n" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "id": "7", 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "cor_da = cat.corine.read().land_cover.compute()" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "id": "8", 102 | "metadata": {}, 103 | "source": [ 104 | "### Colormapping and Encoding\n", 105 | "\n", 106 | "For the different land cover types we use the official color encoding.\n" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "id": "9", 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "# Load encoding\n", 117 | "with cat.corine_cmap.read()[0] as f:\n", 118 | " color_mapping_data = json.load(f)\n", 119 | "\n", 120 | "# Get mapping\n", 121 | "color_mapping = {item[\"value\"]: item for item in color_mapping_data[\"land_cover\"]}\n", 122 | "\n", 123 | "# Create cmap and norm for plotting\n", 124 | "colors = [info[\"color\"] for info in color_mapping.values()]\n", 125 | "categories = [info[\"value\"] for info in color_mapping.values()]\n", 126 | "cmap = ListedColormap(colors)\n", 127 | "norm = BoundaryNorm(categories + [max(categories) + 1], len(categories))" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "id": "10", 133 | "metadata": {}, 134 | "source": [ 135 | "Now we can plot the CORINE Land Cover dataset.\n" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "id": "11", 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "# Get landcover codes present in the image\n", 146 | "present_landcover_codes = np.unique(cor_da.values[~np.isnan(cor_da.values)].astype(int))\n", 147 | "\n", 148 | "# Get colors + text for legend\n", 149 | "handles = [\n", 150 | " mpatches.Patch(color=info[\"color\"], label=(f'{info[\"value\"]} - ' + (info[\"label\"])))\n", 151 | " for info in color_mapping.values()\n", 152 | " if info[\"value\"] in present_landcover_codes\n", 153 | "]\n", 154 | "\n", 155 | "# Create the plot\n", 156 | "cor_da.plot(figsize=(10, 10), cmap=cmap, norm=norm, add_colorbar=False).axes.set_aspect(\n", 157 | " \"equal\"\n", 158 | ")\n", 159 | "\n", 160 | "plt.legend(\n", 161 | " handles=handles,\n", 162 | " bbox_to_anchor=(1.01, 1),\n", 163 | " loc=\"upper left\",\n", 164 | " borderaxespad=0,\n", 165 | " fontsize=7,\n", 166 | ")\n", 167 | "plt.title(\"CORINE Land Cover (EPSG:27704)\")" 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "id": "12", 173 | "metadata": {}, 174 | "source": [ 175 | "Now we are ready to merge the backscatter data (`sig0_da`) with the land cover dataset (`cor_da`) to have one dataset combining all data.\n" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "id": "13", 182 | "metadata": {}, 183 | "outputs": [], 184 | "source": [ 185 | "var_ds = xr.merge([sig0_da, cor_da]).drop_vars(\"band\")\n", 186 | "var_ds" 187 | ] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "id": "14", 192 | "metadata": {}, 193 | "source": [ 194 | "## Backscatter Variability\n", 195 | "\n", 196 | "With this combined dataset we can study backscatter variability in relation to natural media. For example we can look at the backscatter variability for water by clipping the dataset to only contain the land cover class water, like so:\n" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": null, 202 | "id": "15", 203 | "metadata": {}, 204 | "outputs": [], 205 | "source": [ 206 | "# 41 = encoded value for water bodies\n", 207 | "waterbodies_mask = var_ds.land_cover == 41\n", 208 | "waterbodies_mask.plot().axes.set_aspect(\"equal\")" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "id": "16", 214 | "metadata": {}, 215 | "source": [ 216 | "This gives use backscatter values over water only.\n" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "id": "17", 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "waterbodies_sig0 = var_ds.sig0.isel(time=0).where(waterbodies_mask)\n", 227 | "waterbodies_sig0.plot(robust=True, cmap=\"Greys_r\").axes.set_aspect(\"equal\")" 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "id": "18", 233 | "metadata": {}, 234 | "source": [ 235 | "To get an idea of the variability we can create a histogram. Radar backscatter from water bodies fluctuates with surface roughness, which changes with wind conditions, creating spatial and temporal variations in signal intensity.\n" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": null, 241 | "id": "19", 242 | "metadata": {}, 243 | "outputs": [], 244 | "source": [ 245 | "waterbodies_sig0.plot.hist(bins=50, edgecolor=\"black\")" 246 | ] 247 | }, 248 | { 249 | "cell_type": "markdown", 250 | "id": "20", 251 | "metadata": {}, 252 | "source": [ 253 | "## Variability over Time\n", 254 | "\n", 255 | "Next we will look at the changes in variability in backscatter values over time for each of the CORINE Land Cover types. We do this by creating the following interactive plot. We can spot that backscatter in agricultural fields varies due to seasonal cycles like planting, growing, and harvesting, each of which changes vegetation structure. Changes in backscatter are strongly related to soil moisture content from irrigation or rainfall. Ultimately, phenological stages of crops and canopy moisture dynamics can affect the backscatter signal.\n" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "id": "21", 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "robust_min = var_ds.sig0.quantile(0.02).item()\n", 266 | "robust_max = var_ds.sig0.quantile(0.98).item()\n", 267 | "\n", 268 | "bin_edges = [\n", 269 | " i + j * 0.5\n", 270 | " for i in range(int(robust_min) - 2, int(robust_max) + 2)\n", 271 | " for j in range(2)\n", 272 | "]\n", 273 | "\n", 274 | "land_cover = {\"\\xa0\\xa0\\xa0 Complete Land Cover\": 1}\n", 275 | "land_cover.update(\n", 276 | " {\n", 277 | " f\"{int(value): 02} {color_mapping[value]['label']}\": int(value)\n", 278 | " for value in present_landcover_codes\n", 279 | " }\n", 280 | ")\n", 281 | "time = var_ds.sig0[\"time\"].values\n", 282 | "\n", 283 | "rangexy = RangeXY()\n", 284 | "\n", 285 | "\n", 286 | "def load_image(time, land_cover, x_range, y_range):\n", 287 | " \"\"\"\n", 288 | " Callback Function Landcover.\n", 289 | "\n", 290 | " Parameters\n", 291 | " ----------\n", 292 | " time: panda.datatime\n", 293 | " time slice\n", 294 | " landcover: int\n", 295 | " land cover type\n", 296 | " x_range: array_like\n", 297 | " longitude range\n", 298 | " y_range: array_like\n", 299 | " latitude range\n", 300 | "\n", 301 | " Returns\n", 302 | " -------\n", 303 | " holoviews.Image\n", 304 | " \"\"\"\n", 305 | "\n", 306 | " if land_cover == \"\\xa0\\xa0\\xa0 Complete Land Cover\":\n", 307 | " sig0_selected_ds = var_ds.sig0.sel(time=time)\n", 308 | "\n", 309 | " else:\n", 310 | " land_cover_value = int(land_cover.split()[0])\n", 311 | " mask_ds = var_ds.land_cover == land_cover_value\n", 312 | " sig0_selected_ds = var_ds.sig0.sel(time=time).where(mask_ds)\n", 313 | "\n", 314 | " hv_ds = hv.Dataset(sig0_selected_ds)\n", 315 | " img = hv_ds.to(hv.Image, [\"x\", \"y\"])\n", 316 | "\n", 317 | " if x_range and y_range:\n", 318 | " img = img.select(x=x_range, y=y_range)\n", 319 | "\n", 320 | " return hv.Image(img)\n", 321 | "\n", 322 | "\n", 323 | "dmap = (\n", 324 | " hv.DynamicMap(load_image, kdims=[\"Time\", \"Landcover\"], streams=[rangexy])\n", 325 | " .redim.values(Time=time, Landcover=land_cover)\n", 326 | " .hist(normed=True, bins=bin_edges)\n", 327 | ")\n", 328 | "\n", 329 | "image_opts = hv.opts.Image(\n", 330 | " cmap=\"Greys_r\",\n", 331 | " colorbar=True,\n", 332 | " tools=[\"hover\"],\n", 333 | " clim=(robust_min, robust_max),\n", 334 | " aspect=\"equal\",\n", 335 | " framewise=False,\n", 336 | " frame_height=500,\n", 337 | " frame_width=500,\n", 338 | ")\n", 339 | "\n", 340 | "hist_opts = hv.opts.Histogram(width=350, height=555)\n", 341 | "\n", 342 | "dmap.opts(image_opts, hist_opts)" 343 | ] 344 | } 345 | ], 346 | "metadata": { 347 | "kernelspec": { 348 | "display_name": "Python 3 (ipykernel)", 349 | "language": "python", 350 | "name": "python3" 351 | }, 352 | "language_info": { 353 | "codemirror_mode": { 354 | "name": "ipython", 355 | "version": 3 356 | }, 357 | "file_extension": ".py", 358 | "mimetype": "text/x-python", 359 | "name": "python", 360 | "nbconvert_exporter": "python", 361 | "pygments_lexer": "ipython3", 362 | "version": "3.11.11" 363 | } 364 | }, 365 | "nbformat": 4, 366 | "nbformat_minor": 5 367 | } 368 | -------------------------------------------------------------------------------- /notebooks/courses/microwave-remote-sensing/07_in_class_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Speckle Statistics\n", 9 | "\n", 10 | "\n", 11 | "This notebook will provide an empirical demonstration of speckle - how it originates, how it visually and statistically looks like, and some of the most common approaches to filter it.\n", 12 | "\n", 13 | "Speckle is defined as a kind of noise that affects all radar images. Given the multiple scattering contributions originating from the various elementary objects present within a resolution cell, the resulting backscatter signal can be described as a random constructive and destructive interference of wavelets. As a consequence, speckle is the reason why a granular pattern normally affects SAR images, making it more challenging to interpret and analyze them.\n", 14 | "\n", 15 | "![](../../images/speckle_effect.png)\n", 16 | "\n", 17 | "*Credits: ESRI*\n", 18 | "\n" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "id": "1", 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "import json\n", 29 | "from functools import partial\n", 30 | "\n", 31 | "import holoviews as hv\n", 32 | "import intake\n", 33 | "import matplotlib.pyplot as plt\n", 34 | "import numpy as np\n", 35 | "from holoviews.streams import RangeXY\n", 36 | "from scipy.ndimage import uniform_filter\n", 37 | "\n", 38 | "hv.extension(\"bokeh\")" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "id": "2", 44 | "metadata": {}, 45 | "source": [ 46 | "Let's make an example of a cornfield (with a typical backscattering value of about -10 dB). According to the following equation:\n", 47 | "\n", 48 | "$$\n", 49 | "\\sigma^0 = \\frac{1}{\\text{area}} \\sum_{n \\in \\text{area}} \\sigma_n\n", 50 | "$$\n", 51 | "\n", 52 | "We should ideally have a uniform discrete **sigma naught** $\\sigma^0$ value, given that the cornfield pixel is the only individual contributor.\n", 53 | "\n", 54 | "However, since we already learned from the previous notebooks that a pixel's ground size can be in the order of tens of meters (i.e., 10 meters for Sentinel-1), we can imagine that different distributed targets in the scene contribute to the global backscattered information.\n", 55 | "\n", 56 | "Let´s replicate this behavior with an ideal uniform area constituted by 100 pixels and then by adding 30% of speckle.\n" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "id": "3", 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "ideal_backscatter = -10 # in dB, a typical value for cornfields\n", 67 | "width = 12\n", 68 | "size = (width, width)\n", 69 | "ideal_data = np.full(size, ideal_backscatter)\n", 70 | "ideal_data_linear = 10 ** (\n", 71 | " ideal_data / 10\n", 72 | ") # Convert dB to linear scale for speckle addition\n", 73 | "\n", 74 | "speckle_fraction = 0.3\n", 75 | "num_speckled_pixels = int(\n", 76 | " size[0] * size[1] * speckle_fraction\n", 77 | ") # Rayleigh speckle noise\n", 78 | "speckled_indices = np.random.choice(\n", 79 | " width * width, num_speckled_pixels, replace=False\n", 80 | ") # random indices for speckle\n", 81 | "\n", 82 | "# Initialize speckled data as the same as the ideal data\n", 83 | "speckled_data_linear = ideal_data_linear.copy()\n", 84 | "\n", 85 | "speckle_noise = np.random.gumbel(scale=1.0, size=num_speckled_pixels)\n", 86 | "speckled_data_linear.ravel()[\n", 87 | " speckled_indices\n", 88 | "] *= speckle_noise # Add speckle to the selected pixels\n", 89 | "\n", 90 | "ideal_data_dB = 10 * np.log10(ideal_data_linear)\n", 91 | "speckled_data_dB = 10 * np.log10(speckled_data_linear)\n", 92 | "plt.figure(figsize=(16, 10))\n", 93 | "\n", 94 | "# Ideal data\n", 95 | "plt.subplot(2, 2, 1)\n", 96 | "plt.imshow(ideal_data_dB, cmap=\"gray\", vmin=-20, vmax=0)\n", 97 | "plt.title(\"Ideal Backscatter (Cornfield)\")\n", 98 | "plt.colorbar(label=\"Backscatter (dB)\")\n", 99 | "\n", 100 | "# Speckled data\n", 101 | "plt.subplot(2, 2, 2)\n", 102 | "plt.imshow(speckled_data_dB, cmap=\"gray\", vmin=-20, vmax=0)\n", 103 | "plt.title(f\"Speckled Backscatter ({int(speckle_fraction * 100)}% of Pixels)\")\n", 104 | "plt.colorbar(label=\"Backscatter (dB)\")\n", 105 | "\n", 106 | "bins = 25\n", 107 | "hist_ideal, bins_ideal = np.histogram(ideal_data_dB.ravel(), bins=bins, range=(-20, 0))\n", 108 | "hist_speckled, bins_speckled = np.histogram(\n", 109 | " speckled_data_dB.ravel(), bins=bins, range=(-20, 0)\n", 110 | ")\n", 111 | "max_freq = max(\n", 112 | " hist_ideal.max(), hist_speckled.max()\n", 113 | ") # maximum frequency for normalization\n", 114 | "\n", 115 | "# Histogram for ideal data\n", 116 | "plt.subplot(2, 2, 3)\n", 117 | "plt.hist(ideal_data_dB.ravel(), bins=bins, range=(-20, 0), color=\"gray\", alpha=0.7)\n", 118 | "plt.ylim(0, max_freq)\n", 119 | "plt.title(\"Histogram of Ideal Backscatter\")\n", 120 | "plt.xlabel(\"Backscatter (dB)\")\n", 121 | "plt.ylabel(\"Frequency\")\n", 122 | "\n", 123 | "# Histogram for speckled data\n", 124 | "plt.subplot(2, 2, 4)\n", 125 | "plt.hist(speckled_data_dB.ravel(), bins=bins, range=(-20, 0), color=\"gray\", alpha=0.7)\n", 126 | "plt.ylim(0, max_freq)\n", 127 | "plt.title(f\"Histogram of Speckled Backscatter ({int(speckle_fraction * 100)}%)\")\n", 128 | "plt.xlabel(\"Backscatter (dB)\")\n", 129 | "plt.ylabel(\"Frequency\")\n", 130 | "\n", 131 | "plt.tight_layout()" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "id": "4", 137 | "metadata": {}, 138 | "source": [ 139 | "*Figure 1: Synthetic data that emulates speckles in microwave backscattering*\n", 140 | "\n", 141 | "We can imagine that the second plot represents a real SAR acquisition over a cornfield, while the first plot represents an ideal uniform SAR image over a cornfield land (no speckle). The introduction of a simulated 30% speckle noise could be related to the presence of distributed scatterers of any sort present in the scene, which would cause a pixel-to-pixel variation in terms of intensity.\n", 142 | "\n", 143 | "All the random contributions (such as the wind) would result in a different speckle pattern each time a SAR scene is acquired over the same area. Many subpixel contributors build up a complex scattered pattern in any SAR image, making it erroneous to rely on a single pixel intensity for making reliable image analysis. In order to enhance the degree of usability of a SAR image, several techniques have been put in place to mitigate speckle.\n", 144 | "We will now show two of the most common approaches: the temporal and the spatial filter.\n", 145 | "\n", 146 | "## Lake Neusiedl data\n", 147 | "\n", 148 | "We load a dataset that contains the CORINE land cover and Sentinel-1 $\\sigma^0_E$ at a 20 meter resolution. This is the same data presented in notebook 6.\n" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": null, 154 | "id": "5", 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "url = \"https://huggingface.co/datasets/martinschobben/microwave-remote-sensing/resolve/main/microwave-remote-sensing.yml\"\n", 159 | "cat = intake.open_catalog(url)\n", 160 | "fused_ds = cat.speckle.read().compute()\n", 161 | "fused_ds" 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "id": "6", 167 | "metadata": {}, 168 | "source": [ 169 | "We also create the same dashboard for backscatter of different landcover types over time. In order to make this code reusable and adaptable we will define the following function `plot_variability`, which allows the injection of a spatial and/or temporal filter. It is not important to understand all the code of the following cell!\n" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "id": "7", 176 | "metadata": {}, 177 | "outputs": [], 178 | "source": [ 179 | "# Load encoding\n", 180 | "with cat.corine_cmap.read()[0] as f:\n", 181 | " color_mapping_data = json.load(f)\n", 182 | "\n", 183 | "# Get mapping\n", 184 | "color_mapping = {item[\"value\"]: item for item in color_mapping_data[\"land_cover\"]}\n", 185 | "\n", 186 | "# Get landcover codes present in the image\n", 187 | "present_landcover_codes = np.unique(\n", 188 | " fused_ds.land_cover.values[~np.isnan(fused_ds.land_cover.values)].astype(int)\n", 189 | ")\n", 190 | "\n", 191 | "\n", 192 | "def load_image(var_ds, time, land_cover, x_range, y_range, filter_fun_spatial=None):\n", 193 | " \"\"\"\n", 194 | " Callback Function Landcover.\n", 195 | "\n", 196 | " Parameters\n", 197 | " ----------\n", 198 | " time: panda.datetime\n", 199 | " time slice\n", 200 | " landcover: int\n", 201 | " land cover type\n", 202 | " x_range: array_like\n", 203 | " longitude range\n", 204 | " y_range: array_like\n", 205 | " latitude range\n", 206 | "\n", 207 | " Returns\n", 208 | " -------\n", 209 | " holoviews.Image\n", 210 | " \"\"\"\n", 211 | "\n", 212 | " if time is not None:\n", 213 | " var_ds = var_ds.sel(time=time)\n", 214 | "\n", 215 | " if land_cover == \"\\xa0\\xa0\\xa0 Complete Land Cover\":\n", 216 | " sig0_selected_ds = var_ds.sig0\n", 217 | " else:\n", 218 | " land_cover_value = int(land_cover.split()[0])\n", 219 | " mask_ds = var_ds.land_cover == land_cover_value\n", 220 | " sig0_selected_ds = var_ds.sig0.where(mask_ds)\n", 221 | "\n", 222 | " if filter_fun_spatial is not None:\n", 223 | " sig0_np = filter_fun_spatial(sig0_selected_ds.values)\n", 224 | " else:\n", 225 | " sig0_np = sig0_selected_ds.values\n", 226 | "\n", 227 | " # Convert unfiltered data into Holoviews Image\n", 228 | " img = hv.Dataset(\n", 229 | " (sig0_selected_ds[\"x\"], sig0_selected_ds[\"y\"], sig0_np), [\"x\", \"y\"], \"sig0\"\n", 230 | " )\n", 231 | "\n", 232 | " if x_range and y_range:\n", 233 | " img = img.select(x=x_range, y=y_range)\n", 234 | "\n", 235 | " return hv.Image(img)\n", 236 | "\n", 237 | "\n", 238 | "def plot_variability(var_ds, filter_fun_spatial=None, filter_fun_temporal=None):\n", 239 | "\n", 240 | " robust_min = var_ds.sig0.quantile(0.02).item()\n", 241 | " robust_max = var_ds.sig0.quantile(0.98).item()\n", 242 | "\n", 243 | " bin_edges = [\n", 244 | " i + j * 0.5\n", 245 | " for i in range(int(robust_min) - 2, int(robust_max) + 2)\n", 246 | " for j in range(2)\n", 247 | " ]\n", 248 | "\n", 249 | " land_cover = {\"\\xa0\\xa0\\xa0 Complete Land Cover\": 1}\n", 250 | " land_cover.update(\n", 251 | " {\n", 252 | " f\"{int(value): 02} {color_mapping[value]['label']}\": int(value)\n", 253 | " for value in present_landcover_codes\n", 254 | " }\n", 255 | " )\n", 256 | " time = var_ds.sig0[\"time\"].values\n", 257 | "\n", 258 | " rangexy = RangeXY()\n", 259 | "\n", 260 | " if filter_fun_temporal is not None:\n", 261 | " var_ds = filter_fun_temporal(var_ds)\n", 262 | " load_image_ = partial(\n", 263 | " load_image, var_ds=var_ds, filter_fun_spatial=filter_fun_spatial, time=None\n", 264 | " )\n", 265 | " dmap = (\n", 266 | " hv.DynamicMap(load_image_, kdims=[\"Landcover\"], streams=[rangexy])\n", 267 | " .redim.values(Landcover=land_cover)\n", 268 | " .hist(normed=True, bins=bin_edges)\n", 269 | " )\n", 270 | "\n", 271 | " else:\n", 272 | " load_image_ = partial(\n", 273 | " load_image, var_ds=var_ds, filter_fun_spatial=filter_fun_spatial\n", 274 | " )\n", 275 | " dmap = (\n", 276 | " hv.DynamicMap(load_image_, kdims=[\"Time\", \"Landcover\"], streams=[rangexy])\n", 277 | " .redim.values(Time=time, Landcover=land_cover)\n", 278 | " .hist(normed=True, bins=bin_edges)\n", 279 | " )\n", 280 | "\n", 281 | " image_opts = hv.opts.Image(\n", 282 | " cmap=\"Greys_r\",\n", 283 | " colorbar=True,\n", 284 | " tools=[\"hover\"],\n", 285 | " clim=(robust_min, robust_max),\n", 286 | " aspect=\"equal\",\n", 287 | " framewise=False,\n", 288 | " frame_height=500,\n", 289 | " frame_width=500,\n", 290 | " )\n", 291 | "\n", 292 | " hist_opts = hv.opts.Histogram(width=350, height=555)\n", 293 | "\n", 294 | " return dmap.opts(image_opts, hist_opts)" 295 | ] 296 | }, 297 | { 298 | "cell_type": "markdown", 299 | "id": "8", 300 | "metadata": {}, 301 | "source": [ 302 | "Now, lets work on the real-life dataset to see how speckle actually looks like.\n" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "id": "9", 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [ 312 | "plot_variability(fused_ds)" 313 | ] 314 | }, 315 | { 316 | "cell_type": "markdown", 317 | "id": "10", 318 | "metadata": {}, 319 | "source": [ 320 | "*Figure 2: Lake Neusiedl $\\sigma^0_E$ without any filter.*\n", 321 | "\n", 322 | "The speckle noise typically appears as a \"salt-and-pepper\" pattern. Also, please note the distribution of backscatter for each land cover. Even though speckle is known for following non-normal distributions (i.e., Rayleigh distribution for amplitude in the linear domain, and the Gumple for intensity in the log domain), we can assume that due to the Central Limit Theorem, the overall backscatter means (dB) tend to follow a Gaussian distribution.\n", 323 | "\n", 324 | "We can mitigate speckle (it is impossible to remove it completely) by following approaches such as:\n", 325 | "- spatial filtering - taking mean backscatter value over the same land cover, or\n", 326 | "- temporal filtering - taking the average backscatter value over some time period.\n", 327 | "\n", 328 | "Either way, one pixel is never representative of ground truth! Therefore we need to look at samples and distributions.\n", 329 | "\n", 330 | "## Spatial filtering\n", 331 | "\n", 332 | "We first introduce a common spatial filter. The Lee filter is an adaptive speckle filter. The filter works using a kernel window with a configurable size, which refers to the dimensions of the neighborhood over which the filter operates. The kernel slides across the data, applying the smoothing operation at each pixel position of the image. It follows three assumptions:\n", 333 | "\n", 334 | "1) SAR speckle is modeled as a multiplicative noise - the brighter the area the noisier the data.\n", 335 | "2) The noise and the signal are statistically independent of each other.\n", 336 | "3) The sample mean and sample variance of a pixel is equal to its local mean and local variance.\n", 337 | "\n", 338 | "This approach comes with some limitations: it reduces the spatial resolution of the SAR image.\n", 339 | "\n", 340 | "Let's build up a function for applying a Lee filter with a kernel window size of 7 (do not forget to switch back to linear units before doing this computation and to dB after it):\n" 341 | ] 342 | }, 343 | { 344 | "cell_type": "code", 345 | "execution_count": null, 346 | "id": "11", 347 | "metadata": {}, 348 | "outputs": [], 349 | "source": [ 350 | "def lee_filter(raster, size=7):\n", 351 | " \"\"\"\n", 352 | " Parameters:\n", 353 | " raster: ndarray\n", 354 | " 2D array representing the noisy image (e.g., radar image with speckle)\n", 355 | " size: int\n", 356 | " Size of the kernel window for the filter (must be odd, default is 7)\n", 357 | "\n", 358 | " Returns:\n", 359 | " filtered_image (ndarray): The filtered image with reduced speckle noise\n", 360 | " \"\"\"\n", 361 | "\n", 362 | " raster = np.nan_to_num(raster)\n", 363 | " raster = 10 ** (raster / 10)\n", 364 | "\n", 365 | " # Mean and variance over local kernel window\n", 366 | " mean_window = uniform_filter(raster, size=size)\n", 367 | " mean_sq_window = uniform_filter(raster**2, size=size)\n", 368 | " variance_window = mean_sq_window - mean_window**2\n", 369 | "\n", 370 | " # Noise variance estimation (this could also be set manually)\n", 371 | " overall_variance = np.var(raster)\n", 372 | "\n", 373 | " # Compute the Lee filter\n", 374 | " weights = variance_window / (variance_window + overall_variance)\n", 375 | "\n", 376 | " return 10 * np.log10(mean_window + weights * (raster - mean_window))" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": null, 382 | "id": "12", 383 | "metadata": {}, 384 | "outputs": [], 385 | "source": [ 386 | "plot_variability(fused_ds, filter_fun_spatial=lee_filter)" 387 | ] 388 | }, 389 | { 390 | "cell_type": "markdown", 391 | "id": "13", 392 | "metadata": {}, 393 | "source": [ 394 | "*Figure 3: Lake Neusiedl $\\sigma^0_E$ with a Lee filter applied.*\n", 395 | "\n", 396 | "## Temporal filtering\n", 397 | "\n", 398 | "Temporal filtering would involve taking the average of all previous (past) observations for each pixel. This approach comes with some limitations: it takes out the content-rich information tied to the temporal variability of backscatter.\n" 399 | ] 400 | }, 401 | { 402 | "cell_type": "code", 403 | "execution_count": null, 404 | "id": "14", 405 | "metadata": {}, 406 | "outputs": [], 407 | "source": [ 408 | "def temporal_filter(raster):\n", 409 | " \"\"\"\n", 410 | " Parameters:\n", 411 | " raster: ndarray\n", 412 | " 3D array representing the noisy image over time\n", 413 | " (e.g., radar image with speckle)\n", 414 | "\n", 415 | " Returns:\n", 416 | " filtered_image (ndarray): The filtered image with reduced speckle noise\n", 417 | " \"\"\"\n", 418 | "\n", 419 | " return raster.mean(\"time\")" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": null, 425 | "id": "15", 426 | "metadata": {}, 427 | "outputs": [], 428 | "source": [ 429 | "plot_variability(fused_ds, filter_fun_temporal=temporal_filter)" 430 | ] 431 | }, 432 | { 433 | "cell_type": "markdown", 434 | "id": "16", 435 | "metadata": {}, 436 | "source": [ 437 | "*Figure 4: Lake Neusiedl $\\sigma^0_E$ with a temporal filter applied.*\n", 438 | "\n", 439 | "Let´s observe the histograms of the two plots. Especially in the region around the lake, it is clear that the distribution is now less dispersed and more centered around a central value." 440 | ] 441 | } 442 | ], 443 | "metadata": { 444 | "kernelspec": { 445 | "display_name": "Python 3 (ipykernel)", 446 | "language": "python", 447 | "name": "python3" 448 | }, 449 | "language_info": { 450 | "codemirror_mode": { 451 | "name": "ipython", 452 | "version": 3 453 | }, 454 | "file_extension": ".py", 455 | "mimetype": "text/x-python", 456 | "name": "python", 457 | "nbconvert_exporter": "python", 458 | "pygments_lexer": "ipython3", 459 | "version": "3.11.11" 460 | } 461 | }, 462 | "nbformat": 4, 463 | "nbformat_minor": 5 464 | } 465 | -------------------------------------------------------------------------------- /notebooks/how-to-cite.md: -------------------------------------------------------------------------------- 1 | # How to Cite This Cookbook 2 | 3 | The material in this Project Pythia Cookbook is licensed for free and open consumption and reuse. All code is served under [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0), while all non-code content is licensed under [Creative Commons BY 4.0 (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/). Effectively, this means you are free to share and adapt this material so long as you give appropriate credit to the Cookbook authors and the Project Pythia community. 4 | 5 | The source code for the book is [released on GitHub](https://github.com/TUW-GEO/eo-datascience-cookbook) and archived on Zenodo. This DOI will always resolve to the latest release of the book source: 6 | 7 | [![DOI](https://zenodo.org/badge/830421828.svg)](https://zenodo.org/badge/latestdoi/830421828) 8 | -------------------------------------------------------------------------------- /notebooks/images/ProjectPythia_Logo_Final-01-Blue.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /notebooks/images/cmaps/06_color_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "land_cover": [ 3 | {"value": 1, "color": "#e6004d", "label": "Continuous urban fabric"}, 4 | {"value": 2, "color": "#ff0000", "label": "Discontinuous urban fabric"}, 5 | {"value": 3, "color": "#cc4df2", "label": "Industrial or commercial units"}, 6 | {"value": 4, "color": "#cc0000", "label": "Road and rail networks and associated land"}, 7 | {"value": 5, "color": "#e6cccc", "label": "Port areas"}, 8 | {"value": 6, "color": "#e6cce6", "label": "Airports"}, 9 | {"value": 7, "color": "#a600cc", "label": "Mineral extraction sites"}, 10 | {"value": 8, "color": "#a64d00", "label": "Dump sites"}, 11 | {"value": 9, "color": "#ff4dff", "label": "Construction sites"}, 12 | {"value": 10, "color": "#ffa6ff", "label": "Green urban areas"}, 13 | {"value": 11, "color": "#ffe6ff", "label": "Sport and leisure facilities"}, 14 | {"value": 12, "color": "#ffffa8", "label": "Non-irrigated arable land"}, 15 | {"value": 13, "color": "#ffff00", "label": "Permanently irrigated land"}, 16 | {"value": 14, "color": "#e6e600", "label": "Rice fields"}, 17 | {"value": 15, "color": "#e68000", "label": "Vineyards"}, 18 | {"value": 16, "color": "#f2a64d", "label": "Fruit trees and berry plantations"}, 19 | {"value": 17, "color": "#e6a600", "label": "Olive groves"}, 20 | {"value": 18, "color": "#e6e64d", "label": "Pastures"}, 21 | {"value": 19, "color": "#ffe6a6", "label": "Annual crops associated with permanent crops"}, 22 | {"value": 20, "color": "#ffe64d", "label": "Complex cultivation patterns"}, 23 | {"value": 21, "color": "#e6cc4d", "label": "Agricultural land with natural vegetation"}, 24 | {"value": 22, "color": "#f2cca6", "label": "Agro-forestry areas"}, 25 | {"value": 23, "color": "#80ff00", "label": "Broad-leaved forest"}, 26 | {"value": 24, "color": "#00a600", "label": "Coniferous forest"}, 27 | {"value": 25, "color": "#4dff00", "label": "Mixed forest"}, 28 | {"value": 26, "color": "#ccf24d", "label": "Natural grasslands"}, 29 | {"value": 27, "color": "#a6ff80", "label": "Moors and heathland"}, 30 | {"value": 28, "color": "#a6e64d", "label": "Sclerophyllous vegetation"}, 31 | {"value": 29, "color": "#a6f200", "label": "Transitional woodland-shrub"}, 32 | {"value": 30, "color": "#e6e6e6", "label": "Beaches - dunes - sands"}, 33 | {"value": 31, "color": "#cccccc", "label": "Bare rocks"}, 34 | {"value": 32, "color": "#ccffcc", "label": "Sparsely vegetated areas"}, 35 | {"value": 33, "color": "#000000", "label": "Burnt areas"}, 36 | {"value": 34, "color": "#a6e6cc", "label": "Glaciers and perpetual snow"}, 37 | {"value": 35, "color": "#a6a6ff", "label": "Inland marshes"}, 38 | {"value": 36, "color": "#4d4dff", "label": "Peat bogs"}, 39 | {"value": 37, "color": "#ccccff", "label": "Salt marshes"}, 40 | {"value": 38, "color": "#e6e6ff", "label": "Salines"}, 41 | {"value": 39, "color": "#a6a6e6", "label": "Intertidal flats"}, 42 | {"value": 40, "color": "#00ccf2", "label": "Water courses"}, 43 | {"value": 41, "color": "#80f2e6", "label": "Water bodies"}, 44 | {"value": 42, "color": "#00ffa6", "label": "Coastal lagoons"}, 45 | {"value": 43, "color": "#a6ffe6", "label": "Estuaries"}, 46 | {"value": 44, "color": "#e6f2ff", "label": "Sea and ocean"}, 47 | {"value": 48, "color": "#ffffff", "label": "NODATA"} 48 | ] 49 | } 50 | 51 | -------------------------------------------------------------------------------- /notebooks/images/icons/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProjectPythia/eo-datascience-cookbook/4f2eb4968e10868a7b99e152739ec001ace1b3d5/notebooks/images/icons/favicon.ico -------------------------------------------------------------------------------- /notebooks/images/logos/pythia_logo-white-notext.svg: -------------------------------------------------------------------------------- 1 | 2 | 17 | 19 | 20 | 22 | image/svg+xml 23 | 25 | 26 | 27 | 28 | 29 | 55 | 57 | 59 | 60 | 63 | 67 | 71 | 75 | 79 | 83 | 87 | 91 | 95 | 99 | 103 | 107 | 111 | 115 | 119 | 123 | 127 | 128 | 129 | -------------------------------------------------------------------------------- /notebooks/images/logos/pythia_logo-white-rtext.svg: -------------------------------------------------------------------------------- 1 | 2 | 17 | 19 | 20 | 22 | image/svg+xml 23 | 25 | 26 | 27 | 28 | 29 | 55 | 57 | 59 | 60 | 64 | 69 | 74 | 79 | 84 | 89 | 94 | 99 | 104 | 109 | 114 | 119 | 124 | 129 | 134 | 139 | 144 | 145 | 149 | 153 | 158 | 163 | 168 | 173 | 178 | 183 | 188 | 189 | 193 | 198 | 203 | 208 | 213 | 218 | 223 | 224 | 225 | 226 | -------------------------------------------------------------------------------- /notebooks/images/logos/tuw-geo_eodc_logo_horizontal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProjectPythia/eo-datascience-cookbook/4f2eb4968e10868a7b99e152739ec001ace1b3d5/notebooks/images/logos/tuw-geo_eodc_logo_horizontal.png -------------------------------------------------------------------------------- /notebooks/images/ridgecrest.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProjectPythia/eo-datascience-cookbook/4f2eb4968e10868a7b99e152739ec001ace1b3d5/notebooks/images/ridgecrest.gif -------------------------------------------------------------------------------- /notebooks/images/side_looking_image_distortions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProjectPythia/eo-datascience-cookbook/4f2eb4968e10868a7b99e152739ec001ace1b3d5/notebooks/images/side_looking_image_distortions.png -------------------------------------------------------------------------------- /notebooks/images/speckle_effect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProjectPythia/eo-datascience-cookbook/4f2eb4968e10868a7b99e152739ec001ace1b3d5/notebooks/images/speckle_effect.png -------------------------------------------------------------------------------- /notebooks/images/tuw-geo_eodc_logo_vertical.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProjectPythia/eo-datascience-cookbook/4f2eb4968e10868a7b99e152739ec001ace1b3d5/notebooks/images/tuw-geo_eodc_logo_vertical.png -------------------------------------------------------------------------------- /notebooks/references.bib: -------------------------------------------------------------------------------- 1 | 2 | @misc{celik_colors_2015, 3 | title = {Colors, {Methods} and {Mistakes} in {Spatial} {Data} {Visualization}}, 4 | url = {https://medium.com/@acelik/colors-methods-and-mistakes-in-spatial-data-visualization-60d02e7f09fd}, 5 | abstract = {Around ten days ago, I have made a presentation in an event organized by Data Science Istanbul group in Istanbul, Turkey. Seminar focused…}, 6 | language = {en}, 7 | urldate = {2024-05-28}, 8 | journal = {Medium}, 9 | author = {Celik, Anil}, 10 | month = aug, 11 | year = {2015}, 12 | } 13 | 14 | @article{nguyen_examining_2021, 15 | title = {Examining data visualization pitfalls in scientific publications}, 16 | volume = {4}, 17 | issn = {2096-496X}, 18 | url = {https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8556474/}, 19 | doi = {10.1186/s42492-021-00092-y}, 20 | abstract = {Data visualization blends art and science to convey stories from data via graphical representations. Considering different problems, applications, requirements, and design goals, it is challenging to combine these two components at their full force. While the art component involves creating visually appealing and easily interpreted graphics for users, the science component requires accurate representations of a large amount of input data. With a lack of the science component, visualization cannot serve its role of creating correct representations of the actual data, thus leading to wrong perception, interpretation, and decision. It might be even worse if incorrect visual representations were intentionally produced to deceive the viewers. To address common pitfalls in graphical representations, this paper focuses on identifying and understanding the root causes of misinformation in graphical representations. We reviewed the misleading data visualization examples in the scientific publications collected from indexing databases and then projected them onto the fundamental units of visual communication such as color, shape, size, and spatial orientation. Moreover, a text mining technique was applied to extract practical insights from common visualization pitfalls. Cochran’s Q test and McNemar’s test were conducted to examine if there is any difference in the proportions of common errors among color, shape, size, and spatial orientation. The findings showed that the pie chart is the most misused graphical representation, and size is the most critical issue. It was also observed that there were statistically significant differences in the proportion of errors among color, shape, size, and spatial orientation.}, 21 | urldate = {2024-05-29}, 22 | journal = {Visual Computing for Industry, Biomedicine, and Art}, 23 | author = {Nguyen, Vinh T and Jung, Kwanghee and Gupta, Vibhuti}, 24 | month = oct, 25 | year = {2021}, 26 | pmid = {34714412}, 27 | pmcid = {PMC8556474}, 28 | pages = {27}, 29 | file = {PubMed Central Full Text PDF:/home/mschobbe/Zotero/storage/5N6D56NV/Nguyen et al. - 2021 - Examining data visualization pitfalls in scientifi.pdf:application/pdf}, 30 | } 31 | 32 | @article{ware_color_1988, 33 | title = {Color sequences for univariate maps: theory, experiments and principles}, 34 | volume = {8}, 35 | copyright = {https://ieeexplore.ieee.org/Xplorehelp/downloads/license-information/IEEE.html}, 36 | issn = {0272-1716}, 37 | shorttitle = {Color sequences for univariate maps}, 38 | url = {http://ieeexplore.ieee.org/document/7760/}, 39 | doi = {10.1109/38.7760}, 40 | language = {en}, 41 | number = {5}, 42 | urldate = {2024-05-29}, 43 | journal = {IEEE Computer Graphics and Applications}, 44 | author = {Ware, C.}, 45 | month = sep, 46 | year = {1988}, 47 | pages = {41--49}, 48 | file = {Ware - 1988 - Color sequences for univariate maps theory, exper.pdf:/home/mschobbe/Zotero/storage/8WAHXK7V/Ware - 1988 - Color sequences for univariate maps theory, exper.pdf:application/pdf}, 49 | } 50 | 51 | @misc{noauthor_httpsccomunhedusitesdefaultfilespublicationsware_1988_cga_color_sequences_univariate_mapspdf_nodate, 52 | title = {https://ccom.unh.edu/sites/default/files/publications/{Ware}\_1988\_CGA\_Color\_sequences\_univariate\_maps.pdf}, 53 | url = {https://ccom.unh.edu/sites/default/files/publications/Ware_1988_CGA_Color_sequences_univariate_maps.pdf}, 54 | urldate = {2024-05-29}, 55 | } 56 | 57 | @article{sibrel_relation_2020, 58 | title = {The relation between color and spatial structure for interpreting colormap data visualizations}, 59 | volume = {20}, 60 | issn = {1534-7362}, 61 | url = {https://doi.org/10.1167/jov.20.12.7}, 62 | doi = {10.1167/jov.20.12.7}, 63 | abstract = {Interpreting colormap visualizations requires determining how dimensions of color in visualizations map onto quantities in data. People have color-based biases that influence their interpretations of colormaps, such as a dark-is-more bias—darker colors map to larger quantities. Previous studies of color-based biases focused on colormaps with weak data spatial structure, but color-based biases may not generalize to colormaps with strong data spatial structure, like “hotspots” typically found in weather maps and neuroimaging brain maps. There may be a hotspot-is-more bias to infer that colors within hotspots represent larger quantities, which may override the dark-is-more bias. We tested this possibility in four experiments. Participants saw colormaps with hotspots and a legend that specified the color-quantity mapping. Their task was to indicate which side of the colormap depicted larger quantities (left/right). We varied whether the legend specified dark-more mapping or light-more mapping across trials and operationalized a dark-is-more bias as faster response time (RT) when the legend specified dark-more mapping. Experiment 1 demonstrated robust evidence for the dark-is-more bias, without evidence for a hotspot-is-more bias. Experiments 2 to 4 suggest that a hotspot-is-more bias becomes relevant when hotspots are a statistically reliable cue to “more” (i.e., the locus of larger quantities) and when hotspots are more perceptually pronounced. Yet, comparing conditions in which the hotspots were “more,” RTs were always faster for dark hotspots than light hotspots. Thus, in the presence of strong spatial cues to the locus of larger quantities, color-based biases still influenced interpretations of colormap data visualizations.}, 64 | number = {12}, 65 | urldate = {2024-05-29}, 66 | journal = {Journal of Vision}, 67 | author = {Sibrel, Shannon C. and Rathore, Ragini and Lessard, Laurent and Schloss, Karen B.}, 68 | month = nov, 69 | year = {2020}, 70 | pages = {7}, 71 | file = {Full Text:/home/mschobbe/Zotero/storage/BBK94XG6/Sibrel et al. - 2020 - The relation between color and spatial structure f.pdf:application/pdf;Snapshot:/home/mschobbe/Zotero/storage/8669JRUZ/article.html:text/html}, 72 | } 73 | 74 | @article{bauer-marschallinger_satellite-based_2022, 75 | title = {Satellite-{Based} {Flood} {Mapping} through {Bayesian} {Inference} from a {Sentinel}-1 {SAR} {Datacube}}, 76 | volume = {14}, 77 | copyright = {http://creativecommons.org/licenses/by/3.0/}, 78 | issn = {2072-4292}, 79 | url = {https://www.mdpi.com/2072-4292/14/15/3673}, 80 | doi = {10.3390/rs14153673}, 81 | abstract = {Spaceborne Synthetic Aperture Radar (SAR) are well-established systems for flood mapping, thanks to their high sensitivity towards water surfaces and their independence from daylight and cloud cover. Particularly able is the 2014-launched Copernicus Sentinel-1 C-band SAR mission, with its systematic monitoring schedule featuring global land coverage in a short revisit time and a 20 m ground resolution. Yet, variable environment conditions, low-contrasting land cover, and complex terrain pose major challenges to fully automated flood monitoring. To overcome these issues, and aiming for a robust classification, we formulate a datacube-based flood mapping algorithm that exploits the Sentinel-1 orbit repetition and a priori generated probability parameters for flood and non-flood conditions. A globally applicable flood signature is obtained from manually collected wind- and frost-free images. Through harmonic analysis of each pixel’s full time series, we derive a local seasonal non-flood signal comprising the expected backscatter values for each day-of-year. From those predefined probability distributions, we classify incoming Sentinel-1 images by simple Bayes inference, which is computationally slim and hence suitable for near-real-time operations, and also yields uncertainty values. The datacube-based masking of no-sensitivity resulting from impeding land cover and ill-posed SAR configuration enhances the classification robustness. We employed the algorithm on a 6-year Sentinel-1 datacube over Greece, where a major flood hit the region of Thessaly in 2018. In-depth analysis of model parameters and sensitivity, and the evaluation against microwave and optical reference flood maps, suggest excellent flood mapping skill, and very satisfying classification metrics with about 96\% overall accuracy and only few false positives. The presented algorithm is part of the ensemble flood mapping product of the Global Flood Monitoring (GFM) component of the Copernicus Emergency Management Service (CEMS).}, 82 | language = {en}, 83 | number = {15}, 84 | urldate = {2024-06-03}, 85 | journal = {Remote Sensing}, 86 | author = {Bauer-Marschallinger, Bernhard and Cao, Senmao and Tupas, Mark Edwin and Roth, Florian and Navacchi, Claudio and Melzer, Thomas and Freeman, Vahid and Wagner, Wolfgang}, 87 | month = jan, 88 | year = {2022}, 89 | note = {Number: 15 90 | Publisher: Multidisciplinary Digital Publishing Institute}, 91 | keywords = {automatic flood monitoring, Bayes inference, datacube, flood mapping, SAR, Sentinel-1, time series analysis}, 92 | pages = {3673}, 93 | file = {Full Text PDF:/home/mschobbe/Zotero/storage/X4KXFTT7/Bauer-Marschallinger et al. - 2022 - Satellite-Based Flood Mapping through Bayesian Inf.pdf:application/pdf}, 94 | } 95 | 96 | @article{massart_mitigating_2024, 97 | title = {Mitigating the impact of dense vegetation on the {Sentinel}-1 surface soil moisture retrievals over {Europe}}, 98 | volume = {57}, 99 | issn = {null}, 100 | url = {https://doi.org/10.1080/22797254.2023.2300985}, 101 | doi = {10.1080/22797254.2023.2300985}, 102 | abstract = {The C-band Synthetic Aperture Radar (SAR) on board of the Sentinel-1 satellites have a strong potential to retrieve Surface Soil Moisture (SSM). Using a change detection model to Sentinel-1 backscatter, an SSM product at a kilometre scale resolution over Europe could be established in the Copernicus Global Land Service (CGLS). Over areas with dense vegetation and high biomass. The geometry and water content influence the seasonality of the backscatter dynamics and hamper the SSM retrieval quality from Sentinel-1. This study demonstrates the effect of woody vegetation on SSM retrievals and proposes a masking method at the native resolution of Sentinel-1’s Interferometric Wide (IW) swath mode. At a continental 20 m grid, four dense vegetation masks are implemented over Europe in the resampling of the backscatter to a kilometre scale. The resulting backscatter is then used as input for the TUWien (TUW) change detection model and compared to both in-situ and modelled SSM. This paper highlights the potential of high-resolution vegetation datasets to mask for non-soil moisture-sensitive pixels at a sub-kilometre resolution. Results show that both correlation and seasonality of the retrieved SSM are improved by masking the dense vegetation at a 20 m resolution. Dense vegetation reduces the ability to retrieve surface soil moisture at a kilometre scale from Sentinel-1 backscatter which is currently available on the Copernicus Global Land Service portal.Applying selective masking for vegetation during the resampling phase improves Sentinel-1 sensitivity to soil moisture.A novel vegetation-corrected Sentinel-1 surface soil moisture product is processed over Europe for the period 2016–2022 included.The Sentinel-1 forest mask improves the Sentinel-1 SSM product correlation and seasonality compared to both modelled and in-situ datasets. Dense vegetation reduces the ability to retrieve surface soil moisture at a kilometre scale from Sentinel-1 backscatter which is currently available on the Copernicus Global Land Service portal. Applying selective masking for vegetation during the resampling phase improves Sentinel-1 sensitivity to soil moisture. A novel vegetation-corrected Sentinel-1 surface soil moisture product is processed over Europe for the period 2016–2022 included. The Sentinel-1 forest mask improves the Sentinel-1 SSM product correlation and seasonality compared to both modelled and in-situ datasets.}, 103 | number = {1}, 104 | urldate = {2024-06-03}, 105 | journal = {European Journal of Remote Sensing}, 106 | author = {Massart, Samuel and Vreugdenhil, Mariette and Bauer-Marschallinger, Bernhard and Navacchi, Claudio and Raml, Bernhard and Wagner, Wolfgang}, 107 | month = dec, 108 | year = {2024}, 109 | note = {Publisher: Taylor \& Francis 110 | \_eprint: https://doi.org/10.1080/22797254.2023.2300985}, 111 | keywords = {change detection, high-resolution, Sentinel-1, soil moisture, Synthetic Aperture Radar, vegetation}, 112 | pages = {2300985}, 113 | file = {Full Text PDF:/home/mschobbe/Zotero/storage/ME6MG37F/Massart et al. - 2024 - Mitigating the impact of dense vegetation on the S.pdf:application/pdf}, 114 | } 115 | 116 | @article{bauer-marschallinger_toward_2019, 117 | title = {Toward {Global} {Soil} {Moisture} {Monitoring} {With} {Sentinel}-1: {Harnessing} {Assets} and {Overcoming} {Obstacles}}, 118 | volume = {57}, 119 | issn = {1558-0644}, 120 | shorttitle = {Toward {Global} {Soil} {Moisture} {Monitoring} {With} {Sentinel}-1}, 121 | url = {https://ieeexplore.ieee.org/document/8444430}, 122 | doi = {10.1109/TGRS.2018.2858004}, 123 | abstract = {Soil moisture is a key environmental variable, important to, e.g., farmers, meteorologists, and disaster management units. Here, we present a method to retrieve surface soil moisture (SSM) from the Sentinel-1 (S-1) satellites, which carry C-band Synthetic Aperture Radar (CSAR) sensors that provide the richest freely available SAR data source so far, unprecedented in accuracy and coverage. Our SSM retrieval method, adapting well-established change detection algorithms, builds the first globally deployable soil moisture observation data set with 1-km resolution. This paper provides an algorithm formulation to be operated in data cube architectures and high-performance computing environments. It includes the novel dynamic Gaussian upscaling method for spatial upscaling of SAR imagery, harnessing its field-scale information and successfully mitigating effects from the SAR's high signal complexity. Also, a new regression-based approach for estimating the radar slope is defined, coping with Sentinel-1's inhomogeneity in spatial coverage. We employ the S-1 SSM algorithm on a 3-year S-1 data cube over Italy, obtaining a consistent set of model parameters and product masks, unperturbed by coverage discontinuities. An evaluation of therefrom generated S-1 SSM data, involving a 1-km soil water balance model over Umbria, yields high agreement over plains and agricultural areas, with low agreement over forests and strong topography. While positive biases during the growing season are detected, the excellent capability to capture small-scale soil moisture changes as from rainfall or irrigation is evident. The S-1 SSM is currently in preparation toward operational product dissemination in the Copernicus Global Land Service.}, 124 | number = {1}, 125 | urldate = {2024-06-03}, 126 | journal = {IEEE Transactions on Geoscience and Remote Sensing}, 127 | author = {Bauer-Marschallinger, Bernhard and Freeman, Vahid and Cao, Senmao and Paulik, Christoph and Schaufler, Stefan and Stachl, Tobias and Modanesi, Sara and Massari, Christian and Ciabatta, Luca and Brocca, Luca and Wagner, Wolfgang}, 128 | month = jan, 129 | year = {2019}, 130 | note = {Conference Name: IEEE Transactions on Geoscience and Remote Sensing}, 131 | keywords = {Change detection algorithms, Copernicus, Image resolution, image sampling, Monitoring, Sensors, Sentinel-1, Signal resolution, soil moisture, Soil moisture, Synthetic aperture radar}, 132 | pages = {520--539}, 133 | file = {IEEE Xplore Abstract Record:/home/mschobbe/Zotero/storage/5QJ4V6PY/8444430.html:text/html}, 134 | } 135 | 136 | @article{quast_getting_2024, 137 | author = {Raphael Quast}, 138 | title = {EOmaps: A python package to visualize and analyze geographical datasets.}, 139 | doi = {10.5281/zenodo.6459598}, 140 | url = {https://doi.org/10.5281/zenodo.6459598}, 141 | journal = {}, 142 | year = {2024} 143 | } 144 | 145 | @article{rouse1974monitoring, 146 | title={Monitoring vegetation systems in the Great Plains with ERTS}, 147 | author={Rouse, John Wilson and Haas, R{\"u}diger H and Schell, John A and Deering, Donald W and others}, 148 | journal={NASA Spec. Publ}, 149 | volume={351}, 150 | number={1}, 151 | pages={309}, 152 | year={1974} 153 | } 154 | 155 | @online{nasa2020, 156 | title = {Earth Observatory}, 157 | author = {NASA}, 158 | year = {2020}, 159 | url = {https://earthobservatory.nasa.gov/features/MeasuringVegetation/measuring_vegetation_2.php}, 160 | urldate = {2024-07-26} 161 | } 162 | 163 | -------------------------------------------------------------------------------- /notebooks/references.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# References\n", 8 | "```{bibliography}\n", 9 | ":style: plain\n", 10 | "```\n" 11 | ] 12 | } 13 | ], 14 | "metadata": { 15 | "kernelspec": { 16 | "display_name": "Python 3 (ipykernel)", 17 | "language": "python", 18 | "name": "python3" 19 | } 20 | }, 21 | "nbformat": 4, 22 | "nbformat_minor": 4 23 | } 24 | -------------------------------------------------------------------------------- /notebooks/templates/classification.yml: -------------------------------------------------------------------------------- 1 | name: classification 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.12 6 | - mamba 7 | - jupyter 8 | - cmcrameri 9 | - dask 10 | - geopandas 11 | - matplotlib 12 | - nbformat 13 | - numpy 14 | - odc-stac 15 | - openssl 16 | - rasterio 17 | - rioxarray 18 | - scikit-learn 19 | - seaborn 20 | - stackstac 21 | - xarray 22 | - pystac-client -------------------------------------------------------------------------------- /notebooks/templates/prereqs-templates.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Templates\n", 9 | "\n", 10 | "\n", 11 | "This section of the Cookbook covers a wide range of topics. The intent is to\n", 12 | "create templates to showcase workflows that can be used by students as a primer\n", 13 | "for independent research projects.\n", 14 | "\n", 15 | "| Concepts | Importance | Notes |\n", 16 | "|---|---|---|\n", 17 | "| [Intro to xarray](https://foundations.projectpythia.org/core/xarray/xarray-intro.html) | Necessary | |\n", 18 | "| [Dask Arrays](https://foundations.projectpythia.org/core/xarray/dask-arrays-xarray.html)| Necessary| |\n", 19 | "| [Documentation scikit-learn](https://scikit-learn.org/stable/)|Neccesary|Machine Learning in Python|\n", 20 | "| [Documentation Matplotlib](https://matplotlib.org/stable/users/explain/quick_start.html)|Helpful|Ploting in Python|\n", 21 | "| [Documentation odc-stac](https://odc-stac.readthedocs.io/en/latest/)|Helpful|Data access|\n", 22 | "\n", 23 | "- **Time to learn**: 10 min\n" 24 | ] 25 | } 26 | ], 27 | "metadata": { 28 | "kernelspec": { 29 | "display_name": "Python 3 (ipykernel)", 30 | "language": "python", 31 | "name": "python3", 32 | "path": "/opt/hostedtoolcache/Python/3.11.11/x64/share/jupyter/kernels/python3" 33 | }, 34 | "language_info": { 35 | "codemirror_mode": { 36 | "name": "ipython", 37 | "version": 3 38 | }, 39 | "file_extension": ".py", 40 | "mimetype": "text/x-python", 41 | "name": "python", 42 | "nbconvert_exporter": "python", 43 | "pygments_lexer": "ipython3", 44 | "version": "3.11.11" 45 | } 46 | }, 47 | "nbformat": 4, 48 | "nbformat_minor": 5 49 | } 50 | -------------------------------------------------------------------------------- /notebooks/tutorials/floodmapping.yml: -------------------------------------------------------------------------------- 1 | name: floodmapping 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - python=3.12 6 | - mamba 7 | - jupyter 8 | - numpy 9 | - scipy 10 | - xarray 11 | - datashader 12 | - cartopy 13 | - hvplot 14 | - geoviews 15 | - jupyter_bokeh 16 | - dask 17 | - pystac-client 18 | - odc-stac 19 | - rioxarray -------------------------------------------------------------------------------- /notebooks/tutorials/prereqs-tutorials.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0", 6 | "metadata": {}, 7 | "source": [ 8 | "# Tutorials\n", 9 | "\n", 10 | "\n", 11 | "This section of the Cookbook covers a wide range of topics. They showcase the\n", 12 | "creation and usage of data products developed by TU Wien and EODC.\n", 13 | "\n", 14 | "\n", 15 | "| Concepts | Importance | Notes |\n", 16 | "|---|---|---|\n", 17 | "| [Intro to xarray](https://foundations.projectpythia.org/core/xarray/xarray-intro.html) | Necessary | |\n", 18 | "| [Dask Arrays](https://foundations.projectpythia.org/core/xarray/dask-arrays-xarray.html)| Necessary| |\n", 19 | "| [Documentation hvPlot](https://hvplot.holoviz.org/)|Helpful|Interactive plotting|\n", 20 | "| [Documentation odc-stac](https://odc-stac.readthedocs.io/en/latest/)|Helpful|Data access|\n", 21 | "\n", 22 | "- **Time to learn**: 10 min\n" 23 | ] 24 | } 25 | ], 26 | "metadata": { 27 | "kernelspec": { 28 | "display_name": "Python 3 (ipykernel)", 29 | "language": "python", 30 | "name": "python3", 31 | "path": "/opt/hostedtoolcache/Python/3.11.11/x64/share/jupyter/kernels/python3" 32 | }, 33 | "language_info": { 34 | "codemirror_mode": { 35 | "name": "ipython", 36 | "version": 3 37 | }, 38 | "file_extension": ".py", 39 | "mimetype": "text/x-python", 40 | "name": "python", 41 | "nbconvert_exporter": "python", 42 | "pygments_lexer": "ipython3", 43 | "version": "3.11.11" 44 | } 45 | }, 46 | "nbformat": 4, 47 | "nbformat_minor": 5 48 | } 49 | --------------------------------------------------------------------------------