├── .flake8 ├── .github ├── CODEOWNERS ├── pull_request_template.md ├── release-drafter.yml └── workflows │ ├── draft-release.yml │ ├── publish-release.yml │ └── tests.yml ├── .gitignore ├── .readthedocs.yaml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── design ├── checkpoint_01 │ ├── harmony_py.ipynb │ └── harmony_py_mock.py ├── checkpoint_02 │ └── harmony_py.ipynb └── checkpoint_03 │ └── harmony_py.ipynb ├── docs ├── Makefile ├── api.rst ├── conf.py ├── index.rst ├── make.bat └── user │ ├── install.rst │ ├── notebook.html │ └── tutorial.rst ├── examples ├── 2020_01_01_7f00ff_global_regridded_subsetted.png ├── 2020_01_01_7f00ff_global_regridded_subsetted.wld ├── Big_Island_0005.zip ├── asf_example.json ├── basic.ipynb ├── collection_capabilities.ipynb ├── helper.py ├── intro_tutorial.ipynb ├── job_label.ipynb ├── job_pause_resume.ipynb ├── job_results.ipynb ├── job_results_iterator.ipynb ├── job_stac.ipynb ├── job_status.ipynb ├── s3_access.ipynb ├── shapefile_subset.ipynb └── tutorial.ipynb ├── harmony ├── __init__.py ├── auth.py ├── client.py ├── config.py ├── request.py └── util.py ├── internal └── genparams.py ├── pyproject.toml └── tests ├── __init__.py ├── test_auth.py ├── test_client.py ├── test_config.py ├── test_request.py └── test_util.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 99 3 | ignore = F401, W503 4 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @indiejames @chris-durbin @ygliuvt 2 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Jira Issue ID 2 | 3 | 4 | ## Description 5 | 6 | 7 | ## Local Test Steps 8 | 9 | 10 | ## PR Acceptance Checklist 11 | * [ ] Acceptance criteria met 12 | * [ ] Tests added/updated (if needed) and passing 13 | * [ ] Documentation updated (if needed) -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | version-resolver: 2 | major: 3 | labels: 4 | - 'major' 5 | minor: 6 | labels: 7 | - 'minor' 8 | patch: 9 | labels: 10 | - 'patch' 11 | default: patch 12 | name-template: 'v$RESOLVED_VERSION' 13 | tag-template: 'v$RESOLVED_VERSION' 14 | template: | 15 | $CHANGES 16 | -------------------------------------------------------------------------------- /.github/workflows/draft-release.yml: -------------------------------------------------------------------------------- 1 | name: Draft Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Release Drafter 13 | uses: release-drafter/release-drafter@v5.12.1 14 | env: 15 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 16 | -------------------------------------------------------------------------------- /.github/workflows/publish-release.yml: -------------------------------------------------------------------------------- 1 | name: Publish Release 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | with: 13 | fetch-depth: '0' 14 | - uses: actions/setup-python@v5 15 | with: 16 | python-version: '3.11' 17 | - shell: bash 18 | env: 19 | VERSION_TAG: ${{ github.event.release.tag_name }} 20 | BRANCH: ${{ github.event.release.target_commitish }} 21 | run: | 22 | VERSION=$(echo "${VERSION_TAG}" | cut -c2-) make install build 23 | 24 | # Setup git 25 | # https://api.github.com/users/github-actions%5Bbot%5D 26 | git config --global user.name "github-actions[bot]" 27 | git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" 28 | 29 | # Commit and push updated release files 30 | git checkout -b "${BRANCH}" 31 | git add . 32 | git commit -m "Update release version to ${VERSION_TAG}" 33 | git push origin "${BRANCH}" 34 | 35 | git tag --force "${VERSION_TAG}" 36 | git push --force origin "${VERSION_TAG}" 37 | - name: upload dists 38 | uses: actions/upload-artifact@v4 39 | with: 40 | name: release-dists 41 | path: dist/ 42 | 43 | pypi-publish: 44 | runs-on: ubuntu-latest 45 | needs: 46 | - build 47 | permissions: 48 | id-token: write 49 | 50 | steps: 51 | - name: Retrieve release distributions 52 | uses: actions/download-artifact@v4 53 | with: 54 | name: release-dists 55 | path: dist/ 56 | 57 | - name: Publish release distributions to PyPI 58 | uses: pypa/gh-action-pypi-publish@release/v1 59 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - name: Set up Python ${{ matrix.python-version }} 16 | uses: actions/setup-python@v5 17 | with: 18 | python-version: ${{ matrix.python-version }} 19 | 20 | - name: Install dependencies 21 | run: | 22 | make install 23 | 24 | - name: Tests 25 | run: | 26 | make ci 27 | 28 | - name: Archive code coverage results 29 | uses: actions/upload-artifact@v4 30 | with: 31 | name: code-coverage-report ${{ github.event.pull_request.head.sha }} ${{ matrix.python-version }} 32 | path: htmlcov/* 33 | if: ${{ github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize') }} 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | docs/source/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # pipenv 90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 92 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 93 | # install all needed dependencies. 94 | #Pipfile.lock 95 | 96 | # Celery stuff 97 | celerybeat-schedule 98 | celerybeat.pid 99 | 100 | # SageMath parsed files 101 | *.sage.py 102 | 103 | # Environments 104 | .env 105 | .venv 106 | env/ 107 | venv/ 108 | ENV/ 109 | env.bak/ 110 | venv.bak/ 111 | 112 | # Spyder project settings 113 | .spyderproject 114 | .spyproject 115 | 116 | # Rope project settings 117 | .ropeproject 118 | 119 | # mkdocs documentation 120 | /site 121 | 122 | # mypy 123 | .mypy_cache/ 124 | .dmypy.json 125 | dmypy.json 126 | 127 | # Pyre type checker 128 | .pyre/ 129 | 130 | # Microsoft VS Code settings 131 | .vscode/ 132 | 133 | # Notebook generated files 134 | examples/*.tif 135 | examples/*.png 136 | 137 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 138 | __pypackages__/ 139 | 140 | .idea 141 | .virtual_documents 142 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | 18 | # Install Python dependencies using pyproject.toml 19 | python: 20 | install: 21 | - method: pip 22 | path: . 23 | extra_requirements: 24 | - docs 25 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Harmony-Py 2 | 3 | Thanks for contributing! 4 | 5 | ## Making Changes 6 | 7 | To allow us to incorporate your changes, please use the following process: 8 | 9 | 1. Fork this repository to your personal account. 10 | 2. Create a branch and make your changes. 11 | 3. Test the changes locally/in your personal fork. 12 | 4. Submit a pull request to open a discussion about your proposed changes. 13 | 5. The maintainers will talk with you about it and decide to merge or request additional changes. 14 | 15 | ## Commits 16 | 17 | Our ticketing and CI/CD tools are configured to sync statuses amongst each other. Commits play an important role in this process. Please start all commits with the Harmony ticket number associated with your feature, task, or bug. All commit messages should follow the format "HARMONY-XXXX: [Your commit message here]" 18 | 19 | ## Disclaimer 20 | 21 | The Harmony development team will review all pull requests submitted. Only requests that meet the standard of quality set forth by existing code, following the patterns set forth by existing code, and adhering to existing design patterns will be considered and/or accepted. 22 | 23 | For general tips on open source contributions, see [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/). -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2019-2021 United States Government as represented by the Administrator of the National Aeronautics and Space Administration. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 10 | 11 | --- 12 | 13 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 14 | 15 | 1. Definitions. 16 | 17 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 18 | 19 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 20 | 21 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 24 | 25 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 26 | 27 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 28 | 29 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 30 | 31 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 32 | 33 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 34 | 35 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 36 | 37 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 38 | 39 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 40 | 41 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 42 | 43 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 44 | You must cause any modified files to carry prominent notices stating that You changed the files; and 45 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 46 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 47 | 48 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 49 | 50 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 51 | 52 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 53 | 54 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 55 | 56 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 57 | 58 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 59 | 60 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Include the README, et. al. 2 | include *.md 3 | 4 | # Include license and setup files 5 | include LICENSE 6 | include pyproject.toml 7 | 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: install install-examples clean examples lint test test-watch ci docs 2 | 3 | VERSION ?= $(shell git describe --tags | sed 's/-/\+/' | sed 's/-/\./g') 4 | REPO ?= https://upload.pypi.org/legacy/ 5 | REPO_USER ?= __token__ 6 | REPO_PASS ?= unset 7 | 8 | clean: 9 | coverage erase 10 | rm -rf htmlcov 11 | rm -rf build dist *.egg-info || true 12 | 13 | clean-docs: 14 | cd docs && $(MAKE) clean 15 | 16 | install: 17 | python -m pip install --upgrade pip 18 | pip install . 19 | pip install .[dev,docs] 20 | 21 | install-examples: install 22 | pip install .[examples] 23 | 24 | examples: install-examples 25 | jupyter-lab 26 | 27 | lint: 28 | flake8 harmony --show-source --statistics 29 | 30 | test: 31 | pytest --cov=harmony --cov-report=term --cov-report=html --cov-branch tests 32 | 33 | test-watch: 34 | ptw -c -w 35 | 36 | ci: lint test 37 | 38 | docs-notebook = examples/tutorial.ipynb 39 | docs/user/notebook.html: $(docs-notebook) 40 | jupyter nbconvert --execute --to html --output notebook.html --output-dir docs/user $(docs-notebook) 41 | 42 | docs: docs/user/notebook.html 43 | cd docs && $(MAKE) html 44 | 45 | version: 46 | sed -i.bak "s/__version__ .*/__version__ = \"$(VERSION)\"/" harmony/__init__.py && rm harmony/__init__.py.bak 47 | 48 | build: clean version 49 | python -m pip install --upgrade --quiet setuptools wheel twine build 50 | python -m build 51 | 52 | publish: build 53 | python -m twine check dist/* 54 | python -m twine upload --username "$(REPO_USER)" --password "$(REPO_PASS)" --repository-url "$(REPO)" dist/* 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # harmony-py 2 | 3 | [![Documentation Status](https://readthedocs.org/projects/harmony-py/badge/?version=latest)](https://harmony-py.readthedocs.io/en/latest/?badge=latest) 4 | 5 | Harmony-Py is a Python library for integrating with NASA's [Harmony](https://harmony.earthdata.nasa.gov/) Services. 6 | 7 | Harmony-Py provides a Python alternative to directly using [Harmony's OGC Coverage RESTful API](https://harmony.earthdata.nasa.gov/docs/api/) and [Harmony's OGC EDR RESTful API](https://harmony.earthdata.nasa.gov/docs/edr-api/). It handles NASA [Earthdata Login (EDL)](https://urs.earthdata.nasa.gov/home) authentication and optionally integrates with the [CMR Python Wrapper](https://github.com/nasa/eo-metadata-tools) by accepting collection results as a request parameter. It's convenient for scientists who wish to use Harmony from Jupyter notebooks as well as machine-to-machine communication with larger Python applications. 8 | 9 | We welcome feedback on Harmony-Py via [GitHub Issues](https://github.com/nasa/harmony-py/issues) 10 | 11 | # Using Harmony Py 12 | 13 | ## Prerequisites 14 | 15 | * Python 3.9 through 3.13 (other versions are end of life or untested) 16 | 17 | 18 | ## Installing 19 | 20 | The library is available from [PyPI](https://pypi.org/project/harmony-py/) and can be installed with pip: 21 | 22 | $ pip install -U harmony-py 23 | 24 | This will install harmony-py and its dependencies into your current Python environment. It's recommended that you install harmony-py into a virtual environment along with any other dependencies you may have. 25 | 26 | 27 | # Running Examples & Developing on Harmony Py 28 | 29 | ## Prerequisites 30 | 31 | * Python 3.9 through 3.13, ideally installed via a virtual environment 32 | 33 | 34 | ## Installing Development & Example Dependencies 35 | 36 | 1. Install dependencies: 37 | 38 | $ make install 39 | 40 | 2. Optionally register your local copy with pip: 41 | 42 | $ pip install -e ./path/to/harmony_py 43 | 44 | 45 | ## Running the Example Jupyter Notebooks 46 | 47 | Jupyter notebooks in the `examples` subdirectory show how to use the Harmony Py library. Start up the Jupyter Lab notebook server and run these examples: 48 | 49 | The Jupyter Lab server will start and [open in your browser](http://localhost:8888/lab). Double-click on a notebook in the file-browser sidebar and run the notebook. Note that some notebooks may have cells which prompt for your EDL username and password. Be sure to use your UAT credentials since most of the example notebooks use the Harmony UAT environment. 50 | 51 | $ make examples 52 | 53 | 54 | ## Developing 55 | 56 | ### Generating Documentation 57 | 58 | Documentation on the Read The Docs site is generated automatically. It is generated by using `sphinx` with reStructuredText (.rst) and other files in the `docs` directory. To generate the docs locally and see what they look like: 59 | 60 | $ make docs 61 | 62 | You can then view the documentation in a web browser under `./docs/_build/html/index.html`. 63 | 64 | IMPORTANT: The documentation uses a notebook from the `examples` directory rendered as HTML. If you've modified that notebook (see `Makefile` for notebook that is currently rendered), you will need to run `make docs` locally. You will see a change to the `docs/user/notebook.html` file after doing so. This file should be committed to the git repo since it is used when the latest docs are pushed to the Read The Docs site (it can't currently be generated as part of the build). 65 | 66 | ### Running the Linter & Unit Tests 67 | 68 | Run the linter on the project source: 69 | 70 | $ make lint 71 | 72 | Run unit tests and test coverage. This will display terminal output and generate an HTML coverage report in the `htmlcov` directory. 73 | 74 | $ make test 75 | 76 | For development, you may want to run the unit tests continuously as you update tests and the code-under-test: 77 | 78 | $ make test-watch 79 | 80 | 81 | ### Generating Request Parameters 82 | 83 | The `harmony.Request` constructor can accept parameters that are defined in the [Harmony OGC API schema](https://harmony.earthdata.nasa.gov/schemas/ogc-api-coverages/1.0.0/ogc-api-coverages-v1.0.0.yml). If this schema has been changed and the `Request` constructor needs to be updated, you may run the generator utility. This tool reads the Harmony schema and generates a partial constructor signature with docstrings: 84 | 85 | $ python internal/genparams.py ${HARMONY_DIR}/services/harmony/app/schemas/ogc-api-coverages/1.0.0/ogc-api-coverages-v1.0.0.yml 86 | 87 | Either set `HARMONY_DIR` or replace it with your Harmony project directory path. You may then write standard output to a file and then use it to update the `harmony.Request` constructor and code. 88 | 89 | ## CI 90 | 91 | Harmony-py uses [GitHub 92 | Actions](https://github.com/nasa/harmony-py/actions) to run the Linter 93 | & Unit Tests. The test coverage output is saved as a build artifact. 94 | 95 | ## Building and Releasing 96 | 97 | New versions of Harmony-Py will be published to PyPi via a GitHub [action](.github/workflows/publish-release.yml) whenever a draft release is marked as published https://github.com/nasa/harmony-py/releases. 98 | -------------------------------------------------------------------------------- /design/checkpoint_01/harmony_py.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.8.5-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3.8.5 64-bit", 19 | "metadata": { 20 | "interpreter": { 21 | "hash": "31862cc71836a94b0e0781803a3648767fc4cb197cc35bade0ddf231ddce7d7c" 22 | } 23 | } 24 | } 25 | }, 26 | "nbformat": 4, 27 | "nbformat_minor": 2, 28 | "cells": [ 29 | { 30 | "source": [ 31 | "# Usage Overview\n", 32 | "The following is an example mocked outline for `harmony-py`. \n" 33 | ], 34 | "cell_type": "markdown", 35 | "metadata": {} 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 1, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "from harmony_py_mock import HarmonyRequest\n" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 2, 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "req = HarmonyRequest()\n" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 3, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "# Multiple authentication options outlined below.\n", 62 | "req.authenticate()\n" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 4, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "collection_id = 'C1940468263-POCLOUD'\n", 72 | "\n", 73 | "req.params = {\n", 74 | " 'collection_id': collection_id,\n", 75 | " 'ogc-api-coverages_version': '1.0.0',\n", 76 | " 'variable': 'all',\n", 77 | " 'lat':'(40:41)',\n", 78 | " 'lon':'(-107:-105)',\n", 79 | " 'start': '2020-06-01T00:00:00Z',\n", 80 | " 'stop':'2020-06-30T23:59:59Z',\n", 81 | " 'format': 'application/x-zarr',\n", 82 | "}\n" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 5, 88 | "metadata": {}, 89 | "outputs": [ 90 | { 91 | "output_type": "stream", 92 | "name": "stdout", 93 | "text": [ 94 | "Processing request:\n", 95 | "100% (100 of 100) |######################| Elapsed Time: 0:00:08 Time: 0:00:08\n", 96 | "Request processing complete.\n" 97 | ] 98 | } 99 | ], 100 | "source": [ 101 | "# Note: By default the user does not specify sync or async. Optional: specify request type.\n", 102 | "req.submit()\n", 103 | "# OR\n", 104 | "HarmonyRequest.submit(req)\n" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 6, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "output = req.output\n", 114 | "\n" 115 | ] 116 | }, 117 | { 118 | "source": [ 119 | "# Further Examples" 120 | ], 121 | "cell_type": "markdown", 122 | "metadata": {} 123 | }, 124 | { 125 | "source": [ 126 | "## Request Parameters" 127 | ], 128 | "cell_type": "markdown", 129 | "metadata": {} 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 7, 134 | "metadata": {}, 135 | "outputs": [], 136 | "source": [ 137 | "import datetime\n", 138 | "\n", 139 | "collection_id = 'C1940468263-POCLOUD'\n", 140 | "\n", 141 | "req.params = {\n", 142 | " 'collection_id': collection_id,\n", 143 | " 'ogc-api-coverages_version': '1.0.0',\n", 144 | " 'variable': 'all',\n", 145 | " 'lat':'(40:41)',\n", 146 | " 'lon':'(-107:-105)',\n", 147 | " 'start': '2020-06-01T00:00:00Z',\n", 148 | " 'stop':'2020-06-30T23:59:59Z',\n", 149 | " 'format': 'application/x-zarr',\n", 150 | "}\n", 151 | "\n", 152 | "# or\n", 153 | "\n", 154 | "req.params = {\n", 155 | " 'collection_id': collection_id,\n", 156 | " 'lat': (40, 42), # (min, max) format\n", 157 | " 'lon': (-107, -105), # (min, max) format\n", 158 | " 'start': datetime.date(2020, 6, 1), # date object\n", 159 | " 'stop': datetime.date(2020, 6, 30), # date object\n", 160 | " 'format': HarmonyRequest.format.zarr,\n", 161 | "}\n", 162 | "\n", 163 | "# alternative: Keyword attributes or request object w/ methods for setting individually or grouped as warranted\n", 164 | "# - bounding box class? Allow for shape / polygons\n", 165 | "# - maybe namedtuple\n", 166 | "# - enumerated types; leverage typing; structural vs. semantic validation\n", 167 | "# - validate values for each (type based on enum type)\n", 168 | "# - validate sanity (start not after stop, etc.)\n", 169 | "# - validation automatic prior to sending to Harmony and while constructing request (moreso the latter)\n", 170 | "# - think about what the API docs will look like; autocomplete kwargs\n", 171 | "# - is there a commonly used 'bounding box' thing that others use?\n", 172 | "# - are we going to have a multi-option API or specific option?\n", 173 | "# - validate requests before sending to Harmony\n", 174 | "# - instantiating the class would implicitly validate\n", 175 | "# - hide OGC particulars from end-users; make it pythonic\n", 176 | "# - bag of parameters vs. chained method calls (dot notation); builder pattern\n", 177 | "# - incremental approach has advantages for conveying documentation and demos\n", 178 | "# - decouple particular services from subset/transformation\n", 179 | "# - make sure request can be built incrementally, parameter by parameter ?\n", 180 | "# - allow both all at once as well as incremental change/addition\n", 181 | "# - _collection_ should be separate 'building stage'; pulled out as a separate entity\n", 182 | "# - what is the best way to interoperate with CMR python library\n", 183 | "\n", 184 | "# modules for API; modules for CLI\n", 185 | "# - allow either use for harmony python library\n", 186 | "\n", 187 | "# or\n", 188 | "\n", 189 | "req.params = {\n", 190 | " 'collection_id': collection_id,\n", 191 | " 'lat': (40, 42),\n", 192 | " 'lon': (-107, -105),\n", 193 | " 'temporal': '2020-6-1, 30 days', # DSL\n", 194 | " # or\n", 195 | " 'temporal': '2012-6-1 for 2 months every year for 6 years', # DSL for seasonal temporal period\n", 196 | " 'output_format': HarmonyRequest.format.zarr\n", 197 | "}\n", 198 | "\n", 199 | "# recommend or provide examples of third-party library to generate lists of time periods\n", 200 | "# - timedeltas?\n", 201 | "# - arrow https://github.com/vinta/awesome-python#date-and-time\n" 202 | ] 203 | }, 204 | { 205 | "source": [ 206 | "## Authentication" 207 | ], 208 | "cell_type": "markdown", 209 | "metadata": {} 210 | }, 211 | { 212 | "source": [ 213 | "req = HarmonyRequest()\n", 214 | "\n", 215 | "# Authentication options:\n", 216 | "# .) specify `username` and `password`\n", 217 | "# .) specify `username` and receive password prompt\n", 218 | "# .) specify .netrc\n", 219 | "# .) read .netrc in default location\n", 220 | "# .) read .env file\n", 221 | "# .) read ENV vars \n", 222 | "\n", 223 | "req.authenticate(username='myusername', password='supersecret')\n", 224 | "# or\n", 225 | "req.authenticate(username='myusername')\n", 226 | "# or\n", 227 | "req.authenticate(netrc='/usr/local/etc/some/path/.netrc')\n", 228 | "\n", 229 | "# or\n", 230 | "req.authenticate()\n", 231 | "\n", 232 | "\n", 233 | "# Use requests library .netrc support\n", 234 | "# - utils modules in requests\n", 235 | "\n", 236 | "# check out requests usage w/ async in harmony-netcdf-to-zarr\n" 237 | ], 238 | "cell_type": "code", 239 | "metadata": {}, 240 | "execution_count": 8, 241 | "outputs": [ 242 | { 243 | "output_type": "stream", 244 | "name": "stdout", 245 | "text": [ 246 | "'username' and 'password' accepted.\n" 247 | ] 248 | } 249 | ] 250 | }, 251 | { 252 | "source": [ 253 | "## Determine Service Availability and Variables / Working with CMR" 254 | ], 255 | "cell_type": "markdown", 256 | "metadata": {} 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 9, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "# NOTES:\n", 265 | "# - Decide what ServiceOptions to use from UMM-S\n", 266 | "# - The extent to which we wrap the CMR python library OR (preferred) integrate with the CMR python library\n", 267 | "# - What UMM-C metadata to expose?\n", 268 | "# - UMM-C assocation to UMM-S\n", 269 | "# - UMM-Var\n", 270 | "\n", 271 | "\n", 272 | "req = HarmonyRequest()\n", 273 | "req.params = {\n", 274 | " 'collection_id': collection_id,\n", 275 | " 'lat': (40, 42),\n", 276 | " 'lon': (-107, -105),\n", 277 | " 'temporal': '2020-6-1, 30 days',\n", 278 | " 'format': HarmonyRequest.format.zarr\n", 279 | "}\n", 280 | "#req.dataset.info() # similar to icepyx\n", 281 | "#req.dataset.service_availability() # similar to icepyx\n", 282 | "#req.dataset.variables()\n", 283 | "\n", 284 | "# Probably out of scope:\n", 285 | "#req.spatial.visualize() # similar to icepyx - map with bbox overlay\n", 286 | "\n", 287 | "\n", 288 | "# Use CMR library as much as possible; in-scope to make CMR library contributions as needed" 289 | ] 290 | }, 291 | { 292 | "source": [ 293 | "## Retrieve Results in Cloud: In / Out of Region; Internet Retrieval" 294 | ], 295 | "cell_type": "markdown", 296 | "metadata": {} 297 | }, 298 | { 299 | "cell_type": "code", 300 | "execution_count": null, 301 | "metadata": {}, 302 | "outputs": [], 303 | "source": [ 304 | "# Report back number of files and total size of data\n", 305 | "\n", 306 | "# Should the Harmony python library return job results or simply a job id so they may follow up with results outside of the python library?\n", 307 | "# We need a way to expose the job / job id to at least be able to cancel it\n", 308 | "# - block forever for request to finish OR get a job id to retrieve results outside of Harmony python library\n", 309 | "# - provide information on time/size of request\n", 310 | "# - job status\n", 311 | "# - cancellable" 312 | ] 313 | }, 314 | { 315 | "source": [ 316 | "## Error Notification and Handling" 317 | ], 318 | "cell_type": "markdown", 319 | "metadata": {} 320 | }, 321 | { 322 | "source": [ 323 | "## Notes:" 324 | ], 325 | "cell_type": "markdown", 326 | "metadata": {} 327 | }, 328 | { 329 | "cell_type": "code", 330 | "execution_count": null, 331 | "metadata": {}, 332 | "outputs": [], 333 | "source": [ 334 | "# Do we need to support restricted datasets (w/ CMR token)\n", 335 | "# - currently authentication is only grabbing an EDL cookie\n", 336 | "\n", 337 | "# CMR library\n", 338 | "# - we need to understand its capabilities now and near-future; what will we be able to use?\n", 339 | "# - cmr-stac ?\n", 340 | "\n", 341 | "# STAC\n", 342 | "# What STAC information is exposed to the end-user?\n", 343 | "\n", 344 | "# Verbs\n", 345 | "# HTTP or higher level concepts like \"subset\"\n", 346 | "# - used 'submit' instead" 347 | ] 348 | } 349 | ] 350 | } -------------------------------------------------------------------------------- /design/checkpoint_01/harmony_py_mock.py: -------------------------------------------------------------------------------- 1 | import progressbar 2 | import time 3 | from getpass import getpass 4 | from halo import Halo 5 | 6 | 7 | class HarmonyFormat(): 8 | zarr = None 9 | 10 | 11 | class Dataset(): 12 | def __init__(self, dataset): 13 | self.dataset = dataset 14 | 15 | def info(self): 16 | pass 17 | 18 | def summary(self): 19 | pass 20 | 21 | def visualize(self): 22 | pass 23 | 24 | 25 | class Spatial(): 26 | def __init__(self, spatial): 27 | self.spatial = spatial 28 | 29 | def visualize(self): 30 | pass 31 | 32 | 33 | class HarmonyRequest(): 34 | 35 | format = HarmonyFormat 36 | 37 | def __init__(self, params=None): 38 | self.params = params 39 | self.dataset = Dataset(None) 40 | self.spatial = Spatial(None) 41 | 42 | def authenticate(self, username=None, password=None, netrc=None): 43 | if username is not None and password is None: 44 | password = getpass('Password: ') 45 | self._creds = {'username': username, 'password': password} 46 | print('\'username\' and \'password\' accepted.') 47 | 48 | 49 | @property 50 | def params(self): 51 | return self._params 52 | 53 | 54 | @params.setter 55 | def params(self, value): 56 | self._params = value 57 | 58 | 59 | def subset(self): 60 | print('Processing request:') 61 | for i in progressbar.progressbar(range(100)): 62 | time.sleep(0.08) 63 | print('Request processing complete.') 64 | 65 | 66 | @property 67 | def output(self): 68 | pass 69 | -------------------------------------------------------------------------------- /design/checkpoint_02/harmony_py.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.8.5-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3.8.5 64-bit", 19 | "metadata": { 20 | "interpreter": { 21 | "hash": "31862cc71836a94b0e0781803a3648767fc4cb197cc35bade0ddf231ddce7d7c" 22 | } 23 | } 24 | } 25 | }, 26 | "nbformat": 4, 27 | "nbformat_minor": 2, 28 | "cells": [ 29 | { 30 | "source": [ 31 | "# Usage Overview\n", 32 | "The following is example usage for `harmony-py`. \n" 33 | ], 34 | "cell_type": "markdown", 35 | "metadata": {} 36 | }, 37 | { 38 | "source": [ 39 | "\n", 40 | "# Under this proposed interface harmony-py would have the concept of three (3) entities: a Request, a Client, and a Job:\n", 41 | "# - The Request contains request parameters\n", 42 | "# - The Client represents the ability to perform authenticated HTTP with the Harmony endpoint\n", 43 | "# - The Job represents what Harmony is working on as well as retrieving the finished work results. It's referenced via 'job_id' and used by the Client.\n", 44 | "\n", 45 | "# Individual parameters are validated when set.\n", 46 | "# Not all keyword args need be supplied at once. Also, parameters may be replaced.\n", 47 | "request = Request(\n", 48 | " collection=Collection(id='C1940468263-POCLOUD'),\n", 49 | " spatial={'ll': (40, -107),\n", 50 | " 'ur': (42, -105)}\n", 51 | " temporal={'start': datetime.date(2020, 6, 1),\n", 52 | " 'stop': datetime.date(2020, 6, 30)},\n", 53 | " format=Format.ZARR\n", 54 | ")\n", 55 | "\n", 56 | "# Authentication is stored in a client object for subsequent server interaction.\n", 57 | "client = Client(Authentication())\n", 58 | "\n", 59 | "# Validation may be performed prior to job processing; uses Harmony server-side checking.\n", 60 | "client.validate(request)\n", 61 | "\n", 62 | "# Starts job processing; async by default.\n", 63 | "job_id = client.submit(request, verbose=True, async=True)\n", 64 | "\n", 65 | "# Optional\n", 66 | "client.status(job_id)\n", 67 | "\n", 68 | "# Optional\n", 69 | "client.cancel(job_id)\n", 70 | "\n", 71 | "# Retrieve results in-region; returns a generator\n", 72 | "urls = client.result_urls(job_id, region=Region.US_WEST_2)\n", 73 | "# - or -\n", 74 | "# Download files to a local directory\n", 75 | "client.download(job_id, region=Region.US_WEST_2, directory='./research', overwrite=True)\n" 76 | ], 77 | "cell_type": "code", 78 | "metadata": {}, 79 | "execution_count": null, 80 | "outputs": [] 81 | }, 82 | { 83 | "source": [ 84 | "# Further Examples" 85 | ], 86 | "cell_type": "markdown", 87 | "metadata": {} 88 | }, 89 | { 90 | "source": [ 91 | "## Authentication" 92 | ], 93 | "cell_type": "markdown", 94 | "metadata": {} 95 | }, 96 | { 97 | "source": [ 98 | "# Authentication options:\n", 99 | "# .) specify `username` and `password`\n", 100 | "# .) specify `username` and receive password prompt\n", 101 | "# .) specify .netrc\n", 102 | "# .) read .netrc in default location\n", 103 | "# .) read .env file\n", 104 | "# .) read ENV vars \n", 105 | "\n", 106 | "auth = Authenticate(username='myusername', password='supersecret')\n", 107 | "# or\n", 108 | "auth = Authenticate(username='myusername')\n", 109 | "# or\n", 110 | "auth = Authenticate(netrc='/usr/local/etc/some/path/.netrc')\n", 111 | "\n", 112 | "# or\n", 113 | "auth = Authenticate()\n" 114 | ], 115 | "cell_type": "code", 116 | "metadata": {}, 117 | "execution_count": null, 118 | "outputs": [] 119 | }, 120 | { 121 | "source": [ 122 | "## Determine Service Availability and Variables / Working with CMR" 123 | ], 124 | "cell_type": "markdown", 125 | "metadata": {} 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "# Notes from a previous meeting:\n", 134 | "# extend CMR library to UMM-S; nice output\n", 135 | "# when and where do we use Harmony's Capabilities documents?\n", 136 | "# stick with Python data structures (dicts, lists, etc.)\n", 137 | "# output of CMR should be acceptable input to Harmony python lib\n", 138 | "# but also allow a user to submit strings as input for Harmony python lib\n", 139 | "# understand UMM-Var response (coupled ot their metadata format)\n", 140 | "\n", 141 | "# More Notes: We may want to contribute to the CMR python library in order to make feeding data into the Harmony python library easier than what's shown here.\n", 142 | "\n", 143 | "import re\n", 144 | "\n", 145 | "# Import CMR's python library\n", 146 | "import cmr.search.collection as coll\n", 147 | "import cmr.search.granule as gran\n", 148 | "\n", 149 | "cmr_res = coll.search({'keyword':'MOD09*',\n", 150 | " 'archive_center': 'lp daac'})\n", 151 | "\n", 152 | "# regex uses a negative look-around assertion\n", 153 | "brief = [[r['meta']['concept-id'],\n", 154 | " r['umm']['ShortName'],\n", 155 | " r['meta']['native-id']] for r in cmr_res if re.search('^((?!mmt_collection_).)*$', r['meta']['native-id'])]\n", 156 | "[print(b) for b in brief]\n", 157 | "# ['C193529903-LPDAAC_ECS', 'MOD09GQ', 'MODIS/Terra Surface Reflectance Daily L2G Global 250m SIN Grid V006']\n", 158 | "# ['C193529902-LPDAAC_ECS', 'MOD09GA', 'MODIS/Terra Surface Reflectance Daily L2G Global 1km and 500m SIN Grid V006']\n", 159 | "# ['C193529899-LPDAAC_ECS', 'MOD09A1', 'MODIS/Terra Surface Reflectance 8-Day L3 Global 500m SIN Grid V006']\n", 160 | "# ['C193529944-LPDAAC_ECS', 'MOD09Q1', 'MODIS/Terra Surface Reflectance 8-Day L3 Global 250m SIN Grid V006']\n", 161 | "# ['C193529901-LPDAAC_ECS', 'MOD09CMG', 'MODIS/Terra Surface Reflectance Daily L3 Global 0.05Deg CMG V006']\n", 162 | "\n", 163 | "\n", 164 | "####\n", 165 | "# The CMR python library does not support variable browsing at this time.\n", 166 | "####\n", 167 | "\n", 168 | "\n", 169 | "# The output from CMR may be used as input to the Harmony python library\n", 170 | "req = Request(\n", 171 | " collection=cmr_res[0],\n", 172 | " spatial=Bbox(lat=(40, 42), lon=(-107, -105)), # could accept bbox, shapely, geojson (polygon)\n", 173 | " temporal=Temporal(start=datetime.date(2020, 6, 1), stop=datetime.date(2020, 6, 30)),\n", 174 | " format=Format.ZARR\n", 175 | ")\n" 176 | ] 177 | }, 178 | { 179 | "source": [ 180 | "## Async vs. Sync Request Submit()" 181 | ], 182 | "cell_type": "markdown", 183 | "metadata": {} 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "# Sync Request\n", 192 | "res = Request.submit(req, auth, sync=True)\n", 193 | "\n", 194 | "# Async request; default behavior\n", 195 | "res = Request.submit(req, auth)\n", 196 | "\n", 197 | "# Async usage: Poll Harmony status page and display progress updates.\n", 198 | "res.update()\n", 199 | "\n", 200 | "# Async usage: Cancel an ongoing job (Result)\n", 201 | "res.cancel()\n" 202 | ] 203 | }, 204 | { 205 | "source": [ 206 | "## Retrieve Results in Cloud: In / Out of Region; Internet Retrieval" 207 | ], 208 | "cell_type": "markdown", 209 | "metadata": {} 210 | }, 211 | { 212 | "source": [ 213 | "# Notes:\n", 214 | "# require the user to be explicit on style of retrieval: in-cloud/in-region vs. over internet\n", 215 | "# if in same region, collect k/v pairs for parameters to boto3 constructor\n", 216 | "# End-user _must_ specify region keyword argument.\n", 217 | "\n", 218 | "import boto3\n", 219 | "import requests\n", 220 | "\n", 221 | "# Downloads files to a local directory. Easy method.\n", 222 | "Response.download(res, region=Response.IN_REGION, directory='./research', overwrite=True)\n", 223 | "\n", 224 | "# Downloads files to a local directory but skips files which already exist; Note: doesn't verify existing file size. Easy method.\n", 225 | "Response.download(res, region=Response.IN_REGION, directory='./research', overwrite=False)\n", 226 | "\n", 227 | "####\n", 228 | "\n", 229 | "# In-Region; alternative to the above.\n", 230 | "s3 = boto3.client('s3')\n", 231 | "files = Response.files(res, region=Response.IN_REGION)\n", 232 | "for f in files:\n", 233 | " # The parameters for each output file are the inputs to this boto method.\n", 234 | " s3.download_file(f.bucket_name, f.object_name, './research/' + f.filename)\n", 235 | "\n", 236 | "# Out-of-Region; alternative to the above easy methods.\n", 237 | "files = Response.files(res, region=Response.OUT_OF_REGION)\n", 238 | "for f in files:\n", 239 | " r = requests.get(f.url, allow_redirects=True)\n", 240 | " open(f.filename, 'wb').write(r.content)\n" 241 | ], 242 | "cell_type": "code", 243 | "metadata": {}, 244 | "execution_count": null, 245 | "outputs": [] 246 | }, 247 | { 248 | "source": [ 249 | "## Error Notification and Handling\n", 250 | "\n", 251 | "Open for suggestions. We probably should raise exceptions as needed and output friendly messages via logging. The same logging will be used for async operation. Async is futures based so the GIL and cooperative multitasking will handle contention for logging output destinations. STDOUT will be the default logging target." 252 | ], 253 | "cell_type": "markdown", 254 | "metadata": {} 255 | }, 256 | { 257 | "source": [ 258 | "Notes from 2021/2/8 meeting:\n", 259 | "- no Temporal class\n", 260 | " - use datetime objects but not strings; tuple is fine; allow None on either side\n", 261 | " - could also use a NamedTuple or @dataclass\n", 262 | " - possible python construct here\n", 263 | "- for Bbox\n", 264 | " - don't provide just numbers of the form \"12:34\" (OGC standard)\n", 265 | " - follow CMR python library usage? Or send them a PR to align things with harmony-py\n", 266 | "- remove req.action\n", 267 | "- Request.validate(req)\n", 268 | " - throw an exception that contains the individual errors\n", 269 | " - 'validate' should exist in Harmony and be a function\n", 270 | " - perform HEAD request (or something that allows for responses in body) and send to Harmony; use server-side validation\n", 271 | "- Request.submit() lift up\n", 272 | " - will need to get Harmony to have status page even on sync requests\n", 273 | "- Response object could yield a sequence of e.g. URLs; watch for coroutine usage on resetting sequence\n", 274 | " - download() would continue to download files in the background\n", 275 | "- instead of IN_REGION it should specify us-west-2 somehow\n", 276 | "- everything requires auth for server communication; a 'client' or something should have auth as a part of it. Does not need to hold up short-term development.\n", 277 | "\n", 278 | "- UMM-S and UMM-Var browsing via CMR python library\n", 279 | " - ... meeting ended here\n", 280 | "\n" 281 | ], 282 | "cell_type": "markdown", 283 | "metadata": {} 284 | } 285 | ] 286 | } -------------------------------------------------------------------------------- /design/checkpoint_03/harmony_py.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "language_info": { 4 | "codemirror_mode": { 5 | "name": "ipython", 6 | "version": 3 7 | }, 8 | "file_extension": ".py", 9 | "mimetype": "text/x-python", 10 | "name": "python", 11 | "nbconvert_exporter": "python", 12 | "pygments_lexer": "ipython3", 13 | "version": "3.8.5-final" 14 | }, 15 | "orig_nbformat": 2, 16 | "kernelspec": { 17 | "name": "python3", 18 | "display_name": "Python 3.8.5 64-bit", 19 | "metadata": { 20 | "interpreter": { 21 | "hash": "31862cc71836a94b0e0781803a3648767fc4cb197cc35bade0ddf231ddce7d7c" 22 | } 23 | } 24 | } 25 | }, 26 | "nbformat": 4, 27 | "nbformat_minor": 2, 28 | "cells": [ 29 | { 30 | "source": [ 31 | "# Usage Overview\n", 32 | "The following is example usage for `harmony-py`. \n" 33 | ], 34 | "cell_type": "markdown", 35 | "metadata": {} 36 | }, 37 | { 38 | "source": [ 39 | "\n", 40 | "# Under this proposed interface harmony-py would have the concept of three (3) entities: a Request, a Client, and a Job:\n", 41 | "# - The Request contains request parameters\n", 42 | "# - The Client represents the ability to perform authenticated HTTP with the Harmony endpoint\n", 43 | "# - The Job represents what Harmony is working on as well as retrieving the finished work results. It's referenced via 'job_id' and used by the Client.\n", 44 | "\n", 45 | "# Individual parameters are validated when set.\n", 46 | "# Not all keyword args need be supplied at once. Also, parameters may be replaced.\n", 47 | "request = Request(\n", 48 | " collection=Collection(id='C1940468263-POCLOUD'),\n", 49 | " spatial={'ll': (40, -107),\n", 50 | " 'ur': (42, -105)}\n", 51 | " temporal={'start': datetime.date(2020, 6, 1),\n", 52 | " 'stop': datetime.date(2020, 6, 30)},\n", 53 | " format=Format.ZARR\n", 54 | ")\n", 55 | "\n", 56 | "# Authentication is stored in a client object for subsequent server interaction.\n", 57 | "client = Client(auth=Authentication())\n", 58 | "\n", 59 | "# Validation may be performed prior to job processing; uses Harmony server-side checking.\n", 60 | "client.validate(request)\n", 61 | "\n", 62 | "# Starts job processing; async by default.\n", 63 | "job_id = client.submit(request, verbose=True, async=True)\n", 64 | "\n", 65 | "# Optional\n", 66 | "client.status(job_id)\n", 67 | "\n", 68 | "# Optional\n", 69 | "client.cancel(job_id)\n", 70 | "\n", 71 | "# Retrieve results in-region; returns a generator\n", 72 | "urls = client.result_urls(job_id, region=Region.US_WEST_2)\n", 73 | "# - or -\n", 74 | "# Download files to a local directory\n", 75 | "client.download(job_id, region=Region.US_WEST_2, directory='./research', overwrite=True)\n", 76 | "# - or -\n", 77 | "client.download(job_id, region=Region.OUT_OF_REGION, directory='./research', overwrite=True)\n", 78 | "# - or -\n", 79 | "# Download a STAC Catalog of STAC Items; returned as JSON.\n", 80 | "client.stac_catalog(job_id)\n" 81 | ], 82 | "cell_type": "code", 83 | "metadata": {}, 84 | "execution_count": null, 85 | "outputs": [] 86 | }, 87 | { 88 | "source": [ 89 | "# Further Examples" 90 | ], 91 | "cell_type": "markdown", 92 | "metadata": {} 93 | }, 94 | { 95 | "source": [ 96 | "## Authentication" 97 | ], 98 | "cell_type": "markdown", 99 | "metadata": {} 100 | }, 101 | { 102 | "source": [ 103 | "# Authentication options:\n", 104 | "# .) specify `username` and `password`\n", 105 | "# .) specify `username` and receive password prompt\n", 106 | "# .) specify .netrc\n", 107 | "# .) read .netrc in default location\n", 108 | "# .) read .env file\n", 109 | "# .) read ENV vars \n", 110 | "\n", 111 | "auth = Authenticate(username='myusername', password='supersecret')\n", 112 | "# or\n", 113 | "auth = Authenticate(username='myusername')\n", 114 | "# or\n", 115 | "auth = Authenticate(netrc='/usr/local/etc/some/path/.netrc')\n", 116 | "\n", 117 | "# or\n", 118 | "auth = Authenticate()\n" 119 | ], 120 | "cell_type": "code", 121 | "metadata": {}, 122 | "execution_count": null, 123 | "outputs": [] 124 | }, 125 | { 126 | "source": [ 127 | "## Determine Service Availability and Variables / Working with CMR" 128 | ], 129 | "cell_type": "markdown", 130 | "metadata": {} 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "# Notes from a previous meeting:\n", 139 | "# extend CMR library to UMM-S; nice output\n", 140 | "# when and where do we use Harmony's Capabilities documents?\n", 141 | "# stick with Python data structures (dicts, lists, etc.)\n", 142 | "# output of CMR should be acceptable input to Harmony python lib\n", 143 | "# but also allow a user to submit strings as input for Harmony python lib\n", 144 | "# understand UMM-Var response (coupled ot their metadata format)\n", 145 | "\n", 146 | "# More Notes: We may want to contribute to the CMR python library in order to make feeding data into the Harmony python library easier than what's shown here.\n", 147 | "\n", 148 | "import re\n", 149 | "\n", 150 | "# Import CMR's python library\n", 151 | "import cmr.search.collection as coll\n", 152 | "import cmr.search.granule as gran\n", 153 | "\n", 154 | "cmr_res = coll.search({'keyword':'MOD09*',\n", 155 | " 'archive_center': 'lp daac'})\n", 156 | "\n", 157 | "# regex uses a negative look-around assertion\n", 158 | "brief = [[r['meta']['concept-id'],\n", 159 | " r['umm']['ShortName'],\n", 160 | " r['meta']['native-id']] for r in cmr_res if re.search('^((?!mmt_collection_).)*$', r['meta']['native-id'])]\n", 161 | "[print(b) for b in brief]\n", 162 | "# ['C193529903-LPDAAC_ECS', 'MOD09GQ', 'MODIS/Terra Surface Reflectance Daily L2G Global 250m SIN Grid V006']\n", 163 | "# ['C193529902-LPDAAC_ECS', 'MOD09GA', 'MODIS/Terra Surface Reflectance Daily L2G Global 1km and 500m SIN Grid V006']\n", 164 | "# ['C193529899-LPDAAC_ECS', 'MOD09A1', 'MODIS/Terra Surface Reflectance 8-Day L3 Global 500m SIN Grid V006']\n", 165 | "# ['C193529944-LPDAAC_ECS', 'MOD09Q1', 'MODIS/Terra Surface Reflectance 8-Day L3 Global 250m SIN Grid V006']\n", 166 | "# ['C193529901-LPDAAC_ECS', 'MOD09CMG', 'MODIS/Terra Surface Reflectance Daily L3 Global 0.05Deg CMG V006']\n", 167 | "\n", 168 | "\n", 169 | "####\n", 170 | "# The CMR python library does not support variable browsing at this time.\n", 171 | "####\n", 172 | "\n", 173 | "\n", 174 | "# The output from CMR may be used as input to the Harmony python library\n", 175 | "req = Request(\n", 176 | " collection=cmr_res[0],\n", 177 | " spatial=Bbox(lat=(40, 42), lon=(-107, -105)), # could accept bbox, shapely, geojson (polygon)\n", 178 | " temporal=Temporal(start=datetime.date(2020, 6, 1), stop=datetime.date(2020, 6, 30)),\n", 179 | " format=Format.ZARR\n", 180 | ")\n", 181 | "\n", 182 | "#### The following is not yet implemented in the CMR Python Wrapper:\n", 183 | "import cmr.search.services as serv\n", 184 | "cmr_serv_res = serv.search({'collection':cmr_res[0]})\n", 185 | "# 'format' could then be supplied as an input argument to Harmony.Request\n", 186 | "format = cmr_serv_res.SupportedOutputFormats[0]\n", 187 | "\n", 188 | "\n" 189 | ] 190 | }, 191 | { 192 | "source": [ 193 | "## Async vs. Sync Request Submit()" 194 | ], 195 | "cell_type": "markdown", 196 | "metadata": {} 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "# Sync Request\n", 205 | "client.submit(req, auth, sync=True)\n", 206 | "\n", 207 | "# Async request; default behavior\n", 208 | "client.submit(req, auth)\n", 209 | "\n", 210 | "# Async usage: Poll Harmony status page and display progress updates.\n", 211 | "client.status()\n", 212 | "\n", 213 | "# Async usage: Cancel an ongoing job (Result)\n", 214 | "client.cancel()\n" 215 | ] 216 | }, 217 | { 218 | "source": [ 219 | "## Retrieve Results in Cloud: In / Out of Region; Internet Retrieval" 220 | ], 221 | "cell_type": "markdown", 222 | "metadata": {} 223 | }, 224 | { 225 | "source": [ 226 | "# Notes:\n", 227 | "# require the user to be explicit on style of retrieval: in-cloud/in-region vs. over internet\n", 228 | "# if in same region, collect k/v pairs for parameters to boto3 constructor\n", 229 | "# End-user _must_ specify region keyword argument.\n", 230 | "\n", 231 | "import boto3\n", 232 | "import requests\n", 233 | "\n", 234 | "# Downloads files to a local directory. Easy method.\n", 235 | "Response.download(res, region=Response.IN_REGION, directory='./research', overwrite=True)\n", 236 | "\n", 237 | "# Downloads files to a local directory but skips files which already exist; Note: doesn't verify existing file size. Easy method.\n", 238 | "Response.download(res, region=Response.IN_REGION, directory='./research', overwrite=False)\n", 239 | "\n", 240 | "####\n", 241 | "\n", 242 | "# In-Region; alternative to the above.\n", 243 | "s3 = boto3.client('s3')\n", 244 | "files = Response.files(res, region=Response.IN_REGION)\n", 245 | "for f in files:\n", 246 | " # The parameters for each output file are the inputs to this boto method.\n", 247 | " s3.download_file(f.bucket_name, f.object_name, './research/' + f.filename)\n", 248 | "\n", 249 | "# Out-of-Region; alternative to the above easy methods.\n", 250 | "files = Response.files(res, region=Response.OUT_OF_REGION)\n", 251 | "for f in files:\n", 252 | " r = requests.get(f.url, allow_redirects=True)\n", 253 | " open(f.filename, 'wb').write(r.content)\n" 254 | ], 255 | "cell_type": "code", 256 | "metadata": {}, 257 | "execution_count": null, 258 | "outputs": [] 259 | }, 260 | { 261 | "source": [ 262 | "## Error Notification and Handling\n", 263 | "\n", 264 | "Open for suggestions. We probably should raise exceptions as needed and output friendly messages via logging. The same logging will be used for async operation. Async is futures based so the GIL and cooperative multitasking will handle contention for logging output destinations. STDOUT will be the default logging target." 265 | ], 266 | "cell_type": "markdown", 267 | "metadata": {} 268 | }, 269 | { 270 | "source": [ 271 | "## Notes from 2021/2/8 meeting:\n", 272 | "- no Temporal class\n", 273 | " - use datetime objects but not strings; tuple is fine; allow None on either side\n", 274 | " - could also use a NamedTuple or @dataclass\n", 275 | " - possible python construct here\n", 276 | "- for Bbox\n", 277 | " - don't provide just numbers of the form \"12:34\" (OGC standard)\n", 278 | " - follow CMR python library usage? Or send them a PR to align things with harmony-py\n", 279 | "- remove req.action\n", 280 | "- Request.validate(req)\n", 281 | " - throw an exception that contains the individual errors\n", 282 | " - 'validate' should exist in Harmony and be a function\n", 283 | " - perform HEAD request (or something that allows for responses in body) and send to Harmony; use server-side validation\n", 284 | "- Request.submit() lift up\n", 285 | " - will need to get Harmony to have status page even on sync requests\n", 286 | "- Response object could yield a sequence of e.g. URLs; watch for coroutine usage on resetting sequence\n", 287 | " - download() would continue to download files in the background\n", 288 | "- instead of IN_REGION it should specify us-west-2 somehow\n", 289 | "- everything requires auth for server communication; a 'client' or something should have auth as a part of it. Does not need to hold up short-term development.\n", 290 | "\n", 291 | "- UMM-S and UMM-Var browsing via CMR python library\n", 292 | " - ... meeting ended here\n", 293 | "\n" 294 | ], 295 | "cell_type": "markdown", 296 | "metadata": {} 297 | } 298 | ] 299 | } -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | API Documentation 4 | ================= 5 | 6 | .. module:: harmony 7 | :noindex: 8 | 9 | Here we cover the package and its modules, focusing first on the classes normally imported when working with Harmony. 10 | 11 | Top-level Package API 12 | --------------------- 13 | 14 | The classes in the ``harmony`` package that are used for crafting a request, submitting it to Harmony, and getting the results. 15 | 16 | .. autoclass:: harmony.Request 17 | 18 | .. autoclass:: harmony.AddLabelsRequest 19 | 20 | .. autoclass:: harmony.DeleteLabelsRequest 21 | 22 | .. autoclass:: harmony.CapabilitiesRequest 23 | 24 | .. autoclass:: harmony.JobsRequest 25 | 26 | .. autoclass:: harmony.Client 27 | 28 | When creating a request, the ``BBox`` and ``Collection`` classes are used to create a valid request. 29 | 30 | .. autoclass:: harmony.BBox 31 | 32 | .. autoclass:: harmony.Collection 33 | 34 | 35 | Authenticating with Earthdata Login 36 | ----------------------------------- 37 | 38 | HarmonyPy requires that you have a valid `Earthdata Login account `_. There are four ways to use your EDL account with HarmonyPy: 39 | 40 | 1. Provide EDL token when creating a HarmonyPy ``Client`` :: 41 | 42 | harmony_client = Client(token='myEDLTokenValue') 43 | 44 | 2. Provide your credentials when creating a HarmonyPy ``Client`` :: 45 | 46 | harmony_client = Client(auth=('captainmarvel', 'marve10u5')) 47 | 48 | 3. Set your credentials using environment variables :: 49 | 50 | $ export EDL_USERNAME='captainmarvel' 51 | $ export EDL_PASSWORD='marve10u5' 52 | 53 | 4. Use a ``.netrc`` file: 54 | 55 | Create a ``.netrc`` file in your home directory, using the example below :: 56 | 57 | machine urs.earthdata.nasa.gov 58 | login captainmarvel 59 | password marve10u5 60 | 61 | Exceptions 62 | ---------- 63 | 64 | Exceptions that may be raised when authenticating with Earthdata Login. 65 | 66 | .. autoexception:: harmony.auth.MalformedCredentials 67 | :noindex: 68 | 69 | .. autoexception:: harmony.auth.BadAuthentication 70 | :noindex: 71 | 72 | Developer Documentation 73 | ----------------------- 74 | 75 | Here we show the full API documentation. This will most often be used when developing on the HarmonyPy package, and will not likely be needed if you are using HarmonyPy to make requests. 76 | 77 | Submodules 78 | ---------- 79 | 80 | harmony.auth module 81 | ------------------- 82 | 83 | .. automodule:: harmony.auth 84 | :members: 85 | :undoc-members: 86 | :show-inheritance: 87 | 88 | harmony.config module 89 | --------------------- 90 | 91 | .. automodule:: harmony.config 92 | :members: 93 | :undoc-members: 94 | :show-inheritance: 95 | 96 | harmony.request module 97 | ---------------------- 98 | 99 | .. automodule:: harmony.request 100 | :members: 101 | :undoc-members: 102 | :show-inheritance: 103 | 104 | harmony.client module 105 | ---------------------- 106 | 107 | .. automodule:: harmony.client 108 | :members: 109 | :undoc-members: 110 | :show-inheritance: 111 | 112 | Module contents 113 | --------------- 114 | 115 | .. automodule:: harmony 116 | :members: 117 | :undoc-members: 118 | :show-inheritance: 119 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | import os 17 | import sys 18 | 19 | sys.path.insert(0, os.path.abspath('../')) 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = 'harmony-py' 24 | copyright = '2021, NASA Harmony Project' 25 | author = 'NASA Harmony Project' 26 | 27 | # The full version, including alpha/beta/rc tags 28 | release = '0.0.1' 29 | 30 | 31 | # -- General configuration --------------------------------------------------- 32 | 33 | # Add any Sphinx extension module names here, as strings. They can be 34 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 35 | # ones. 36 | # 37 | # use this to build API docs: sphinx-apidoc -f -o docs/source projectdir 38 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx_rtd_theme'] 39 | 40 | # Add any paths that contain templates here, relative to this directory. 41 | templates_path = ['_templates'] 42 | 43 | # List of patterns, relative to source directory, that match files and 44 | # directories to ignore when looking for source files. 45 | # This pattern also affects html_static_path and html_extra_path. 46 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 47 | 48 | 49 | # -- Options for HTML output ------------------------------------------------- 50 | 51 | # The theme to use for HTML and HTML Help pages. See the documentation for 52 | # a list of builtin themes. 53 | # 54 | html_theme = 'sphinx_rtd_theme' 55 | 56 | # Add any paths that contain custom static files (such as style sheets) here, 57 | # relative to this directory. They are copied after the builtin static files, 58 | # so a file named "default.css" will overwrite the builtin "default.css". 59 | # html_static_path = ['_static'] 60 | 61 | source_suffix = ['.rst', '.md'] 62 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | HarmonyPy: NASA Harmony Python Client 2 | ===================================== 3 | 4 | Harmony-Py provides a Python alternative to directly using `Harmony's RESTful API `_. It handles NASA `Earthdata Login (EDL) `_ authentication and optionally integrates with the `CMR Python Wrapper `_ by accepting collection results as a request parameter. It's convenient for scientists who wish to use Harmony from Jupyter notebooks as well as machine-to-machine communication with larger Python applications. 5 | 6 | Harmony-Py is a work-in-progress, is not feature complete, and should only be used if you would like to test its functionality. We welcome feedback on Harmony-Py via `GitHub Issues `_. 7 | 8 | .. image:: https://readthedocs.org/projects/harmony-py/badge/?version=latest)](https://harmony-py.readthedocs.io/en/latest/?badge=latest 9 | :target: https://github.com/nasa/harmony-py 10 | 11 | ------------------- 12 | 13 | **Harmony In Action** :: 14 | 15 | >>> harmony_client = Client(auth=('captainmarvel', 'marve10u5')) 16 | 17 | >>> request = Request( 18 | collection=Collection(id='C1234088182-EEDTEST'), 19 | spatial=BBox(-140, 20, -50, 60), 20 | crs='EPSG:3995', 21 | format='image/tiff', 22 | height=512, 23 | width=512 24 | ) 25 | 26 | >>> job_id = harmony_client.submit(request) 27 | 28 | >>> harmony_client.download_all(job_id) 29 | 30 | ------------------- 31 | 32 | User Guide 33 | ---------- 34 | 35 | How to install HarmonyPy, and a quick tutorial to get you started with your own Harmony requests. 36 | 37 | .. toctree:: 38 | :maxdepth: 2 39 | 40 | user/install 41 | user/tutorial 42 | 43 | API Documentation 44 | ----------------- 45 | 46 | Specific documentation on the HarmonyPy package, its modules, and their functions, classes, and methods. 47 | 48 | .. toctree:: 49 | :maxdepth: 2 50 | 51 | api 52 | 53 | Indices and Tables 54 | ================== 55 | 56 | * :ref:`genindex` 57 | * :ref:`modindex` 58 | * :ref:`search` 59 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/user/install.rst: -------------------------------------------------------------------------------- 1 | .. _install: 2 | 3 | Installing HarmonyPy 4 | ==================== 5 | 6 | Before we can start using HarmonyPy, we need to install it. 7 | 8 | 9 | Install from PyPI using pip 10 | --------------------------- 11 | 12 | To install HarmonyPy, use pip to pull the latest version from `PyPI `_. Note that you may not want to install this in your Python system directory. See below for options. 13 | 14 | To install using pip :: 15 | 16 | $ pip install harmony-py 17 | 18 | To install into your user directory :: 19 | 20 | $ pip install --user harmony-py 21 | 22 | Other options include making a virtual environment using ``venv``, or creating a ``conda`` environment and installing it there. We'll show how to create a virtualenv here, but see the conda documentation for creating a conda env and installing it there. 23 | 24 | Using venv :: 25 | 26 | $ python -m venv env 27 | $ source env/bin/activate 28 | $ pip install harmony-py 29 | 30 | This will create a directory named ``env`` in the current working directory, and by activating it, pip will install ``harmony-py`` in the ``env`` directory tree, isolating it from other projects. See the `Python Packaging site `_ for more details about creating and using Python virtual environments. 31 | 32 | Getting the Code 33 | ---------------- 34 | 35 | HarmonyPy is actively developed on GitHub and the code is 36 | `publicly available `. 37 | 38 | Clone the repository :: 39 | 40 | $ git clone https://github.com/nasa/harmony-py.git 41 | -------------------------------------------------------------------------------- /docs/user/tutorial.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial: 2 | 3 | HarmonyPy Tutorial 4 | ================== 5 | 6 | .. raw:: html 7 | :file: notebook.html 8 | 9 | -------------------------------------------------------------------------------- /examples/2020_01_01_7f00ff_global_regridded_subsetted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nasa/harmony-py/b19f073306aec021746cb4c7b360f8a9242f8e29/examples/2020_01_01_7f00ff_global_regridded_subsetted.png -------------------------------------------------------------------------------- /examples/2020_01_01_7f00ff_global_regridded_subsetted.wld: -------------------------------------------------------------------------------- 1 | 7258.5225700055 2 | 0.0000000000 3 | 0.0000000000 4 | -30562.8903733433 5 | -8675639.6730617080 6 | 6633677.4642865239 7 | -------------------------------------------------------------------------------- /examples/Big_Island_0005.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nasa/harmony-py/b19f073306aec021746cb4c7b360f8a9242f8e29/examples/Big_Island_0005.zip -------------------------------------------------------------------------------- /examples/asf_example.json: -------------------------------------------------------------------------------- 1 | {"type":"FeatureCollection","features":[{"type":"Feature","id":0,"geometry":{"type":"Polygon","coordinates":[[[-50.713065120637054,0.050686902349725629],[-50.746947744580282,0.024355797981540993],[-50.745758864214125,-0.03249575430991828],[-50.698204371913604,-0.053440845439199872],[-50.658972213544466,-0.034889227400078715],[-50.645300429437476,0.029741453062155507],[-50.685721388533004,0.049489906114758959],[-50.713065120637054,0.050686902349725629]]]},"properties":{"FID":0,"Id":0}}]} -------------------------------------------------------------------------------- /examples/basic.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "romantic-atmosphere", 6 | "metadata": {}, 7 | "source": [ 8 | "## Harmony Py Library\n", 9 | "\n", 10 | "### Basic Workflow Example\n", 11 | "\n", 12 | "This notebook shows three basic examples of Harmony jobs, each using a Harmony test Collection. The first example requests a spatial subset of Alaska, the second a temporal subset (a single-month timespan), and the third shows a combination of both spatial and temporal subsetting.\n", 13 | "\n", 14 | "First, we import a helper module for the notebook, but then import the Harmony Py classes we need to make a request." 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "id": "59eaa108-53e0-40dd-83f7-1c81d21a1e42", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "# Install notebook requirements\n", 25 | "import sys\n", 26 | "import helper\n", 27 | "# Install the project and 'examples' dependencies\n", 28 | "helper.install_project_and_dependencies('..', libs=['examples'])" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "id": "impaired-transfer", 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "import datetime as dt\n", 39 | "from harmony import BBox, Client, Collection, Request, Environment" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "id": "motivated-pressing", 45 | "metadata": {}, 46 | "source": [ 47 | "First let's prompt for your CMR credentials (UAT). Your credentials are stored without needing to hit enter in either field." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "id": "attractive-german", 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "username = helper.Text(placeholder='captainmarvel', description='Username')\n", 58 | "helper.display(username)\n", 59 | "password = helper.Password(placeholder='Password', description='Password')\n", 60 | "helper.display(password)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "electoral-update", 66 | "metadata": {}, 67 | "source": [ 68 | "Now we create a Harmony Client object, passing in the `auth` tuple containing the username and password entered above." 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "id": "peripheral-cattle", 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "harmony_client = Client(auth=(username.value, password.value), env=Environment.UAT)" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "id": "dominican-serial", 84 | "metadata": {}, 85 | "source": [ 86 | "Next, we create a Collection object with the CMR collection id for our test collection. We then create a Request which specifies the collection, and a `spatial` `BBox` describing the bounding box for the area we're interested in. We'll see later in the notebook how to make sure the request we have is valid." 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "id": "charming-wheat", 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "collection = Collection(id='C1234088182-EEDTEST')\n", 97 | "\n", 98 | "request = Request(\n", 99 | " collection=collection,\n", 100 | " spatial=BBox(-165, 52, -140, 77),\n", 101 | " format='image/tiff'\n", 102 | ")" 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "id": "tracked-distributor", 108 | "metadata": {}, 109 | "source": [ 110 | "Now that we have a request, we can submit it to Harmony using the Harmony Client object we created earlier. We'll get back a job id belonging to our Harmony request.\n", 111 | "\n", 112 | "By default the job will have a 'harmony_py' label. This can be disabled by setting the `EXCLUDE_DEFAULT_LABEL` environment variable to \"true\" before making the request." 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": null, 118 | "id": "entertaining-romania", 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "job1_id = harmony_client.submit(request)" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "id": "confirmed-affair", 128 | "metadata": {}, 129 | "source": [ 130 | "If we want to, we can retrieve the job's status which includes information about the processing Harmony job." 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "id": "sufficient-pleasure", 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "helper.JSON(harmony_client.status(job1_id))" 141 | ] 142 | }, 143 | { 144 | "cell_type": "markdown", 145 | "id": "threatened-complement", 146 | "metadata": {}, 147 | "source": [ 148 | "There are a number of options available for downloading results. We'll start with the 'download_all()' method which uses a multithreaded downloader and quickly returns with a \"future\" (specifically a python conccurrent.futures future).\n", 149 | "\n", 150 | "If you're unfamiliar with futures, at their most basic level they represent an eventual value. In our case, once a file is downloaded its future will contain the name of the local file. We can then hand the name off to other functions which open files based on their name to perform further operations. Work performed on behalf of each future takes place in a \"thread pool\" created for each Client instantiation.\n", 151 | "\n", 152 | "To extract the eventual value of a future, call its 'result()' method. By using futures we can process downloaded files as soon as they're ready while the rest of the files are still downloading in the background. Because of how we're working with the futures, the order of our results are maintained even though the files will likely be downloaded out of order." 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "id": "global-ground", 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "print(f'\\nHarmony job ID: {job1_id}')\n", 163 | "\n", 164 | "print('\\nWaiting for the job to finish')\n", 165 | "results = harmony_client.result_json(job1_id, show_progress=True)\n", 166 | "\n", 167 | "print('\\nDownloading results:')\n", 168 | "futures = harmony_client.download_all(job1_id)\n", 169 | "\n", 170 | "for f in futures:\n", 171 | " print(f.result()) # f.result() is a filename, in this case\n", 172 | "\n", 173 | "print('\\nDone downloading.')" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "id": "documented-membrane", 179 | "metadata": {}, 180 | "source": [ 181 | "Now using our helper module, we can view the files. Note that we're calling download_all() again here. Because the overwrite option is set to False (the default value), the method will see each of the files are already downloaded and will not do so again. It'll return quickly because it avoids the unnecessary work." 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "id": "outer-syndrome", 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "futures = harmony_client.download_all(job1_id, overwrite=False)\n", 192 | "filenames = [f.result() for f in futures]\n", 193 | "\n", 194 | "for filename in filenames:\n", 195 | " helper.show_result(filename)" 196 | ] 197 | }, 198 | { 199 | "cell_type": "markdown", 200 | "id": "former-quick", 201 | "metadata": {}, 202 | "source": [ 203 | "Now we show a Harmony request for a temporal range: one month in 2020. As before, we create a Request, and submit it with the same Harmony Client we used above." 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "id": "medieval-appliance", 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [ 213 | "request = Request(\n", 214 | " collection=collection,\n", 215 | " temporal={\n", 216 | " 'start': dt.datetime(2020, 6, 1),\n", 217 | " 'stop': dt.datetime(2020, 6, 30)\n", 218 | " },\n", 219 | " format='image/tiff'\n", 220 | ")\n", 221 | "\n", 222 | "job2_id = harmony_client.submit(request)" 223 | ] 224 | }, 225 | { 226 | "cell_type": "markdown", 227 | "id": "9e1a5fd5", 228 | "metadata": {}, 229 | "source": [ 230 | "With our second request, we've chosen to call 'wait_for_processing()'. This is optional as the other results oriented methods like downloading will implicitly wait for processing but this method can provide visual feedback to let us know if Harmony is still working on our submitted job." 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "id": "minus-hampton", 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "harmony_client.wait_for_processing(job2_id, show_progress=True)\n", 241 | "\n", 242 | "for filename in [f.result() for f in harmony_client.download_all(job2_id)]:\n", 243 | " helper.show_result(filename)" 244 | ] 245 | }, 246 | { 247 | "cell_type": "markdown", 248 | "id": "premium-initial", 249 | "metadata": {}, 250 | "source": [ 251 | "Finally, we show a Harmony request for both a spatial and temporal range. We create the Request and simply specify both a `spatial` bounds and a `temporal` range, submitting it with the Harmony Client." 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": null, 257 | "id": "broadband-lobby", 258 | "metadata": {}, 259 | "outputs": [], 260 | "source": [ 261 | "request = Request(\n", 262 | " collection=collection,\n", 263 | " spatial=BBox(-165, 52, -140, 77),\n", 264 | " temporal={\n", 265 | " 'start': dt.datetime(2010, 1, 1),\n", 266 | " 'stop': dt.datetime(2020, 12, 30)\n", 267 | " },\n", 268 | " format='image/tiff'\n", 269 | ")\n", 270 | "\n", 271 | "job3_id = harmony_client.submit(request)" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": null, 277 | "id": "martial-radar", 278 | "metadata": {}, 279 | "outputs": [], 280 | "source": [ 281 | "for filename in [f.result() for f in harmony_client.download_all(job3_id)]:\n", 282 | " helper.show_result(filename)" 283 | ] 284 | }, 285 | { 286 | "cell_type": "markdown", 287 | "id": "northern-smith", 288 | "metadata": {}, 289 | "source": [ 290 | "If we're just interested in the json Harmony produces we can retrieve that also." 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "id": "requested-reset", 297 | "metadata": {}, 298 | "outputs": [], 299 | "source": [ 300 | "helper.JSON(harmony_client.result_json(job3_id))" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "id": "speaking-archive", 306 | "metadata": {}, 307 | "source": [ 308 | "Now that we know how to make a request, let's investigate how the Harmony Py library can help us make sure we have a valid request. Recall that we used the Harmony `BBox` type to provide a spatial constraint in our request. If we investigate its help text, we see that we create a `BBox` by providing the western, southern, eastern, and northern latitude/longitude bounds for a bounding box." 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": null, 314 | "id": "discrete-excitement", 315 | "metadata": {}, 316 | "outputs": [], 317 | "source": [ 318 | "help(BBox)" 319 | ] 320 | }, 321 | { 322 | "cell_type": "markdown", 323 | "id": "arranged-advisory", 324 | "metadata": {}, 325 | "source": [ 326 | "Now let's create an invalid bounding box by specifying a longitude less than -180 and a northern latitude less than its southern bounds:" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "id": "circular-motion", 333 | "metadata": {}, 334 | "outputs": [], 335 | "source": [ 336 | "collection = Collection(id='C1234088182-EEDTEST')\n", 337 | "\n", 338 | "request = Request(\n", 339 | " collection=collection,\n", 340 | " spatial=BBox(-183, 40, 10, 30),\n", 341 | " format='image/tiff'\n", 342 | ")\n", 343 | "\n", 344 | "print(f'Request valid? {request.is_valid()}')\n", 345 | "\n", 346 | "for m in request.error_messages():\n", 347 | " print(f' * {m}')" 348 | ] 349 | }, 350 | { 351 | "cell_type": "markdown", 352 | "id": "democratic-combining", 353 | "metadata": {}, 354 | "source": [ 355 | "Similarly, we can see errors in the temporal parameter:" 356 | ] 357 | }, 358 | { 359 | "cell_type": "code", 360 | "execution_count": null, 361 | "id": "incredible-tuning", 362 | "metadata": {}, 363 | "outputs": [], 364 | "source": [ 365 | "collection = Collection(id='C1234088182-EEDTEST')\n", 366 | "\n", 367 | "request = Request(\n", 368 | " collection=collection,\n", 369 | " temporal={\n", 370 | " 'start': dt.datetime(2020, 12, 30),\n", 371 | " 'stop': dt.datetime(2010, 1, 1)\n", 372 | " },\n", 373 | " format='image/tiff'\n", 374 | ")\n", 375 | "\n", 376 | "print(f'Request valid? {request.is_valid()}')\n", 377 | "\n", 378 | "for m in request.error_messages():\n", 379 | " print(f' * {m}')" 380 | ] 381 | }, 382 | { 383 | "cell_type": "markdown", 384 | "id": "imported-choir", 385 | "metadata": {}, 386 | "source": [ 387 | "So before submitting a Harmony Request, you can test your request to see if it's valid and how to fix it if not:" 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": null, 393 | "id": "parallel-difference", 394 | "metadata": {}, 395 | "outputs": [], 396 | "source": [ 397 | "collection = Collection(id='C1234088182-EEDTEST')\n", 398 | "\n", 399 | "request = Request(\n", 400 | " collection=collection,\n", 401 | " spatial=BBox(-183, 40, 10, 30),\n", 402 | " temporal={\n", 403 | " 'start': dt.datetime(2020, 12, 30),\n", 404 | " 'stop': dt.datetime(2010, 1, 1)\n", 405 | " },\n", 406 | " format='image/tiff'\n", 407 | ")\n", 408 | "\n", 409 | "print(f'Request valid? {request.is_valid()}')\n", 410 | "\n", 411 | "for m in request.error_messages():\n", 412 | " print(f' * {m}')" 413 | ] 414 | }, 415 | { 416 | "cell_type": "markdown", 417 | "id": "overall-census", 418 | "metadata": {}, 419 | "source": [ 420 | "If we don't validate the request first, Harmony Py will validate it automatically and raise an exception with a message indicating the errors that need to be fixed:" 421 | ] 422 | }, 423 | { 424 | "cell_type": "code", 425 | "execution_count": null, 426 | "id": "hired-force", 427 | "metadata": {}, 428 | "outputs": [], 429 | "source": [ 430 | "try:\n", 431 | " harmony_client.submit(request)\n", 432 | "except Exception as e:\n", 433 | " print('Harmony Py raised an exception:\\n')\n", 434 | " print(e)" 435 | ] 436 | }, 437 | { 438 | "cell_type": "markdown", 439 | "id": "nutritional-position", 440 | "metadata": {}, 441 | "source": [ 442 | "Now let's look at some examples of some of the other parameters that you can use when submitting a Harmony request:\n", 443 | "\n", 444 | "First, let's start by specifying a couple ways to limit how many granules of data we're interested in. When creating the Request, you can add the `max_results` argument. This is useful if we eventually want to run a bigger request, but we're experimenting and would like to get some sample results first:" 445 | ] 446 | }, 447 | { 448 | "cell_type": "code", 449 | "execution_count": null, 450 | "id": "offshore-mainland", 451 | "metadata": {}, 452 | "outputs": [], 453 | "source": [ 454 | "collection = Collection(id='C1234088182-EEDTEST')\n", 455 | "\n", 456 | "request = Request(\n", 457 | " collection=collection,\n", 458 | " spatial=BBox(-10, 0, 10, 10),\n", 459 | " temporal={\n", 460 | " 'start': dt.datetime(2021, 1, 1),\n", 461 | " 'stop': dt.datetime(2021, 1, 10)\n", 462 | " },\n", 463 | " max_results=2,\n", 464 | " format='image/tiff'\n", 465 | ")\n", 466 | "request.is_valid()" 467 | ] 468 | }, 469 | { 470 | "cell_type": "markdown", 471 | "id": "personal-watts", 472 | "metadata": {}, 473 | "source": [ 474 | "Or maybe you'd like to operate on some specific granules. In that case, passing the `granule_id` argument allows you to list the granule IDs (one or more) to operate upon. Let's try this in combination with another parameter: `crs`, the coordinate reference system we'd like to reproject our results into. In addition we show other options which specify what output format we'd like, the resulting image height and width." 475 | ] 476 | }, 477 | { 478 | "cell_type": "code", 479 | "execution_count": null, 480 | "id": "coated-denial", 481 | "metadata": {}, 482 | "outputs": [], 483 | "source": [ 484 | "collection = Collection(id='C1234088182-EEDTEST')\n", 485 | "\n", 486 | "request = Request(\n", 487 | " collection=collection,\n", 488 | " spatial=BBox(-140, 20, -50, 60),\n", 489 | " granule_id=['G1234088196-EEDTEST'],\n", 490 | " crs='EPSG:3995',\n", 491 | " format='image/tiff',\n", 492 | " height=400,\n", 493 | " width=900\n", 494 | ")\n", 495 | "request.is_valid()" 496 | ] 497 | }, 498 | { 499 | "cell_type": "code", 500 | "execution_count": null, 501 | "id": "humanitarian-caution", 502 | "metadata": {}, 503 | "outputs": [], 504 | "source": [ 505 | "job_id = harmony_client.submit(request)\n", 506 | "\n", 507 | "for filename in [f.result() for f in harmony_client.download_all(job_id)]:\n", 508 | " helper.show_result(filename)" 509 | ] 510 | }, 511 | { 512 | "cell_type": "markdown", 513 | "id": "polyphonic-digit", 514 | "metadata": {}, 515 | "source": [ 516 | "Now we'll craft the same request, but this time instead of getting all the variables in the granule--the default--we'll select just the red, green, and blue variables." 517 | ] 518 | }, 519 | { 520 | "cell_type": "code", 521 | "execution_count": null, 522 | "id": "configured-leave", 523 | "metadata": {}, 524 | "outputs": [], 525 | "source": [ 526 | "collection = Collection(id='C1234088182-EEDTEST')\n", 527 | "\n", 528 | "request = Request(\n", 529 | " collection=collection,\n", 530 | " spatial=BBox(-140, 20, -50, 60),\n", 531 | " granule_id=['G1234088196-EEDTEST'],\n", 532 | " crs='EPSG:3995',\n", 533 | " format='image/tiff',\n", 534 | " height=400,\n", 535 | " width=900,\n", 536 | " variables=['red_var', 'green_var', 'blue_var']\n", 537 | ")\n", 538 | "request.is_valid()" 539 | ] 540 | }, 541 | { 542 | "cell_type": "code", 543 | "execution_count": null, 544 | "id": "initial-enemy", 545 | "metadata": {}, 546 | "outputs": [], 547 | "source": [ 548 | "job_id = harmony_client.submit(request)\n", 549 | "\n", 550 | "for filename in [f.result() for f in harmony_client.download_all(job_id)]:\n", 551 | " helper.show_result(filename)" 552 | ] 553 | }, 554 | { 555 | "cell_type": "markdown", 556 | "id": "729f6b35", 557 | "metadata": {}, 558 | "source": [ 559 | "We can also use the `granule_name` parameter to to select (one or more) granules. This corresponds to the CMR `readable_granule_name` parameter and matches either the granule ur or the producer granule id." 560 | ] 561 | }, 562 | { 563 | "cell_type": "code", 564 | "execution_count": null, 565 | "id": "8d0ff231", 566 | "metadata": {}, 567 | "outputs": [], 568 | "source": [ 569 | "collection = Collection(id='C1233800302-EEDTEST')\n", 570 | "\n", 571 | "request = Request(\n", 572 | " collection=collection,\n", 573 | " spatial=BBox(-140, 20, -50, 60),\n", 574 | " granule_name=['001_00_7f00ff_global'],\n", 575 | " crs='EPSG:3995',\n", 576 | " format='image/tiff',\n", 577 | " height=400,\n", 578 | " width=900,\n", 579 | " variables=['red_var', 'green_var', 'blue_var']\n", 580 | ")\n", 581 | "request.is_valid()" 582 | ] 583 | }, 584 | { 585 | "cell_type": "code", 586 | "execution_count": null, 587 | "id": "f703951f", 588 | "metadata": {}, 589 | "outputs": [], 590 | "source": [ 591 | "job_id = harmony_client.submit(request)\n", 592 | "\n", 593 | "for filename in [f.result() for f in harmony_client.download_all(job_id)]:\n", 594 | " helper.show_result(filename)" 595 | ] 596 | }, 597 | { 598 | "cell_type": "markdown", 599 | "id": "2fe8ba31", 600 | "metadata": {}, 601 | "source": [ 602 | "We can pass multiple values to `granule_name` or use wildcards `*` (multi character match) or `?` (single character match)." 603 | ] 604 | }, 605 | { 606 | "cell_type": "code", 607 | "execution_count": null, 608 | "id": "620653f1", 609 | "metadata": {}, 610 | "outputs": [], 611 | "source": [ 612 | "collection = Collection(id='C1233800302-EEDTEST')\n", 613 | "\n", 614 | "request = Request(\n", 615 | " collection=collection,\n", 616 | " spatial=BBox(-180, -90, 180, 90),\n", 617 | " granule_name=['001_08*', '001_05_7f00ff_?ustralia'],\n", 618 | " crs='EPSG:3995',\n", 619 | " format='image/tiff',\n", 620 | " height=400,\n", 621 | " width=900,\n", 622 | " variables=['red_var', 'green_var', 'blue_var']\n", 623 | ")\n", 624 | "request.is_valid()" 625 | ] 626 | }, 627 | { 628 | "cell_type": "code", 629 | "execution_count": null, 630 | "id": "cb9d69b1", 631 | "metadata": {}, 632 | "outputs": [], 633 | "source": [ 634 | "job_id = harmony_client.submit(request)\n", 635 | "\n", 636 | "for filename in [f.result() for f in harmony_client.download_all(job_id)]:\n", 637 | " helper.show_result(filename)" 638 | ] 639 | } 640 | ], 641 | "metadata": { 642 | "kernelspec": { 643 | "display_name": "Python 3 (ipykernel)", 644 | "language": "python", 645 | "name": "python3" 646 | }, 647 | "language_info": { 648 | "codemirror_mode": { 649 | "name": "ipython", 650 | "version": 3 651 | }, 652 | "file_extension": ".py", 653 | "mimetype": "text/x-python", 654 | "name": "python", 655 | "nbconvert_exporter": "python", 656 | "pygments_lexer": "ipython3", 657 | "version": "3.11.9" 658 | }, 659 | "vscode": { 660 | "interpreter": { 661 | "hash": "341b7d2e50a6ee8d836f143dcf87119dfe72f0053ce895c8752bf7a40b324b52" 662 | } 663 | } 664 | }, 665 | "nbformat": 4, 666 | "nbformat_minor": 5 667 | } 668 | -------------------------------------------------------------------------------- /examples/collection_capabilities.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Harmony Py Library\n", 8 | "### Collection Capabilities Example" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "metadata": { 15 | "tags": [] 16 | }, 17 | "outputs": [], 18 | "source": [ 19 | "import helper\n", 20 | "import json\n", 21 | "import sys\n", 22 | "sys.path.append('..')\n", 23 | "\n", 24 | "# Install harmony-py requirements. Not necessary if you ran `pip install harmony-py` in your kernel \n", 25 | "helper.install_project_and_dependencies('..')\n", 26 | "\n", 27 | "from harmony import BBox, Client, Collection, Request, CapabilitiesRequest, Environment" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "#### Get collection capabilities" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": { 41 | "tags": [] 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "harmony_client = Client(env=Environment.UAT)\n", 46 | "\n", 47 | "capabilities_request = CapabilitiesRequest(collection_id='C1234088182-EEDTEST')\n", 48 | "\n", 49 | "capabilities = harmony_client.submit(capabilities_request)\n", 50 | "print(json.dumps(capabilities, indent=2))" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "metadata": {}, 56 | "source": [ 57 | "#### Get collection capabilities for a specific api version" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "metadata": { 64 | "tags": [] 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "harmony_client = Client(env=Environment.UAT)\n", 69 | "\n", 70 | "capabilities_request = CapabilitiesRequest(\n", 71 | " collection_id='C1234088182-EEDTEST',\n", 72 | " capabilities_version='2'\n", 73 | ")\n", 74 | "\n", 75 | "capabilities = harmony_client.submit(capabilities_request)\n", 76 | "print(json.dumps(capabilities, indent=2))" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "#### Get collection capabilities with collection short name" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": { 90 | "tags": [] 91 | }, 92 | "outputs": [], 93 | "source": [ 94 | "harmony_client = Client(env=Environment.UAT)\n", 95 | "\n", 96 | "capabilities_request = CapabilitiesRequest(short_name='harmony_example')\n", 97 | "\n", 98 | "capabilities = harmony_client.submit(capabilities_request)\n", 99 | "print(json.dumps(capabilities, indent=2))" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "#### From the returned capabilities, we can see the transfromation supported (variable subsetting, bounding box subsetting, shapefile subsetting, concatenation and reprojection, etc.) and the supported services and variables. \n", 107 | "\n", 108 | "#### Based on the returned capabilites info, we can submit a variable subsetting request (the granule_id is added to make the request small).\n" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": { 115 | "tags": [] 116 | }, 117 | "outputs": [], 118 | "source": [ 119 | "collection = Collection(id='C1234088182-EEDTEST')\n", 120 | "\n", 121 | "request = Request(\n", 122 | " collection=collection,\n", 123 | " spatial=BBox(-140, 20, -50, 60),\n", 124 | " granule_id=['G1234088196-EEDTEST'],\n", 125 | " crs='EPSG:3995',\n", 126 | " format='image/png',\n", 127 | " height=400,\n", 128 | " width=900,\n", 129 | " variables=['red_var', 'green_var', 'blue_var']\n", 130 | ")\n", 131 | "job_id = harmony_client.submit(request)" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "metadata": {}, 137 | "source": [ 138 | "#### Download and show the png file in result" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": { 145 | "tags": [] 146 | }, 147 | "outputs": [], 148 | "source": [ 149 | "for filename in [f.result() for f in harmony_client.download_all(job_id)]:\n", 150 | " if filename.endswith(\"png\"):\n", 151 | " helper.show_result(filename)" 152 | ] 153 | } 154 | ], 155 | "metadata": { 156 | "kernelspec": { 157 | "display_name": "Python 3 (ipykernel)", 158 | "language": "python", 159 | "name": "python3" 160 | }, 161 | "language_info": { 162 | "codemirror_mode": { 163 | "name": "ipython", 164 | "version": 3 165 | }, 166 | "file_extension": ".py", 167 | "mimetype": "text/x-python", 168 | "name": "python", 169 | "nbconvert_exporter": "python", 170 | "pygments_lexer": "ipython3", 171 | "version": "3.11.9" 172 | } 173 | }, 174 | "nbformat": 4, 175 | "nbformat_minor": 4 176 | } 177 | -------------------------------------------------------------------------------- /examples/helper.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | sys.path.append('..') 4 | 5 | import datetime as dt 6 | from getpass import getpass 7 | from glob import glob 8 | from time import sleep 9 | 10 | import ipyplot 11 | from ipywidgets import IntSlider, Password, Text 12 | from IPython.display import display, JSON 13 | import rasterio 14 | from rasterio.plot import show 15 | import requests 16 | 17 | def install_project_and_dependencies(project_root, libs=None): 18 | """ 19 | Change to the project root, install the project and its optional dependencies, 20 | then switch back to the original directory. 21 | 22 | :param project_root: Path to the project root directory where pyproject.toml is located. 23 | :param libs: List of optional pip extra dependencies (e.g., ['examples', 'dev']). 24 | """ 25 | # Save the current working directory 26 | original_dir = os.getcwd() 27 | 28 | try: 29 | # Change directory to the project root 30 | os.chdir(project_root) 31 | 32 | # If libs are specified, install them 33 | if libs: 34 | libs_str = ','.join(libs) 35 | os.system(f'{sys.executable} -m pip install -q .[{libs_str}]') 36 | 37 | # Install the project itself 38 | os.system(f'{sys.executable} -m pip install -q .') 39 | finally: 40 | # Switch back to the original directory after installation 41 | os.chdir(original_dir) 42 | 43 | 44 | def show_result(filename): 45 | print (f'\n {filename}') 46 | show(rasterio.open(filename)) 47 | -------------------------------------------------------------------------------- /examples/job_label.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Harmony Py Library\n", 8 | "### Job and Label Examples" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "metadata": { 15 | "tags": [] 16 | }, 17 | "outputs": [], 18 | "source": [ 19 | "import helper\n", 20 | "import json\n", 21 | "import sys\n", 22 | "sys.path.append('..')\n", 23 | "\n", 24 | "# Install harmony-py requirements. Not necessary if you ran `pip install harmony-py` in your kernel\n", 25 | "helper.install_project_and_dependencies('..')\n", 26 | "\n", 27 | "from harmony import BBox, Client, Collection, Request, AddLabelsRequest, DeleteLabelsRequest, JobsRequest, Environment" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "#### List jobs" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "harmony_client = Client(env=Environment.UAT)\n", 44 | "request = JobsRequest()\n", 45 | "jobs = harmony_client.submit(request)\n", 46 | "print(json.dumps(jobs, indent=2))" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "#### List jobs with page and limit" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "request = JobsRequest(\n", 63 | " page=2,\n", 64 | " limit=1)\n", 65 | "jobs = harmony_client.submit(request)\n", 66 | "print(json.dumps(jobs, indent=2))" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "#### Submit a couple harmony requests" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "metadata": { 80 | "tags": [] 81 | }, 82 | "outputs": [], 83 | "source": [ 84 | "harmony_client = Client(env=Environment.UAT)\n", 85 | "\n", 86 | "collection = Collection(id='C1234208438-POCLOUD')\n", 87 | "request = Request(\n", 88 | " collection=collection,\n", 89 | " spatial=BBox(-160, -80, 160, 80),\n", 90 | " granule_id=['G1234495188-POCLOUD'],\n", 91 | " variables=['bathymetry']\n", 92 | ")\n", 93 | "job_1 = harmony_client.submit(request)\n", 94 | "\n", 95 | "request = Request(\n", 96 | " collection=collection,\n", 97 | " concatenate=True,\n", 98 | " spatial=BBox(-160, -80, 160, 80),\n", 99 | " granule_id=['G1234515613-POCLOUD', 'G1234515574-POCLOUD'],\n", 100 | " variables=['bathymetry'],\n", 101 | " ignore_errors=True\n", 102 | ")\n", 103 | "job_2 = harmony_client.submit(request)" 104 | ] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "metadata": {}, 109 | "source": [ 110 | "#### Check labels on the job status" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": null, 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "job_1_status = harmony_client.result_json(job_1)\n", 120 | "print(json.dumps(job_1_status, indent=2))" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": {}, 126 | "source": [ 127 | "#### Add labels on jobs" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": { 134 | "tags": [] 135 | }, 136 | "outputs": [], 137 | "source": [ 138 | "request = AddLabelsRequest(\n", 139 | " labels=['foo', 'bar'],\n", 140 | " job_ids=[job_1, job_2])\n", 141 | "\n", 142 | "response = harmony_client.submit(request)\n", 143 | "print(json.dumps(response, indent=2))" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "#### Check added labels are in the job status" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": { 157 | "tags": [] 158 | }, 159 | "outputs": [], 160 | "source": [ 161 | "job_1_status = harmony_client.result_json(job_1)\n", 162 | "print(json.dumps(job_1_status, indent=2))" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "#### Check job 2 status and labels" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "metadata": { 176 | "tags": [] 177 | }, 178 | "outputs": [], 179 | "source": [ 180 | "response = harmony_client.result_json(job_2)\n", 181 | "print(json.dumps(response, indent=2))" 182 | ] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": {}, 187 | "source": [ 188 | "#### Search jobs by labels (multiple labels are ORed)" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": null, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "harmony_client = Client(env=Environment.UAT)\n", 198 | "request = JobsRequest(labels=['foo', 'bar'])\n", 199 | "jobs = harmony_client.submit(request)\n", 200 | "print(json.dumps(jobs, indent=2))" 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "metadata": {}, 206 | "source": [ 207 | "#### Delete labels from jobs" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": null, 213 | "metadata": {}, 214 | "outputs": [], 215 | "source": [ 216 | "request = DeleteLabelsRequest(\n", 217 | " labels=['foo', 'bar'],\n", 218 | " job_ids=[job_1, job_2])\n", 219 | "\n", 220 | "harmony_client.submit(request)" 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "metadata": {}, 226 | "source": [ 227 | "#### Search jobs by labels again, job_1 and job_2 are no longer in the result" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": null, 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [ 236 | "harmony_client = Client(env=Environment.UAT)\n", 237 | "request = JobsRequest(labels=['foo', 'bar'])\n", 238 | "jobs = harmony_client.submit(request)\n", 239 | "print(json.dumps(jobs, indent=2))" 240 | ] 241 | } 242 | ], 243 | "metadata": { 244 | "kernelspec": { 245 | "display_name": "Python 3 (ipykernel)", 246 | "language": "python", 247 | "name": "python3" 248 | }, 249 | "language_info": { 250 | "codemirror_mode": { 251 | "name": "ipython", 252 | "version": 3 253 | }, 254 | "file_extension": ".py", 255 | "mimetype": "text/x-python", 256 | "name": "python", 257 | "nbconvert_exporter": "python", 258 | "pygments_lexer": "ipython3", 259 | "version": "3.11.9" 260 | } 261 | }, 262 | "nbformat": 4, 263 | "nbformat_minor": 4 264 | } 265 | -------------------------------------------------------------------------------- /examples/job_pause_resume.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Harmony Py Library\n", 8 | "### Job Pause/Resume Example" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "import sys\n", 18 | "import helper\n", 19 | "helper.install_project_and_dependencies('..')\n", 20 | "\n", 21 | "from harmony import BBox, Client, Collection, Request, Environment" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 31 | "\n", 32 | "collection = Collection(id='C1234724470-POCLOUD')\n", 33 | "request = Request(\n", 34 | " collection=collection,\n", 35 | " spatial=BBox(0, -45, 180, 45),\n", 36 | " max_results=101\n", 37 | ")" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "#### submit an async request for processing and return the job_id, big requests get automatically paused after generating a preview of the results" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "job_id = harmony_client.submit(request)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "#### checking the status of the job we see that it is 'previewing'" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "harmony_client.status(job_id)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "#### 'wait_for_processing()' will wait while the job is in the 'previewing' state then warns that the job is paused before exiting" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "harmony_client.wait_for_processing(job_id, show_progress=True)" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": {}, 91 | "source": [ 92 | "#### checking the status we see that the job is paused" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "harmony_client.status(job_id)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "#### 'result_json()' will not wait for paused jobs and just returns any available results." 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "name": "stdout", 118 | "output_type": "stream", 119 | "text": [ 120 | "/tmp/5405885_20020704124006-JPL-L2P_GHRSST-SSTskin-MODIS_A-D-v02.0-fv01.0_subsetted.nc4\n" 121 | ] 122 | } 123 | ], 124 | "source": [ 125 | "results = harmony_client.download_all(job_id, directory='/tmp', overwrite=True)\n", 126 | "count = 0\n", 127 | "for r in results:\n", 128 | " count += 1\n", 129 | "print(f'Got {count} results')" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "#### we can resume the job with 'resume()'" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "harmony_client.resume(job_id)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "metadata": {}, 151 | "source": [ 152 | "#### checking the status we see that the job is running again" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "harmony_client.status(job_id)" 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": {}, 167 | "source": [ 168 | "#### we can pause the job with 'pause()'." 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "metadata": {}, 175 | "outputs": [], 176 | "source": [ 177 | "harmony_client.pause(job_id)" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "#### checking the status we see that the job is paused again" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": null, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "harmony_client.status(job_id)" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": {}, 199 | "source": [ 200 | "#### We can resume the job again" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [ 209 | "harmony_client.resume(job_id)" 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "metadata": {}, 215 | "source": [ 216 | "#### 'wait_for_processing()' will show resumed progress" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "metadata": {}, 223 | "outputs": [], 224 | "source": [ 225 | "harmony_client.wait_for_processing(job_id, show_progress=True)" 226 | ] 227 | }, 228 | { 229 | "cell_type": "markdown", 230 | "metadata": {}, 231 | "source": [ 232 | "#### 'download_all()' now has access to the full results" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [ 241 | "results = harmony_client.download_all(job_id, directory='/tmp', overwrite=True)\n", 242 | "count = 0\n", 243 | "for r in results:\n", 244 | " count += 1\n", 245 | "print(f'Got {count} results')" 246 | ] 247 | }, 248 | { 249 | "cell_type": "markdown", 250 | "metadata": {}, 251 | "source": [ 252 | "#### Attempting to pause a completed job will result in an error" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": null, 258 | "metadata": {}, 259 | "outputs": [], 260 | "source": [ 261 | "try: \n", 262 | " harmony_client.pause(job_id)\n", 263 | "except Exception as e:\n", 264 | " print(e)" 265 | ] 266 | }, 267 | { 268 | "cell_type": "markdown", 269 | "metadata": {}, 270 | "source": [ 271 | "#### Attempting to resume a completed job will also result in an error" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": null, 277 | "metadata": {}, 278 | "outputs": [], 279 | "source": [ 280 | "try: \n", 281 | " harmony_client.resume(job_id)\n", 282 | "except Exception as e:\n", 283 | " print(e)" 284 | ] 285 | }, 286 | { 287 | "cell_type": "markdown", 288 | "metadata": {}, 289 | "source": [ 290 | "#### we can use the 'skip_preview' parameter to tell Harmony to skip the auto-pause/preview and just start running" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "metadata": {}, 297 | "outputs": [], 298 | "source": [ 299 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 300 | "\n", 301 | "collection = Collection(id='C1234724470-POCLOUD')\n", 302 | "request = Request(\n", 303 | " collection=collection,\n", 304 | " spatial=BBox(0, -45, 180, 45),\n", 305 | " max_results=101,\n", 306 | " skip_preview=True\n", 307 | ")" 308 | ] 309 | }, 310 | { 311 | "cell_type": "markdown", 312 | "metadata": {}, 313 | "source": [ 314 | "#### submit an async request for processing and return the job_id, big requests get automatically paused after generating a preview of the results" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": null, 320 | "metadata": {}, 321 | "outputs": [], 322 | "source": [ 323 | "job_id = harmony_client.submit(request)" 324 | ] 325 | }, 326 | { 327 | "cell_type": "markdown", 328 | "metadata": {}, 329 | "source": [ 330 | "#### checking the status we see that the job is running" 331 | ] 332 | }, 333 | { 334 | "cell_type": "code", 335 | "execution_count": null, 336 | "metadata": {}, 337 | "outputs": [], 338 | "source": [ 339 | "harmony_client.status(job_id)" 340 | ] 341 | }, 342 | { 343 | "cell_type": "markdown", 344 | "metadata": {}, 345 | "source": [ 346 | "#### we can now use'wait_for_processing()' to wait until the job completes" 347 | ] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "execution_count": null, 352 | "metadata": {}, 353 | "outputs": [], 354 | "source": [ 355 | "harmony_client.wait_for_processing(job_id, show_progress=True)" 356 | ] 357 | }, 358 | { 359 | "cell_type": "markdown", 360 | "metadata": {}, 361 | "source": [ 362 | "### Cancel Job Example\n", 363 | "\n", 364 | "#### Submit a job to be canceled" 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": null, 370 | "metadata": {}, 371 | "outputs": [], 372 | "source": [ 373 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 374 | "\n", 375 | "collection = Collection(id='C1234724470-POCLOUD')\n", 376 | "request = Request(\n", 377 | " collection=collection,\n", 378 | " spatial=BBox(0, -45, 180, 45),\n", 379 | " max_results=101\n", 380 | ")\n", 381 | "job_id = harmony_client.submit(request)\n", 382 | "harmony_client.status(job_id)" 383 | ] 384 | }, 385 | { 386 | "cell_type": "markdown", 387 | "metadata": {}, 388 | "source": [ 389 | "#### Cancel the job and check for job status" 390 | ] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "execution_count": null, 395 | "metadata": {}, 396 | "outputs": [], 397 | "source": [ 398 | "harmony_client.cancel(job_id)" 399 | ] 400 | }, 401 | { 402 | "cell_type": "code", 403 | "execution_count": null, 404 | "metadata": {}, 405 | "outputs": [], 406 | "source": [ 407 | "harmony_client.status(job_id)" 408 | ] 409 | } 410 | ], 411 | "metadata": { 412 | "kernelspec": { 413 | "display_name": "Python 3 (ipykernel)", 414 | "language": "python", 415 | "name": "python3" 416 | }, 417 | "language_info": { 418 | "codemirror_mode": { 419 | "name": "ipython", 420 | "version": 3 421 | }, 422 | "file_extension": ".py", 423 | "mimetype": "text/x-python", 424 | "name": "python", 425 | "nbconvert_exporter": "python", 426 | "pygments_lexer": "ipython3", 427 | "version": "3.12.4" 428 | } 429 | }, 430 | "nbformat": 4, 431 | "nbformat_minor": 4 432 | } 433 | -------------------------------------------------------------------------------- /examples/job_results.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Harmony Py Library\n", 8 | "### Job Results Example" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "import sys\n", 18 | "import helper\n", 19 | "helper.install_project_and_dependencies('..')\n", 20 | "\n", 21 | "from harmony import BBox, Client, Collection, Request, Environment\n" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 31 | "\n", 32 | "collection = Collection(id='C1234088182-EEDTEST')\n", 33 | "request = Request(\n", 34 | " collection=collection,\n", 35 | " spatial=BBox(-165, 52, -140, 77)\n", 36 | ")\n" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "# submit an async request for processing and return the job_id\n", 46 | "job_id = harmony_client.submit(request)\n", 47 | "job_id\n" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "# We can check on the progress of a processing job with 'status()'.\n", 57 | "# This method blocks while communicating with the server but returns quickly.\n", 58 | "harmony_client.status(job_id)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "# 'wait_for_processing()'\n", 68 | "# Optionally shows progress bar.\n", 69 | "# Blocking.\n", 70 | "harmony_client.wait_for_processing(job_id, show_progress=True)" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "# 'result_json()' calls 'wait_for_processing()' and returns the complete job json once processing is complete.\n", 80 | "# Optionally shows progress bar.\n", 81 | "# Blocking.\n", 82 | "data = harmony_client.result_json(job_id)\n" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "# 'result_urls()' calls 'wait_for_processing()' and returns the job's data urls once processing is complete.\n", 92 | "# Optionally shows progress bar.\n", 93 | "# Blocking. Returns a generator to support returning many pages of results.\n", 94 | "urls = harmony_client.result_urls(job_id)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": {}, 101 | "outputs": [], 102 | "source": [ 103 | "# 'download_all()' downloads all data urls and returns immediately with a list of concurrent.futures.\n", 104 | "# Optionally shows progress bar for processing only.\n", 105 | "# Non-blocking during download but blocking while waitinig for job processing to finish.\n", 106 | "# Call 'result()' on future objects to realize them. A call to 'result()' blocks until that particular future finishes downloading. Other futures will download in the background, in parallel, up to the number of workers assigned to the thread pool (thread pool not publicly available).\n", 107 | "# Downloading on any unfinished futures can be cancelled early.\n", 108 | "# When downloading is complete the futures will return the file path string of the file that was just downloaded. This file path can then be fed into other libraries that may read the data files and perform other operations.\n", 109 | "futures = harmony_client.download_all(job_id)\n", 110 | "file_names = [f.result() for f in futures]" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": null, 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "# 'download()' will download only the url specified, in case a person would like more control over individual files.\n", 120 | "# Returns a future containing the file path string of the file downloaded.\n", 121 | "# Blocking upon calling result()\n", 122 | "file_name = harmony_client.download(next(urls), overwrite=True).result()\n" 123 | ] 124 | } 125 | ], 126 | "metadata": { 127 | "kernelspec": { 128 | "display_name": "Python 3 (ipykernel)", 129 | "language": "python", 130 | "name": "python3" 131 | }, 132 | "language_info": { 133 | "codemirror_mode": { 134 | "name": "ipython", 135 | "version": 3 136 | }, 137 | "file_extension": ".py", 138 | "mimetype": "text/x-python", 139 | "name": "python", 140 | "nbconvert_exporter": "python", 141 | "pygments_lexer": "ipython3", 142 | "version": "3.11.9" 143 | } 144 | }, 145 | "nbformat": 4, 146 | "nbformat_minor": 4 147 | } 148 | -------------------------------------------------------------------------------- /examples/job_results_iterator.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Harmony Py Library\n", 8 | "### Job Results Iterator Example" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "import sys\n", 18 | "import helper\n", 19 | "helper.install_project_and_dependencies('..')\n", 20 | "\n", 21 | "import concurrent.futures\n", 22 | "import os\n", 23 | "from harmony import BBox, Client, Collection, Request, Environment\n" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 33 | "\n", 34 | "collection = Collection(id='C1234088182-EEDTEST')\n", 35 | "request = Request(\n", 36 | " collection=collection,\n", 37 | " spatial=BBox(-165, 52, -140, 77)\n", 38 | ")\n" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "# submit an async request for processing and return the job_id\n", 48 | "job_id = harmony_client.submit(request)\n", 49 | "job_id\n" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "# We can iterate over the results which will cause them to download in parallel while the job is running\n", 59 | "# the first argument to `iterator()` is the ID of the job we want\n", 60 | "# the second argument is the path where we want to download the files\n", 61 | "# the third argument is whether or not we want to overwrite files - this defaults to False which\n", 62 | "# speeds up processing by not re-downloading files that have already been downloaded\n", 63 | "iter = harmony_client.iterator(job_id, '/tmp', True)\n", 64 | "# use verbose output so we can see the file names as they are downloaded\n", 65 | "os.environ['VERBOSE'] = 'true'\n", 66 | "# grab all the futures that are downloading the files\n", 67 | "futures = list(map(lambda x: x['path'], iter))\n", 68 | "# wait for all the futures to complete\n", 69 | "(done_futures, _) = concurrent.futures.wait(futures)" 70 | ] 71 | } 72 | ], 73 | "metadata": { 74 | "kernelspec": { 75 | "display_name": "Python 3 (ipykernel)", 76 | "language": "python", 77 | "name": "python3" 78 | }, 79 | "language_info": { 80 | "codemirror_mode": { 81 | "name": "ipython", 82 | "version": 3 83 | }, 84 | "file_extension": ".py", 85 | "mimetype": "text/x-python", 86 | "name": "python", 87 | "nbconvert_exporter": "python", 88 | "pygments_lexer": "ipython3", 89 | "version": "3.11.9" 90 | }, 91 | "vscode": { 92 | "interpreter": { 93 | "hash": "691e87652ceac3c5099314ecb0538ffe7a8fdf4c87c30f290964dc685c42c47b" 94 | } 95 | } 96 | }, 97 | "nbformat": 4, 98 | "nbformat_minor": 4 99 | } 100 | -------------------------------------------------------------------------------- /examples/job_stac.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Harmony Py Library\n", 8 | "### Job Results STAC data" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "import sys\n", 18 | "import helper\n", 19 | "# Install the project and 'examples' dependencies\n", 20 | "helper.install_project_and_dependencies('..', libs=['examples'])\n", 21 | "\n", 22 | "from harmony import BBox, Client, Collection, Request, Environment" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "First, let's get a job processing in Harmony." 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 39 | "\n", 40 | "collection = Collection(id='C1234088182-EEDTEST')\n", 41 | "request = Request(\n", 42 | " collection=collection,\n", 43 | " spatial=BBox(-165, 52, -140, 77),\n", 44 | " max_results=5\n", 45 | ")\n", 46 | "\n", 47 | "job_id = harmony_client.submit(request)\n", 48 | "job_id" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "Harmony-py can return the STAC Catalog URL for a completed job." 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "stac_catalog_url = harmony_client.stac_catalog_url(job_id, show_progress=True)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "Following the directions for PySTAC (https://pystac.readthedocs.io/en/latest/quickstart.html), we can hook our harmony-py client into STAC_IO." 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "from urllib.parse import urlparse\n", 81 | "import requests\n", 82 | "from pystac import stac_io\n", 83 | "\n", 84 | "\n", 85 | "def requests_read_method(uri):\n", 86 | " parsed = urlparse(uri)\n", 87 | " if parsed.hostname.startswith('harmony.') or parsed.hostname.startswith('localhost'):\n", 88 | " return harmony_client.read_text(uri)\n", 89 | " else:\n", 90 | " return stac_io.default_read_text_method(uri)\n", 91 | "\n", 92 | "stac_io.read_text_method = requests_read_method" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "For each STAC item in the catalog list its date, when it expires, and its access url" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [ 108 | "from pystac import Catalog\n", 109 | "\n", 110 | "\n", 111 | "cat = Catalog.from_file(stac_catalog_url)\n", 112 | "print(cat.title)\n", 113 | "\n", 114 | "for item in cat.get_all_items():\n", 115 | " print(item.datetime, item.properties.get('expires'), [asset.href for asset in item.assets.values()])" 116 | ] 117 | } 118 | ], 119 | "metadata": { 120 | "kernelspec": { 121 | "display_name": "Python 3 (ipykernel)", 122 | "language": "python", 123 | "name": "python3" 124 | }, 125 | "language_info": { 126 | "codemirror_mode": { 127 | "name": "ipython", 128 | "version": 3 129 | }, 130 | "file_extension": ".py", 131 | "mimetype": "text/x-python", 132 | "name": "python", 133 | "nbconvert_exporter": "python", 134 | "pygments_lexer": "ipython3", 135 | "version": "3.11.9" 136 | } 137 | }, 138 | "nbformat": 4, 139 | "nbformat_minor": 4 140 | } 141 | -------------------------------------------------------------------------------- /examples/job_status.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Harmony Py Library\n", 8 | "### Job Status Example" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "metadata": { 15 | "tags": [] 16 | }, 17 | "outputs": [], 18 | "source": [ 19 | "import datetime as dt\n", 20 | "import sys\n", 21 | "import helper\n", 22 | "helper.install_project_and_dependencies('..')\n", 23 | "\n", 24 | "from harmony import BBox, Client, Collection, Request, Environment" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "### Successful Job Status Example" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": null, 37 | "metadata": { 38 | "tags": [] 39 | }, 40 | "outputs": [], 41 | "source": [ 42 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 43 | "\n", 44 | "collection = Collection(id='C1234088182-EEDTEST')\n", 45 | "request = Request(\n", 46 | " collection=collection,\n", 47 | " spatial=BBox(-165, 52, -140, 77)\n", 48 | ")" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": { 55 | "tags": [] 56 | }, 57 | "outputs": [], 58 | "source": [ 59 | "success_job_id = harmony_client.submit(request)" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": null, 65 | "metadata": { 66 | "tags": [] 67 | }, 68 | "outputs": [], 69 | "source": [ 70 | "harmony_client.status(success_job_id)" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "### ignore_errors Example\n", 78 | "\n", 79 | "#### Submit a request with partial failure with ignore_errors false, the job status will be 'failed'" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": { 86 | "tags": [] 87 | }, 88 | "outputs": [], 89 | "source": [ 90 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 91 | "\n", 92 | "collection = Collection(id='C1256476536-NSIDC_CUAT')\n", 93 | "request = Request(\n", 94 | " collection=collection,\n", 95 | " spatial=BBox(50.227106, -13.003536, 54.544869, -12.374467),\n", 96 | " temporal={\n", 97 | " 'start': dt.datetime(2020, 1, 4),\n", 98 | " 'stop': dt.datetime(2020, 1, 5),\n", 99 | " },\n", 100 | " variables=['all'],\n", 101 | " ignore_errors=False\n", 102 | ")\n", 103 | "fail_job_id = harmony_client.submit(request)" 104 | ] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "metadata": { 109 | "tags": [] 110 | }, 111 | "source": [ 112 | "The request is failed as a whole. The request status is 'Failed' and there is no result url generated." 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": null, 118 | "metadata": { 119 | "tags": [] 120 | }, 121 | "outputs": [], 122 | "source": [ 123 | "urls = harmony_client.result_urls(fail_job_id, show_progress=True)\n", 124 | "try: \n", 125 | " next(urls)\n", 126 | "except Exception as e:\n", 127 | " print(e)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "execution_count": null, 133 | "metadata": { 134 | "tags": [] 135 | }, 136 | "outputs": [], 137 | "source": [ 138 | "harmony_client.status(fail_job_id)" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "#### Submit a request with partial failure without ignore_errors (default to true), the job status will be 'complete_with_errors'" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": { 152 | "tags": [] 153 | }, 154 | "outputs": [], 155 | "source": [ 156 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 157 | "\n", 158 | "collection = Collection(id='C1256476536-NSIDC_CUAT')\n", 159 | "request = Request(\n", 160 | " collection=collection,\n", 161 | " spatial=BBox(50.227106, -13.003536, 54.544869, -12.374467),\n", 162 | " temporal={\n", 163 | " 'start': dt.datetime(2020, 1, 4),\n", 164 | " 'stop': dt.datetime(2020, 1, 5),\n", 165 | " },\n", 166 | " variables=['all']\n", 167 | ")\n", 168 | "ignore_errors_job_id = harmony_client.submit(request)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "The request is processed to completion. The request status is 'complete_with_errors' and there is result url generated." 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "metadata": { 182 | "tags": [] 183 | }, 184 | "outputs": [], 185 | "source": [ 186 | "urls = harmony_client.result_urls(ignore_errors_job_id, show_progress=True)\n", 187 | "print(*urls, sep='\\n')" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "metadata": { 194 | "tags": [] 195 | }, 196 | "outputs": [], 197 | "source": [ 198 | "harmony_client.status(ignore_errors_job_id)" 199 | ] 200 | } 201 | ], 202 | "metadata": { 203 | "kernelspec": { 204 | "display_name": "Python 3 (ipykernel)", 205 | "language": "python", 206 | "name": "python3" 207 | }, 208 | "language_info": { 209 | "codemirror_mode": { 210 | "name": "ipython", 211 | "version": 3 212 | }, 213 | "file_extension": ".py", 214 | "mimetype": "text/x-python", 215 | "name": "python", 216 | "nbconvert_exporter": "python", 217 | "pygments_lexer": "ipython3", 218 | "version": "3.11.9" 219 | } 220 | }, 221 | "nbformat": 4, 222 | "nbformat_minor": 4 223 | } 224 | -------------------------------------------------------------------------------- /examples/s3_access.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Harmony Py Library\n", 8 | "### Getting data using AWS credentials, S3 URLs, and downloading with AWS boto3" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "metadata": {}, 15 | "outputs": [], 16 | "source": [ 17 | "import datetime as dt\n", 18 | "import sys\n", 19 | "import helper\n", 20 | "# Install the project and 'examples' dependencies\n", 21 | "helper.install_project_and_dependencies('..', libs=['examples'])\n", 22 | "\n", 23 | "from harmony import BBox, Client, Collection, LinkType, Request, s3_components, Environment" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "request = Request(\n", 33 | " collection=Collection(id='C1234088182-EEDTEST'),\n", 34 | " spatial=BBox(-165, 52, -140, 77),\n", 35 | " temporal={\n", 36 | " 'start': dt.datetime(2010, 1, 1),\n", 37 | " 'stop': dt.datetime(2020, 12, 30)\n", 38 | " },\n", 39 | " variables=['blue_var'],\n", 40 | " max_results=2,\n", 41 | " crs='EPSG:3995',\n", 42 | " format='image/tiff',\n", 43 | " height=512,\n", 44 | " width=512,\n", 45 | " # If desired, deliver results to a custom destination bucket. Note the bucket must reside in AWS us-west-2 region.\n", 46 | " # destination_url='s3://my-bucket'\n", 47 | ")\n", 48 | "\n", 49 | "request.is_valid()" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "Cloud access credentials can be retrieved from an instantiated Client." 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "harmony_client = Client(env=Environment.UAT) # assumes .netrc usage\n", 66 | "job_id = harmony_client.submit(request)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "Now let's get the data URLs from Harmony, but request S3 URLs instead of the default HTTPS. We also request temporary AWS credentials that we can use to authenticate and download the data." 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [ 82 | "results = harmony_client.result_urls(job_id, link_type=LinkType.s3)\n", 83 | "print(results)\n", 84 | "# NOTE: if you specified destination_url you'll have to retrieve your credentials in another manner\n", 85 | "# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html\n", 86 | "creds = harmony_client.aws_credentials()" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "Now we'll use the AWS Python library boto3 to download our results from the S3 bucket, providing boto3 with our temporary credentials that Harmony supplied:" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "#\n", 103 | "# NOTE: Execution of this cell will only succeed within the AWS us-west-2 region.\n", 104 | "#\n", 105 | "\n", 106 | "import boto3\n", 107 | "\n", 108 | "s3 = boto3.client('s3', **creds)\n", 109 | "for url in results:\n", 110 | " bucket, obj, fn = s3_components(url)\n", 111 | " with open(fn, 'wb') as f:\n", 112 | " s3.download_fileobj(bucket, obj, f)" 113 | ] 114 | } 115 | ], 116 | "metadata": { 117 | "kernelspec": { 118 | "display_name": "Python 3 (ipykernel)", 119 | "language": "python", 120 | "name": "python3" 121 | }, 122 | "language_info": { 123 | "codemirror_mode": { 124 | "name": "ipython", 125 | "version": 3 126 | }, 127 | "file_extension": ".py", 128 | "mimetype": "text/x-python", 129 | "name": "python", 130 | "nbconvert_exporter": "python", 131 | "pygments_lexer": "ipython3", 132 | "version": "3.11.9" 133 | }, 134 | "vscode": { 135 | "interpreter": { 136 | "hash": "bc748110a6ec18982109b2289f9c506ebfe86428d3e48ddecc746f3969e698e0" 137 | } 138 | } 139 | }, 140 | "nbformat": 4, 141 | "nbformat_minor": 4 142 | } 143 | -------------------------------------------------------------------------------- /examples/shapefile_subset.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Harmony Py Library\n", 8 | "### Shapefile Subsetting Example" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "Set up a harmony client pointing to UAT" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": null, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "import sys\n", 25 | "import helper\n", 26 | "# Install the project and 'examples' dependencies\n", 27 | "helper.install_project_and_dependencies('..', libs=['examples'])\n", 28 | "\n", 29 | "import datetime as dt\n", 30 | "from harmony import BBox, Client, Collection, Request\n", 31 | "from harmony.config import Environment\n", 32 | "\n", 33 | "harmony_client = Client()" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "Perform a shapefile subsetting request on a supported collection by passing the path to a GeoJSON file (*.json or *.geojson), an ESRI Shapefile (*.zip or *.shz), or a kml file (*.kml) as the \"shape\" parameter" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "The example utilized in this tutorial demonstrates a shapefile subset of the Big Island of Hawaii on February 24, 2020. A bounding box subset over the Mauna Kea and Mauna Loa volcanoes is also commented out below to show a similar subsetting option. The SENTINEL-1_INTERFEROGRAMS dataset, distributed by the ASF DAAC, is a prototype Level 2 NISAR-Format product. See https://asf.alaska.edu/data-sets/derived-data-sets/sentinel-1-interferograms/ for more information. \n", 48 | "\n", 49 | "This request specifies a subset of the unwrappedPhase variable, in TIFF format, with a maximum file result capped at 2 for demonstration purposes. \n", 50 | "\n", 51 | "#### ___Note that a Sentinel-3 End-User License Agreement (EULA) is required to access these data.___\n", 52 | "#### ___Please go to https://grfn.asf.alaska.edu/door/download/S1-GUNW-D-R-021-tops-20201029_20191029-033636-28753N_27426N-PP-2dde-v2_0_3.nc to initiate a file download, which will first prompt you to accept the required EULA if you have not already done so. If you do not accept this EULA, you will receive an error when submitting your Harmony request.___" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "shapefile_path = 'Big_Island_0005.zip' \n", 62 | "\n", 63 | "request = Request(\n", 64 | " collection=Collection(id='SENTINEL-1_INTERFEROGRAMS'),\n", 65 | " #spatial=BBox(-155.75, 19.26, -155.3, 19.94), # bounding box example that can be used as an alternative to shapefile input\n", 66 | " shape=shapefile_path,\n", 67 | " temporal={\n", 68 | " 'start': dt.datetime(2020, 2, 24),\n", 69 | " 'stop': dt.datetime(2020, 2, 25),\n", 70 | " },\n", 71 | " variables=['science/grids/data/unwrappedPhase'],\n", 72 | " format='image/tiff',\n", 73 | " max_results=2,\n", 74 | ")" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "Wait for processing and then view the output" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "job_id = harmony_client.submit(request)\n", 91 | "\n", 92 | "print(f'jobID = {job_id}')\n", 93 | "harmony_client.wait_for_processing(job_id, show_progress=True)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "for filename in [f.result() for f in harmony_client.download_all(job_id)]:\n", 103 | " helper.show_result(filename)" 104 | ] 105 | } 106 | ], 107 | "metadata": { 108 | "kernelspec": { 109 | "display_name": "Python 3 (ipykernel)", 110 | "language": "python", 111 | "name": "python3" 112 | }, 113 | "language_info": { 114 | "codemirror_mode": { 115 | "name": "ipython", 116 | "version": 3 117 | }, 118 | "file_extension": ".py", 119 | "mimetype": "text/x-python", 120 | "name": "python", 121 | "nbconvert_exporter": "python", 122 | "pygments_lexer": "ipython3", 123 | "version": "3.11.9" 124 | } 125 | }, 126 | "nbformat": 4, 127 | "nbformat_minor": 4 128 | } 129 | -------------------------------------------------------------------------------- /examples/tutorial.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "artificial-botswana", 6 | "metadata": {}, 7 | "source": [ 8 | "## Harmony Py Tutorial\n", 9 | "\n", 10 | "This notebook shows a basic example of a Harmony job using a Harmony test Collection to perform a combination of both spatial and temporal subsetting.\n", 11 | "\n", 12 | "First, we import a few things that will help us create a request and display images. We then import the Harmony Py classes we need to make a request." 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "id": "cross-sample", 19 | "metadata": { 20 | "tags": [] 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "import helper\n", 25 | "# Install the project and 'examples' dependencies\n", 26 | "helper.install_project_and_dependencies('..', libs=['examples'])\n", 27 | "\n", 28 | "import sys\n", 29 | "import datetime as dt\n", 30 | "from IPython.display import display, JSON\n", 31 | "import rasterio\n", 32 | "import rasterio.plot\n", 33 | "import netCDF4 as nc4\n", 34 | "from matplotlib import pyplot as plt\n", 35 | "import numpy as np\n", 36 | "\n", 37 | "from harmony import BBox, WKT, Client, Collection, Request, Environment" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "still-casting", 43 | "metadata": {}, 44 | "source": [ 45 | "Now we create a Harmony Client object, letting it pick up our credentials from a `.netrc` file." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "id": "cellular-differential", 52 | "metadata": { 53 | "tags": [] 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "harmony_client = Client(env=Environment.UAT)" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "id": "mental-marble", 63 | "metadata": {}, 64 | "source": [ 65 | "Next, let's create a Collection object with the [CMR](https://cmr.earthdata.nasa.gov/) collection id for the CMR collection we'd like to look at.\n", 66 | "\n", 67 | "We then create a Request which specifies that collection, a `spatial` `BBox` describing the bounding box for the area we're interested in (we'll look at the ``BBox`` in other tutorials). In this case we're interested in looking at Alaska (and who wouldn't be?). We also include a date/time range to narrow down the data.\n", 68 | "\n", 69 | "Because this data includes a lot of different variables, we limit it by passing in a list of `variable`s we're interested in; in this test collection we'll look at the blue variable. We include a `max_results` parameter to limit the results to the first 10 images just to get a sample of what things look like.\n", 70 | "\n", 71 | "Next, we include a coordinate reference system (CRS) indicating we'd like to reproject the data into the [Arctic Polar Stereographic projection](https://epsg.io/3995). We also specify that we'd like the output to be in the GeoTIFF format with a resolution of 512x512 pixels.\n", 72 | "\n", 73 | "Finally we check if the request we've created is valid." 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "id": "affecting-colors", 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "collection = Collection(id='C1234088182-EEDTEST')\n", 84 | "\n", 85 | "request = Request(\n", 86 | " collection=collection,\n", 87 | " spatial=BBox(-165, 52, -140, 77),\n", 88 | " temporal={\n", 89 | " 'start': dt.datetime(2010, 1, 1),\n", 90 | " 'stop': dt.datetime(2020, 12, 30)\n", 91 | " },\n", 92 | " variables=['blue_var'],\n", 93 | " max_results=10,\n", 94 | " crs='EPSG:3995',\n", 95 | " format='image/tiff',\n", 96 | " height=512,\n", 97 | " width=512\n", 98 | ")\n", 99 | "\n", 100 | "request.is_valid()" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "id": "defensive-defeat", 106 | "metadata": {}, 107 | "source": [ 108 | "Now that we have a request, we can submit it to Harmony using the Harmony Client object we created earlier. We'll get back an id for our request which we can use to find the job's status and get the results." 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "id": "heard-moldova", 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "job_id = harmony_client.submit(request)" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "id": "restricted-context", 124 | "metadata": {}, 125 | "source": [ 126 | "Let's see how it's going. This will show the percentage complete in the `progress` field. (We use the JSON helper function to show the results in a nicer-to-look-at format)." 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "id": "smooth-soviet", 133 | "metadata": {}, 134 | "outputs": [], 135 | "source": [ 136 | "JSON(harmony_client.status(job_id))" 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "id": "improved-cowboy", 142 | "metadata": {}, 143 | "source": [ 144 | "Let's download the results to our system temp directory, overwriting files if they already exist. This returns us a list of `Future` objects. Each of these \"stand in\" for a file in our set of results. We can ask a `Future` for its result and when it's available, it will return the filename to us." 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "id": "living-associate", 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "results = harmony_client.download_all(job_id, directory='/tmp', overwrite=True)" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "id": "wanted-detective", 160 | "metadata": {}, 161 | "source": [ 162 | "Allright, now let's show some colorful Alaska images! Here we iterate over the results, asking each `Future` for its result, and then using `rasterio` to open the file and display the image." 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": null, 168 | "id": "innovative-cambodia", 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "for r in results:\n", 173 | " rasterio.plot.show(rasterio.open(r.result()))" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "id": "f97fa92f", 179 | "metadata": {}, 180 | "source": [ 181 | "We can also get a URL corresponding to our request that we can use in a browser. **Note:** This will not work if the request includes a shapefile." 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "id": "f03e22a8", 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "url = harmony_client.request_as_url(request)\n", 192 | "print(url)" 193 | ] 194 | }, 195 | { 196 | "cell_type": "markdown", 197 | "id": "45a14473", 198 | "metadata": {}, 199 | "source": [ 200 | "Let's build another request. This time, we'll request a specific granule and grid using the UMM grid name from the CMR. You can find grids in the CMR at https://cmr.uat.earthdata.nasa.gov/search/grids.umm_json. Include a query parameter `?grid=LambertExample` to list detailed information about a specific grid." 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "id": "6bead4b7", 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "collection = Collection(id='C1233860183-EEDTEST')\n", 211 | "\n", 212 | "request = Request(\n", 213 | " collection=collection,\n", 214 | " granule_id='G1233860486-EEDTEST',\n", 215 | " interpolation='near',\n", 216 | " grid='GEOS1x1test'\n", 217 | ")\n", 218 | "\n", 219 | "request.is_valid()" 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "id": "5cb4496b", 225 | "metadata": {}, 226 | "source": [ 227 | "Submit the job and check on the status." 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": null, 233 | "id": "f3f030a6", 234 | "metadata": {}, 235 | "outputs": [], 236 | "source": [ 237 | "job_id = harmony_client.submit(request)\n", 238 | "JSON(harmony_client.status(job_id))" 239 | ] 240 | }, 241 | { 242 | "cell_type": "markdown", 243 | "id": "d76b6104", 244 | "metadata": {}, 245 | "source": [ 246 | "Download and plot the file using pyplot and numpy." 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": null, 252 | "id": "b182ee17", 253 | "metadata": { 254 | "tags": [] 255 | }, 256 | "outputs": [], 257 | "source": [ 258 | "results = harmony_client.download_all(job_id, directory='/tmp', overwrite=True)\n", 259 | "nc4_file=nc4.Dataset(list(results)[0].result())\n", 260 | "arrays = []\n", 261 | "for var in ['red_var', 'green_var', 'blue_var', 'alpha_var']:\n", 262 | " ds = nc4_file.variables[var][0,:]\n", 263 | " arrays.append(ds)\n", 264 | "plt.imshow(np.dstack(arrays))" 265 | ] 266 | }, 267 | { 268 | "cell_type": "markdown", 269 | "id": "b9e8d8d7-03c2-4fb3-b10e-6dee510676e2", 270 | "metadata": {}, 271 | "source": [ 272 | "For request that completes directly without creating a job, the submit call returns the harmony JSON response including the direct download links." 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": null, 278 | "id": "9f498daa-7522-45e2-b726-cd9d18a18f61", 279 | "metadata": { 280 | "tags": [] 281 | }, 282 | "outputs": [], 283 | "source": [ 284 | "collection = Collection(id='C1233800302-EEDTEST')\n", 285 | "\n", 286 | "request = Request(\n", 287 | " collection=collection,\n", 288 | " max_results=1,\n", 289 | " variables=['all']\n", 290 | ")\n", 291 | "response = harmony_client.submit(request)\n", 292 | "response" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "id": "bb0f4124-488a-477c-8cf4-b72b46d2b0fa", 298 | "metadata": {}, 299 | "source": [ 300 | "Download the result and show the file can be opened with NetCDF library." 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": null, 306 | "id": "77aee47d-3679-49db-ac0e-05a7d9f15bff", 307 | "metadata": { 308 | "tags": [] 309 | }, 310 | "outputs": [], 311 | "source": [ 312 | "results = harmony_client.download_all(response, directory='/tmp', overwrite=True)\n", 313 | "file_names = [f.result() for f in results]\n", 314 | "for filename in file_names:\n", 315 | " if filename.endswith(\"nc\"):\n", 316 | " print(nc4.Dataset(filename))" 317 | ] 318 | }, 319 | { 320 | "cell_type": "markdown", 321 | "id": "a22a34f6-36a5-4554-b248-1b3e599dc4c2", 322 | "metadata": {}, 323 | "source": [ 324 | "Example of submitting a request with WKT spatial. The supported WKT geometry types are listed at: https://harmony-py.readthedocs.io/en/latest/api.html#harmony.harmony.WKT" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": null, 330 | "id": "f7a3308e-e655-4d78-97a5-87e81aab508d", 331 | "metadata": {}, 332 | "outputs": [], 333 | "source": [ 334 | "collection = Collection(id='C1233800302-EEDTEST')\n", 335 | "\n", 336 | "request = Request(\n", 337 | " collection=collection,\n", 338 | " spatial=WKT('POLYGON((-140 20, -50 20, -50 60, -140 60, -140 20))'),\n", 339 | " granule_id=['C1233800302-EEDTEST'],\n", 340 | " max_results=1,\n", 341 | " temporal={\n", 342 | " 'start': dt.datetime(1980, 1, 1),\n", 343 | " 'stop': dt.datetime(2020, 12, 30)\n", 344 | " },\n", 345 | " variables=['blue_var'],\n", 346 | " crs='EPSG:31975',\n", 347 | " format='image/png'\n", 348 | ")\n", 349 | "\n", 350 | "request.is_valid()" 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": null, 356 | "id": "6a8909d4-133a-4f51-bec4-c5698971a96a", 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [ 360 | "response = harmony_client.submit(request)\n", 361 | "response" 362 | ] 363 | }, 364 | { 365 | "cell_type": "markdown", 366 | "id": "74d54eb6-54b8-4b08-a469-326770d7910f", 367 | "metadata": {}, 368 | "source": [ 369 | "Show the result:" 370 | ] 371 | }, 372 | { 373 | "cell_type": "code", 374 | "execution_count": null, 375 | "id": "6d920033-4ba2-493c-ae15-3cc504a0fca5", 376 | "metadata": {}, 377 | "outputs": [], 378 | "source": [ 379 | "for filename in [f.result() for f in harmony_client.download_all(response, directory='/tmp', overwrite=True)]:\n", 380 | " if filename.endswith(\"png\"):\n", 381 | " helper.show_result(filename)" 382 | ] 383 | } 384 | ], 385 | "metadata": { 386 | "kernelspec": { 387 | "display_name": "Python 3 (ipykernel)", 388 | "language": "python", 389 | "name": "python3" 390 | }, 391 | "language_info": { 392 | "codemirror_mode": { 393 | "name": "ipython", 394 | "version": 3 395 | }, 396 | "file_extension": ".py", 397 | "mimetype": "text/x-python", 398 | "name": "python", 399 | "nbconvert_exporter": "python", 400 | "pygments_lexer": "ipython3", 401 | "version": "3.11.9" 402 | }, 403 | "vscode": { 404 | "interpreter": { 405 | "hash": "bc748110a6ec18982109b2289f9c506ebfe86428d3e48ddecc746f3969e698e0" 406 | } 407 | } 408 | }, 409 | "nbformat": 4, 410 | "nbformat_minor": 5 411 | } 412 | -------------------------------------------------------------------------------- /harmony/__init__.py: -------------------------------------------------------------------------------- 1 | # Automatically updated by `make build` 2 | __version__ = "1.2.0" 3 | 4 | from harmony.config import Environment 5 | from harmony.request import BBox, WKT, Collection, LinkType, Dimension, Request, \ 6 | CapabilitiesRequest, AddLabelsRequest, DeleteLabelsRequest, JobsRequest 7 | from harmony.client import Client 8 | from harmony.util import s3_components 9 | -------------------------------------------------------------------------------- /harmony/auth.py: -------------------------------------------------------------------------------- 1 | """Earthdata Login Authorization extensions to the ``requests`` package. 2 | 3 | This module defines two functions that enable seamless integration between 4 | the ``requests`` module and NASA Earthdata Login. The ``create_session`` function 5 | constructs a ``requests.Session`` that will correctly handle the OAuth redirect 6 | 'dance' that is necessary to authenticate a user. The ``validate_auth`` function 7 | checks that the authentication credentials are valid, and can be used before 8 | attempting to download data, for example. 9 | 10 | The ``SessionWithHeaderRedirection``--a ``requests.Session`` subclass--is used 11 | to perform authentication with Earthdata Login. The ``create_session`` function 12 | uses this class and clients of the Harmony Py package do not need to use this 13 | explicitly. 14 | """ 15 | import re 16 | from typing import Optional, Tuple, cast 17 | from urllib.parse import urlparse 18 | 19 | from requests import Session 20 | from requests.models import PreparedRequest, Response 21 | from requests.utils import get_netrc_auth 22 | 23 | from harmony.config import Config 24 | 25 | 26 | def _is_edl_hostname(hostname: str) -> bool: 27 | """ 28 | Determine if a hostname matches an EDL hostname. 29 | 30 | Args: 31 | hostname: A fully-qualified domain name (FQDN). 32 | 33 | Returns: 34 | True if the hostname is an EDL hostname, else False. 35 | """ 36 | edl_hostname_pattern = r'.*urs\.earthdata\.nasa\.gov$' 37 | return re.fullmatch(edl_hostname_pattern, hostname, flags=re.IGNORECASE) is not None 38 | 39 | 40 | class MalformedCredentials(Exception): 41 | """The provided Earthdata Login credentials were not correctly specified.""" 42 | pass 43 | 44 | 45 | class BadAuthentication(Exception): 46 | """The provided Earthdata Login credentials were invalid.""" 47 | pass 48 | 49 | 50 | class SessionWithHeaderRedirection(Session): 51 | """A ``requests.Session`` that modifies HTTP Authorization headers in accordance 52 | with Earthdata Login (EDL) common usage. 53 | 54 | Example:: 55 | 56 | session = SessionWithHeaderRedirection(username, password) 57 | 58 | Args: 59 | auth: A tuple of the form ('edl_username', 'edl_password') 60 | """ 61 | 62 | def __init__(self, auth: Optional[Tuple[str, str]] = None, token: str = None) -> None: 63 | super().__init__() 64 | if token: 65 | self.headers.update({'Authorization': f'Bearer {token}'}) 66 | elif auth: 67 | self.auth = auth 68 | else: 69 | self.auth = None 70 | 71 | def rebuild_auth(self, prepared_request: PreparedRequest, response: Response) -> None: 72 | """ 73 | Override Session.rebuild_auth. Strips the Authorization header if neither 74 | original URL nor redirected URL belong to an Earthdata Login (EDL) host. Also 75 | allows the default requests behavior of searching for relevant .netrc 76 | credentials if and only if a username and password weren't provided during 77 | object instantiation. 78 | 79 | Args: 80 | prepared_request: Object for the redirection destination. 81 | response: Object for the where we just came from. 82 | """ 83 | 84 | headers = prepared_request.headers 85 | redirect_hostname = cast(str, urlparse(prepared_request.url).hostname) 86 | original_hostname = cast(str, urlparse(response.request.url).hostname) 87 | 88 | if ('Authorization' in headers 89 | and (original_hostname != redirect_hostname) 90 | and not _is_edl_hostname(redirect_hostname)): 91 | del headers['Authorization'] 92 | 93 | if self.auth is None: 94 | # .netrc might have more auth for us on our new host. 95 | new_auth = get_netrc_auth(prepared_request.url) if self.trust_env else None 96 | if new_auth is not None: 97 | prepared_request.prepare_auth(new_auth) 98 | 99 | return 100 | 101 | 102 | def create_session(config: Config, auth: Tuple[str, str] = None, token: str = None) -> Session: 103 | """Creates a configured ``requests`` session. 104 | 105 | Attempts to create an authenticated session in the following order: 106 | 107 | 1) If ``auth`` is a tuple of (username, password), create a session. 108 | 2) Attempt to read a username and password from environment variables, either from the system 109 | or from a .env file to return a session. 110 | 3) Return a session that attempts to read credentials from a .netrc file. 111 | 112 | Args: 113 | config: Configuration object with EDL and authentication context 114 | auth: A tuple of the form ('edl_username', 'edl_password') 115 | 116 | Returns: 117 | The authenticated ``requests`` session. 118 | 119 | Raises: 120 | MalformedCredentials: ``auth`` credential not in the correct format. 121 | BadAuthentication: Incorrect credentials or unknown error. 122 | """ 123 | edl_username = config.EDL_USERNAME 124 | edl_password = config.EDL_PASSWORD 125 | 126 | if token: 127 | session = SessionWithHeaderRedirection(token=token) 128 | elif isinstance(auth, tuple) and len(auth) == 2 and all([isinstance(x, str) for x in auth]): 129 | session = SessionWithHeaderRedirection(auth=auth) 130 | elif auth is not None: 131 | raise MalformedCredentials('Authentication: `auth` argument requires tuple of ' 132 | '(username, password).') 133 | elif edl_username and edl_password: 134 | session = SessionWithHeaderRedirection(auth=(edl_username, edl_password)) 135 | else: 136 | session = SessionWithHeaderRedirection() 137 | 138 | return session 139 | 140 | 141 | def validate_auth(config: Config, session: Session): 142 | """Validates the credentials against the EDL authentication URL.""" 143 | if session.headers.get('Authorization') is None: 144 | url = config.edl_validation_url 145 | response = session.get(url) 146 | 147 | if response.status_code == 200: 148 | return 149 | elif response.status_code == 401: 150 | raise BadAuthentication('Authentication: incorrect or missing credentials during ' 151 | 'credential validation.') 152 | else: 153 | raise BadAuthentication(f'Authentication: An unknown error occurred during credential ' 154 | f'validation: HTTP {response.status_code}') 155 | -------------------------------------------------------------------------------- /harmony/config.py: -------------------------------------------------------------------------------- 1 | """Provides a Config class for conveniently specifying the environment for Harmony Py. 2 | 3 | The ``Config`` class can be instantiated without parameters and will default to 4 | the Harmony production environment. To create a configuration for the 5 | testing (UAT) environment, for example:: 6 | 7 | cfg = Config(Environment.UAT) 8 | 9 | This configuration object can then be passed as an argument when creating 10 | the ``harmony.Client``. 11 | """ 12 | import os 13 | from enum import Enum 14 | from typing import cast 15 | 16 | from dotenv import load_dotenv 17 | 18 | Environment = Enum('Environment', ['LOCAL', 'SIT', 'UAT', 'PROD']) 19 | 20 | HOSTNAMES = { 21 | Environment.LOCAL: 'localhost', 22 | Environment.SIT: 'harmony.sit.earthdata.nasa.gov', 23 | Environment.UAT: 'harmony.uat.earthdata.nasa.gov', 24 | Environment.PROD: 'harmony.earthdata.nasa.gov', 25 | } 26 | 27 | 28 | class Config: 29 | """Runtime configuration variables including defaults and environment vars. 30 | 31 | Example:: 32 | 33 | >>> cfg = Config() 34 | >>> cfg.foo 35 | 'bar' 36 | 37 | Parameters: 38 | None 39 | """ 40 | 41 | config = { 42 | 'NUM_REQUESTS_WORKERS': '3', # increase for servers 43 | 'DOWNLOAD_CHUNK_SIZE': str(4 * 1024 * 1024) # recommend 16MB for servers 44 | } 45 | 46 | def __init__(self, 47 | environment: Environment = Environment.PROD, 48 | localhost_port: int = 3000) -> None: 49 | """Creates a new Config instance for the specified Environment.""" 50 | load_dotenv() 51 | for k, v in Config.config.items(): 52 | setattr(self, k, v) 53 | self.environment = environment 54 | self.localhost_port = localhost_port 55 | 56 | @property 57 | def harmony_hostname(self): 58 | """Returns the hostname for this Config object's Environment.""" 59 | return HOSTNAMES[self.environment] 60 | 61 | @property 62 | def url_scheme(self) -> str: 63 | return 'http' if self.environment == Environment.LOCAL else 'https' 64 | 65 | @property 66 | def root_url(self) -> str: 67 | if self.environment == Environment.LOCAL: 68 | return f'{self.url_scheme}://{self.harmony_hostname}:{self.localhost_port}' 69 | else: 70 | return f'{self.url_scheme}://{self.harmony_hostname}' 71 | 72 | @property 73 | def edl_validation_url(self): 74 | """Returns the full URL to a Harmony endpoint used to validate the 75 | user's Earthdata Login credentials for this Config's Environment. 76 | """ 77 | return f'{self.root_url}/jobs' 78 | 79 | def __getattribute__(self, name: str) -> str: 80 | """Overrides attribute retrieval for instances of this class. 81 | 82 | Attribute lookup follow this order: 83 | 1. .env file variables 84 | 2. OS environment variables 85 | 3. object attributes that match ``name`` 86 | 87 | This dunder method is not called directly. 88 | 89 | Args: 90 | name: An EDL username. 91 | 92 | Returns: 93 | The value of the referenced attribute 94 | """ 95 | var = os.getenv(name.upper()) 96 | if var is None: 97 | try: 98 | var = object.__getattribute__(self, name) 99 | except AttributeError: 100 | var = None 101 | return cast(str, var) 102 | -------------------------------------------------------------------------------- /harmony/request.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from enum import Enum 3 | import os 4 | from shapely.lib import ShapelyError 5 | from shapely.wkt import loads 6 | from typing import Any, ContextManager, IO, Iterator, List, Mapping, NamedTuple, Optional, \ 7 | Tuple, Generator, Union 8 | 9 | 10 | class Collection: 11 | """The identity of a CMR Collection.""" 12 | 13 | def __init__(self, id: str): 14 | """Constructs a Collection instance from a CMR Collection ID. 15 | 16 | Args: 17 | id: CMR Collection ID 18 | 19 | Returns: 20 | A Collection instance 21 | """ 22 | self.id = id 23 | 24 | 25 | class BBox(NamedTuple): 26 | """A bounding box specified by western & eastern longitude, 27 | southern & northern latitude constraints in degrees. 28 | 29 | Example: 30 | An area bounded by latitudes 30N and 60N and longitudes 31 | 130W and 100W:: 32 | 33 | >>> spatial = BBox(-130, 30, -100, 60) 34 | 35 | Important: When specified positionally, the parameters must 36 | be given in order: west, south, east, north. 37 | 38 | Alternatively, one can explicitly set each bound using the 39 | single-letter for each bound:: 40 | 41 | >>> spatial = BBox(n=60, s=30, e=-100, w=-130) 42 | 43 | Print a readable representation of the spatial bounds:: 44 | 45 | >>> print(spatial) 46 | BBox: West:-130, South:30, East:-100, North:60 47 | 48 | Args: 49 | w: The western longitude bounds (degrees) 50 | s: The souther latitude bounds (degrees) 51 | e: The easter longitude bounds (degrees) 52 | n: The northern latitude bounds (degrees) 53 | 54 | Returns: 55 | A BBox instance with the provided bounds. 56 | """ 57 | w: float 58 | s: float 59 | e: float 60 | n: float 61 | 62 | def __repr__(self) -> str: 63 | return f'BBox: West:{self.w}, South:{self.s}, East:{self.e}, North:{self.n}' 64 | 65 | 66 | class WKT: 67 | """The Well Known Text (WKT) representation of Spatial. 68 | Supported WKT geometry types are: 69 | POINT, MULTIPOINT, POLYGON, MULTIPOLYGON, LINESTRING and MULTILINESTRING. 70 | 71 | Example: 72 | spatial=WKT('POINT(-40 10)') 73 | 74 | spatial=WKT('MULTIPOINT((-77 38.9),(-40 10))') 75 | 76 | spatial=WKT('POLYGON((-140 20, -50 20, -50 60, -140 60, -140 20))') 77 | 78 | spatial=WKT('MULTIPOLYGON(((10 10, 20 20, 30 10, 10 10)),((40 40, 50 50, 60 40, 40 40)))') 79 | 80 | spatial=WKT('LINESTRING(-155.75 19.26, -155.3 19.94)') 81 | 82 | spatial=WKT('MULTILINESTRING((-155.75 19.26, -155.3 19.94),(10 1, 10 30))') 83 | """ 84 | 85 | def __init__(self, wkt: str): 86 | """Constructs a WKT instance of spatial area. 87 | 88 | Args: 89 | wkt: the WKT string 90 | 91 | Returns: 92 | A WKT instance 93 | """ 94 | self.wkt = wkt 95 | 96 | 97 | class Dimension: 98 | """An arbitrary dimension to subset against. A dimension can take a minimum value and a 99 | maximum value to to subset against. 100 | 101 | Example: 102 | Requesting the data to be subset by the dimension lev with a minimum value of 10.0 103 | and maximum value of 20.0:: 104 | 105 | >>> dimension = Dimension('lev', 10.0, 20.0) 106 | 107 | Important: When specified positionally, the parameters must be given in the order: 108 | dimension name, minimum value, maximum value. 109 | 110 | Alternatively, one can explicitly set each value used named parameters: 111 | 112 | >>> dimension = Dimension(name='lev', min=10.0, max=20.0) 113 | 114 | Print a readable representation of the dimension:: 115 | 116 | >>> print(dimension) 117 | Dimension: Name: lev, Minimum: 10.0, Maximum: 20.0 118 | 119 | Args: 120 | name: The dimension name 121 | min: The minimum value for the given dimension to subset against (optional) 122 | max: The maximum value for the given dimension to subset against (optional) 123 | 124 | Returns: 125 | A Dimension instance with the provided dimension subset values. 126 | """ 127 | name: str 128 | min: float 129 | max: float 130 | 131 | def __init__(self, name: str, min: float = None, max: float = None): 132 | self.name = name 133 | self.min = min 134 | self.max = max 135 | 136 | def __repr__(self) -> str: 137 | return f'Dimension: Name: {self.name}, Minimum:{self.min}, Maximum:{self.max}' 138 | 139 | 140 | _shapefile_exts_to_mimes = { 141 | 'json': 'application/geo+json', 142 | 'geojson': 'application/geo+json', 143 | 'kml': 'application/vnd.google-earth.kml+xml', 144 | 'shz': 'application/shapefile+zip', 145 | 'zip': 'application/shapefile+zip', 146 | } 147 | _valid_shapefile_exts = ', '.join((_shapefile_exts_to_mimes.keys())) 148 | 149 | 150 | class HttpMethod(Enum): 151 | """Enumeration of HTTP methods used in Harmony requests. 152 | 153 | This enum defines the standard HTTP methods that can be used when making requests 154 | to the Harmony API. 155 | """ 156 | GET = "GET" 157 | PUT = "PUT" 158 | POST = "POST" 159 | DELETE = "DELETE" 160 | 161 | 162 | class BaseRequest: 163 | """A Harmony base request for all client requests. It is the base class of all harmony 164 | requests. 165 | 166 | Args: 167 | collection: The CMR collection that should be queried 168 | 169 | Returns: 170 | A Harmony Request instance 171 | """ 172 | 173 | def __init__(self, 174 | *, 175 | http_method: HttpMethod = HttpMethod.GET): 176 | self.http_method = http_method 177 | 178 | def error_messages(self) -> List[str]: 179 | """A list of error messages, if any, for the request. 180 | Validation of request parameters should go here. 181 | Returns the list of validation error messages back if any""" 182 | return [] 183 | 184 | def is_valid(self) -> bool: 185 | """Determines if the request and its parameters are valid.""" 186 | return len(self.error_messages()) == 0 187 | 188 | def parameter_values(self) -> List[Tuple[str, Any]]: 189 | """Returns tuples of each query parameter that has been set and its value.""" 190 | pvs = [(param, getattr(self, variable)) 191 | for variable, param in self.variable_name_to_query_param.items()] 192 | return [(p, v) for p, v in pvs if v is not None] 193 | 194 | 195 | class OgcBaseRequest(BaseRequest): 196 | """A Harmony OGC base request with the CMR collection. It is the base class of OGC harmony 197 | requests. 198 | 199 | Args: 200 | collection: The CMR collection that should be queried 201 | 202 | Returns: 203 | A Harmony Request instance 204 | """ 205 | 206 | def __init__(self, 207 | *, 208 | collection: Collection): 209 | super().__init__(http_method=HttpMethod.POST) 210 | self.collection = collection 211 | self.variable_name_to_query_param = {} 212 | 213 | 214 | def is_wkt_valid(wkt_string: str) -> bool: 215 | try: 216 | # Attempt to load the WKT string 217 | loads(wkt_string) 218 | return True 219 | except (ShapelyError, ValueError) as e: 220 | # Handle WKT reading errors and invalid WKT strings 221 | print(f"Invalid WKT: {e}") 222 | return False 223 | 224 | 225 | class Request(OgcBaseRequest): 226 | """A Harmony request with the CMR collection and various parameters expressing 227 | how the data is to be transformed. 228 | 229 | Args: 230 | collection: The CMR collection that should be queried 231 | 232 | spatial: Bounding box spatial constraints on the data or Well Known Text (WKT) string 233 | describing the spatial constraints. 234 | 235 | temporal: Date/time constraints on the data provided as a dict mapping "start" and "stop" 236 | keys to corresponding start/stop datetime.datetime objects 237 | 238 | dimensions: A list of dimensions to use for subsetting the data 239 | 240 | extend: A list of dimensions to extend 241 | 242 | crs: reproject the output coverage to the given CRS. Recognizes CRS types that can be 243 | inferred by gdal, including EPSG codes, Proj4 strings, and OGC URLs 244 | (http://www.opengis.net/def/crs/...) 245 | 246 | interpolation: specify the interpolation method used during reprojection and scaling 247 | 248 | scale_extent: scale the resulting coverage either among one axis to a given extent 249 | 250 | scale_size: scale the resulting coverage either among one axis to a given size 251 | 252 | shape: a file path to an ESRI Shapefile zip, GeoJSON file, or KML file to use for 253 | spatial subsetting. Note: not all collections support shapefile subsetting 254 | 255 | variables: The list of variables to subset 256 | 257 | granule_id: The CMR Granule ID for the granule which should be retrieved 258 | 259 | granule_name: The granule ur or provider id for the granule(s) to be retrieved 260 | wildcards * (multi character match) and ? (single character match) are supported 261 | 262 | width: number of columns to return in the output coverage 263 | 264 | height: number of rows to return in the output coverage 265 | 266 | format: the output mime type to return 267 | 268 | max_results: limits the number of input granules processed in the request 269 | 270 | concatenate: Whether to invoke a service that supports concatenation 271 | 272 | skip_preview: Whether Harmony should skip auto-pausing and generating a preview for 273 | large jobs 274 | 275 | ignore_errors: if "true", continue processing a request to completion 276 | even if some items fail 277 | 278 | destination_url: Destination URL specified by the client 279 | (only S3 is supported, e.g. s3://my-bucket-name/mypath) 280 | 281 | grid: The name of the output grid to use for regridding requests. The name must 282 | match the UMM grid name in the CMR. 283 | 284 | labels: The list of labels to include for the request. By default a 'harmony-py' 285 | label is added to all requests unless the environment variable EXCLUDE_DEFAULT_LABEL 286 | is set to 'true'. 287 | 288 | pixel_subset: Whether to perform pixel subset 289 | 290 | service_id: The CMR UMM-S concept ID or service chain name to invoke. Only supported in 291 | test environments for testing collections not yet associated with a service. 292 | 293 | Returns: 294 | A Harmony Transformation Request instance 295 | """ 296 | 297 | def __init__(self, 298 | collection: Collection, 299 | *, 300 | spatial: Union[BBox, WKT] = None, 301 | temporal: Mapping[str, datetime] = None, 302 | dimensions: List[Dimension] = None, 303 | extend: List[str] = None, 304 | crs: str = None, 305 | destination_url: str = None, 306 | format: str = None, 307 | granule_id: List[str] = None, 308 | granule_name: List[str] = None, 309 | height: int = None, 310 | interpolation: str = None, 311 | max_results: int = None, 312 | scale_extent: List[float] = None, 313 | scale_size: List[float] = None, 314 | shape: Optional[Tuple[IO, str]] = None, 315 | variables: List[str] = ['all'], 316 | width: int = None, 317 | concatenate: bool = None, 318 | skip_preview: bool = None, 319 | ignore_errors: bool = None, 320 | grid: str = None, 321 | labels: List[str] = None, 322 | pixel_subset: bool = None, 323 | service_id: str = None): 324 | """Creates a new Request instance from all specified criteria.' 325 | """ 326 | super().__init__(collection=collection) 327 | self.spatial = spatial 328 | self.temporal = temporal 329 | self.dimensions = dimensions 330 | self.extend = extend 331 | self.crs = crs 332 | self.destination_url = destination_url 333 | self.format = format 334 | self.granule_id = granule_id 335 | self.granule_name = granule_name 336 | self.height = height 337 | self.interpolation = interpolation 338 | self.max_results = max_results 339 | self.scale_extent = scale_extent 340 | self.scale_size = scale_size 341 | self.shape = shape 342 | self.variables = variables 343 | self.width = width 344 | self.concatenate = concatenate 345 | self.skip_preview = skip_preview 346 | self.ignore_errors = ignore_errors 347 | self.grid = grid 348 | self.labels = labels 349 | self.pixel_subset = pixel_subset 350 | self.service_id = service_id 351 | 352 | if self.is_edr_request(): 353 | self.variable_name_to_query_param = { 354 | 'crs': 'crs', 355 | 'destination_url': 'destinationUrl', 356 | 'interpolation': 'interpolation', 357 | 'scale_extent': 'scaleExtent', 358 | 'scale_size': 'scaleSize', 359 | 'shape': 'shapefile', 360 | 'granule_id': 'granuleId', 361 | 'granule_name': 'granuleName', 362 | 'width': 'width', 363 | 'height': 'height', 364 | 'format': 'f', 365 | 'max_results': 'maxResults', 366 | 'concatenate': 'concatenate', 367 | 'skip_preview': 'skipPreview', 368 | 'ignore_errors': 'ignoreErrors', 369 | 'grid': 'grid', 370 | 'extend': 'extend', 371 | 'variables': 'parameter-name', 372 | 'labels': 'label', 373 | 'pixel_subset': 'pixelSubset', 374 | 'service_id': 'serviceId', 375 | } 376 | self.spatial_validations = [ 377 | (lambda s: is_wkt_valid(s.wkt), f'WKT {spatial.wkt} is invalid'), 378 | ] 379 | else: 380 | self.variable_name_to_query_param = { 381 | 'crs': 'outputcrs', 382 | 'destination_url': 'destinationUrl', 383 | 'interpolation': 'interpolation', 384 | 'scale_extent': 'scaleExtent', 385 | 'scale_size': 'scaleSize', 386 | 'shape': 'shapefile', 387 | 'granule_id': 'granuleId', 388 | 'granule_name': 'granuleName', 389 | 'width': 'width', 390 | 'height': 'height', 391 | 'format': 'format', 392 | 'max_results': 'maxResults', 393 | 'concatenate': 'concatenate', 394 | 'skip_preview': 'skipPreview', 395 | 'ignore_errors': 'ignoreErrors', 396 | 'grid': 'grid', 397 | 'extend': 'extend', 398 | 'variables': 'variable', 399 | 'labels': 'label', 400 | 'pixel_subset': 'pixelSubset', 401 | 'service_id': 'serviceId' 402 | } 403 | 404 | self.spatial_validations = [ 405 | (lambda bb: bb.s <= bb.n, ('Southern latitude must be less than ' 406 | 'or equal to Northern latitude')), 407 | (lambda bb: bb.s >= -90.0, 'Southern latitude must be greater than -90.0'), 408 | (lambda bb: bb.n >= -90.0, 'Northern latitude must be greater than -90.0'), 409 | (lambda bb: bb.s <= 90.0, 'Southern latitude must be less than 90.0'), 410 | (lambda bb: bb.n <= 90.0, 'Northern latitude must be less than 90.0'), 411 | (lambda bb: bb.w >= -180.0, 'Western longitude must be greater than -180.0'), 412 | (lambda bb: bb.e >= -180.0, 'Eastern longitude must be greater than -180.0'), 413 | (lambda bb: bb.w <= 180.0, 'Western longitude must be less than 180.0'), 414 | (lambda bb: bb.e <= 180.0, 'Eastern longitude must be less than 180.0'), 415 | ] 416 | 417 | self.temporal_validations = [ 418 | (lambda tr: 'start' in tr or 'stop' in tr, 419 | ('When included in the request, the temporal range should include a ' 420 | 'start or stop attribute.')), 421 | (lambda tr: tr['start'] < tr['stop'] if 'start' in tr and 'stop' in tr else True, 422 | 'The temporal range\'s start must be earlier than its stop datetime.') 423 | ] 424 | self.shape_validations = [ 425 | (lambda s: os.path.isfile(s), 'The provided shape path is not a file'), 426 | (lambda s: s.split('.').pop().lower() in _shapefile_exts_to_mimes, 427 | 'The provided shape file is not a recognized type. Valid file extensions: ' 428 | + f'[{_valid_shapefile_exts}]'), 429 | ] 430 | self.dimension_validations = [ 431 | (lambda dim: dim.min is None or dim.max is None or dim.min <= dim.max, 432 | ('Dimension minimum value must be less than or equal to the maximum value')) 433 | ] 434 | self.parameter_validations = [ # for simple, one-off validations 435 | (True if self.destination_url is None else self.destination_url.startswith('s3://'), 436 | 'Destination URL must be an S3 location'), 437 | (self.concatenate is None or isinstance(self.concatenate, bool), 438 | 'concatenate must be a boolean (True or False)'), 439 | (self.ignore_errors is None or isinstance(self.ignore_errors, bool), 440 | 'ignore_errors must be a boolean (True or False)'), 441 | (self.skip_preview is None or isinstance(self.skip_preview, bool), 442 | 'skip_preview must be a boolean (True or False)'), 443 | (self.pixel_subset is None or isinstance(self.pixel_subset, bool), 444 | 'pixel_subset must be a boolean (True or False)') 445 | ] 446 | 447 | def _shape_error_messages(self, shape) -> List[str]: 448 | """Returns a list of error message for the provided shape.""" 449 | if not shape: 450 | return [] 451 | if not os.path.exists(shape): 452 | return [f'The provided shape path "{shape}" does not exist'] 453 | if not os.path.isfile(shape): 454 | return [f'The provided shape path "{shape}" is not a file'] 455 | ext = shape.split('.').pop().lower() 456 | if ext not in _shapefile_exts_to_mimes: 457 | return [f'The provided shape path "{shape}" has extension "{ext}" which is not ' 458 | + f'recognized. Valid file extensions: [{_valid_shapefile_exts}]'] 459 | return [] 460 | 461 | def is_edr_request(self) -> bool: 462 | """Return true if the request needs to be submitted as an EDR request, 463 | i.e. Spatial is WKT.""" 464 | return isinstance(self.spatial, WKT) 465 | 466 | def error_messages(self) -> List[str]: 467 | """A list of error messages, if any, for the request.""" 468 | spatial_msgs = [] 469 | temporal_msgs = [] 470 | dimension_msgs = [] 471 | parameter_msgs = [m for v, m in self.parameter_validations if not v] 472 | shape_msgs = self._shape_error_messages(self.shape) 473 | if self.spatial: 474 | spatial_msgs = [m for v, m in self.spatial_validations if not v(self.spatial)] 475 | if self.temporal: 476 | temporal_msgs = [m for v, m in self.temporal_validations if not v(self.temporal)] 477 | if self.dimensions: 478 | for dim in self.dimensions: 479 | msgs = [m for v, m in self.dimension_validations if not v(dim)] 480 | if msgs: 481 | dimension_msgs += msgs 482 | 483 | return spatial_msgs + temporal_msgs + shape_msgs + dimension_msgs + parameter_msgs 484 | 485 | 486 | class CapabilitiesRequest(BaseRequest): 487 | """A Harmony request to retrieve the capabilities of a CMR collection. 488 | 489 | This request queries the Harmony API for the capabilities of a specified CMR 490 | (Common Metadata Repository) collection, allowing users to retrieve metadata 491 | about available processing and data access options. 492 | 493 | Args: 494 | collection_id (str, optional): The CMR collection ID to query. 495 | short_name (str, optional): The short name of the CMR collection. 496 | capabilities_version (str, optional): The version of the collection 497 | capabilities request API. 498 | 499 | Returns: 500 | CapabilitiesRequest: An instance of the request configured with 501 | the provided parameters. 502 | """ 503 | 504 | def __init__(self, 505 | **request_params 506 | ): 507 | 508 | super().__init__() 509 | self.collection_id = request_params.get('collection_id') 510 | self.short_name = request_params.get('short_name') 511 | self.capabilities_version = request_params.get('capabilities_version') 512 | 513 | self.variable_name_to_query_param = { 514 | 'collection_id': 'collectionid', 515 | 'short_name': 'shortname', 516 | 'capabilities_version': 'version', 517 | } 518 | 519 | def error_messages(self) -> List[str]: 520 | """A list of error messages, if any, for the request.""" 521 | error_msgs = [] 522 | if self.collection_id is None and self.short_name is None: 523 | error_msgs = [ 524 | 'Must specify either collection_id or short_name for CapabilitiesRequest' 525 | ] 526 | elif self.collection_id and self.short_name: 527 | error_msgs = [ 528 | 'CapabilitiesRequest cannot have both collection_id and short_name values' 529 | ] 530 | 531 | return error_msgs 532 | 533 | 534 | class AddLabelsRequest(BaseRequest): 535 | """A Harmony request to add labels on jobs. 536 | 537 | Args: 538 | labels (List[str]): A list of labels to be added. 539 | job_ids (List[str]): A list of job IDs to which the labels apply. 540 | 541 | Returns: 542 | AddLabelsRequest: An instance of the request configured with the provided parameters. 543 | """ 544 | 545 | def __init__(self, 546 | *, 547 | labels: List[str], 548 | job_ids: List[str] 549 | ): 550 | super().__init__(http_method=HttpMethod.PUT) 551 | self.labels = labels 552 | self.job_ids = job_ids 553 | 554 | self.variable_name_to_query_param = { 555 | 'labels': 'label', 556 | 'job_ids': 'jobID', 557 | } 558 | 559 | 560 | class DeleteLabelsRequest(BaseRequest): 561 | """A Harmony request to delete labels from jobs. 562 | 563 | Args: 564 | labels (List[str]): A list of labels to be removed. 565 | job_ids (List[str]): A list of job IDs to which the labels apply. 566 | 567 | Returns: 568 | DeleteLabelsRequest: An instance of the request configured with the provided parameters. 569 | """ 570 | 571 | def __init__(self, 572 | *, 573 | labels: List[str], 574 | job_ids: List[str] 575 | ): 576 | super().__init__(http_method=HttpMethod.DELETE) 577 | self.labels = labels 578 | self.job_ids = job_ids 579 | 580 | self.variable_name_to_query_param = { 581 | 'labels': 'label', 582 | 'job_ids': 'jobID', 583 | } 584 | 585 | 586 | class JobsRequest(BaseRequest): 587 | """A Harmony request to list or search for jobs. 588 | 589 | Args: 590 | page (int): The current page number. 591 | limit (int): The number of jobs in each page. 592 | labels (List[str]): A list of labels to search jobs with. 593 | 594 | Returns: 595 | JobsRequest: An instance of the jobs request configured with the provided parameters. 596 | """ 597 | 598 | def __init__(self, 599 | *, 600 | page: int = None, 601 | limit: int = None, 602 | labels: List[str] = None, 603 | ): 604 | super().__init__() 605 | self.page = page 606 | self.limit = limit 607 | self.labels = labels 608 | 609 | self.variable_name_to_query_param = { 610 | 'page': 'page', 611 | 'limit': 'limit', 612 | 'labels': 'label', 613 | } 614 | 615 | 616 | class LinkType(Enum): 617 | """The type of URL to provide when returning links to data.""" 618 | s3 = 's3' 619 | http = 'http' 620 | https = 'https' 621 | -------------------------------------------------------------------------------- /harmony/util.py: -------------------------------------------------------------------------------- 1 | from os.path import basename 2 | from urllib.parse import urlparse 3 | 4 | 5 | def s3_components(url: str) -> tuple: 6 | """Returns a tuple with the S3 bucket, key, and file names parsed from the S3 URL. 7 | 8 | Note: the url should be a valid S3 url. For example :: 9 | 10 | s3://bucket-name/full/path/data.nc 11 | 12 | In this case, the function will return the tuple: 13 | 14 | ('bucket-name', 'full/path', 'data.nc') 15 | 16 | This tuple may be used to more easily download data using the boto3 API. 17 | 18 | Args: 19 | url: a valid S3 URL 20 | 21 | Returns: 22 | A tuple containing the strings (bucket, object, filename). 23 | """ 24 | parsed = urlparse(url) 25 | return (parsed.netloc, parsed.path.lstrip('/'), basename(parsed.path)) 26 | -------------------------------------------------------------------------------- /internal/genparams.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from camel_case_switcher import camel_case_to_underscore 4 | import yaml 5 | 6 | 7 | def canonical_name(yaml_param): 8 | name_map = { 9 | 'outputcrs': 'crs' 10 | } 11 | 12 | print(yaml_param) 13 | name = yaml_param['name'] 14 | return name_map.get(name, camel_case_to_underscore(name)) 15 | 16 | 17 | def canonical_type(yaml_param): 18 | type_map = { 19 | 'string': 'str', 20 | 'boolean': 'bool', 21 | 'integer': 'int', 22 | 'number': 'float' 23 | } 24 | 25 | py_type = None 26 | yaml_type = yaml_param['schema']['type'] 27 | if yaml_type == 'array': 28 | item_type = type_map[yaml_param['schema']['items']['type']] 29 | py_type = f"list[{item_type}]" 30 | else: 31 | py_type = type_map[yaml_type] 32 | 33 | return py_type 34 | 35 | 36 | def param_docstring(yaml_param): 37 | name = canonical_name(yaml_param) 38 | descr = yaml_param['description'] 39 | return f'{name}: {descr}' 40 | 41 | 42 | def main(schema_filename: str): 43 | with open(schema_filename, 'r') as schema: 44 | api = yaml.load(schema, Loader=yaml.Loader) 45 | 46 | do_not_generate = ['collectionId', 'subset'] 47 | 48 | params = api['paths']['/collections/{collectionId}/coverage/rangeset']['get']['parameters'] 49 | refs = [p.get('$ref').split('/')[-1] for p in params] 50 | param_types = [api['components']['parameters'][r] for 51 | r in refs if r not in do_not_generate] 52 | 53 | params = [f'{canonical_name(pt)}: {canonical_type(pt)}' for pt in param_types] 54 | param_docstrings = [param_docstring(pt) for pt in param_types] 55 | 56 | print("def __init__(self, *, " + ", ".join(params) + "):") 57 | print(' """') 58 | print(' Parameters:') 59 | print(' -----------') 60 | for pds in param_docstrings: 61 | print(' ' + pds) 62 | print() 63 | print() 64 | print(' """') 65 | 66 | 67 | if __name__ == '__main__': 68 | if len(sys.argv) != 2: 69 | print("Usage:") 70 | print(" python internal/genparams.py harmony_ogc_schema_filename") 71 | sys.exit(1) 72 | 73 | main(sys.argv[1]) 74 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # License and classifier list: 2 | # https://pypi.org/pypi?%3Aaction=list_classifiers 3 | 4 | [build-system] 5 | requires = ["setuptools"] 6 | build-backend = "setuptools.build_meta" 7 | 8 | [project] 9 | name = "harmony-py" 10 | description = "Python library for integrating with NASA's Harmony Services." 11 | authors = [ 12 | {name = "NASA EOSDIS Harmony Team", email = "christopher.d.durbin@nasa.gov"} 13 | ] 14 | readme = "README.md" 15 | license = {file = "LICENSE"} 16 | requires-python = ">=3.9, <4" 17 | keywords = ["nasa", "harmony", "remote-sensing", "science", "geoscience"] 18 | classifiers = [ 19 | 'Development Status :: 3 - Alpha', 20 | "Intended Audience :: Developers", 21 | "Intended Audience :: Science/Research", 22 | "Programming Language :: Python :: 3", 23 | "Programming Language :: Python :: 3.9", 24 | "Programming Language :: Python :: 3.10", 25 | "Programming Language :: Python :: 3.11", 26 | "Programming Language :: Python :: 3.12", 27 | "Programming Language :: Python :: 3.13", 28 | "Programming Language :: Python :: 3 :: Only", 29 | ] 30 | dynamic = ["version"] 31 | dependencies = [ 32 | "python-dateutil ~= 2.9", 33 | "python-dotenv ~= 0.20.0", 34 | "progressbar2 ~= 4.2.0", 35 | "requests ~= 2.32.3", 36 | "sphinxcontrib-napoleon ~= 0.7", 37 | "curlify ~= 2.2.1", 38 | "shapely ~= 2.0.4" 39 | ] 40 | 41 | [project.urls] 42 | Homepage = "https://github.com/nasa/harmony-py" 43 | Documentation = "https://harmony-py.readthedocs.io/en/main/" 44 | Repository = "https://github.com/nasa/harmony-py.git" 45 | 46 | [project.optional-dependencies] 47 | dev = [ 48 | "coverage ~= 7.4", 49 | "flake8 ~= 7.1.1", 50 | "hypothesis ~= 6.103", 51 | "PyYAML ~= 6.0.1", 52 | "pytest ~= 8.2", 53 | "pytest-cov ~= 5.0", 54 | "pytest-mock ~= 3.14", 55 | "pytest-watch ~= 4.2", 56 | "responses ~= 0.25.6" 57 | ] 58 | docs = [ 59 | "curlify ~= 2.2.1", 60 | "Jinja2 ~= 3.1.2", 61 | "load-dotenv ~=0.1.0", 62 | "nbconvert ~= 7.10.0", 63 | "progressbar2 ~= 4.2.0", 64 | "sphinx ~= 7.1.2", 65 | "sphinx-rtd-theme ~= 1.3.0", 66 | "shapely ~= 2.0.4" 67 | ] 68 | examples = [ 69 | "boto3 ~= 1.28", 70 | "intake-stac ~= 0.4.0", 71 | "ipyplot ~= 1.1", 72 | "ipywidgets ~= 8.1", 73 | "jupyterlab ~= 4.0", 74 | "matplotlib ~= 3.8", 75 | "netCDF4 ~= 1.6", 76 | "numpy ~= 1.26", 77 | "pillow ~= 10.1", # A dependency of ipyplot, pinned to avoid critical vulnerability. 78 | "pystac ~= 1.9.0", 79 | "rasterio ~= 1.3" 80 | ] 81 | 82 | [tool.setuptools.dynamic] 83 | # Will read __version__ from harmony.__init__.py 84 | version = {attr = "harmony.__version__"} 85 | 86 | [tool.setuptools.packages.find] 87 | exclude = ["contrib", "docs", "tests*"] 88 | 89 | [tool.flake8] 90 | max-line-length = 99 91 | ignore = ["F401", "W503"] 92 | 93 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nasa/harmony-py/b19f073306aec021746cb4c7b360f8a9242f8e29/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_auth.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import responses 3 | 4 | from harmony.auth import (_is_edl_hostname, create_session, validate_auth, 5 | BadAuthentication, MalformedCredentials, SessionWithHeaderRedirection) 6 | from harmony.config import Config 7 | 8 | 9 | class Object(object): 10 | pass 11 | 12 | 13 | @pytest.fixture 14 | def config(): 15 | return Config() 16 | 17 | 18 | def test_authentication_no_args_no_validate(): 19 | fake_config = Object() 20 | fake_config.EDL_USERNAME = None 21 | fake_config.EDL_PASSWORD = None 22 | session = create_session(fake_config) 23 | assert session.auth is None 24 | 25 | 26 | @pytest.mark.parametrize('hostname,expected', [ 27 | ('uat.urs.earthdata.nasa.gov', True), 28 | ('urs.earthdata.nasa.gov', True), 29 | ('example.gov', False), 30 | ('earthdata.nasa.gov', False), 31 | ('urs.earthdata.nasa.gov.badactor.com', False) 32 | ]) 33 | def test__is_edl_hostname(hostname, expected): 34 | assert _is_edl_hostname(hostname) is expected 35 | 36 | 37 | @pytest.mark.parametrize('auth', [ 38 | (None,), 39 | ('username'), 40 | ('username',), 41 | ('username', 333), 42 | (999, 'secret'), 43 | ]) 44 | def test_authentication_with_malformed_auth(auth, config, mocker): 45 | with pytest.raises(MalformedCredentials) as exc_info: 46 | session = create_session(config, auth=auth) 47 | validate_auth(config, session) 48 | assert 'Authentication: `auth` argument requires tuple' in str(exc_info.value) 49 | 50 | 51 | @responses.activate 52 | @pytest.mark.parametrize('status_code,should_error', 53 | [(200, False), (401, True), (500, True)]) 54 | def test_authentication(status_code, should_error, config, mocker): 55 | auth_url = 'https://harmony.earthdata.nasa.gov/jobs' 56 | responses.add( 57 | responses.GET, 58 | auth_url, 59 | status=status_code 60 | ) 61 | if should_error: 62 | with pytest.raises(BadAuthentication) as exc_info: 63 | actual_session = create_session(config) 64 | validate_auth(config, actual_session) 65 | if status_code == 401: 66 | assert 'Authentication: incorrect or missing credentials' in str(exc_info.value) 67 | elif status_code == 500: 68 | assert 'Authentication: An unknown error occurred' in str(exc_info.value) 69 | else: 70 | actual_session = create_session(config) 71 | validate_auth(config, actual_session) 72 | assert actual_session is not None 73 | 74 | 75 | def test_SessionWithHeaderRedirection_with_no_edl(mocker): 76 | preparedrequest_mock = mocker.PropertyMock() 77 | preparedrequest_props = {'url': 'https://www.example.gov', 78 | 'headers': {'Authorization': 'lorem ipsum'}} 79 | preparedrequest_mock.configure_mock(**preparedrequest_props) 80 | 81 | response_mock = mocker.PropertyMock() 82 | response_mock.request.configure_mock(url='https://www.othersite.gov') 83 | 84 | mocker.patch('harmony.auth.PreparedRequest', return_value=preparedrequest_mock) 85 | mocker.patch('harmony.auth.Response', return_value=response_mock) 86 | 87 | session_with_creds = SessionWithHeaderRedirection(auth=('foo', 'bar')) 88 | session_with_creds.rebuild_auth(preparedrequest_mock, response_mock) 89 | 90 | assert preparedrequest_mock.url == preparedrequest_props['url'] \ 91 | and 'Authorization' not in preparedrequest_mock.headers 92 | 93 | 94 | def test_SessionWithHeaderRedirection_with_edl(mocker): 95 | preparedrequest_mock = mocker.PropertyMock() 96 | preparedrequest_props = {'url': 'https://uat.urs.earthdata.nasa.gov', 97 | 'headers': {'Authorization': 'lorem ipsum'}} 98 | preparedrequest_mock.configure_mock(**preparedrequest_props) 99 | 100 | response_mock = mocker.PropertyMock() 101 | response_mock.request.configure_mock(url='https://www.othersite.gov') 102 | 103 | mocker.patch('harmony.auth.PreparedRequest', return_value=preparedrequest_mock) 104 | mocker.patch('harmony.auth.Response', return_value=response_mock) 105 | 106 | session_with_creds = SessionWithHeaderRedirection(auth=('foo', 'bar')) 107 | session_with_creds.rebuild_auth(preparedrequest_mock, response_mock) 108 | 109 | assert preparedrequest_mock.url == preparedrequest_props['url'] \ 110 | and 'Authorization' in preparedrequest_mock.headers 111 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from harmony.config import Config, Environment 4 | 5 | 6 | def test_config_from_env(mocker): 7 | expected_value = 'bar' 8 | mocker.patch('harmony.config.os.getenv', return_value=expected_value) 9 | config = Config() 10 | assert config.FOO == expected_value 11 | 12 | 13 | def test_config_not_there(): 14 | config = Config() 15 | assert config.ASDF is None 16 | 17 | 18 | def test_config_built_in(): 19 | config = Config() 20 | assert config.NUM_REQUESTS_WORKERS is not None 21 | 22 | 23 | @pytest.mark.parametrize('env,url', [ 24 | (Environment.LOCAL, 'localhost'), 25 | (Environment.SIT, 'harmony.sit.earthdata.nasa.gov'), 26 | (Environment.UAT, 'harmony.uat.earthdata.nasa.gov'), 27 | (Environment.PROD, 'harmony.earthdata.nasa.gov') 28 | ]) 29 | def test_harmony_hostname_matches_environment(env, url): 30 | config = Config(env) 31 | 32 | assert config.harmony_hostname == url 33 | 34 | 35 | @pytest.mark.parametrize('env,url', [ 36 | (Environment.LOCAL, 'http'), 37 | (Environment.SIT, 'https'), 38 | (Environment.UAT, 'https'), 39 | (Environment.PROD, 'https') 40 | ]) 41 | def test_url_scheme_matches_environment(env, url): 42 | config = Config(env) 43 | 44 | assert config.url_scheme == url 45 | 46 | 47 | @pytest.mark.parametrize('env,url', [ 48 | (Environment.LOCAL, 'http://localhost:3000'), 49 | (Environment.SIT, 'https://harmony.sit.earthdata.nasa.gov'), 50 | (Environment.UAT, 'https://harmony.uat.earthdata.nasa.gov'), 51 | (Environment.PROD, 'https://harmony.earthdata.nasa.gov') 52 | ]) 53 | def test_root_url_matches_environment(env, url): 54 | config = Config(env) 55 | 56 | assert config.root_url == url 57 | 58 | 59 | @pytest.mark.parametrize('env,url', [ 60 | (Environment.LOCAL, 'http://localhost:9999'), 61 | (Environment.SIT, 'https://harmony.sit.earthdata.nasa.gov'), 62 | (Environment.UAT, 'https://harmony.uat.earthdata.nasa.gov'), 63 | (Environment.PROD, 'https://harmony.earthdata.nasa.gov') 64 | ]) 65 | def test_localhost_port_is_overridable(env, url): 66 | config = Config(env, localhost_port=9999) 67 | 68 | assert config.root_url == url 69 | 70 | 71 | @pytest.mark.parametrize('env,url', [ 72 | (Environment.LOCAL, 'http://localhost:3000/jobs'), 73 | (Environment.SIT, 'https://harmony.sit.earthdata.nasa.gov/jobs'), 74 | (Environment.UAT, 'https://harmony.uat.earthdata.nasa.gov/jobs'), 75 | (Environment.PROD, 'https://harmony.earthdata.nasa.gov/jobs') 76 | ]) 77 | def test_edl_validation_url_matches_environment(env, url): 78 | config = Config(env) 79 | 80 | assert config.edl_validation_url == url 81 | -------------------------------------------------------------------------------- /tests/test_request.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | 3 | from hypothesis import given, settings, strategies as st 4 | import pytest 5 | 6 | from harmony.request import BBox, WKT, Collection, OgcBaseRequest, Request, Dimension, \ 7 | CapabilitiesRequest, AddLabelsRequest, DeleteLabelsRequest, JobsRequest 8 | 9 | 10 | def test_request_has_collection_with_id(): 11 | collection = Collection('foobar') 12 | request = OgcBaseRequest(collection=collection) 13 | assert request.collection.id == 'foobar' 14 | assert request.is_valid() 15 | 16 | 17 | def test_transformation_request_has_collection_with_id(): 18 | collection = Collection('foobar') 19 | request = Request(collection) 20 | assert request.collection.id == 'foobar' 21 | 22 | 23 | def test_request_with_only_a_collection(): 24 | request = Request(collection=Collection('foobar')) 25 | assert request.is_valid() 26 | 27 | 28 | def test_request_with_skip_preview_false(): 29 | request = Request(collection=Collection('foobar'), skip_preview=False) 30 | assert request.is_valid() 31 | assert request.skip_preview is not None and request.skip_preview == False 32 | 33 | 34 | def test_request_with_skip_preview_true(): 35 | request = Request(collection=Collection('foobar'), skip_preview=True) 36 | assert request.is_valid() 37 | assert request.skip_preview is not None and request.skip_preview == True 38 | 39 | 40 | def test_request_defaults_to_skip_preview_false(): 41 | request = Request(collection=Collection('foobar')) 42 | assert not request.skip_preview 43 | 44 | 45 | def test_request_with_ignore_errors_false(): 46 | request = Request(collection=Collection('foobar'), ignore_errors=False) 47 | assert request.is_valid() 48 | assert request.ignore_errors is not None and request.ignore_errors == False 49 | 50 | 51 | def test_request_with_ignore_errors_true(): 52 | request = Request(collection=Collection('foobar'), ignore_errors=True) 53 | assert request.is_valid() 54 | assert request.ignore_errors is not None and request.ignore_errors == True 55 | 56 | 57 | def test_request_defaults_to_ignore_errors_false(): 58 | request = Request(collection=Collection('foobar')) 59 | assert not request.ignore_errors 60 | 61 | 62 | @settings(max_examples=100) 63 | @given(west=st.floats(allow_infinity=True), 64 | south=st.floats(allow_infinity=True), 65 | east=st.floats(allow_infinity=True), 66 | north=st.floats(allow_infinity=True)) 67 | def test_request_spatial_bounding_box(west, south, east, north): 68 | spatial = BBox(west, south, east, north) 69 | request = Request( 70 | collection=Collection('foobar'), 71 | spatial=spatial, 72 | ) 73 | 74 | if request.is_valid(): 75 | assert request.spatial is not None 76 | w, s, e, n = request.spatial 77 | assert w == west 78 | assert s == south 79 | assert e == east 80 | assert n == north 81 | 82 | assert south <= north 83 | 84 | assert south >= -90.0 85 | assert north >= -90.0 86 | assert south <= 90.0 87 | assert north <= 90.0 88 | 89 | assert west >= -180.0 90 | assert east >= -180.0 91 | assert west <= 180.0 92 | assert east <= 180.0 93 | 94 | 95 | @pytest.mark.parametrize('key, value', [ 96 | ('spatial', WKT('POINT(0 51.48)')), 97 | ('spatial', WKT('LINESTRING(30 10, 10 30, 40 40)')), 98 | ('spatial', WKT('POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))')), 99 | ('spatial', WKT('POLYGON((35 10, 45 45, 15 40, 10 20, 35 10),(20 30, 35 35, 30 20, 20 30))')), 100 | ('spatial', WKT('MULTIPOINT((10 40), (40 30), (20 20), (30 10))')), 101 | ('spatial', WKT('MULTILINESTRING((10 10, 20 20, 10 40),(40 40, 30 30, 40 20, 30 10))')), 102 | ('spatial', WKT('MULTIPOLYGON(((30 20, 45 40, 10 40, 30 20)),((15 5, 40 10, 10 20, 5 10, 15 5)))')), 103 | ]) 104 | def test_request_spatial_wkt(key, value): 105 | request = Request(Collection('foo'), **{key: value}) 106 | assert request.is_valid() 107 | 108 | 109 | @settings(max_examples=100) 110 | @given(key_a=st.one_of(st.none(), st.sampled_from(['start', 'stop']), st.text()), 111 | key_b=st.one_of(st.none(), st.sampled_from(['start', 'stop']), st.text()), 112 | datetime_a=st.datetimes(), 113 | datetime_b=st.datetimes()) 114 | def test_request_temporal_range(key_a, key_b, datetime_a, datetime_b): 115 | temporal = { 116 | key_a: datetime_a, 117 | key_b: datetime_b 118 | } 119 | request = Request( 120 | collection=Collection('foobar'), 121 | temporal=temporal 122 | ) 123 | 124 | if request.is_valid(): 125 | assert request.temporal is not None 126 | assert 'start' in request.temporal or 'stop' in request.temporal 127 | if 'start' in request.temporal and 'stop' in request.temporal: 128 | assert request.temporal['start'] < request.temporal['stop'] 129 | 130 | 131 | @settings(max_examples=100) 132 | @given(min=st.one_of(st.floats(allow_infinity=True), st.integers()), 133 | max=st.one_of(st.floats(allow_infinity=True), st.integers())) 134 | def test_request_dimensions(min, max): 135 | dimension = Dimension('foo', min, max) 136 | request = Request( 137 | collection=Collection('foobar'), 138 | dimensions=[dimension], 139 | ) 140 | 141 | if request.is_valid(): 142 | assert len(request.dimensions) == 1 143 | min_actual = request.dimensions[0].min 144 | max_actual = request.dimensions[0].max 145 | assert min == min_actual 146 | assert max == max_actual 147 | 148 | 149 | @pytest.mark.parametrize('key, value, message', [ 150 | ('spatial', BBox(10, -10, 20, -20), 'Southern latitude must be less than or equal to Northern latitude'), 151 | ('spatial', BBox(10, -100, 20, 20), 'Southern latitude must be greater than -90.0'), 152 | ('spatial', BBox(10, -110, 20, -100), 'Northern latitude must be greater than -90.0'), 153 | ('spatial', BBox(10, 100, 20, 110), 'Southern latitude must be less than 90.0'), 154 | ('spatial', BBox(10, 10, 20, 100), 'Northern latitude must be less than 90.0'), 155 | ('spatial', BBox(-190, 10, 20, 20), 'Western longitude must be greater than -180.0'), 156 | ('spatial', BBox(-200, 10, -190, 20), 'Eastern longitude must be greater than -180.0'), 157 | ('spatial', BBox(10, 10, 190, 20), 'Eastern longitude must be less than 180.0'), 158 | ('spatial', BBox(190, 10, 200, 20), 'Western longitude must be less than 180.0'), 159 | ]) 160 | def test_request_spatial_error_messages(key, value, message): 161 | request = Request(Collection('foo'), **{key: value}) 162 | messages = request.error_messages() 163 | 164 | assert not request.is_valid() 165 | assert message in messages 166 | 167 | 168 | @pytest.mark.parametrize('value', [ 169 | [Dimension('foo', 0, -100.0)], 170 | [Dimension('foo', 0, -100.0), Dimension('bar', 50.0, 0)], 171 | [Dimension('foo', -100.0, 0), Dimension('bar', 50.0, 0)], 172 | [Dimension(name='bar', max=25, min=125.0)] 173 | ]) 174 | def test_request_dimensions_error_messages(value): 175 | message = 'Dimension minimum value must be less than or equal to the maximum value' 176 | request = Request(Collection('foo'), **{'dimensions': value}) 177 | messages = request.error_messages() 178 | 179 | assert not request.is_valid() 180 | assert message in messages 181 | 182 | 183 | @pytest.mark.parametrize('key, value, message', [ 184 | ('spatial', WKT('BBOX(-140,20,-50,60)'), 'WKT BBOX(-140,20,-50,60) is invalid'), 185 | ('spatial', WKT('APOINT(0 51.48)'), 'WKT APOINT(0 51.48) is invalid'), 186 | ('spatial', WKT('CIRCULARSTRING(0 0, 1 1, 1 0)'), 'WKT CIRCULARSTRING(0 0, 1 1, 1 0) is invalid'), 187 | ]) 188 | def test_request_spatial_error_messages(key, value, message): 189 | request = Request(Collection('foo'), **{key: value}) 190 | messages = request.error_messages() 191 | 192 | assert not request.is_valid() 193 | assert message in messages 194 | 195 | 196 | @pytest.mark.parametrize('key, value, message', [ 197 | ( 198 | 'temporal', { 199 | 'foo': None 200 | }, 201 | ('When included in the request, the temporal range should include a ' 202 | 'start or stop attribute.') 203 | ), ( 204 | 'temporal', { 205 | 'start': dt.datetime(1969, 7, 20), 206 | 'stop': dt.datetime(1941, 12, 7) 207 | }, 208 | 'The temporal range\'s start must be earlier than its stop datetime.' 209 | ) 210 | ]) 211 | def test_request_temporal_error_messages(key, value, message): 212 | request = Request(Collection('foo'), **{key: value}) 213 | messages = request.error_messages() 214 | 215 | assert not request.is_valid() 216 | assert message in messages 217 | 218 | 219 | def test_request_valid_shape(): 220 | request = Request(Collection('foo'), shape='./examples/asf_example.json') 221 | messages = request.error_messages() 222 | assert request.is_valid() 223 | assert messages == [] 224 | 225 | 226 | @pytest.mark.parametrize('key, value, messages', [ 227 | ('shape', './tests/', ['The provided shape path "./tests/" is not a file']), 228 | ('shape', './pyproject.toml', 229 | ['The provided shape path "./pyproject.toml" has extension "toml" which is not recognized. ' 230 | + 'Valid file extensions: [json, geojson, kml, shz, zip]']), 231 | ]) 232 | def test_request_shape_file_error_message(key, value, messages): 233 | request = Request(Collection('foo'), **{key: value}) 234 | 235 | assert not request.is_valid() 236 | assert request.error_messages() == messages 237 | 238 | 239 | def test_request_destination_url_error_message(): 240 | request = Request(Collection('foo'), destination_url='http://somesite.com') 241 | messages = request.error_messages() 242 | 243 | assert not request.is_valid() 244 | assert 'Destination URL must be an S3 location' in messages 245 | 246 | 247 | def test_collection_capabilities_without_coll_identifier(): 248 | request = CapabilitiesRequest(capabilities_version='2') 249 | messages = request.error_messages() 250 | 251 | assert not request.is_valid() 252 | assert 'Must specify either collection_id or short_name for CapabilitiesRequest' in messages 253 | 254 | 255 | def test_collection_capabilities_two_coll_identifier(): 256 | request = CapabilitiesRequest(collection_id='C1234-PROV', 257 | short_name='foobar', 258 | capabilities_version='2') 259 | messages = request.error_messages() 260 | 261 | assert not request.is_valid() 262 | assert 'CapabilitiesRequest cannot have both collection_id and short_name values' in messages 263 | 264 | 265 | def test_collection_capabilities_request_coll_id(): 266 | request = CapabilitiesRequest(collection_id='C1234-PROV') 267 | assert request.is_valid() 268 | 269 | 270 | def test_collection_capabilities_request_shortname(): 271 | request = CapabilitiesRequest(short_name='foobar') 272 | assert request.is_valid() 273 | 274 | 275 | def test_collection_capabilities_request_coll_id_version(): 276 | request = CapabilitiesRequest(collection_id='C1234-PROV', 277 | capabilities_version='2') 278 | assert request.is_valid() 279 | 280 | 281 | def test_collection_capabilities_request_shortname_version(): 282 | request = CapabilitiesRequest(short_name='foobar', 283 | capabilities_version='2') 284 | assert request.is_valid() 285 | 286 | 287 | def test_valid_add_labels_request(): 288 | request = AddLabelsRequest(labels=['label1', 'label2'], 289 | job_ids=['job_1', 'job_2'],) 290 | assert request.is_valid() 291 | 292 | 293 | def test_add_labels_request_invalid_arguments(): 294 | with pytest.raises(TypeError, match=".*got an unexpected keyword argument 'job_labels'"): 295 | AddLabelsRequest(job_labels=['label1']) 296 | 297 | 298 | def test_add_labels_request_missing_labels(): 299 | with pytest.raises(TypeError, match=".*missing 1 required keyword-only argument: 'labels'"): 300 | AddLabelsRequest(job_ids=['job_123']) 301 | 302 | 303 | def test_add_labels_request_missing_job_ids(): 304 | with pytest.raises(TypeError, match=".*missing 1 required keyword-only argument: 'job_ids'"): 305 | AddLabelsRequest(labels=['label1']) 306 | 307 | 308 | def test_add_labels_request_missing_all_arguments(): 309 | with pytest.raises(TypeError, match=".*missing 2 required keyword-only arguments: 'labels' and 'job_ids'"): 310 | AddLabelsRequest() 311 | 312 | 313 | def test_valid_delete_labels_request(): 314 | request = DeleteLabelsRequest(labels=['label1', 'label2'], 315 | job_ids=['job_1', 'job_2'],) 316 | assert request.is_valid() 317 | 318 | 319 | def test_delete_labels_request_invalid_arguments(): 320 | with pytest.raises(TypeError, match=".*got an unexpected keyword argument 'job_labels'"): 321 | DeleteLabelsRequest(job_labels=['label1']) 322 | 323 | 324 | def test_delete_labels_request_missing_labels(): 325 | with pytest.raises(TypeError, match=".*missing 1 required keyword-only argument: 'labels'"): 326 | DeleteLabelsRequest(job_ids=['job_123']) 327 | 328 | 329 | def test_delete_labels_request_missing_job_ids(): 330 | with pytest.raises(TypeError, match=".*missing 1 required keyword-only argument: 'job_ids'"): 331 | DeleteLabelsRequest(labels=['label1']) 332 | 333 | 334 | def test_delete_labels_request_missing_all_arguments(): 335 | with pytest.raises(TypeError, match=".*missing 2 required keyword-only arguments: 'labels' and 'job_ids'"): 336 | DeleteLabelsRequest() 337 | 338 | 339 | def test_valid_get_jobs_request(): 340 | request = JobsRequest() 341 | assert request.is_valid() 342 | 343 | 344 | def test_valid_get_jobs_request_with_page_limit(): 345 | request = JobsRequest(page=1, limit=10) 346 | assert request.is_valid() 347 | 348 | 349 | def test_valid_get_jobs_request_with_all_params(): 350 | request = JobsRequest(page=1, limit=10, labels=['foo']) 351 | assert request.is_valid() 352 | 353 | 354 | def test_valid_get_jobs_request_with_labels_in_string(): 355 | request = JobsRequest(labels='foo') 356 | assert request.is_valid() 357 | 358 | 359 | def test_valid_get_jobs_request_with_multiple_labels(): 360 | request = JobsRequest(labels=['foo', 'bar']) 361 | assert request.is_valid() 362 | 363 | 364 | def test_valid_get_jobs_request_with_multiple_labels_in_string(): 365 | request = JobsRequest(labels='foo,bar') 366 | assert request.is_valid() 367 | 368 | 369 | def test_get_jobs_request_with_invalid_argument(): 370 | with pytest.raises(TypeError, match=".*got an unexpected keyword argument 'page_num'"): 371 | JobsRequest(page_num=1) 372 | 373 | def test_request_with_pixel_subset_false(): 374 | request = Request(collection=Collection('foobar'), pixel_subset=False) 375 | assert request.is_valid() 376 | assert request.pixel_subset is not None and request.pixel_subset == False 377 | 378 | 379 | def test_request_with_pixel_subset_true(): 380 | request = Request(collection=Collection('foobar'), pixel_subset=True) 381 | assert request.is_valid() 382 | assert request.pixel_subset is not None and request.pixel_subset == True 383 | 384 | 385 | def test_request_defaults_to_pixel_subset_none(): 386 | request = Request(collection=Collection('foobar')) 387 | assert request.pixel_subset is None 388 | 389 | def test_request_with_pixel_subset_invalid(): 390 | request = Request(collection=Collection('foobar'), pixel_subset='invalid') 391 | messages = request.error_messages() 392 | assert not request.is_valid() 393 | assert 'pixel_subset must be a boolean (True or False)' in messages 394 | 395 | 396 | def test_request_with_service_id(): 397 | request = Request(collection=Collection('foobar'), service_id='S123-PROV') 398 | assert request.is_valid() 399 | assert request.service_id == 'S123-PROV' 400 | -------------------------------------------------------------------------------- /tests/test_util.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from harmony.util import s3_components 4 | 5 | 6 | @pytest.mark.parametrize('bucket, path, fn', [ 7 | ('harmony-uat-staging', 'public/harmony/gdal/aed38eeb-f01a-41bb-a790-affbb2ab2bd6', 8 | '2020_01_01_7f00ff_global_blue_var_regridded_subsetted.nc.tif'), 9 | ('foo', 'bar', 'xyzzy.txt'), 10 | ('foo', '', 'xyzzy.nc') 11 | ]) 12 | def test_s3_url_parts(bucket, path, fn): 13 | key = f'{path}/{fn}' if path else fn 14 | url = f's3://{bucket}/{key}' 15 | 16 | assert (bucket, key, fn) == s3_components(url) 17 | --------------------------------------------------------------------------------